diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index ce16604..813c8a7 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -40,9 +40,11 @@ jobs: env: SAFE_TARGET: ${{ matrix.target }} VERSION: ${{ matrix.version }} + AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }} steps: - uses: builtin:checkout - - name: Prepare persistent cache and bounded builder + - name: Prepare persistent builder run: sh ci/setup-builder.sh - name: Build SDK image run: | @@ -51,7 +53,10 @@ jobs: --build-arg SAFE_TARGET="$SAFE_TARGET" \ --build-arg VERSION="$VERSION" \ --build-arg JOBS=8 \ - --build-arg SCCACHE_REDIS_ENDPOINT=redis://127.0.0.1:6379 \ + --build-arg SCCACHE_REDIS_ENDPOINT=redis://127.0.0.1:6380 \ + --build-arg SCCACHE_ENDPOINT=http://127.0.0.1:9000 \ + --secret id=aws-access-key-id,env=AWS_ACCESS_KEY_ID \ + --secret id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY \ --tag "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}" . - name: Verify SDK image run: | diff --git a/Dockerfile b/Dockerfile index 41470ec..2860c09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 ARG SAFE_TARGET ARG VERSION @@ -16,8 +17,15 @@ FROM tooling AS configured ARG SAFE_TARGET ARG VERSION ENV SCCACHE_REDIS_KEY_PREFIX=openwrt-rust:${VERSION}: \ + SCCACHE_S3_KEY_PREFIX=openwrt-rust/${VERSION}/ \ + SCCACHE_BUCKET=sccache \ + SCCACHE_REGION=us-east-1 \ + SCCACHE_S3_USE_SSL=false \ + SCCACHE_MULTILEVEL_CHAIN=redis,s3 \ + SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all \ CARGO_INCREMENTAL=0 COPY --chmod=0755 ci/build-rust.sh /usr/local/bin/build-rust +COPY --chmod=0755 ci/sccache-stats.py /usr/local/bin/sccache-stats RUN ./scripts/feeds update packages && \ ./scripts/feeds install rust && \ printf '\nCONFIG_RUST_SCCACHE=y\n' >> .config && \ @@ -30,6 +38,9 @@ RUN ./scripts/feeds update packages && \ FROM configured AS sdk ARG JOBS=8 -# A non-secret endpoint on the runner; do not persist it in the published image. +# Non-secret runner endpoints apply during the build, not as runtime defaults. ARG SCCACHE_REDIS_ENDPOINT -RUN build-rust +ARG SCCACHE_ENDPOINT +RUN --mount=type=secret,id=aws-access-key-id,env=AWS_ACCESS_KEY_ID,required=true \ + --mount=type=secret,id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY,required=true \ + build-rust diff --git a/README.md b/README.md index 2536545..58ed7f7 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,11 @@ triggers. `publish=false` is the safe validation default. Dispatch `publish=true` only after publication is approved. Login uses a temporary job directory, and each successfully built and checked image is pushed under its original tag. Before any push, `ci/verify-cache.sh` uses the installed SDK rustc -in two disposable containers and requires a Redis miss followed by a hit. +in fresh containers and requires writes to both tiers, Redis/S3 hits, and backfilling. + +Also set `SCCACHE_S3_ACCESS_KEY_ID` and `SCCACHE_S3_SECRET_ACCESS_KEY` from the +runner cache service in Gitea repository or organization secrets. These are needed +for non-publishing builds too; Docker Hub credentials are only used for publishing. Run a non-publishing dispatch for the default six combinations, then an approved publishing dispatch. Check the pulled image's Rust/Cargo installation @@ -33,27 +37,51 @@ plugins or credentials are used by the new workflow. ## Persistent compilation cache -`ci/setup-builder.sh` provisions one Redis service and one reusable BuildKit -builder on that daemon. Redis uses the named volume `openwrt-rust-sccache`, AOF -persistence (`appendfsync everysec`), a 6 GiB LRU cache, an 8 GiB memory limit, -and one CPU. Jobs and build containers can disappear without losing it. A -machine crash can lose up to the most recent second of cache writes; eviction -or lost entries cause recompilation. Do not run `docker compose down -v` or -prune this volume. BuildKit's layer cache is separate and may be garbage -collected without clearing sccache. +The Gitea runner's `homelab-stacks/stacks/gitea-runner/compose.yaml` owns +Redis and the SSD-backed RustFS S3 service. This repository only prepares its +reusable BuildKit builder and consumes those services; jobs never deploy them. +Redis is a 1 GiB LRU hot cache with a 1.5 GiB memory limit. It is deliberately +non-persistent: S3 holds the persistent copies on a quota-limited 100 GiB Incus/ZFS +volume. Jobs, Redis restarts, and RustFS container recreation do not clear it. +The cache bucket expires objects after 30 days; the filesystem quota provides +an independent hard limit. See the runner stack's README for allocation and setup. -The cache joins the existing `gitea-runner-jobs` network and exposes port 6379 -only on loopback. BuildKit uses host networking to reach it; package containers -use its Docker DNS name. This is a trusted-runner cache: do not attach untrusted -PR containers or public networks to it. No password or endpoint is baked into -published SDK images. Consumers outside this runner can use their own Redis -endpoint. Each OpenWrt version has its own key prefix, shared across targets; -sccache also -hashes Rust compiler version and shared libraries, compilation arguments -(including target and flags), source and dependency contents. Different Rust -versions/toolchains therefore miss rather than reuse incompatible objects. -Previously populated target-specific namespaces are not reused by the new -prefixes and will eventually be evicted by Redis. +Each job runs its own pinned sccache 0.18.0 daemon with +`SCCACHE_MULTILEVEL_CHAIN=redis,s3` in the standard server-side mode. +Writes go to both tiers; Redis misses fall through to S3 and backfill Redis. +`SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all` makes each cache write await both tiers. +The installed `sccache-stats` helper waits up to 30 seconds for outstanding writes +and backfills before printing statistics and stopping the job's daemon. +BuildKit's layer cache remains separate. + +Both services join `gitea-runner-jobs`. SDK/package containers use +`redis://sccache-redis:6379` and `http://sccache-s3:9000`; BuildKit's host network +uses the loopback ports 6380 and 9000. There is no public ingress. S3 credentials +are supplied through required BuildKit secret mounts for compilation and runtime +environment variables for disposable verification/package containers. Credentials are not stored in the published SDK image. Runner endpoints are +build arguments rather than default runtime settings. The image +contains only public backend configuration: bucket `sccache`, region `us-east-1`, +path-style HTTP, and cache prefixes. Other environments can override these values. + +Each OpenWrt release has a shared prefix across targets: `openwrt-rust::` +in Redis and `openwrt-rust//` in S3. Compiler identities, target, +flags, sources, and dependencies still determine cache keys, preventing +incompatible reuse. The old Redis service and its volume remain untouched during +this change; their entries are not copied into the new SSD cache. + +A single persistent sccache daemon was tested across containers with client-side +compilation. The second container reported a cache error and recompiled: the +daemon's disk backend returns an absolute cache-file path that the client must +open, but the disposable container cannot see that filesystem. Version 0.18.0 +also fixes TCP connections to loopback, and client-side raw reads bypass +multi-level backfilling and per-tier read statistics. Per-job daemons with remote +storage avoid filesystem sharing, socket proxies, and job-network changes. + +RustFS and VersityGW were both tested with real Rust compilation, S3 fallback, +Redis backfilling, container restarts, and differing targets/toolchains. Both +passed. RustFS was selected following the owner's preference; the small storage +benchmark does not establish a full-build speedup. Deploy the runner cache and +initialize its bucket before running this workflow. The shared BuildKit worker executes one build step at a time, even across simultaneous dispatches. It is limited to eight CPUs and 20 GiB RAM with no @@ -84,24 +112,28 @@ OpenWrt integration was inspected in the SDK's pinned packages feed: Sources: [OpenWrt Rust configuration](https://github.com/openwrt/packages/blob/openwrt-24.10/lang/rust/rust-values.mk), [Rust bootstrap](https://github.com/rust-lang/rust/blob/1.90.0/src/bootstrap/src/core/build_steps/compile.rs), -[sccache Redis](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Redis.md), +[sccache multi-level cache](https://github.com/mozilla/sccache/blob/v0.18.0/docs/MultiLevel.md), +[sccache client-side architecture](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Architecture.md), [Rust cache limitations](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Rust.md). ## Downstream OpenWrt packages -Use the same persistent daemon and target/version image. For example, after +Use the same cache services and target/version image. Export `AWS_ACCESS_KEY_ID` +and `AWS_SECRET_ACCESS_KEY` from CI secrets before starting the container. For example, after installing the `sqm-autorate-rust` feed/package in the SDK: ```sh docker run --rm --network gitea-runner-jobs --cpus 8 \ --memory 20g --memory-swap 20g \ - -e SCCACHE_REDIS_ENDPOINT=redis://openwrt-rust-cache:6379 \ + -e SCCACHE_REDIS_ENDPOINT=redis://sccache-redis:6379 \ + -e SCCACHE_ENDPOINT=http://sccache-s3:9000 \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ lochnair/openwrt-sdk-rust:x86-64-24.10.8 sh -ec ' # Install/update the downstream feed here before defconfig. make defconfig grep -qx "CONFIG_RUST_SCCACHE=y" .config sccache --start-server - trap "sccache --show-stats; sccache --stop-server" EXIT + trap "sccache-stats; sccache --stop-server" EXIT sccache --zero-stats make package/sqm-autorate-rust/compile -j8 V=sc ' @@ -115,7 +147,8 @@ OpenWrt SDK package workflow; that unrelated pipeline is unchanged. The package Makefile in `openwrt-feeds` already uses OpenWrt's `rust-package.mk`, so an SDK package job can use this cache without patching the package. -For a local SDK build on the prepared runner: +For a local SDK build on the prepared runner, export `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY` from the cache credentials in your environment first: ```sh sh ci/setup-builder.sh @@ -123,7 +156,10 @@ docker buildx build --builder openwrt-sdk-rust --load --progress plain \ --network host --allow network.host \ --build-arg SAFE_TARGET=x86-64 --build-arg VERSION=24.10.8 \ --build-arg JOBS=8 \ - --build-arg SCCACHE_REDIS_ENDPOINT=redis://127.0.0.1:6379 \ + --build-arg SCCACHE_REDIS_ENDPOINT=redis://127.0.0.1:6380 \ + --build-arg SCCACHE_ENDPOINT=http://127.0.0.1:9000 \ + --secret id=aws-access-key-id,env=AWS_ACCESS_KEY_ID \ + --secret id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY \ -t lochnair/openwrt-sdk-rust:x86-64-24.10.8 . ``` diff --git a/ci/VALIDATION.md b/ci/VALIDATION.md index 013c0e8..7fa4023 100644 --- a/ci/VALIDATION.md +++ b/ci/VALIDATION.md @@ -109,3 +109,161 @@ expanded the actual OpenWrt `Host/Compile` recipe at both job counts: it invokes against the 24.10.8 feed in a disposable container. ShellCheck, shell syntax, actionlint and diff checks pass. Full source compilation after this change has not been rerun, and no images were published or infrastructure limits changed. + +## Follow-up: runner-owned Redis and SSD cache — 2026-10-11 + +Initially prototyped a 1 GiB Redis LRU hot cache and Apache WebDAV backed by a +persistent Docker volume on `fastpool`. This prototype was superseded by the +S3 setup below after testing both requested alternatives. The SDK repository +consumes these services and no longer deploys Redis. Production services have +not been redeployed; the original Redis container and volume remain untouched. + +Inspected the pinned sccache 0.18.0 source and ran a cross-container central +server experiment. With client-side compilation and an unshared disk backend, +the first compile missed; the second reported a cache error and recompiled, +because `IpcStorage` attempts to open the daemon's returned absolute cache path. +TCP addresses are fixed to loopback in `net::SocketAddr::with_port`. In addition, +client-side `get_raw` bypasses multi-level read statistics and backfilling. The +standard per-job daemon mode with Redis/WebDAV avoids these restrictions. + +Writes in standard mode can still be outstanding when the compiler client +returns. `SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all` awaits both tiers within a +write; `sccache-stats` then drains writes and backfills for up to 30 seconds before +printing statistics and stopping the job daemon. Its completion checks use the +pinned version's cache-miss/write and per-tier backfill counters. + +Passed on TheBeast using an isolated `sccache-probe` Compose project: + +- Built the WebDAV image and validated Apache configuration. Real writes exposed + the base Alpine image's missing APR DBM driver; the added Dockerfile installs + `apr-util-dbm_gdbm`. Both services then passed health checks and real writes. +- Retained CI verification: one miss writing both tiers, one Redis hit in a + fresh container, then one WebDAV-only hit in a third fresh container. Zero + cache errors, read/write errors, or per-tier write failures. +- Real Rust compilation inside Docker BuildKit followed by a fresh SDK runtime + container: miss, then hit, using host-loopback and Docker-DNS endpoints. +- Repeated the unchanged BuildKit build: the actual compilation RUN step was + cached and did not invoke sccache again. +- Recreated both test service containers, retaining only the WebDAV volume: + Redis started empty. A fresh SDK container hit WebDAV and backfilled Redis; + the following fresh container hit Redis. +- Identical source and prefix with GNU versus musl targets: musl missed, then + hit. Rust 1.90.0 versus 1.94.0 under the same test prefix: 1.94.0 missed, then + hit. All cache-error counters remained zero. +- A disposable OpenWrt package compiled via the real `rust-package.mk` + `Build/Compile/Cargo` helper and SDK musl linker in two fresh SDK containers: + dependency miss, then hit. This used the actual completed 24.10.8 SDK image, + not the earlier validation toolchain. The binary remained non-cacheable. +- Proposed full runner Compose configuration, YAML parsing, Apache config, + ShellCheck, shell syntax, Python syntax, actionlint, and diff checks passed. + actionlint excludes only Gitea's built-in checkout syntax. Dockerfile checks + exclude the public `KEY_PREFIX` secret-name heuristic and mandatory base + image argument warning, as in the original validation. +- Publishing remains manual, with the original image tags and verification + before registry login/push. No login or publication was performed. Matrix, + runner labels/capacity, BuildKit caching, and serial verbose retry are unchanged. + +Full SDK source bootstrap and a Gitea dispatch were not rerun for this cache +change. Deploy/build the updated runner stack before using the changed workflow. +The prototype WebDAV tier had no automatic LRU or capacity limit; the final +S3 configuration below adds an independent hard filesystem bound and retention. +Old Redis entries are not migrated; retiring the old service/volume is a separate +approved cutover step. Test logs are under `/tmp/gitea-sccache-stack` and +`/tmp/gitea-sccache-tests` on the runner; isolated test services are cleaned up. + +Sources: [sccache architecture](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Architecture.md), +[pinned IPC storage](https://github.com/mozilla/sccache/blob/v0.18.0/src/cache/ipc_storage.rs), +[pinned multi-level implementation](https://github.com/mozilla/sccache/blob/v0.18.0/src/cache/multilevel.rs). + + +## Follow-up: RustFS and VersityGW comparison — 2026-10-11 + +Selected **RustFS 1.0.1** after testing it and **VersityGW 1.8.0**, following the +owner's preference. The final runner Compose configuration uses Redis/S3 and +removes the uncommitted Apache image/config prototype. Services were tested in +isolated projects, not deployed over the existing runner/cache services. + +Both candidates passed real sccache 0.18.0 compilation tests using the completed +SDK images on TheBeast: + +- Cold miss wrote both tiers; fresh-container Redis hit; S3-only hit. +- Service restart plus empty isolated Redis: S3 hit backfilled Redis, then the + next fresh container hit Redis. Cache read/write/error counters stayed zero. +- Same prefix/input with GNU versus musl targets: musl missed then hit. Rust + 1.90.0 versus 1.94.0: the different toolchain missed then hit. + +The objects live on a separate Incus ZFS volume with a 16 GiB hard quota for +these tests. A separate 64 MiB quota probe rejected an 80 MiB incompressible write +with `Disk quota exceeded`. No production data was used for the quota test. +Production setup requires a new **100 GiB** quota-backed Incus volume; Compose +requires an existing data directory and rejects a filesystem reporting more than +100 GiB capacity. This guard passed on the quota-backed mount and rejected the +unbounded SSD filesystem. Missing BuildKit credentials failed before compilation, +and test logs contained no credential values. RustFS log files use a bounded tmpfs; Docker logs rotate. + +Storage microbenchmark: 64 objects of 1 MiB incompressible data, three passes, +median aggregate MiB/s over host-loopback HTTP on the same SSD pool. Every GET +was compared with its source bytes. These include client/protocol overhead and +warm filesystem cache; they do not measure full SDK build speed or sustained +cold-SSD throughput. + +| Backend | PUT, 1 client | GET, 1 client | PUT, 4 clients | GET, 4 clients | +| --- | ---: | ---: | ---: | ---: | +| Apache WebDAV prototype | 110.2 | 660.8 | failed | not run | +| RustFS 1.0.1 | 101.5 | 314.7 | 231.2 | 471.2 | +| VersityGW 1.8.0 | 99.1 | 329.3 | 232.2 | 410.2 | + +WebDAV's four-client PUT test returned HTTP 500 lock-database errors, despite +passing sequential compiler-cache tests. Both S3 backends handled that load. +Observed Docker memory usage after the probes was about 202 MiB for RustFS and +28 MiB for VersityGW; these are small test workloads, not worst-case bounds. +RustFS was chosen for preference and compatibility, not a proven performance +advantage over VersityGW. Sequential S3 GETs were slower than the WebDAV baseline. +Redis remains the primary hot tier. + +Final RustFS setup and SDK integration checks: + +- The entire proposed runner Compose configuration validates with dummy + registration credentials. Only the cache services ran in the isolated project; + both passed their real health checks. Bucket initialization with pinned official + AWS CLI 2.37.12 passed. RustFS accepted and returned the exact configured + 30-day object expiration and one-day abandoned multipart-upload policy. + Expiration execution itself was not tested by waiting 30 days. +- Replaced the original test RustFS container with the proposed Compose service + over the same retained quota-backed data: an earlier Rust artifact was still an + S3 hit and backfilled a new Redis namespace in a fresh SDK container. +- The retained `ci/verify-cache.sh` passed all five probes (cold writes, Redis hit, + S3-only hit, backfill into a fresh Redis namespace, subsequent Redis hit) on + x86-64/24.10.8 and mediatek-filogic/25.12.5. It never flushes shared Redis data. +- Real BuildKit Rust compilation using the proposed required secret mounts, + then a fresh runtime-container cache hit, passed. Secret values were absent + from exported image configuration/history; subsequent layers and a fresh + container had no credential environment variables or secret files. The repeated + unchanged compilation layer was cached. Only a local disposable image was + produced; no registry login/push occurred. +- Actual OpenWrt `Build/Compile/Cargo` and SDK musl linker compiled a disposable + package twice: dependency miss followed by hit in a fresh container. This + confirms inherited S3 settings/credentials reach OpenWrt's sccache wrapper. +- BuildKit checked the exact changed SDK Dockerfile. Only the known public + `KEY_PREFIX` secret-name heuristic and required base-image argument warning were + excluded. Workflow/actionlint, ShellCheck, shell/Python/JSON/YAML parsing and + whitespace checks passed. + +The SDK image retains public S3 settings and version-only prefixes. Credentials are not stored in image configuration/history, and runner endpoints +are build arguments rather than runtime defaults. Gitea secrets feed ephemeral BuildKit +mounts and runtime verification containers. The runner service's registration, +labels/capacity, build matrix, image tags, manual publication gate, BuildKit layer +caching and serial verbose fallback are unchanged. + +Manual setup: allocate/mount the 100 GiB volume, set the two cache credentials in +Komodo and Gitea, deploy the cache services, initialize the bucket, and run a +non-publishing workflow before cutover. Full SDK bootstrap and live Gitea dispatch +were not repeated for this backend change. Production Redis, its volume, and +Jenkins remain untouched pending acceptance/approved retirement. Isolated test +containers and volumes are removed after verification; logs remain under +`/tmp/s3-cache-tests` on the runner. + +Sources: [RustFS deployment](https://docs.rustfs.com/en/installation/container/docker), +[VersityGW Docker setup](https://github.com/versity/versitygw/wiki/Docker), +[sccache S3](https://github.com/mozilla/sccache/blob/v0.18.0/docs/S3.md), +[BuildKit secrets](https://docs.docker.com/build/building/secrets/). diff --git a/ci/build-rust.sh b/ci/build-rust.sh index 4bd0842..144662a 100644 --- a/ci/build-rust.sh +++ b/ci/build-rust.sh @@ -2,11 +2,14 @@ set -eu : "${SCCACHE_REDIS_ENDPOINT:?Set the persistent sccache Redis endpoint}" +: "${SCCACHE_ENDPOINT:?Set the persistent sccache S3 endpoint}" +: "${AWS_ACCESS_KEY_ID:?Supply the S3 access key through a BuildKit secret}" +: "${AWS_SECRET_ACCESS_KEY:?Supply the S3 secret key through a BuildKit secret}" : "${JOBS:=8}" case "$JOBS" in ''|*[!0-9]*|0) echo 'JOBS must be a positive integer' >&2; exit 1 ;; esac sccache --start-server -trap 'sccache --show-stats; sccache --stop-server' EXIT +trap 'sccache-stats; sccache --stop-server' EXIT sccache --zero-stats # Bootstrap/Ninja does not inherit Make's job count. Bound both explicitly. make package/rust/host/compile -j"$JOBS" RUST_BUILD_JOBS="$JOBS" || \ diff --git a/ci/cache.compose.yml b/ci/cache.compose.yml deleted file mode 100644 index 21eb7a6..0000000 --- a/ci/cache.compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -services: - cache: - image: redis:7.4.6-alpine - container_name: openwrt-rust-cache - restart: unless-stopped - command: - - redis-server - - --appendonly - - 'yes' - - --appendfsync - - everysec - - --maxmemory - - 6gb - - --maxmemory-policy - - allkeys-lru - - --save - - '' - mem_limit: 8g - memswap_limit: 8g - cpus: 1 - ports: - - '127.0.0.1:6379:6379' - volumes: - - cache:/data - networks: - - jobs - healthcheck: - test: ['CMD', 'redis-cli', 'ping'] - interval: 5s - timeout: 3s - retries: 12 - -volumes: - cache: - name: openwrt-rust-sccache - -networks: - jobs: - external: true - name: gitea-runner-jobs diff --git a/ci/sccache-stats.py b/ci/sccache-stats.py new file mode 100644 index 0000000..2e60399 --- /dev/null +++ b/ci/sccache-stats.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Wait for this job's cache writes/backfills, then print complete statistics.""" +import json +import subprocess +import time + +end = time.monotonic() + 30 +while True: + data = json.loads(subprocess.check_output( + ["sccache", "--show-stats", "--stats-format", "json"], + timeout=max(0.1, end - time.monotonic()), + )) + stats = data["stats"] + misses = sum(stats["cache_misses"]["counts"].values()) + levels = stats.get("multi_level") or [] + writes_done = stats["cache_writes"] + stats["cache_write_errors"] >= misses + backfills_done = sum(x["backfills_to"] for x in levels) >= sum( + x["backfills_from"] for x in levels + ) + if writes_done and backfills_done: + break + if time.monotonic() >= end: + subprocess.run(["sccache", "--show-stats"], check=True, timeout=5) + raise SystemExit("Timed out waiting for sccache writes/backfills") + time.sleep(0.1) +subprocess.run(["sccache", "--show-stats"], check=True, timeout=5) diff --git a/ci/setup-builder.sh b/ci/setup-builder.sh index 1be03b5..820e32d 100644 --- a/ci/setup-builder.sh +++ b/ci/setup-builder.sh @@ -1,6 +1,5 @@ #!/bin/sh set -eu -docker compose -p openwrt-rust -f ci/cache.compose.yml up -d --wait if ! docker buildx inspect openwrt-sdk-rust >/dev/null 2>&1; then docker buildx create --name openwrt-sdk-rust --driver docker-container \ --driver-opt network=host,memory=20g,memory-swap=20g,cpu-period=100000,cpu-quota=800000 \ diff --git a/ci/verify-cache.sh b/ci/verify-cache.sh index 854e895..fee614b 100644 --- a/ci/verify-cache.sh +++ b/ci/verify-cache.sh @@ -1,16 +1,34 @@ #!/bin/sh -# Compile in two disposable containers: the second must read the first's cache. +# Verify writes, Redis/S3 hits, and Redis backfilling in fresh containers. set -eu image=${1:?Usage: verify-cache.sh IMAGE} +: "${SCCACHE_REDIS_ENDPOINT:=redis://sccache-redis:6379}" +: "${SCCACHE_ENDPOINT:=http://sccache-s3:9000}" +: "${AWS_ACCESS_KEY_ID:?Set the S3 access key}" +: "${AWS_SECRET_ACCESS_KEY:?Set the S3 secret key}" +stats_script=$(cat ci/sccache-stats.py) nonce=$(mktemp -d) trap 'rmdir "$nonce"' EXIT -prefix="validation:${image##*:}:$(basename "$nonce"):" -for expected in miss hit; do +prefix="validation/${image##*:}/$(basename "$nonce")/" +for expected in miss hit s3-hit backfill backfill-hit; do + redis_prefix=$prefix + case "$expected" in backfill*) redis_prefix="${prefix}backfill/" ;; esac + chain=redis,s3 + if [ "$expected" = s3-hit ]; then chain=s3; fi docker run --rm -i --network gitea-runner-jobs --cpus 2 \ --memory 2g --memory-swap 2g \ - -e SCCACHE_REDIS_ENDPOINT=redis://openwrt-rust-cache:6379 \ - -e SCCACHE_REDIS_KEY_PREFIX="$prefix" -e EXPECTED="$expected" \ - "$image" sh -es <<'PROBE' + -e SCCACHE_REDIS_ENDPOINT="$SCCACHE_REDIS_ENDPOINT" \ + -e SCCACHE_ENDPOINT="$SCCACHE_ENDPOINT" \ + -e SCCACHE_REDIS_KEY_PREFIX="$redis_prefix" \ + -e SCCACHE_S3_KEY_PREFIX="$prefix" \ + -e SCCACHE_BUCKET=sccache -e SCCACHE_REGION=us-east-1 \ + -e SCCACHE_S3_USE_SSL=false \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + -e SCCACHE_MULTILEVEL_CHAIN="$chain" \ + -e SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all \ + -e SCCACHE_CLIENT_SIDE=0 \ + -e SCCACHE_STATS_SCRIPT="$stats_script" \ + -e EXPECTED="$expected" "$image" sh -es <<'PROBE' rustc=$(find /builder/staging_dir -type f -path '*/host/bin/rustc') test -x "$rustc" mkdir /tmp/sccache-probe @@ -21,19 +39,29 @@ trap 'sccache --stop-server' EXIT sccache --zero-stats sccache "$rustc" --crate-name ci_sccache_probe \ --crate-type rlib --emit link,dep-info --out-dir . probe.rs -sccache --show-stats +python3 -c "$SCCACHE_STATS_SCRIPT" sccache --show-stats --stats-format json > stats.json python3 - <<'PY' import json import os data = json.load(open("stats.json")) -assert data["cache_location"].startswith("redis,"), data +assert data["cache_location"].startswith("Multi-level"), data stats = data["stats"] -field = "cache_hits" if os.environ["EXPECTED"] == "hit" else "cache_misses" +expected = os.environ["EXPECTED"] +field = "cache_misses" if expected == "miss" else "cache_hits" assert sum(stats[field]["counts"].values()) == 1, data assert stats["cache_read_errors"] == stats["cache_write_errors"] == 0, data -print("Verified persistent cache", os.environ["EXPECTED"]) +assert sum(stats["cache_errors"]["counts"].values()) == 0, data +levels = stats["multi_level"] +assert all(level["write_failures"] == 0 for level in levels), data +if expected == "miss": + assert len(levels) == 2 and all(level["writes"] == 1 for level in levels), data +elif expected == "backfill": + assert levels[1]["hits"] == levels[0]["backfills_to"] == 1, data +else: + assert levels[0]["hits"] == 1, data +print("Verified persistent cache", expected) PY PROBE done