ci(matrix): Dynamically discover all Rust-supported OpenWrt SDK targets with fail-fast: false and ensure local image cleanup

Centralize cache service management in the Gitea runner for improved persistence and reliability using an SSD-backed S3 backend. This ensures robust verification of multi-tier cache operations.

.dockerignore

@@ -0,0 +1,5 @@
+.git
+.gitea
+Jenkinsfile
+README.md
+**/__pycache__

.gitea/workflows/build.yml

@@ -0,0 +1,93 @@
+name: Build OpenWrt Rust SDK images
+
+on:
+  workflow_dispatch:
+    inputs:
+      targets:
+        description: 'Comma-separated targets, or all'
+        default: 'all'
+        required: true
+        type: string
+      publish:
+        description: 'Publish the original Docker Hub tags (requires approval)'
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+
+jobs:
+  matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.resolve.outputs.matrix }}
+    steps:
+      - uses: builtin:checkout
+      - name: Resolve targets and OpenWrt versions
+        id: resolve
+        env:
+          TARGETS: ${{ inputs.targets }}
+        run: |
+          python3 -m unittest discover -s ci -p 'test_matrix.py'
+          python3 ci/matrix.py
+
+  build:
+    needs: matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }}
+    env:
+      SAFE_TARGET: ${{ matrix.target }}
+      VERSION: ${{ matrix.version }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: builtin:checkout
+      - name: Prepare persistent builder
+        run: sh ci/setup-builder.sh
+      - name: Build SDK image
+        run: |
+          docker buildx build --builder openwrt-sdk-rust --load --progress plain \
+            --network host --allow network.host \
+            --build-arg SAFE_TARGET="$SAFE_TARGET" \
+            --build-arg VERSION="$VERSION" \
+            --build-arg JOBS=8 \
+            --build-arg SCCACHE_REDIS_ENDPOINT=redis://127.0.0.1:6380 \
+            --build-arg SCCACHE_ENDPOINT=http://127.0.0.1:9000 \
+            --secret id=aws-access-key-id,env=AWS_ACCESS_KEY_ID \
+            --secret id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY \
+            --tag "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}" .
+      - name: Verify SDK image
+        run: |
+          docker run --rm "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}" sh -ec '
+            sccache --version
+            grep -qx "CONFIG_RUST_SCCACHE=y" .config
+            rustc=$(find staging_dir -type f -path "*/host/bin/rustc")
+            test -x "$rustc"
+            "$rustc" -vV
+            "$(dirname "$rustc")/cargo" --version
+          '
+          sh ci/verify-cache.sh "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}"
+      - name: Publish Docker Hub tag
+        if: ${{ success() && inputs.publish == true }}
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          DOCKER_CONFIG=$(mktemp -d)
+          export DOCKER_CONFIG
+          trap 'rm -rf "$DOCKER_CONFIG"' EXIT
+          printf '%s' "$DOCKERHUB_TOKEN" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
+          docker push "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}"
+      - name: Remove local SDK image
+        if: always()
+        run: |
+          image="lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}"
+          if docker image inspect "$image" >/dev/null 2>&1; then
+            docker image rm "$image"
+          fi

Dockerfile

@@ -1,13 +1,46 @@
+# syntax=docker/dockerfile:1
 ARG SAFE_TARGET
 ARG VERSION
 
-FROM ghcr.io/openwrt/sdk:${SAFE_TARGET}-${VERSION}
+FROM ghcr.io/openwrt/sdk:${SAFE_TARGET}-${VERSION} AS tooling
 
+# The SDK runs on amd64 even when compiling for another OpenWrt target.
+USER root
+RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.18.0/sccache-v0.18.0-x86_64-unknown-linux-musl.tar.gz -o /tmp/sccache.tar.gz && \
+    echo '45f1447fbe231e3037bde351ef70677dd212216c8d62ae7ca409fecc4d6acc89  /tmp/sccache.tar.gz' | sha256sum -c - && \
+    tar -xzf /tmp/sccache.tar.gz -C /tmp && \
+    install -m 0755 /tmp/sccache-v0.18.0-x86_64-unknown-linux-musl/sccache /usr/local/bin/sccache && \
+    rm -rf /tmp/sccache.tar.gz /tmp/sccache-v0.18.0-x86_64-unknown-linux-musl
+USER buildbot
+
+FROM tooling AS configured
+ARG SAFE_TARGET
+ARG VERSION
+ENV SCCACHE_REDIS_KEY_PREFIX=openwrt-rust:${VERSION}: \
+    SCCACHE_S3_KEY_PREFIX=openwrt-rust/${VERSION}/ \
+    SCCACHE_BUCKET=sccache \
+    SCCACHE_REGION=us-east-1 \
+    SCCACHE_S3_USE_SSL=false \
+    SCCACHE_MULTILEVEL_CHAIN=redis,s3 \
+    SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all \
+    CARGO_INCREMENTAL=0
+COPY --chmod=0755 ci/build-rust.sh /usr/local/bin/build-rust
+COPY --chmod=0755 ci/sccache-stats.py /usr/local/bin/sccache-stats
 RUN ./scripts/feeds update packages && \
     ./scripts/feeds install rust && \
+    printf '\nCONFIG_RUST_SCCACHE=y\n' >> .config && \
     make defconfig && \
-    sed -i 's/--set=llvm.download-ci-llvm=true/--set=llvm.download-ci-llvm=false/g' feeds/packages/lang/rust/Makefile; \
+    grep -qx 'CONFIG_RUST_SCCACHE=y' .config && \
-    { make package/rust/host/compile -j$(nproc) || \
+    sed -i 's/--set=llvm.download-ci-llvm=true/--set=llvm.download-ci-llvm=false/g' feeds/packages/lang/rust/Makefile && \
-      make package/rust/host/compile -j1 V=sc; } && \
+    sed -i '/--set=llvm.download-ci-llvm=false/a\	--set=build.ccache=sccache \\' feeds/packages/lang/rust/Makefile && \
-    rm -rf build_dir/host/rustc-*/rustc-*/build \
+    sed -i '/^define Host\/Compile/i RUST_BUILD_JOBS ?= 8' feeds/packages/lang/rust/Makefile && \
-           dl/rustc-*.tar.*
+    sed -i 's/--build-dir /--jobs=$(RUST_BUILD_JOBS) --build-dir /' feeds/packages/lang/rust/Makefile
+
+FROM configured AS sdk
+ARG JOBS=8
+# Non-secret runner endpoints apply during the build, not as runtime defaults.
+ARG SCCACHE_REDIS_ENDPOINT
+ARG SCCACHE_ENDPOINT
+RUN --mount=type=secret,id=aws-access-key-id,env=AWS_ACCESS_KEY_ID,required=true \
+    --mount=type=secret,id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY,required=true \
+    build-rust

README.md

@@ -1 +1,176 @@
 # openwrt-sdk-rust
+
+Builds the OpenWrt SDK with its Rust host toolchain installed. Gitea Actions
+resolves `stable_version` and `oldstable_version` from OpenWrt's `.versions.json`.
+The default `targets=all` selects the SDK tags actually available in GHCR for
+each release, excluding architectures unsupported by OpenWrt's Rust package.
+Release metadata supplies architectures, with release source definitions for
+SDK-only targets omitted from `.targets.json`. Alternatively, supply a
+comma-separated target list; unavailable or unsupported combinations are skipped
+and reported, and an empty matrix fails. Images still use
+`ghcr.io/openwrt/sdk:<target-with-dashes>-<version>` and
+publish to `lochnair/openwrt-sdk-rust:<target-with-dashes>-<version>` on Docker Hub.
+No `latest` tag or additional registry is introduced.
+
+## Runner setup and cutover
+
+The workflow uses `ubuntu-latest` on the existing TheBeast runner, currently
+the only runner. Its job image and access to the persistent Docker daemon were
+verified; no additional runner, label, capacity change, or restart is required.
+`builtin:checkout` uses the existing Runner 5 checkout support.
+
+Enable repository Actions and set `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN`
+secrets with push permission to `lochnair/openwrt-sdk-rust`. Jenkins' credential
+ID is not reusable in Gitea. The workflow is manually dispatched, matching the
+parameterized Jenkins job; there are no PR, push, or scheduled publication
+triggers. `publish=false` is the safe validation default. Dispatch
+`publish=true` only after publication is approved. Login uses a temporary job
+directory, and each successfully built and checked image is pushed under its
+original tag. Before any push, `ci/verify-cache.sh` uses the installed SDK rustc
+in fresh containers and requires writes to both tiers, Redis/S3 hits, and backfilling.
+Each matrix job publishes independently after its own verification succeeds.
+Its local SDK image is removed at the end, including after verification or push
+failure and during non-publishing runs. Cleanup leaves the shared builder and
+compilation caches intact.
+
+Also set `SCCACHE_S3_ACCESS_KEY_ID` and `SCCACHE_S3_SECRET_ACCESS_KEY` from the
+runner cache service in Gitea repository or organization secrets. These are needed
+for non-publishing builds too; Docker Hub credentials are only used for publishing.
+
+Run a non-publishing dispatch with
+`targets=x86/64,mediatek/filogic,octeon/generic` for the six validation combinations, then an
+approved publishing dispatch. Check the pulled image's Rust/Cargo installation
+and downstream package build before disabling the old Jenkins job and deleting
+`Jenkinsfile`. Do not retire Jenkins before this acceptance run. No Jenkins
+plugins or credentials are used by the new workflow.
+
+## Persistent compilation cache
+
+The Gitea runner's `homelab-stacks/stacks/gitea-runner/compose.yaml` owns
+Redis and the SSD-backed RustFS S3 service. This repository only prepares its
+reusable BuildKit builder and consumes those services; jobs never deploy them.
+Redis is a 1 GiB LRU hot cache with a 1.5 GiB memory limit. It is deliberately
+non-persistent: S3 holds the persistent copies on a quota-limited 100 GiB Incus/ZFS
+volume. Jobs, Redis restarts, and RustFS container recreation do not clear it.
+The cache bucket expires objects after 30 days; the filesystem quota provides
+an independent hard limit. See the runner stack's README for allocation and setup.
+
+Each job runs its own pinned sccache 0.18.0 daemon with
+`SCCACHE_MULTILEVEL_CHAIN=redis,s3` in the standard server-side mode.
+Writes go to both tiers; Redis misses fall through to S3 and backfill Redis.
+`SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all` makes each cache write await both tiers.
+The installed `sccache-stats` helper waits up to 30 seconds for outstanding writes
+and backfills before printing statistics and stopping the job's daemon.
+BuildKit's layer cache remains separate.
+
+Both services join `gitea-runner-jobs`. SDK/package containers use
+`redis://sccache-redis:6379` and `http://sccache-s3:9000`; BuildKit's host network
+uses the loopback ports 6380 and 9000. There is no public ingress. S3 credentials
+are supplied through required BuildKit secret mounts for compilation and runtime
+environment variables for disposable verification/package containers. Credentials are not stored in the published SDK image. Runner endpoints are
+build arguments rather than default runtime settings. The image
+contains only public backend configuration: bucket `sccache`, region `us-east-1`,
+path-style HTTP, and cache prefixes. Other environments can override these values.
+
+Each OpenWrt release has a shared prefix across targets: `openwrt-rust:<version>:`
+in Redis and `openwrt-rust/<version>/` in S3. Compiler identities, target,
+flags, sources, and dependencies still determine cache keys, preventing
+incompatible reuse. The old Redis service and its volume remain untouched during
+this change; their entries are not copied into the new SSD cache.
+
+A single persistent sccache daemon was tested across containers with client-side
+compilation. The second container reported a cache error and recompiled: the
+daemon's disk backend returns an absolute cache-file path that the client must
+open, but the disposable container cannot see that filesystem. Version 0.18.0
+also fixes TCP connections to loopback, and client-side raw reads bypass
+multi-level backfilling and per-tier read statistics. Per-job daemons with remote
+storage avoid filesystem sharing, socket proxies, and job-network changes.
+
+RustFS and VersityGW were both tested with real Rust compilation, S3 fallback,
+Redis backfilling, container restarts, and differing targets/toolchains. Both
+passed. RustFS was selected following the owner's preference; the small storage
+benchmark does not establish a full-build speedup. Deploy the runner cache and
+initialize its bucket before running this workflow.
+
+The shared BuildKit worker executes one build step at a time, even across
+simultaneous dispatches. It is limited to eight CPUs and 20 GiB RAM with no
+swap. Make and Rust bootstrap both receive eight jobs; the original `-j1 V=sc`
+diagnostic retry also reduces bootstrap/Ninja to one job. Rust
+build-directory/archive cleanup remains. BuildKit
+layer storage has a 50 GiB GC budget. Matrix jobs also run one at a time, with
+`fail-fast: false` so a failed target does not stop the remaining builds.
+
+OpenWrt integration was inspected in the SDK's pinned packages feed:
+
+- `rust-values.mk` **unexports `RUSTC_WRAPPER`**. Setting it in the workflow
+  alone does not enable caching. `CONFIG_RUST_SCCACHE=y` makes OpenWrt inject
+  `CARGO_INCREMENTAL=0`, `RUSTC_WRAPPER=sccache`, and `SCCACHE_DIR` into host
+  and package Cargo invocations. The Dockerfile enables and checks that option.
+- The base SDK has no sccache binary. We install the pinned 0.18.0 amd64-musl
+  release and verify its SHA-256 before using it.
+- Rust bootstrap wraps rustc itself. We also set `build.ccache=sccache` in
+  its configure arguments so stage-zero compilation and LLVM use sccache,
+  retaining `llvm.download-ci-llvm=false`. The inherited wrapper is forwarded
+  by bootstrap through `RUSTC_WRAPPER_REAL` where applicable.
+- sccache is a compiler cache, not a Cargo target-directory cache. Linked
+  binaries, proc macros, some bootstrap work and incremental compilation are
+  not cacheable. SDK bootstrap and downstream dependencies can share storage
+  without necessarily sharing every compiled object. Every actual SDK build
+  prints statistics on success or failure. BuildKit can reuse unchanged layers;
+  a cached compilation layer does not run sccache or produce fresh statistics.
+
+Sources: [OpenWrt Rust configuration](https://github.com/openwrt/packages/blob/openwrt-24.10/lang/rust/rust-values.mk),
+[Rust bootstrap](https://github.com/rust-lang/rust/blob/1.90.0/src/bootstrap/src/core/build_steps/compile.rs),
+[sccache multi-level cache](https://github.com/mozilla/sccache/blob/v0.18.0/docs/MultiLevel.md),
+[sccache client-side architecture](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Architecture.md),
+[Rust cache limitations](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Rust.md).
+
+## Downstream OpenWrt packages
+
+Use the same cache services and target/version image. Export `AWS_ACCESS_KEY_ID`
+and `AWS_SECRET_ACCESS_KEY` from CI secrets before starting the container. For example, after
+installing the `sqm-autorate-rust` feed/package in the SDK:
+
+```sh
+docker run --rm --network gitea-runner-jobs --cpus 8 \
+  --memory 20g --memory-swap 20g \
+  -e SCCACHE_REDIS_ENDPOINT=redis://sccache-redis:6379 \
+  -e SCCACHE_ENDPOINT=http://sccache-s3:9000 \
+  -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
+  lochnair/openwrt-sdk-rust:x86-64-24.10.8 sh -ec '
+    # Install/update the downstream feed here before defconfig.
+    make defconfig
+    grep -qx "CONFIG_RUST_SCCACHE=y" .config
+    sccache --start-server
+    trap "sccache-stats; sccache --stop-server" EXIT
+    sccache --zero-stats
+    make package/sqm-autorate-rust/compile -j8 V=sc
+  '
+```
+
+Keep the image's cache prefix and enable `CONFIG_RUST_SCCACHE` after any
+downstream configuration replacement. Cache statistics belong to the current
+container's sccache server; `--zero-stats` does not erase cached objects.
+`sqm-autorate-rust` currently uses cross-rs to publish static binaries, not an
+OpenWrt SDK package workflow; that unrelated pipeline is unchanged. The package
+Makefile in `openwrt-feeds` already uses OpenWrt's `rust-package.mk`, so an SDK
+package job can use this cache without patching the package.
+
+For a local SDK build on the prepared runner, export `AWS_ACCESS_KEY_ID` and
+`AWS_SECRET_ACCESS_KEY` from the cache credentials in your environment first:
+
+```sh
+sh ci/setup-builder.sh
+docker buildx build --builder openwrt-sdk-rust --load --progress plain \
+  --network host --allow network.host \
+  --build-arg SAFE_TARGET=x86-64 --build-arg VERSION=24.10.8 \
+  --build-arg JOBS=8 \
+  --build-arg SCCACHE_REDIS_ENDPOINT=redis://127.0.0.1:6380 \
+  --build-arg SCCACHE_ENDPOINT=http://127.0.0.1:9000 \
+  --secret id=aws-access-key-id,env=AWS_ACCESS_KEY_ID \
+  --secret id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY \
+  -t lochnair/openwrt-sdk-rust:x86-64-24.10.8 .
+```
+
+See [validation results](ci/VALIDATION.md) for the checks performed and the
+remaining cutover acceptance.

ci/VALIDATION.md

@@ -0,0 +1,319 @@
+# All-target matrix validation — 2026-10-13
+
+The Gitea workflow now defaults to `targets=all`. Target discovery reads every
+page of the public GHCR SDK tag list and selects each release's target tags,
+using release directory names to exclude architecture aliases. Architecture
+metadata comes from the release's `.targets.json`, with the matching OpenWrt
+source tag supplying `ARCH` for SDK-only targets absent from that JSON.
+
+Live generation and registry manifest checks produced:
+
+| OpenWrt release | Available target SDK images | Rust-supported matrix targets |
+| --- | ---: | ---: |
+| 25.12.5 (stable) | 94 | 93 |
+| 24.10.8 (oldstable) | 90 | 89 |
+| Total | 184 | 182 |
+
+Both releases exclude `ixp4xx/generic` (`armeb`). The architecture allowlist
+matches `RUST_ARCH_DEPENDS` at the released packages feed revisions:
+[25.12.5](https://github.com/openwrt/packages/blob/5caa62e0bc9f7fb9b0c12a23267bceb7724214dd/lang/rust/rust-values.mk)
+and [24.10.8](https://github.com/openwrt/packages/blob/23abaa6f3b0fdfd76b570031107e5718476ff0c8/lang/rust/rust-values.mk).
+All four Malta subtargets remain included for both releases. The older release
+also retains SDK-only `lantiq/ase`, `lantiq/xway_legacy`, `ramips/rt288x` and
+`realtek/rtl931x`.
+
+Validation passed:
+
+- All 182 generated base-image tags returned HTTP 200 and a manifest digest
+  from GHCR. Matrix entries are unique; `GITHUB_OUTPUT` matches JSON stdout.
+- The explicit `x86/64,mediatek/filogic,octeon/generic` subset produces six
+  jobs. Selecting `siflower/sf21` produces only its 25.12.5 job.
+- Six offline regression tests cover per-release availability, unsupported
+  architectures, explicit selections and duplicates, invalid/empty selections,
+  architecture aliases, SDK-only architecture fallback, missing metadata and
+  registry pagination. The matrix job runs these tests before live discovery.
+- YAML parsing, actionlint 1.7.12 (excluding only the existing Gitea
+  `builtin:checkout` syntax), ShellCheck and `git diff --check` pass.
+- Workflow assertions confirm `fail-fast: false`, unchanged `max-parallel: 1`,
+  timeout, runner, build/verification commands and cache environment. Each job
+  pushes after its own successful verification when `publish=true`.
+- Mock Docker execution checks publication and cleanup after a successful push,
+  a failed push and an absent image. Cleanup runs with `always()` and removes
+  only the job's SDK tag; it does not prune caches, volumes or other images.
+
+The Redis/S3 infrastructure, sccache configuration, eight-job compilation limit,
+eight-CPU/20 GiB builder limits and 50 GiB BuildKit GC budget are unchanged.
+This validation did not compile all SDKs or dispatch/publish images. Architecture
+eligibility and available base images do not guarantee every Rust bootstrap will
+succeed. Live workflow/publication acceptance remains pending; the legacy
+Jenkinsfile is retained unchanged until cutover acceptance.
+
+# Migration validation — 2026-10-10
+
+Inspected the original Jenkinsfile, Dockerfile, downstream `sqm-autorate-rust`
+workflows and package Makefile, Terraform runner definition, and live TheBeast
+runner configuration before changing the repository.
+
+## Passed
+
+- YAML parsing; TOML parsing; `sh -n`; ShellCheck; `git diff --check`.
+- actionlint 1.7.12, excluding only its unsupported `builtin:checkout` action
+  syntax. The workflow uses the existing `ubuntu-latest` label. Gitea's documentation and the
+  existing downstream workflow confirm built-in checkout support.
+- Default matrix resolves six entries for stable 25.12.5 and oldstable 24.10.8.
+  `targets=all` ran upstream target discovery inside the actual job image and
+  resolved 194 entries. Empty/invalid targets fail. Every default base-image
+  tag was read successfully from GHCR.
+- Docker Compose configuration accepted; Redis healthy with its named volume,
+  AOF enabled and successful AOF writes. Service and bounded builder were
+  provisioned on TheBeast without changing the runner configuration.
+- Builder inspected: eight-CPU quota, 20 GiB memory/swap limit, one BuildKit
+  worker step, 50 GiB layer GC budget. A second disposable job container with
+  fresh Docker client configuration reused the existing daemon and builder.
+- Dockerfile build checks completed. Two expected advisory warnings: mandatory
+  base-image arguments have no defaults, and Docker's secret-name heuristic
+  mistakes the public `SCCACHE_REDIS_KEY_PREFIX` for a secret.
+- `configured` Dockerfile stage built for x86/64 on both OpenWrt releases.
+  Pinned feeds select Rust 1.94.0 on 25.12.5 and 1.90.0 on 24.10.8. Both retain
+  `CONFIG_RUST_SCCACHE=y` through defconfig and contain the explicit bootstrap
+  `build.ccache=sccache` setting with LLVM CI downloads disabled.
+- Actual OpenWrt make-variable expansion injects sccache into host and package
+  compilation, despite unexporting an inherited wrapper.
+- Real Cargo library compilation in Docker BuildKit: one Redis miss. A fresh
+  runtime container using the Docker-network endpoint: one hit, zero misses.
+- Recreated the newly provisioned Redis container using the same named volume;
+  a fresh compiler container still recorded one hit, zero misses. No existing
+  infrastructure containers were restarted.
+- Same source/cache namespace with GNU versus musl target: first musl build
+  missed; repetition hit. Rust 1.90.0 versus 1.91.0: first 1.91 build missed;
+  repetition hit. All probes reported zero cache read/write errors.
+- A disposable OpenWrt package used the real `Build/Compile/Cargo` helper,
+  SDK GCC/musl linker, OpenWrt LTO/optimization flags and a local Rust library
+  dependency. It built successfully in two separate containers: one dependency
+  miss, then one hit. The linked executable correctly remained non-cacheable.
+  This used a prebuilt Rust 1.90 toolchain for the probe, not a freshly compiled
+  SDK toolchain; it is not a full SDK/source-bootstrap acceptance result.
+- Retained `ci/verify-cache.sh` passed its separate-container miss/hit checks
+  against that prebuilt validation toolchain. Its original hardcoded SDK rustc
+  path was later corrected after the live mediatek/filogic build exposed the
+  target-specific staging layout; see the follow-up below.
+- Publication configuration checked: original Docker Hub namespace/tag format,
+  verification before push, explicit publish input, temporary credential
+  directory, no automatic publication trigger. No registry login or push was
+  attempted.
+
+## Remaining acceptance
+
+1. Enable repository Actions and configure the two Docker Hub secrets. No
+   runner configuration changes are required.
+2. Complete a non-publishing Gitea dispatch for the full default matrix.
+   The mediatek/filogic 24.10.8 job completed Rust source bootstrap and image
+   export; its corrected verification passed on the built image. Acceptance
+   for the other combinations remains pending.
+3. With explicit publication approval, run the publishing dispatch, pull the
+   resulting tags and build the downstream OpenWrt package with the new image.
+   Actual Docker Hub authorization and publication remain unverified.
+4. After acceptance, disable the old Jenkins job and remove `Jenkinsfile`.
+   It is deliberately retained until the replacement passes these checks.
+
+Remote test logs and disposable probe sources are under
+`/tmp/openwrt-sdk-rust-migration` in TheBeast's `gitea-runner` instance. The
+Redis volume/service and BuildKit builder remain available for the replacement
+pipeline; no cache volume or existing infrastructure was deleted.
+
+## Follow-up: mediatek/filogic 24.10.8 verification
+
+The supplied job 1602 log shows full SDK compilation and image export completed
+successfully. Verification failed with exit 127 because it assumed
+`staging_dir/host/bin/rustc`. Inspection of that exact built image reproduced
+the failure and located Rust/Cargo in
+`staging_dir/target-aarch64_cortex-a53_musl/host/bin` instead. Both the workflow
+and retained cache probe now resolve the unique executable compiler from the
+SDK staging tree, rather than assuming a shared host directory.
+
+Reran the corrected verification on that image: Rust 1.90.0 and Cargo 1.90.0
+execute successfully. The retained cache probe reports one Redis miss in its
+first container and one hit in a fresh second container, with zero cache
+read/write errors. ShellCheck, actionlint (excluding only Gitea's built-in
+checkout syntax), and `git diff --check` pass. No rebuild or publication was
+performed for this fix.
+
+## Follow-up: mediatek/filogic 25.12.5 bootstrap concurrency
+
+Job 1650 failed inside LLVM compilation with
+`c++: fatal error: Killed signal terminated program cc1plus`. Its diagnostic
+Make retry still configured `LLVM_PARALLEL_COMPILE_JOBS=32`. The persistent
+builder's cgroup reports three OOM kills and its memory limit remains 20 GiB;
+the counter is cumulative, while the supplied log identifies the killed
+compiler in this job. Outer Make parallelism had not bounded Rust bootstrap's
+independent job count.
+
+The SDK Rust Makefile now passes `RUST_BUILD_JOBS` to `x.py --jobs`, defaulting
+to eight for subsequent SDK use. `build-rust` supplies eight (or the requested
+`JOBS`) on the first attempt and one on the serial diagnostic retry. Rust's
+bootstrap uses this job count for LLVM's compile pool and CMake build command.
+
+Built the corrected `configured` image for mediatek/filogic 25.12.5, and
+expanded the actual OpenWrt `Host/Compile` recipe at both job counts: it invokes
+`x.py --jobs=8` and `x.py --jobs=1` respectively. The same recipe checks pass
+against the 24.10.8 feed in a disposable container. ShellCheck, shell syntax,
+actionlint and diff checks pass. Full source compilation after this change has
+not been rerun, and no images were published or infrastructure limits changed.
+
+## Follow-up: runner-owned Redis and SSD cache — 2026-10-11
+
+Initially prototyped a 1 GiB Redis LRU hot cache and Apache WebDAV backed by a
+persistent Docker volume on `fastpool`. This prototype was superseded by the
+S3 setup below after testing both requested alternatives. The SDK repository
+consumes these services and no longer deploys Redis. Production services have
+not been redeployed; the original Redis container and volume remain untouched.
+
+Inspected the pinned sccache 0.18.0 source and ran a cross-container central
+server experiment. With client-side compilation and an unshared disk backend,
+the first compile missed; the second reported a cache error and recompiled,
+because `IpcStorage` attempts to open the daemon's returned absolute cache path.
+TCP addresses are fixed to loopback in `net::SocketAddr::with_port`. In addition,
+client-side `get_raw` bypasses multi-level read statistics and backfilling. The
+standard per-job daemon mode with Redis/WebDAV avoids these restrictions.
+
+Writes in standard mode can still be outstanding when the compiler client
+returns. `SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all` awaits both tiers within a
+write; `sccache-stats` then drains writes and backfills for up to 30 seconds before
+printing statistics and stopping the job daemon. Its completion checks use the
+pinned version's cache-miss/write and per-tier backfill counters.
+
+Passed on TheBeast using an isolated `sccache-probe` Compose project:
+
+- Built the WebDAV image and validated Apache configuration. Real writes exposed
+  the base Alpine image's missing APR DBM driver; the added Dockerfile installs
+  `apr-util-dbm_gdbm`. Both services then passed health checks and real writes.
+- Retained CI verification: one miss writing both tiers, one Redis hit in a
+  fresh container, then one WebDAV-only hit in a third fresh container. Zero
+  cache errors, read/write errors, or per-tier write failures.
+- Real Rust compilation inside Docker BuildKit followed by a fresh SDK runtime
+  container: miss, then hit, using host-loopback and Docker-DNS endpoints.
+- Repeated the unchanged BuildKit build: the actual compilation RUN step was
+  cached and did not invoke sccache again.
+- Recreated both test service containers, retaining only the WebDAV volume:
+  Redis started empty. A fresh SDK container hit WebDAV and backfilled Redis;
+  the following fresh container hit Redis.
+- Identical source and prefix with GNU versus musl targets: musl missed, then
+  hit. Rust 1.90.0 versus 1.94.0 under the same test prefix: 1.94.0 missed, then
+  hit. All cache-error counters remained zero.
+- A disposable OpenWrt package compiled via the real `rust-package.mk`
+  `Build/Compile/Cargo` helper and SDK musl linker in two fresh SDK containers:
+  dependency miss, then hit. This used the actual completed 24.10.8 SDK image,
+  not the earlier validation toolchain. The binary remained non-cacheable.
+- Proposed full runner Compose configuration, YAML parsing, Apache config,
+  ShellCheck, shell syntax, Python syntax, actionlint, and diff checks passed.
+  actionlint excludes only Gitea's built-in checkout syntax. Dockerfile checks
+  exclude the public `KEY_PREFIX` secret-name heuristic and mandatory base
+  image argument warning, as in the original validation.
+- Publishing remains manual, with the original image tags and verification
+  before registry login/push. No login or publication was performed. Matrix,
+  runner labels/capacity, BuildKit caching, and serial verbose retry are unchanged.
+
+Full SDK source bootstrap and a Gitea dispatch were not rerun for this cache
+change. Deploy/build the updated runner stack before using the changed workflow.
+The prototype WebDAV tier had no automatic LRU or capacity limit; the final
+S3 configuration below adds an independent hard filesystem bound and retention.
+Old Redis entries are not migrated; retiring the old service/volume is a separate
+approved cutover step. Test logs are under `/tmp/gitea-sccache-stack` and
+`/tmp/gitea-sccache-tests` on the runner; isolated test services are cleaned up.
+
+Sources: [sccache architecture](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Architecture.md),
+[pinned IPC storage](https://github.com/mozilla/sccache/blob/v0.18.0/src/cache/ipc_storage.rs),
+[pinned multi-level implementation](https://github.com/mozilla/sccache/blob/v0.18.0/src/cache/multilevel.rs).
+
+
+## Follow-up: RustFS and VersityGW comparison — 2026-10-11
+
+Selected **RustFS 1.0.1** after testing it and **VersityGW 1.8.0**, following the
+owner's preference. The final runner Compose configuration uses Redis/S3 and
+removes the uncommitted Apache image/config prototype. Services were tested in
+isolated projects, not deployed over the existing runner/cache services.
+
+Both candidates passed real sccache 0.18.0 compilation tests using the completed
+SDK images on TheBeast:
+
+- Cold miss wrote both tiers; fresh-container Redis hit; S3-only hit.
+- Service restart plus empty isolated Redis: S3 hit backfilled Redis, then the
+  next fresh container hit Redis. Cache read/write/error counters stayed zero.
+- Same prefix/input with GNU versus musl targets: musl missed then hit. Rust
+  1.90.0 versus 1.94.0: the different toolchain missed then hit.
+
+The objects live on a separate Incus ZFS volume with a 16 GiB hard quota for
+these tests. A separate 64 MiB quota probe rejected an 80 MiB incompressible write
+with `Disk quota exceeded`. No production data was used for the quota test.
+Production setup requires a new **100 GiB** quota-backed Incus volume; Compose
+requires an existing data directory and rejects a filesystem reporting more than
+100 GiB capacity. This guard passed on the quota-backed mount and rejected the
+unbounded SSD filesystem. Missing BuildKit credentials failed before compilation,
+and test logs contained no credential values. RustFS log files use a bounded tmpfs; Docker logs rotate.
+
+Storage microbenchmark: 64 objects of 1 MiB incompressible data, three passes,
+median aggregate MiB/s over host-loopback HTTP on the same SSD pool. Every GET
+was compared with its source bytes. These include client/protocol overhead and
+warm filesystem cache; they do not measure full SDK build speed or sustained
+cold-SSD throughput.
+
+| Backend | PUT, 1 client | GET, 1 client | PUT, 4 clients | GET, 4 clients |
+| --- | ---: | ---: | ---: | ---: |
+| Apache WebDAV prototype | 110.2 | 660.8 | failed | not run |
+| RustFS 1.0.1 | 101.5 | 314.7 | 231.2 | 471.2 |
+| VersityGW 1.8.0 | 99.1 | 329.3 | 232.2 | 410.2 |
+
+WebDAV's four-client PUT test returned HTTP 500 lock-database errors, despite
+passing sequential compiler-cache tests. Both S3 backends handled that load.
+Observed Docker memory usage after the probes was about 202 MiB for RustFS and
+28 MiB for VersityGW; these are small test workloads, not worst-case bounds.
+RustFS was chosen for preference and compatibility, not a proven performance
+advantage over VersityGW. Sequential S3 GETs were slower than the WebDAV baseline.
+Redis remains the primary hot tier.
+
+Final RustFS setup and SDK integration checks:
+
+- The entire proposed runner Compose configuration validates with dummy
+  registration credentials. Only the cache services ran in the isolated project;
+  both passed their real health checks. Bucket initialization with pinned official
+  AWS CLI 2.37.12 passed. RustFS accepted and returned the exact configured
+  30-day object expiration and one-day abandoned multipart-upload policy.
+  Expiration execution itself was not tested by waiting 30 days.
+- Replaced the original test RustFS container with the proposed Compose service
+  over the same retained quota-backed data: an earlier Rust artifact was still an
+  S3 hit and backfilled a new Redis namespace in a fresh SDK container.
+- The retained `ci/verify-cache.sh` passed all five probes (cold writes, Redis hit,
+  S3-only hit, backfill into a fresh Redis namespace, subsequent Redis hit) on
+  x86-64/24.10.8 and mediatek-filogic/25.12.5. It never flushes shared Redis data.
+- Real BuildKit Rust compilation using the proposed required secret mounts,
+  then a fresh runtime-container cache hit, passed. Secret values were absent
+  from exported image configuration/history; subsequent layers and a fresh
+  container had no credential environment variables or secret files. The repeated
+  unchanged compilation layer was cached. Only a local disposable image was
+  produced; no registry login/push occurred.
+- Actual OpenWrt `Build/Compile/Cargo` and SDK musl linker compiled a disposable
+  package twice: dependency miss followed by hit in a fresh container. This
+  confirms inherited S3 settings/credentials reach OpenWrt's sccache wrapper.
+- BuildKit checked the exact changed SDK Dockerfile. Only the known public
+  `KEY_PREFIX` secret-name heuristic and required base-image argument warning were
+  excluded. Workflow/actionlint, ShellCheck, shell/Python/JSON/YAML parsing and
+  whitespace checks passed.
+
+The SDK image retains public S3 settings and version-only prefixes. Credentials are not stored in image configuration/history, and runner endpoints
+are build arguments rather than runtime defaults. Gitea secrets feed ephemeral BuildKit
+mounts and runtime verification containers. The runner service's registration,
+labels/capacity, build matrix, image tags, manual publication gate, BuildKit layer
+caching and serial verbose fallback are unchanged.
+
+Manual setup: allocate/mount the 100 GiB volume, set the two cache credentials in
+Komodo and Gitea, deploy the cache services, initialize the bucket, and run a
+non-publishing workflow before cutover. Full SDK bootstrap and live Gitea dispatch
+were not repeated for this backend change. Production Redis, its volume, and
+Jenkins remain untouched pending acceptance/approved retirement. Isolated test
+containers and volumes are removed after verification; logs remain under
+`/tmp/s3-cache-tests` on the runner.
+
+Sources: [RustFS deployment](https://docs.rustfs.com/en/installation/container/docker),
+[VersityGW Docker setup](https://github.com/versity/versitygw/wiki/Docker),
+[sccache S3](https://github.com/mozilla/sccache/blob/v0.18.0/docs/S3.md),
+[BuildKit secrets](https://docs.docker.com/build/building/secrets/).

ci/build-rust.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -eu
+
+: "${SCCACHE_REDIS_ENDPOINT:?Set the persistent sccache Redis endpoint}"
+: "${SCCACHE_ENDPOINT:?Set the persistent sccache S3 endpoint}"
+: "${AWS_ACCESS_KEY_ID:?Supply the S3 access key through a BuildKit secret}"
+: "${AWS_SECRET_ACCESS_KEY:?Supply the S3 secret key through a BuildKit secret}"
+: "${JOBS:=8}"
+case "$JOBS" in ''|*[!0-9]*|0) echo 'JOBS must be a positive integer' >&2; exit 1 ;; esac
+
+sccache --start-server
+trap 'sccache-stats; sccache --stop-server' EXIT
+sccache --zero-stats
+# Bootstrap/Ninja does not inherit Make's job count. Bound both explicitly.
+make package/rust/host/compile -j"$JOBS" RUST_BUILD_JOBS="$JOBS" || \
+    make package/rust/host/compile -j1 RUST_BUILD_JOBS=1 V=sc
+rm -rf build_dir/host/rustc-*/rustc-*/build dl/rustc-*.tar.*

ci/buildkitd.toml

@@ -0,0 +1,6 @@
+# Shared by all SDK jobs on this daemon, including simultaneous dispatches.
+[worker.oci]
+  max-parallelism = 1
+  gc = true
+  reservedSpace = "10GB"
+  maxUsedSpace = "50GB"

ci/matrix.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Resolve Rust-supported SDK tags for each stable/oldstable release."""
+import json
+import os
+import re
+import sys
+import urllib.parse
+import urllib.request
+
+
+# OpenWrt packages/lang/rust/rust-values.mk: RUST_ARCH_DEPENDS.
+RUST_ARCHES = {
+    "aarch64", "arm", "i386", "loongarch64", "mips", "mips64",
+    "mips64el", "mipsel", "powerpc", "powerpc64", "riscv64", "x86_64",
+}
+
+
+def fetch(url):
+    with urllib.request.urlopen(url, timeout=60) as response:
+        return response.read().decode()
+
+
+def sdk_tags():
+    token = json.loads(fetch("https://ghcr.io/token?service=ghcr.io&scope=repository:openwrt/sdk:pull"))["token"]
+    url = "https://ghcr.io/v2/openwrt/sdk/tags/list?n=1000"
+    tags = set()
+    while url:
+        request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
+        with urllib.request.urlopen(request, timeout=60) as response:
+            tags.update(json.load(response)["tags"])
+            link = response.headers.get("Link", "")
+        next_page = re.search(r'<([^>]+)>;\s*rel="next"', link)
+        url = urllib.parse.urljoin(url, next_page[1]) if next_page else None
+    return tags
+
+
+def release_targets(version, tags):
+    base = f"https://downloads.openwrt.org/releases/{version}"
+    # Directory names distinguish target/subtarget tags from architecture aliases.
+    boards = re.findall(r'href="([a-z0-9_+-]+)/"', fetch(f"{base}/targets/"))
+    suffix = f"-{version}"
+    targets = sorted({f"{board}/{tag[len(board) + 1:-len(suffix)]}"
+                      for board in boards for tag in tags
+                      if tag.startswith(f"{board}-") and tag.endswith(suffix)})
+    if not targets:
+        raise ValueError(f"No SDK images available for OpenWrt {version}")
+    architectures = json.loads(fetch(f"{base}/.targets.json"))
+    result = {}
+    for target in targets:
+        if not re.fullmatch(r"[a-z0-9_+-]+/[a-z0-9_+-]+", target):
+            raise ValueError(f"Invalid SDK target: {target!r}")
+        if target in architectures:
+            arch = architectures[target]
+            arch = arch if arch in RUST_ARCHES else arch.split("_", 1)[0]
+        else:
+            # SDK-only targets (e.g. Malta) have no profiles in .targets.json.
+            board, subtarget = target.split("/")
+            source = f"https://raw.githubusercontent.com/openwrt/openwrt/v{version}/target/linux"
+            definition = fetch(f"{source}/{board}/{subtarget}/target.mk")
+            match = re.search(r"^ARCH\s*:?=\s*([a-z0-9_]+)\s*$", definition, re.MULTILINE)
+            if not match:
+                definition = fetch(f"{source}/{board}/Makefile")
+                match = re.search(r"^ARCH\s*:?=\s*([a-z0-9_]+)\s*$", definition, re.MULTILINE)
+            if not match:
+                raise ValueError(f"Cannot determine architecture for {target} on {version}")
+            arch = match[1]
+        result[target] = arch
+    return result
+
+
+def resolve(versions, targets, tags):
+    versions = [versions[k] for k in ("stable_version", "oldstable_version") if versions.get(k)]
+    if not versions:
+        raise ValueError("OpenWrt returned no stable or oldstable version")
+    targets = [t.strip() for t in targets.split(",") if t.strip()]
+    if not targets:
+        raise ValueError("No targets selected")
+    if "all" in targets and targets != ["all"]:
+        raise ValueError("Select all or a comma-separated target list")
+    for target in targets:
+        if target != "all" and not re.fullmatch(r"[a-z0-9_+-]+/[a-z0-9_+-]+", target):
+            raise ValueError(f"Invalid target: {target!r}")
+    for version in versions:
+        if not re.fullmatch(r"[a-zA-Z0-9_.-]+", version):
+            raise ValueError(f"Invalid version: {version!r}")
+    matrix = []
+    for version in dict.fromkeys(versions):
+        available = release_targets(version, tags)
+        selected = available if targets == ["all"] else dict.fromkeys(targets)
+        supported = [t for t in selected if available.get(t) in RUST_ARCHES]
+        skipped = [t for t in selected if t not in supported]
+        print(f"OpenWrt {version}: {len(supported)} targets selected; "
+              f"{len(available)} SDK images available", file=sys.stderr)
+        if skipped:
+            print(f"Skipped unavailable or Rust-unsupported targets: {', '.join(skipped)}", file=sys.stderr)
+        matrix.extend({"target": t.replace("/", "-"), "version": version} for t in supported)
+    if not matrix:
+        raise ValueError("No available Rust-supported SDK targets selected")
+    return {"include": matrix}
+
+
+if __name__ == "__main__":
+    versions = json.loads(fetch("https://downloads.openwrt.org/.versions.json"))
+    targets = os.environ.get("TARGETS", "all")
+    matrix = json.dumps(resolve(versions, targets, sdk_tags()), separators=(",", ":"))
+    print(matrix)
+    with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+        output.write(f"matrix={matrix}\n")

ci/sccache-stats.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Wait for this job's cache writes/backfills, then print complete statistics."""
+import json
+import subprocess
+import time
+
+end = time.monotonic() + 30
+while True:
+    data = json.loads(subprocess.check_output(
+        ["sccache", "--show-stats", "--stats-format", "json"],
+        timeout=max(0.1, end - time.monotonic()),
+    ))
+    stats = data["stats"]
+    misses = sum(stats["cache_misses"]["counts"].values())
+    levels = stats.get("multi_level") or []
+    writes_done = stats["cache_writes"] + stats["cache_write_errors"] >= misses
+    backfills_done = sum(x["backfills_to"] for x in levels) >= sum(
+        x["backfills_from"] for x in levels
+    )
+    if writes_done and backfills_done:
+        break
+    if time.monotonic() >= end:
+        subprocess.run(["sccache", "--show-stats"], check=True, timeout=5)
+        raise SystemExit("Timed out waiting for sccache writes/backfills")
+    time.sleep(0.1)
+subprocess.run(["sccache", "--show-stats"], check=True, timeout=5)

ci/setup-builder.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+if ! docker buildx inspect openwrt-sdk-rust >/dev/null 2>&1; then
+    docker buildx create --name openwrt-sdk-rust --driver docker-container \
+        --driver-opt network=host,memory=20g,memory-swap=20g,cpu-period=100000,cpu-quota=800000 \
+        --buildkitd-config ci/buildkitd.toml \
+        --buildkitd-flags '--allow-insecure-entitlement network.host' || \
+        docker buildx inspect openwrt-sdk-rust
+fi
+docker buildx inspect openwrt-sdk-rust --bootstrap

ci/test_matrix.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Offline checks for release-specific SDK selection and registry pagination."""
+import contextlib
+import io
+import json
+import unittest
+from unittest.mock import patch
+
+import matrix
+
+
+class MatrixTests(unittest.TestCase):
+    versions = {"stable_version": "25.12.5", "oldstable_version": "24.10.8"}
+
+    def resolve(self, targets):
+        releases = {
+            "25.12.5": {"x86/64": "x86_64", "siflower/sf21": "riscv64", "ixp4xx/generic": "armeb"},
+            "24.10.8": {"x86/64": "x86_64", "malta/be64": "mips64", "arc/generic": "arc"},
+        }
+        with patch.object(matrix, "release_targets", side_effect=lambda v, tags: releases[v]):
+            with contextlib.redirect_stderr(io.StringIO()):
+                return matrix.resolve(self.versions, targets, set())["include"]
+
+    def test_all_uses_each_releases_available_supported_sdks(self):
+        self.assertEqual(self.resolve("all"), [
+            {"target": "x86-64", "version": "25.12.5"},
+            {"target": "siflower-sf21", "version": "25.12.5"},
+            {"target": "x86-64", "version": "24.10.8"},
+            {"target": "malta-be64", "version": "24.10.8"},
+        ])
+
+    def test_explicit_targets_deduplicate_and_skip_unavailable_combinations(self):
+        self.assertEqual(self.resolve("siflower/sf21,x86/64,x86/64,ixp4xx/generic"), [
+            {"target": "siflower-sf21", "version": "25.12.5"},
+            {"target": "x86-64", "version": "25.12.5"},
+            {"target": "x86-64", "version": "24.10.8"},
+        ])
+
+    def test_invalid_or_empty_selection_fails(self):
+        for targets in ("", "../bad", "x86/64;echo", "all,x86/64", "ixp4xx/generic", "missing/generic"):
+            with self.subTest(targets=targets), self.assertRaises(ValueError):
+                self.resolve(targets)
+        for versions in ({}, {"stable_version": "../bad"}):
+            with self.assertRaises(ValueError):
+                matrix.resolve(versions, "all", set())
+
+    def test_discovery_excludes_aliases_and_other_releases_and_keeps_sdk_only_targets(self):
+        base = "https://downloads.openwrt.org/releases/25.12.5"
+        source = "https://raw.githubusercontent.com/openwrt/openwrt/v25.12.5/target/linux"
+        responses = {
+            f"{base}/targets/": '<a href="x86/">x86</a><a href="ixp4xx/">ixp4xx</a>'
+                                 '<a href="malta/">malta</a><a href="ramips/">ramips</a>',
+            f"{base}/.targets.json": json.dumps({"x86/64": "x86_64", "ixp4xx/generic": "armeb_xscale"}),
+            f"{source}/malta/be64/target.mk": "ARCH:=mips64\n",
+            f"{source}/ramips/rt288x/target.mk": "SUBTARGET:=rt288x\n",
+            f"{source}/ramips/Makefile": "ARCH:=mipsel\n",
+        }
+        tags = {"x86-64-25.12.5", "ixp4xx-generic-25.12.5", "malta-be64-25.12.5",
+                "ramips-rt288x-25.12.5", "x86_64-25.12.5", "arm_cortex-a7-25.12.5",
+                "x86-generic-24.10.8", "x86-64-v25.12.5"}
+        with patch.object(matrix, "fetch", side_effect=lambda url: responses[url]):
+            self.assertEqual(matrix.release_targets("25.12.5", tags), {
+                "ixp4xx/generic": "armeb", "malta/be64": "mips64",
+                "ramips/rt288x": "mipsel", "x86/64": "x86_64",
+            })
+
+    def test_missing_images_or_unknown_architecture_fails(self):
+        with patch.object(matrix, "fetch", return_value='<a href="malta/">malta</a>'):
+            with self.assertRaisesRegex(ValueError, "No SDK images"):
+                matrix.release_targets("25.12.5", set())
+        def response(url):
+            if url.endswith("/targets/"):
+                return '<a href="malta/">malta</a>'
+            return "{}" if url.endswith(".json") else "SUBTARGET:=be\n"
+        with patch.object(matrix, "fetch", side_effect=response):
+            with self.assertRaisesRegex(ValueError, "Cannot determine architecture"):
+                matrix.release_targets("25.12.5", {"malta-be-25.12.5"})
+
+    def test_registry_pagination(self):
+        first = io.BytesIO(b'{"tags":["x86-64-25.12.5"]}')
+        first.headers = {"Link": '</v2/openwrt/sdk/tags/list?last=x86-64-25.12.5&n=1000>; rel="next"'}
+        second = io.BytesIO(b'{"tags":["malta-be-24.10.8"]}')
+        second.headers = {}
+        with patch.object(matrix, "fetch", return_value='{"token":"public-pull-token"}'):
+            with patch.object(matrix.urllib.request, "urlopen", side_effect=[first, second]) as opened:
+                self.assertEqual(matrix.sdk_tags(), {"x86-64-25.12.5", "malta-be-24.10.8"})
+        self.assertEqual(opened.call_args_list[1].args[0].full_url,
+                         "https://ghcr.io/v2/openwrt/sdk/tags/list?last=x86-64-25.12.5&n=1000")
+
+
+if __name__ == "__main__":
+    unittest.main()

ci/verify-cache.sh

@@ -0,0 +1,67 @@
+#!/bin/sh
+# Verify writes, Redis/S3 hits, and Redis backfilling in fresh containers.
+set -eu
+image=${1:?Usage: verify-cache.sh IMAGE}
+: "${SCCACHE_REDIS_ENDPOINT:=redis://sccache-redis:6379}"
+: "${SCCACHE_ENDPOINT:=http://sccache-s3:9000}"
+: "${AWS_ACCESS_KEY_ID:?Set the S3 access key}"
+: "${AWS_SECRET_ACCESS_KEY:?Set the S3 secret key}"
+stats_script=$(cat ci/sccache-stats.py)
+nonce=$(mktemp -d)
+trap 'rmdir "$nonce"' EXIT
+prefix="validation/${image##*:}/$(basename "$nonce")/"
+for expected in miss hit s3-hit backfill backfill-hit; do
+    redis_prefix=$prefix
+    case "$expected" in backfill*) redis_prefix="${prefix}backfill/" ;; esac
+    chain=redis,s3
+    if [ "$expected" = s3-hit ]; then chain=s3; fi
+    docker run --rm -i --network gitea-runner-jobs --cpus 2 \
+        --memory 2g --memory-swap 2g \
+        -e SCCACHE_REDIS_ENDPOINT="$SCCACHE_REDIS_ENDPOINT" \
+        -e SCCACHE_ENDPOINT="$SCCACHE_ENDPOINT" \
+        -e SCCACHE_REDIS_KEY_PREFIX="$redis_prefix" \
+        -e SCCACHE_S3_KEY_PREFIX="$prefix" \
+        -e SCCACHE_BUCKET=sccache -e SCCACHE_REGION=us-east-1 \
+        -e SCCACHE_S3_USE_SSL=false \
+        -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
+        -e SCCACHE_MULTILEVEL_CHAIN="$chain" \
+        -e SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all \
+        -e SCCACHE_CLIENT_SIDE=0 \
+        -e SCCACHE_STATS_SCRIPT="$stats_script" \
+        -e EXPECTED="$expected" "$image" sh -es <<'PROBE'
+rustc=$(find /builder/staging_dir -type f -path '*/host/bin/rustc')
+test -x "$rustc"
+mkdir /tmp/sccache-probe
+cd /tmp/sccache-probe
+printf 'pub fn probe(n: u64) -> u64 { n.wrapping_mul(17) }\n' > probe.rs
+sccache --start-server
+trap 'sccache --stop-server' EXIT
+sccache --zero-stats
+sccache "$rustc" --crate-name ci_sccache_probe \
+    --crate-type rlib --emit link,dep-info --out-dir . probe.rs
+python3 -c "$SCCACHE_STATS_SCRIPT"
+sccache --show-stats --stats-format json > stats.json
+python3 - <<'PY'
+import json
+import os
+
+data = json.load(open("stats.json"))
+assert data["cache_location"].startswith("Multi-level"), data
+stats = data["stats"]
+expected = os.environ["EXPECTED"]
+field = "cache_misses" if expected == "miss" else "cache_hits"
+assert sum(stats[field]["counts"].values()) == 1, data
+assert stats["cache_read_errors"] == stats["cache_write_errors"] == 0, data
+assert sum(stats["cache_errors"]["counts"].values()) == 0, data
+levels = stats["multi_level"]
+assert all(level["write_failures"] == 0 for level in levels), data
+if expected == "miss":
+    assert len(levels) == 2 and all(level["writes"] == 1 for level in levels), data
+elif expected == "backfill":
+    assert levels[1]["hits"] == levels[0]["backfills_to"] == 1, data
+else:
+    assert levels[0]["hits"] == 1, data
+print("Verified persistent cache", expected)
+PY
+PROBE
+done

renovate.json

@@ -1,6 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:recommended"
-  ]
-}