Add renovate.json

2026-10-09 18:07:30 +00:00
13 changed files with 13 additions and 958 deletions
@@ -1,5 +0,0 @@
 .git
 .gitea
 Jenkinsfile
 README.md
 **/__pycache__
@@ -1,93 +0,0 @@
 name: Build OpenWrt Rust SDK images
 on:
  workflow_dispatch:
    inputs:
      targets:
        description: 'Comma-separated targets, or all'
        default: 'all'
        required: true
        type: string
      publish:
        description: 'Publish the original Docker Hub tags (requires approval)'
        default: false
        type: boolean
 permissions:
  contents: read
 jobs:
  matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.resolve.outputs.matrix }}
    steps:
      - uses: builtin:checkout
      - name: Resolve targets and OpenWrt versions
        id: resolve
        env:
          TARGETS: ${{ inputs.targets }}
        run: |
          python3 -m unittest discover -s ci -p 'test_matrix.py'
          python3 ci/matrix.py
  build:
    needs: matrix
    runs-on: ubuntu-latest
    timeout-minutes: 360
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }}
    env:
      SAFE_TARGET: ${{ matrix.target }}
      VERSION: ${{ matrix.version }}
      AWS_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
    steps:
      - uses: builtin:checkout
      - name: Prepare persistent builder
        run: sh ci/setup-builder.sh
      - name: Build SDK image
        run: |
          docker buildx build --builder openwrt-sdk-rust --load --progress plain \
            --network host --allow network.host \
            --build-arg SAFE_TARGET="$SAFE_TARGET" \
            --build-arg VERSION="$VERSION" \
            --build-arg JOBS=8 \
            --build-arg SCCACHE_REDIS_ENDPOINT=redis://127.0.0.1:6380 \
            --build-arg SCCACHE_ENDPOINT=http://127.0.0.1:9000 \
            --secret id=aws-access-key-id,env=AWS_ACCESS_KEY_ID \
            --secret id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY \
            --tag "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}" .
      - name: Verify SDK image
        run: |
          docker run --rm "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}" sh -ec '
            sccache --version
            grep -qx "CONFIG_RUST_SCCACHE=y" .config
            rustc=$(find staging_dir -type f -path "*/host/bin/rustc")
            test -x "$rustc"
            "$rustc" -vV
            "$(dirname "$rustc")/cargo" --version
          '
          sh ci/verify-cache.sh "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}"
      - name: Publish Docker Hub tag
        if: ${{ success() && inputs.publish == true }}
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
        shell: bash
        run: |
          set -euo pipefail
          DOCKER_CONFIG=$(mktemp -d)
          export DOCKER_CONFIG
          trap 'rm -rf "$DOCKER_CONFIG"' EXIT
          printf '%s' "$DOCKERHUB_TOKEN" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
          docker push "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}"
      - name: Remove local SDK image
        if: always()
        run: |
          image="lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}"
          if docker image inspect "$image" >/dev/null 2>&1; then
            docker image rm "$image"
          fi
@@ -1,46 +1,13 @@
 # syntax=docker/dockerfile:1
 ARG SAFE_TARGET
 ARG VERSION
-FROM ghcr.io/openwrt/sdk:${SAFE_TARGET}-${VERSION} AS tooling
+FROM ghcr.io/openwrt/sdk:${SAFE_TARGET}-${VERSION}
 # The SDK runs on amd64 even when compiling for another OpenWrt target.
 USER root
 RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.18.0/sccache-v0.18.0-x86_64-unknown-linux-musl.tar.gz -o /tmp/sccache.tar.gz && \
    echo '45f1447fbe231e3037bde351ef70677dd212216c8d62ae7ca409fecc4d6acc89  /tmp/sccache.tar.gz' | sha256sum -c - && \
    tar -xzf /tmp/sccache.tar.gz -C /tmp && \
    install -m 0755 /tmp/sccache-v0.18.0-x86_64-unknown-linux-musl/sccache /usr/local/bin/sccache && \
    rm -rf /tmp/sccache.tar.gz /tmp/sccache-v0.18.0-x86_64-unknown-linux-musl
 USER buildbot
 FROM tooling AS configured
 ARG SAFE_TARGET
 ARG VERSION
 ENV SCCACHE_REDIS_KEY_PREFIX=openwrt-rust:${VERSION}: \
    SCCACHE_S3_KEY_PREFIX=openwrt-rust/${VERSION}/ \
    SCCACHE_BUCKET=sccache \
    SCCACHE_REGION=us-east-1 \
    SCCACHE_S3_USE_SSL=false \
    SCCACHE_MULTILEVEL_CHAIN=redis,s3 \
    SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all \
    CARGO_INCREMENTAL=0
 COPY --chmod=0755 ci/build-rust.sh /usr/local/bin/build-rust
 COPY --chmod=0755 ci/sccache-stats.py /usr/local/bin/sccache-stats
 RUN ./scripts/feeds update packages && \
    ./scripts/feeds install rust && \
    printf '\nCONFIG_RUST_SCCACHE=y\n' >> .config && \
    make defconfig && \
-    grep -qx 'CONFIG_RUST_SCCACHE=y' .config && \
+    sed -i 's/--set=llvm.download-ci-llvm=true/--set=llvm.download-ci-llvm=false/g' feeds/packages/lang/rust/Makefile; \
-    sed -i 's/--set=llvm.download-ci-llvm=true/--set=llvm.download-ci-llvm=false/g' feeds/packages/lang/rust/Makefile && \
+    { make package/rust/host/compile -j$(nproc) || \
-    sed -i '/--set=llvm.download-ci-llvm=false/a\	--set=build.ccache=sccache \\' feeds/packages/lang/rust/Makefile && \
+      make package/rust/host/compile -j1 V=sc; } && \
-    sed -i '/^define Host\/Compile/i RUST_BUILD_JOBS ?= 8' feeds/packages/lang/rust/Makefile && \
+    rm -rf build_dir/host/rustc-*/rustc-*/build \
-    sed -i 's/--build-dir /--jobs=$(RUST_BUILD_JOBS) --build-dir /' feeds/packages/lang/rust/Makefile
+           dl/rustc-*.tar.*
 FROM configured AS sdk
 ARG JOBS=8
 # Non-secret runner endpoints apply during the build, not as runtime defaults.
 ARG SCCACHE_REDIS_ENDPOINT
 ARG SCCACHE_ENDPOINT
 RUN --mount=type=secret,id=aws-access-key-id,env=AWS_ACCESS_KEY_ID,required=true \
    --mount=type=secret,id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY,required=true \
    build-rust
@@ -1,176 +1 @@
-# openwrt-sdk-rust
+# openwrt-sdk-rust
 Builds the OpenWrt SDK with its Rust host toolchain installed. Gitea Actions
 resolves `stable_version` and `oldstable_version` from OpenWrt's `.versions.json`.
 The default `targets=all` selects the SDK tags actually available in GHCR for
 each release, excluding architectures unsupported by OpenWrt's Rust package.
 Release metadata supplies architectures, with release source definitions for
 SDK-only targets omitted from `.targets.json`. Alternatively, supply a
 comma-separated target list; unavailable or unsupported combinations are skipped
 and reported, and an empty matrix fails. Images still use
 `ghcr.io/openwrt/sdk:<target-with-dashes>-<version>` and
 publish to `lochnair/openwrt-sdk-rust:<target-with-dashes>-<version>` on Docker Hub.
 No `latest` tag or additional registry is introduced.
 ## Runner setup and cutover
 The workflow uses `ubuntu-latest` on the existing TheBeast runner, currently
 the only runner. Its job image and access to the persistent Docker daemon were
 verified; no additional runner, label, capacity change, or restart is required.
 `builtin:checkout` uses the existing Runner 5 checkout support.
 Enable repository Actions and set `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN`
 secrets with push permission to `lochnair/openwrt-sdk-rust`. Jenkins' credential
 ID is not reusable in Gitea. The workflow is manually dispatched, matching the
 parameterized Jenkins job; there are no PR, push, or scheduled publication
 triggers. `publish=false` is the safe validation default. Dispatch
 `publish=true` only after publication is approved. Login uses a temporary job
 directory, and each successfully built and checked image is pushed under its
 original tag. Before any push, `ci/verify-cache.sh` uses the installed SDK rustc
 in fresh containers and requires writes to both tiers, Redis/S3 hits, and backfilling.
 Each matrix job publishes independently after its own verification succeeds.
 Its local SDK image is removed at the end, including after verification or push
 failure and during non-publishing runs. Cleanup leaves the shared builder and
 compilation caches intact.
 Also set `SCCACHE_S3_ACCESS_KEY_ID` and `SCCACHE_S3_SECRET_ACCESS_KEY` from the
 runner cache service in Gitea repository or organization secrets. These are needed
 for non-publishing builds too; Docker Hub credentials are only used for publishing.
 Run a non-publishing dispatch with
 `targets=x86/64,mediatek/filogic,octeon/generic` for the six validation combinations, then an
 approved publishing dispatch. Check the pulled image's Rust/Cargo installation
 and downstream package build before disabling the old Jenkins job and deleting
 `Jenkinsfile`. Do not retire Jenkins before this acceptance run. No Jenkins
 plugins or credentials are used by the new workflow.
 ## Persistent compilation cache
 The Gitea runner's `homelab-stacks/stacks/gitea-runner/compose.yaml` owns
 Redis and the SSD-backed RustFS S3 service. This repository only prepares its
 reusable BuildKit builder and consumes those services; jobs never deploy them.
 Redis is a 1 GiB LRU hot cache with a 1.5 GiB memory limit. It is deliberately
 non-persistent: S3 holds the persistent copies on a quota-limited 100 GiB Incus/ZFS
 volume. Jobs, Redis restarts, and RustFS container recreation do not clear it.
 The cache bucket expires objects after 30 days; the filesystem quota provides
 an independent hard limit. See the runner stack's README for allocation and setup.
 Each job runs its own pinned sccache 0.18.0 daemon with
 `SCCACHE_MULTILEVEL_CHAIN=redis,s3` in the standard server-side mode.
 Writes go to both tiers; Redis misses fall through to S3 and backfill Redis.
 `SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all` makes each cache write await both tiers.
 The installed `sccache-stats` helper waits up to 30 seconds for outstanding writes
 and backfills before printing statistics and stopping the job's daemon.
 BuildKit's layer cache remains separate.
 Both services join `gitea-runner-jobs`. SDK/package containers use
 `redis://sccache-redis:6379` and `http://sccache-s3:9000`; BuildKit's host network
 uses the loopback ports 6380 and 9000. There is no public ingress. S3 credentials
 are supplied through required BuildKit secret mounts for compilation and runtime
 environment variables for disposable verification/package containers. Credentials are not stored in the published SDK image. Runner endpoints are
 build arguments rather than default runtime settings. The image
 contains only public backend configuration: bucket `sccache`, region `us-east-1`,
 path-style HTTP, and cache prefixes. Other environments can override these values.
 Each OpenWrt release has a shared prefix across targets: `openwrt-rust:<version>:`
 in Redis and `openwrt-rust/<version>/` in S3. Compiler identities, target,
 flags, sources, and dependencies still determine cache keys, preventing
 incompatible reuse. The old Redis service and its volume remain untouched during
 this change; their entries are not copied into the new SSD cache.
 A single persistent sccache daemon was tested across containers with client-side
 compilation. The second container reported a cache error and recompiled: the
 daemon's disk backend returns an absolute cache-file path that the client must
 open, but the disposable container cannot see that filesystem. Version 0.18.0
 also fixes TCP connections to loopback, and client-side raw reads bypass
 multi-level backfilling and per-tier read statistics. Per-job daemons with remote
 storage avoid filesystem sharing, socket proxies, and job-network changes.
 RustFS and VersityGW were both tested with real Rust compilation, S3 fallback,
 Redis backfilling, container restarts, and differing targets/toolchains. Both
 passed. RustFS was selected following the owner's preference; the small storage
 benchmark does not establish a full-build speedup. Deploy the runner cache and
 initialize its bucket before running this workflow.
 The shared BuildKit worker executes one build step at a time, even across
 simultaneous dispatches. It is limited to eight CPUs and 20 GiB RAM with no
 swap. Make and Rust bootstrap both receive eight jobs; the original `-j1 V=sc`
 diagnostic retry also reduces bootstrap/Ninja to one job. Rust
 build-directory/archive cleanup remains. BuildKit
 layer storage has a 50 GiB GC budget. Matrix jobs also run one at a time, with
 `fail-fast: false` so a failed target does not stop the remaining builds.
 OpenWrt integration was inspected in the SDK's pinned packages feed:
 - `rust-values.mk` **unexports `RUSTC_WRAPPER`**. Setting it in the workflow
  alone does not enable caching. `CONFIG_RUST_SCCACHE=y` makes OpenWrt inject
  `CARGO_INCREMENTAL=0`, `RUSTC_WRAPPER=sccache`, and `SCCACHE_DIR` into host
  and package Cargo invocations. The Dockerfile enables and checks that option.
 - The base SDK has no sccache binary. We install the pinned 0.18.0 amd64-musl
  release and verify its SHA-256 before using it.
 - Rust bootstrap wraps rustc itself. We also set `build.ccache=sccache` in
  its configure arguments so stage-zero compilation and LLVM use sccache,
  retaining `llvm.download-ci-llvm=false`. The inherited wrapper is forwarded
  by bootstrap through `RUSTC_WRAPPER_REAL` where applicable.
 - sccache is a compiler cache, not a Cargo target-directory cache. Linked
  binaries, proc macros, some bootstrap work and incremental compilation are
  not cacheable. SDK bootstrap and downstream dependencies can share storage
  without necessarily sharing every compiled object. Every actual SDK build
  prints statistics on success or failure. BuildKit can reuse unchanged layers;
  a cached compilation layer does not run sccache or produce fresh statistics.
 Sources: [OpenWrt Rust configuration](https://github.com/openwrt/packages/blob/openwrt-24.10/lang/rust/rust-values.mk),
 [Rust bootstrap](https://github.com/rust-lang/rust/blob/1.90.0/src/bootstrap/src/core/build_steps/compile.rs),
 [sccache multi-level cache](https://github.com/mozilla/sccache/blob/v0.18.0/docs/MultiLevel.md),
 [sccache client-side architecture](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Architecture.md),
 [Rust cache limitations](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Rust.md).
 ## Downstream OpenWrt packages
 Use the same cache services and target/version image. Export `AWS_ACCESS_KEY_ID`
 and `AWS_SECRET_ACCESS_KEY` from CI secrets before starting the container. For example, after
 installing the `sqm-autorate-rust` feed/package in the SDK:
 ```sh
 docker run --rm --network gitea-runner-jobs --cpus 8 \
  --memory 20g --memory-swap 20g \
  -e SCCACHE_REDIS_ENDPOINT=redis://sccache-redis:6379 \
  -e SCCACHE_ENDPOINT=http://sccache-s3:9000 \
  -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
  lochnair/openwrt-sdk-rust:x86-64-24.10.8 sh -ec '
    # Install/update the downstream feed here before defconfig.
    make defconfig
    grep -qx "CONFIG_RUST_SCCACHE=y" .config
    sccache --start-server
    trap "sccache-stats; sccache --stop-server" EXIT
    sccache --zero-stats
    make package/sqm-autorate-rust/compile -j8 V=sc
  '
 ```
 Keep the image's cache prefix and enable `CONFIG_RUST_SCCACHE` after any
 downstream configuration replacement. Cache statistics belong to the current
 container's sccache server; `--zero-stats` does not erase cached objects.
 `sqm-autorate-rust` currently uses cross-rs to publish static binaries, not an
 OpenWrt SDK package workflow; that unrelated pipeline is unchanged. The package
 Makefile in `openwrt-feeds` already uses OpenWrt's `rust-package.mk`, so an SDK
 package job can use this cache without patching the package.
 For a local SDK build on the prepared runner, export `AWS_ACCESS_KEY_ID` and
 `AWS_SECRET_ACCESS_KEY` from the cache credentials in your environment first:
 ```sh
 sh ci/setup-builder.sh
 docker buildx build --builder openwrt-sdk-rust --load --progress plain \
  --network host --allow network.host \
  --build-arg SAFE_TARGET=x86-64 --build-arg VERSION=24.10.8 \
  --build-arg JOBS=8 \
  --build-arg SCCACHE_REDIS_ENDPOINT=redis://127.0.0.1:6380 \
  --build-arg SCCACHE_ENDPOINT=http://127.0.0.1:9000 \
  --secret id=aws-access-key-id,env=AWS_ACCESS_KEY_ID \
  --secret id=aws-secret-access-key,env=AWS_SECRET_ACCESS_KEY \
  -t lochnair/openwrt-sdk-rust:x86-64-24.10.8 .
 ```
 See [validation results](ci/VALIDATION.md) for the checks performed and the
 remaining cutover acceptance.
@@ -1,319 +0,0 @@
 # All-target matrix validation — 2026-10-13
 The Gitea workflow now defaults to `targets=all`. Target discovery reads every
 page of the public GHCR SDK tag list and selects each release's target tags,
 using release directory names to exclude architecture aliases. Architecture
 metadata comes from the release's `.targets.json`, with the matching OpenWrt
 source tag supplying `ARCH` for SDK-only targets absent from that JSON.
 Live generation and registry manifest checks produced:
 | OpenWrt release | Available target SDK images | Rust-supported matrix targets |
 | --- | ---: | ---: |
 | 25.12.5 (stable) | 94 | 93 |
 | 24.10.8 (oldstable) | 90 | 89 |
 | Total | 184 | 182 |
 Both releases exclude `ixp4xx/generic` (`armeb`). The architecture allowlist
 matches `RUST_ARCH_DEPENDS` at the released packages feed revisions:
 [25.12.5](https://github.com/openwrt/packages/blob/5caa62e0bc9f7fb9b0c12a23267bceb7724214dd/lang/rust/rust-values.mk)
 and [24.10.8](https://github.com/openwrt/packages/blob/23abaa6f3b0fdfd76b570031107e5718476ff0c8/lang/rust/rust-values.mk).
 All four Malta subtargets remain included for both releases. The older release
 also retains SDK-only `lantiq/ase`, `lantiq/xway_legacy`, `ramips/rt288x` and
 `realtek/rtl931x`.
 Validation passed:
 - All 182 generated base-image tags returned HTTP 200 and a manifest digest
  from GHCR. Matrix entries are unique; `GITHUB_OUTPUT` matches JSON stdout.
 - The explicit `x86/64,mediatek/filogic,octeon/generic` subset produces six
  jobs. Selecting `siflower/sf21` produces only its 25.12.5 job.
 - Six offline regression tests cover per-release availability, unsupported
  architectures, explicit selections and duplicates, invalid/empty selections,
  architecture aliases, SDK-only architecture fallback, missing metadata and
  registry pagination. The matrix job runs these tests before live discovery.
 - YAML parsing, actionlint 1.7.12 (excluding only the existing Gitea
  `builtin:checkout` syntax), ShellCheck and `git diff --check` pass.
 - Workflow assertions confirm `fail-fast: false`, unchanged `max-parallel: 1`,
  timeout, runner, build/verification commands and cache environment. Each job
  pushes after its own successful verification when `publish=true`.
 - Mock Docker execution checks publication and cleanup after a successful push,
  a failed push and an absent image. Cleanup runs with `always()` and removes
  only the job's SDK tag; it does not prune caches, volumes or other images.
 The Redis/S3 infrastructure, sccache configuration, eight-job compilation limit,
 eight-CPU/20 GiB builder limits and 50 GiB BuildKit GC budget are unchanged.
 This validation did not compile all SDKs or dispatch/publish images. Architecture
 eligibility and available base images do not guarantee every Rust bootstrap will
 succeed. Live workflow/publication acceptance remains pending; the legacy
 Jenkinsfile is retained unchanged until cutover acceptance.
 # Migration validation — 2026-10-10
 Inspected the original Jenkinsfile, Dockerfile, downstream `sqm-autorate-rust`
 workflows and package Makefile, Terraform runner definition, and live TheBeast
 runner configuration before changing the repository.
 ## Passed
 - YAML parsing; TOML parsing; `sh -n`; ShellCheck; `git diff --check`.
 - actionlint 1.7.12, excluding only its unsupported `builtin:checkout` action
  syntax. The workflow uses the existing `ubuntu-latest` label. Gitea's documentation and the
  existing downstream workflow confirm built-in checkout support.
 - Default matrix resolves six entries for stable 25.12.5 and oldstable 24.10.8.
  `targets=all` ran upstream target discovery inside the actual job image and
  resolved 194 entries. Empty/invalid targets fail. Every default base-image
  tag was read successfully from GHCR.
 - Docker Compose configuration accepted; Redis healthy with its named volume,
  AOF enabled and successful AOF writes. Service and bounded builder were
  provisioned on TheBeast without changing the runner configuration.
 - Builder inspected: eight-CPU quota, 20 GiB memory/swap limit, one BuildKit
  worker step, 50 GiB layer GC budget. A second disposable job container with
  fresh Docker client configuration reused the existing daemon and builder.
 - Dockerfile build checks completed. Two expected advisory warnings: mandatory
  base-image arguments have no defaults, and Docker's secret-name heuristic
  mistakes the public `SCCACHE_REDIS_KEY_PREFIX` for a secret.
 - `configured` Dockerfile stage built for x86/64 on both OpenWrt releases.
  Pinned feeds select Rust 1.94.0 on 25.12.5 and 1.90.0 on 24.10.8. Both retain
  `CONFIG_RUST_SCCACHE=y` through defconfig and contain the explicit bootstrap
  `build.ccache=sccache` setting with LLVM CI downloads disabled.
 - Actual OpenWrt make-variable expansion injects sccache into host and package
  compilation, despite unexporting an inherited wrapper.
 - Real Cargo library compilation in Docker BuildKit: one Redis miss. A fresh
  runtime container using the Docker-network endpoint: one hit, zero misses.
 - Recreated the newly provisioned Redis container using the same named volume;
  a fresh compiler container still recorded one hit, zero misses. No existing
  infrastructure containers were restarted.
 - Same source/cache namespace with GNU versus musl target: first musl build
  missed; repetition hit. Rust 1.90.0 versus 1.91.0: first 1.91 build missed;
  repetition hit. All probes reported zero cache read/write errors.
 - A disposable OpenWrt package used the real `Build/Compile/Cargo` helper,
  SDK GCC/musl linker, OpenWrt LTO/optimization flags and a local Rust library
  dependency. It built successfully in two separate containers: one dependency
  miss, then one hit. The linked executable correctly remained non-cacheable.
  This used a prebuilt Rust 1.90 toolchain for the probe, not a freshly compiled
  SDK toolchain; it is not a full SDK/source-bootstrap acceptance result.
 - Retained `ci/verify-cache.sh` passed its separate-container miss/hit checks
  against that prebuilt validation toolchain. Its original hardcoded SDK rustc
  path was later corrected after the live mediatek/filogic build exposed the
  target-specific staging layout; see the follow-up below.
 - Publication configuration checked: original Docker Hub namespace/tag format,
  verification before push, explicit publish input, temporary credential
  directory, no automatic publication trigger. No registry login or push was
  attempted.
 ## Remaining acceptance
 1. Enable repository Actions and configure the two Docker Hub secrets. No
   runner configuration changes are required.
 2. Complete a non-publishing Gitea dispatch for the full default matrix.
   The mediatek/filogic 24.10.8 job completed Rust source bootstrap and image
   export; its corrected verification passed on the built image. Acceptance
   for the other combinations remains pending.
 3. With explicit publication approval, run the publishing dispatch, pull the
   resulting tags and build the downstream OpenWrt package with the new image.
   Actual Docker Hub authorization and publication remain unverified.
 4. After acceptance, disable the old Jenkins job and remove `Jenkinsfile`.
   It is deliberately retained until the replacement passes these checks.
 Remote test logs and disposable probe sources are under
 `/tmp/openwrt-sdk-rust-migration` in TheBeast's `gitea-runner` instance. The
 Redis volume/service and BuildKit builder remain available for the replacement
 pipeline; no cache volume or existing infrastructure was deleted.
 ## Follow-up: mediatek/filogic 24.10.8 verification
 The supplied job 1602 log shows full SDK compilation and image export completed
 successfully. Verification failed with exit 127 because it assumed
 `staging_dir/host/bin/rustc`. Inspection of that exact built image reproduced
 the failure and located Rust/Cargo in
 `staging_dir/target-aarch64_cortex-a53_musl/host/bin` instead. Both the workflow
 and retained cache probe now resolve the unique executable compiler from the
 SDK staging tree, rather than assuming a shared host directory.
 Reran the corrected verification on that image: Rust 1.90.0 and Cargo 1.90.0
 execute successfully. The retained cache probe reports one Redis miss in its
 first container and one hit in a fresh second container, with zero cache
 read/write errors. ShellCheck, actionlint (excluding only Gitea's built-in
 checkout syntax), and `git diff --check` pass. No rebuild or publication was
 performed for this fix.
 ## Follow-up: mediatek/filogic 25.12.5 bootstrap concurrency
 Job 1650 failed inside LLVM compilation with
 `c++: fatal error: Killed signal terminated program cc1plus`. Its diagnostic
 Make retry still configured `LLVM_PARALLEL_COMPILE_JOBS=32`. The persistent
 builder's cgroup reports three OOM kills and its memory limit remains 20 GiB;
 the counter is cumulative, while the supplied log identifies the killed
 compiler in this job. Outer Make parallelism had not bounded Rust bootstrap's
 independent job count.
 The SDK Rust Makefile now passes `RUST_BUILD_JOBS` to `x.py --jobs`, defaulting
 to eight for subsequent SDK use. `build-rust` supplies eight (or the requested
 `JOBS`) on the first attempt and one on the serial diagnostic retry. Rust's
 bootstrap uses this job count for LLVM's compile pool and CMake build command.
 Built the corrected `configured` image for mediatek/filogic 25.12.5, and
 expanded the actual OpenWrt `Host/Compile` recipe at both job counts: it invokes
 `x.py --jobs=8` and `x.py --jobs=1` respectively. The same recipe checks pass
 against the 24.10.8 feed in a disposable container. ShellCheck, shell syntax,
 actionlint and diff checks pass. Full source compilation after this change has
 not been rerun, and no images were published or infrastructure limits changed.
 ## Follow-up: runner-owned Redis and SSD cache — 2026-10-11
 Initially prototyped a 1 GiB Redis LRU hot cache and Apache WebDAV backed by a
 persistent Docker volume on `fastpool`. This prototype was superseded by the
 S3 setup below after testing both requested alternatives. The SDK repository
 consumes these services and no longer deploys Redis. Production services have
 not been redeployed; the original Redis container and volume remain untouched.
 Inspected the pinned sccache 0.18.0 source and ran a cross-container central
 server experiment. With client-side compilation and an unshared disk backend,
 the first compile missed; the second reported a cache error and recompiled,
 because `IpcStorage` attempts to open the daemon's returned absolute cache path.
 TCP addresses are fixed to loopback in `net::SocketAddr::with_port`. In addition,
 client-side `get_raw` bypasses multi-level read statistics and backfilling. The
 standard per-job daemon mode with Redis/WebDAV avoids these restrictions.
 Writes in standard mode can still be outstanding when the compiler client
 returns. `SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all` awaits both tiers within a
 write; `sccache-stats` then drains writes and backfills for up to 30 seconds before
 printing statistics and stopping the job daemon. Its completion checks use the
 pinned version's cache-miss/write and per-tier backfill counters.
 Passed on TheBeast using an isolated `sccache-probe` Compose project:
 - Built the WebDAV image and validated Apache configuration. Real writes exposed
  the base Alpine image's missing APR DBM driver; the added Dockerfile installs
  `apr-util-dbm_gdbm`. Both services then passed health checks and real writes.
 - Retained CI verification: one miss writing both tiers, one Redis hit in a
  fresh container, then one WebDAV-only hit in a third fresh container. Zero
  cache errors, read/write errors, or per-tier write failures.
 - Real Rust compilation inside Docker BuildKit followed by a fresh SDK runtime
  container: miss, then hit, using host-loopback and Docker-DNS endpoints.
 - Repeated the unchanged BuildKit build: the actual compilation RUN step was
  cached and did not invoke sccache again.
 - Recreated both test service containers, retaining only the WebDAV volume:
  Redis started empty. A fresh SDK container hit WebDAV and backfilled Redis;
  the following fresh container hit Redis.
 - Identical source and prefix with GNU versus musl targets: musl missed, then
  hit. Rust 1.90.0 versus 1.94.0 under the same test prefix: 1.94.0 missed, then
  hit. All cache-error counters remained zero.
 - A disposable OpenWrt package compiled via the real `rust-package.mk`
  `Build/Compile/Cargo` helper and SDK musl linker in two fresh SDK containers:
  dependency miss, then hit. This used the actual completed 24.10.8 SDK image,
  not the earlier validation toolchain. The binary remained non-cacheable.
 - Proposed full runner Compose configuration, YAML parsing, Apache config,
  ShellCheck, shell syntax, Python syntax, actionlint, and diff checks passed.
  actionlint excludes only Gitea's built-in checkout syntax. Dockerfile checks
  exclude the public `KEY_PREFIX` secret-name heuristic and mandatory base
  image argument warning, as in the original validation.
 - Publishing remains manual, with the original image tags and verification
  before registry login/push. No login or publication was performed. Matrix,
  runner labels/capacity, BuildKit caching, and serial verbose retry are unchanged.
 Full SDK source bootstrap and a Gitea dispatch were not rerun for this cache
 change. Deploy/build the updated runner stack before using the changed workflow.
 The prototype WebDAV tier had no automatic LRU or capacity limit; the final
 S3 configuration below adds an independent hard filesystem bound and retention.
 Old Redis entries are not migrated; retiring the old service/volume is a separate
 approved cutover step. Test logs are under `/tmp/gitea-sccache-stack` and
 `/tmp/gitea-sccache-tests` on the runner; isolated test services are cleaned up.
 Sources: [sccache architecture](https://github.com/mozilla/sccache/blob/v0.18.0/docs/Architecture.md),
 [pinned IPC storage](https://github.com/mozilla/sccache/blob/v0.18.0/src/cache/ipc_storage.rs),
 [pinned multi-level implementation](https://github.com/mozilla/sccache/blob/v0.18.0/src/cache/multilevel.rs).
 ## Follow-up: RustFS and VersityGW comparison — 2026-10-11
 Selected **RustFS 1.0.1** after testing it and **VersityGW 1.8.0**, following the
 owner's preference. The final runner Compose configuration uses Redis/S3 and
 removes the uncommitted Apache image/config prototype. Services were tested in
 isolated projects, not deployed over the existing runner/cache services.
 Both candidates passed real sccache 0.18.0 compilation tests using the completed
 SDK images on TheBeast:
 - Cold miss wrote both tiers; fresh-container Redis hit; S3-only hit.
 - Service restart plus empty isolated Redis: S3 hit backfilled Redis, then the
  next fresh container hit Redis. Cache read/write/error counters stayed zero.
 - Same prefix/input with GNU versus musl targets: musl missed then hit. Rust
  1.90.0 versus 1.94.0: the different toolchain missed then hit.
 The objects live on a separate Incus ZFS volume with a 16 GiB hard quota for
 these tests. A separate 64 MiB quota probe rejected an 80 MiB incompressible write
 with `Disk quota exceeded`. No production data was used for the quota test.
 Production setup requires a new **100 GiB** quota-backed Incus volume; Compose
 requires an existing data directory and rejects a filesystem reporting more than
 100 GiB capacity. This guard passed on the quota-backed mount and rejected the
 unbounded SSD filesystem. Missing BuildKit credentials failed before compilation,
 and test logs contained no credential values. RustFS log files use a bounded tmpfs; Docker logs rotate.
 Storage microbenchmark: 64 objects of 1 MiB incompressible data, three passes,
 median aggregate MiB/s over host-loopback HTTP on the same SSD pool. Every GET
 was compared with its source bytes. These include client/protocol overhead and
 warm filesystem cache; they do not measure full SDK build speed or sustained
 cold-SSD throughput.
 | Backend | PUT, 1 client | GET, 1 client | PUT, 4 clients | GET, 4 clients |
 | --- | ---: | ---: | ---: | ---: |
 | Apache WebDAV prototype | 110.2 | 660.8 | failed | not run |
 | RustFS 1.0.1 | 101.5 | 314.7 | 231.2 | 471.2 |
 | VersityGW 1.8.0 | 99.1 | 329.3 | 232.2 | 410.2 |
 WebDAV's four-client PUT test returned HTTP 500 lock-database errors, despite
 passing sequential compiler-cache tests. Both S3 backends handled that load.
 Observed Docker memory usage after the probes was about 202 MiB for RustFS and
 28 MiB for VersityGW; these are small test workloads, not worst-case bounds.
 RustFS was chosen for preference and compatibility, not a proven performance
 advantage over VersityGW. Sequential S3 GETs were slower than the WebDAV baseline.
 Redis remains the primary hot tier.
 Final RustFS setup and SDK integration checks:
 - The entire proposed runner Compose configuration validates with dummy
  registration credentials. Only the cache services ran in the isolated project;
  both passed their real health checks. Bucket initialization with pinned official
  AWS CLI 2.37.12 passed. RustFS accepted and returned the exact configured
  30-day object expiration and one-day abandoned multipart-upload policy.
  Expiration execution itself was not tested by waiting 30 days.
 - Replaced the original test RustFS container with the proposed Compose service
  over the same retained quota-backed data: an earlier Rust artifact was still an
  S3 hit and backfilled a new Redis namespace in a fresh SDK container.
 - The retained `ci/verify-cache.sh` passed all five probes (cold writes, Redis hit,
  S3-only hit, backfill into a fresh Redis namespace, subsequent Redis hit) on
  x86-64/24.10.8 and mediatek-filogic/25.12.5. It never flushes shared Redis data.
 - Real BuildKit Rust compilation using the proposed required secret mounts,
  then a fresh runtime-container cache hit, passed. Secret values were absent
  from exported image configuration/history; subsequent layers and a fresh
  container had no credential environment variables or secret files. The repeated
  unchanged compilation layer was cached. Only a local disposable image was
  produced; no registry login/push occurred.
 - Actual OpenWrt `Build/Compile/Cargo` and SDK musl linker compiled a disposable
  package twice: dependency miss followed by hit in a fresh container. This
  confirms inherited S3 settings/credentials reach OpenWrt's sccache wrapper.
 - BuildKit checked the exact changed SDK Dockerfile. Only the known public
  `KEY_PREFIX` secret-name heuristic and required base-image argument warning were
  excluded. Workflow/actionlint, ShellCheck, shell/Python/JSON/YAML parsing and
  whitespace checks passed.
 The SDK image retains public S3 settings and version-only prefixes. Credentials are not stored in image configuration/history, and runner endpoints
 are build arguments rather than runtime defaults. Gitea secrets feed ephemeral BuildKit
 mounts and runtime verification containers. The runner service's registration,
 labels/capacity, build matrix, image tags, manual publication gate, BuildKit layer
 caching and serial verbose fallback are unchanged.
 Manual setup: allocate/mount the 100 GiB volume, set the two cache credentials in
 Komodo and Gitea, deploy the cache services, initialize the bucket, and run a
 non-publishing workflow before cutover. Full SDK bootstrap and live Gitea dispatch
 were not repeated for this backend change. Production Redis, its volume, and
 Jenkins remain untouched pending acceptance/approved retirement. Isolated test
 containers and volumes are removed after verification; logs remain under
 `/tmp/s3-cache-tests` on the runner.
 Sources: [RustFS deployment](https://docs.rustfs.com/en/installation/container/docker),
 [VersityGW Docker setup](https://github.com/versity/versitygw/wiki/Docker),
 [sccache S3](https://github.com/mozilla/sccache/blob/v0.18.0/docs/S3.md),
 [BuildKit secrets](https://docs.docker.com/build/building/secrets/).
@@ -1,17 +0,0 @@
 #!/bin/sh
 set -eu
 : "${SCCACHE_REDIS_ENDPOINT:?Set the persistent sccache Redis endpoint}"
 : "${SCCACHE_ENDPOINT:?Set the persistent sccache S3 endpoint}"
 : "${AWS_ACCESS_KEY_ID:?Supply the S3 access key through a BuildKit secret}"
 : "${AWS_SECRET_ACCESS_KEY:?Supply the S3 secret key through a BuildKit secret}"
 : "${JOBS:=8}"
 case "$JOBS" in ''|*[!0-9]*|0) echo 'JOBS must be a positive integer' >&2; exit 1 ;; esac
 sccache --start-server
 trap 'sccache-stats; sccache --stop-server' EXIT
 sccache --zero-stats
 # Bootstrap/Ninja does not inherit Make's job count. Bound both explicitly.
 make package/rust/host/compile -j"$JOBS" RUST_BUILD_JOBS="$JOBS" || \
    make package/rust/host/compile -j1 RUST_BUILD_JOBS=1 V=sc
 rm -rf build_dir/host/rustc-*/rustc-*/build dl/rustc-*.tar.*
@@ -1,6 +0,0 @@
 # Shared by all SDK jobs on this daemon, including simultaneous dispatches.
 [worker.oci]
  max-parallelism = 1
  gc = true
  reservedSpace = "10GB"
  maxUsedSpace = "50GB"
@@ -1,108 +0,0 @@
 #!/usr/bin/env python3
 """Resolve Rust-supported SDK tags for each stable/oldstable release."""
 import json
 import os
 import re
 import sys
 import urllib.parse
 import urllib.request
 # OpenWrt packages/lang/rust/rust-values.mk: RUST_ARCH_DEPENDS.
 RUST_ARCHES = {
    "aarch64", "arm", "i386", "loongarch64", "mips", "mips64",
    "mips64el", "mipsel", "powerpc", "powerpc64", "riscv64", "x86_64",
 }
 def fetch(url):
    with urllib.request.urlopen(url, timeout=60) as response:
        return response.read().decode()
 def sdk_tags():
    token = json.loads(fetch("https://ghcr.io/token?service=ghcr.io&scope=repository:openwrt/sdk:pull"))["token"]
    url = "https://ghcr.io/v2/openwrt/sdk/tags/list?n=1000"
    tags = set()
    while url:
        request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
        with urllib.request.urlopen(request, timeout=60) as response:
            tags.update(json.load(response)["tags"])
            link = response.headers.get("Link", "")
        next_page = re.search(r'<([^>]+)>;\s*rel="next"', link)
        url = urllib.parse.urljoin(url, next_page[1]) if next_page else None
    return tags
 def release_targets(version, tags):
    base = f"https://downloads.openwrt.org/releases/{version}"
    # Directory names distinguish target/subtarget tags from architecture aliases.
    boards = re.findall(r'href="([a-z0-9_+-]+)/"', fetch(f"{base}/targets/"))
    suffix = f"-{version}"
    targets = sorted({f"{board}/{tag[len(board) + 1:-len(suffix)]}"
                      for board in boards for tag in tags
                      if tag.startswith(f"{board}-") and tag.endswith(suffix)})
    if not targets:
        raise ValueError(f"No SDK images available for OpenWrt {version}")
    architectures = json.loads(fetch(f"{base}/.targets.json"))
    result = {}
    for target in targets:
        if not re.fullmatch(r"[a-z0-9_+-]+/[a-z0-9_+-]+", target):
            raise ValueError(f"Invalid SDK target: {target!r}")
        if target in architectures:
            arch = architectures[target]
            arch = arch if arch in RUST_ARCHES else arch.split("_", 1)[0]
        else:
            # SDK-only targets (e.g. Malta) have no profiles in .targets.json.
            board, subtarget = target.split("/")
            source = f"https://raw.githubusercontent.com/openwrt/openwrt/v{version}/target/linux"
            definition = fetch(f"{source}/{board}/{subtarget}/target.mk")
            match = re.search(r"^ARCH\s*:?=\s*([a-z0-9_]+)\s*$", definition, re.MULTILINE)
            if not match:
                definition = fetch(f"{source}/{board}/Makefile")
                match = re.search(r"^ARCH\s*:?=\s*([a-z0-9_]+)\s*$", definition, re.MULTILINE)
            if not match:
                raise ValueError(f"Cannot determine architecture for {target} on {version}")
            arch = match[1]
        result[target] = arch
    return result
 def resolve(versions, targets, tags):
    versions = [versions[k] for k in ("stable_version", "oldstable_version") if versions.get(k)]
    if not versions:
        raise ValueError("OpenWrt returned no stable or oldstable version")
    targets = [t.strip() for t in targets.split(",") if t.strip()]
    if not targets:
        raise ValueError("No targets selected")
    if "all" in targets and targets != ["all"]:
        raise ValueError("Select all or a comma-separated target list")
    for target in targets:
        if target != "all" and not re.fullmatch(r"[a-z0-9_+-]+/[a-z0-9_+-]+", target):
            raise ValueError(f"Invalid target: {target!r}")
    for version in versions:
        if not re.fullmatch(r"[a-zA-Z0-9_.-]+", version):
            raise ValueError(f"Invalid version: {version!r}")
    matrix = []
    for version in dict.fromkeys(versions):
        available = release_targets(version, tags)
        selected = available if targets == ["all"] else dict.fromkeys(targets)
        supported = [t for t in selected if available.get(t) in RUST_ARCHES]
        skipped = [t for t in selected if t not in supported]
        print(f"OpenWrt {version}: {len(supported)} targets selected; "
              f"{len(available)} SDK images available", file=sys.stderr)
        if skipped:
            print(f"Skipped unavailable or Rust-unsupported targets: {', '.join(skipped)}", file=sys.stderr)
        matrix.extend({"target": t.replace("/", "-"), "version": version} for t in supported)
    if not matrix:
        raise ValueError("No available Rust-supported SDK targets selected")
    return {"include": matrix}
 if __name__ == "__main__":
    versions = json.loads(fetch("https://downloads.openwrt.org/.versions.json"))
    targets = os.environ.get("TARGETS", "all")
    matrix = json.dumps(resolve(versions, targets, sdk_tags()), separators=(",", ":"))
    print(matrix)
    with open(os.environ["GITHUB_OUTPUT"], "a") as output:
        output.write(f"matrix={matrix}\n")
@@ -1,26 +0,0 @@
 #!/usr/bin/env python3
 """Wait for this job's cache writes/backfills, then print complete statistics."""
 import json
 import subprocess
 import time
 end = time.monotonic() + 30
 while True:
    data = json.loads(subprocess.check_output(
        ["sccache", "--show-stats", "--stats-format", "json"],
        timeout=max(0.1, end - time.monotonic()),
    ))
    stats = data["stats"]
    misses = sum(stats["cache_misses"]["counts"].values())
    levels = stats.get("multi_level") or []
    writes_done = stats["cache_writes"] + stats["cache_write_errors"] >= misses
    backfills_done = sum(x["backfills_to"] for x in levels) >= sum(
        x["backfills_from"] for x in levels
    )
    if writes_done and backfills_done:
        break
    if time.monotonic() >= end:
        subprocess.run(["sccache", "--show-stats"], check=True, timeout=5)
        raise SystemExit("Timed out waiting for sccache writes/backfills")
    time.sleep(0.1)
 subprocess.run(["sccache", "--show-stats"], check=True, timeout=5)
@@ -1,10 +0,0 @@
 #!/bin/sh
 set -eu
 if ! docker buildx inspect openwrt-sdk-rust >/dev/null 2>&1; then
    docker buildx create --name openwrt-sdk-rust --driver docker-container \
        --driver-opt network=host,memory=20g,memory-swap=20g,cpu-period=100000,cpu-quota=800000 \
        --buildkitd-config ci/buildkitd.toml \
        --buildkitd-flags '--allow-insecure-entitlement network.host' || \
        docker buildx inspect openwrt-sdk-rust
 fi
 docker buildx inspect openwrt-sdk-rust --bootstrap
@@ -1,92 +0,0 @@
 #!/usr/bin/env python3
 """Offline checks for release-specific SDK selection and registry pagination."""
 import contextlib
 import io
 import json
 import unittest
 from unittest.mock import patch
 import matrix
 class MatrixTests(unittest.TestCase):
    versions = {"stable_version": "25.12.5", "oldstable_version": "24.10.8"}
    def resolve(self, targets):
        releases = {
            "25.12.5": {"x86/64": "x86_64", "siflower/sf21": "riscv64", "ixp4xx/generic": "armeb"},
            "24.10.8": {"x86/64": "x86_64", "malta/be64": "mips64", "arc/generic": "arc"},
        }
        with patch.object(matrix, "release_targets", side_effect=lambda v, tags: releases[v]):
            with contextlib.redirect_stderr(io.StringIO()):
                return matrix.resolve(self.versions, targets, set())["include"]
    def test_all_uses_each_releases_available_supported_sdks(self):
        self.assertEqual(self.resolve("all"), [
            {"target": "x86-64", "version": "25.12.5"},
            {"target": "siflower-sf21", "version": "25.12.5"},
            {"target": "x86-64", "version": "24.10.8"},
            {"target": "malta-be64", "version": "24.10.8"},
        ])
    def test_explicit_targets_deduplicate_and_skip_unavailable_combinations(self):
        self.assertEqual(self.resolve("siflower/sf21,x86/64,x86/64,ixp4xx/generic"), [
            {"target": "siflower-sf21", "version": "25.12.5"},
            {"target": "x86-64", "version": "25.12.5"},
            {"target": "x86-64", "version": "24.10.8"},
        ])
    def test_invalid_or_empty_selection_fails(self):
        for targets in ("", "../bad", "x86/64;echo", "all,x86/64", "ixp4xx/generic", "missing/generic"):
            with self.subTest(targets=targets), self.assertRaises(ValueError):
                self.resolve(targets)
        for versions in ({}, {"stable_version": "../bad"}):
            with self.assertRaises(ValueError):
                matrix.resolve(versions, "all", set())
    def test_discovery_excludes_aliases_and_other_releases_and_keeps_sdk_only_targets(self):
        base = "https://downloads.openwrt.org/releases/25.12.5"
        source = "https://raw.githubusercontent.com/openwrt/openwrt/v25.12.5/target/linux"
        responses = {
            f"{base}/targets/": '<a href="x86/">x86</a><a href="ixp4xx/">ixp4xx</a>'
                                 '<a href="malta/">malta</a><a href="ramips/">ramips</a>',
            f"{base}/.targets.json": json.dumps({"x86/64": "x86_64", "ixp4xx/generic": "armeb_xscale"}),
            f"{source}/malta/be64/target.mk": "ARCH:=mips64\n",
            f"{source}/ramips/rt288x/target.mk": "SUBTARGET:=rt288x\n",
            f"{source}/ramips/Makefile": "ARCH:=mipsel\n",
        }
        tags = {"x86-64-25.12.5", "ixp4xx-generic-25.12.5", "malta-be64-25.12.5",
                "ramips-rt288x-25.12.5", "x86_64-25.12.5", "arm_cortex-a7-25.12.5",
                "x86-generic-24.10.8", "x86-64-v25.12.5"}
        with patch.object(matrix, "fetch", side_effect=lambda url: responses[url]):
            self.assertEqual(matrix.release_targets("25.12.5", tags), {
                "ixp4xx/generic": "armeb", "malta/be64": "mips64",
                "ramips/rt288x": "mipsel", "x86/64": "x86_64",
            })
    def test_missing_images_or_unknown_architecture_fails(self):
        with patch.object(matrix, "fetch", return_value='<a href="malta/">malta</a>'):
            with self.assertRaisesRegex(ValueError, "No SDK images"):
                matrix.release_targets("25.12.5", set())
        def response(url):
            if url.endswith("/targets/"):
                return '<a href="malta/">malta</a>'
            return "{}" if url.endswith(".json") else "SUBTARGET:=be\n"
        with patch.object(matrix, "fetch", side_effect=response):
            with self.assertRaisesRegex(ValueError, "Cannot determine architecture"):
                matrix.release_targets("25.12.5", {"malta-be-25.12.5"})
    def test_registry_pagination(self):
        first = io.BytesIO(b'{"tags":["x86-64-25.12.5"]}')
        first.headers = {"Link": '</v2/openwrt/sdk/tags/list?last=x86-64-25.12.5&n=1000>; rel="next"'}
        second = io.BytesIO(b'{"tags":["malta-be-24.10.8"]}')
        second.headers = {}
        with patch.object(matrix, "fetch", return_value='{"token":"public-pull-token"}'):
            with patch.object(matrix.urllib.request, "urlopen", side_effect=[first, second]) as opened:
                self.assertEqual(matrix.sdk_tags(), {"x86-64-25.12.5", "malta-be-24.10.8"})
        self.assertEqual(opened.call_args_list[1].args[0].full_url,
                         "https://ghcr.io/v2/openwrt/sdk/tags/list?last=x86-64-25.12.5&n=1000")
 if __name__ == "__main__":
    unittest.main()
@@ -1,67 +0,0 @@
 #!/bin/sh
 # Verify writes, Redis/S3 hits, and Redis backfilling in fresh containers.
 set -eu
 image=${1:?Usage: verify-cache.sh IMAGE}
 : "${SCCACHE_REDIS_ENDPOINT:=redis://sccache-redis:6379}"
 : "${SCCACHE_ENDPOINT:=http://sccache-s3:9000}"
 : "${AWS_ACCESS_KEY_ID:?Set the S3 access key}"
 : "${AWS_SECRET_ACCESS_KEY:?Set the S3 secret key}"
 stats_script=$(cat ci/sccache-stats.py)
 nonce=$(mktemp -d)
 trap 'rmdir "$nonce"' EXIT
 prefix="validation/${image##*:}/$(basename "$nonce")/"
 for expected in miss hit s3-hit backfill backfill-hit; do
    redis_prefix=$prefix
    case "$expected" in backfill*) redis_prefix="${prefix}backfill/" ;; esac
    chain=redis,s3
    if [ "$expected" = s3-hit ]; then chain=s3; fi
    docker run --rm -i --network gitea-runner-jobs --cpus 2 \
        --memory 2g --memory-swap 2g \
        -e SCCACHE_REDIS_ENDPOINT="$SCCACHE_REDIS_ENDPOINT" \
        -e SCCACHE_ENDPOINT="$SCCACHE_ENDPOINT" \
        -e SCCACHE_REDIS_KEY_PREFIX="$redis_prefix" \
        -e SCCACHE_S3_KEY_PREFIX="$prefix" \
        -e SCCACHE_BUCKET=sccache -e SCCACHE_REGION=us-east-1 \
        -e SCCACHE_S3_USE_SSL=false \
        -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
        -e SCCACHE_MULTILEVEL_CHAIN="$chain" \
        -e SCCACHE_MULTILEVEL_WRITE_ERROR_POLICY=all \
        -e SCCACHE_CLIENT_SIDE=0 \
        -e SCCACHE_STATS_SCRIPT="$stats_script" \
        -e EXPECTED="$expected" "$image" sh -es <<'PROBE'
 rustc=$(find /builder/staging_dir -type f -path '*/host/bin/rustc')
 test -x "$rustc"
 mkdir /tmp/sccache-probe
 cd /tmp/sccache-probe
 printf 'pub fn probe(n: u64) -> u64 { n.wrapping_mul(17) }\n' > probe.rs
 sccache --start-server
 trap 'sccache --stop-server' EXIT
 sccache --zero-stats
 sccache "$rustc" --crate-name ci_sccache_probe \
    --crate-type rlib --emit link,dep-info --out-dir . probe.rs
 python3 -c "$SCCACHE_STATS_SCRIPT"
 sccache --show-stats --stats-format json > stats.json
 python3 - <<'PY'
 import json
 import os
 data = json.load(open("stats.json"))
 assert data["cache_location"].startswith("Multi-level"), data
 stats = data["stats"]
 expected = os.environ["EXPECTED"]
 field = "cache_misses" if expected == "miss" else "cache_hits"
 assert sum(stats[field]["counts"].values()) == 1, data
 assert stats["cache_read_errors"] == stats["cache_write_errors"] == 0, data
 assert sum(stats["cache_errors"]["counts"].values()) == 0, data
 levels = stats["multi_level"]
 assert all(level["write_failures"] == 0 for level in levels), data
 if expected == "miss":
    assert len(levels) == 2 and all(level["writes"] == 1 for level in levels), data
 elif expected == "backfill":
    assert levels[1]["hits"] == levels[0]["backfills_to"] == 1, data
 else:
    assert levels[0]["hits"] == 1, data
 print("Verified persistent cache", expected)
 PY
 PROBE
 done
@@ -0,0 +1,6 @@
 {
  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
  "extends": [
    "config:recommended"
  ]
 }