diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 813c8a7..ec6d991 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -5,7 +5,7 @@ on: inputs: targets: description: 'Comma-separated targets, or all' - default: 'x86/64,mediatek/filogic,octeon/generic' + default: 'all' required: true type: string publish: @@ -27,14 +27,16 @@ jobs: id: resolve env: TARGETS: ${{ inputs.targets }} - run: python3 ci/matrix.py + run: | + python3 -m unittest discover -s ci -p 'test_matrix.py' + python3 ci/matrix.py build: needs: matrix runs-on: ubuntu-latest timeout-minutes: 360 strategy: - fail-fast: true + fail-fast: false max-parallel: 1 matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} env: @@ -70,7 +72,7 @@ jobs: ' sh ci/verify-cache.sh "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}" - name: Publish Docker Hub tag - if: inputs.publish == true + if: ${{ success() && inputs.publish == true }} env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} @@ -82,3 +84,10 @@ jobs: trap 'rm -rf "$DOCKER_CONFIG"' EXIT printf '%s' "$DOCKERHUB_TOKEN" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin docker push "lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}" + - name: Remove local SDK image + if: always() + run: | + image="lochnair/openwrt-sdk-rust:${SAFE_TARGET}-${VERSION}" + if docker image inspect "$image" >/dev/null 2>&1; then + docker image rm "$image" + fi diff --git a/README.md b/README.md index 58ed7f7..26498dc 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,13 @@ Builds the OpenWrt SDK with its Rust host toolchain installed. Gitea Actions resolves `stable_version` and `oldstable_version` from OpenWrt's `.versions.json`. -The default targets remain `x86/64,mediatek/filogic,octeon/generic`; dispatch with -`targets=all` to use upstream `dump-target-info.pl`, or supply a comma-separated -list. Images still use `ghcr.io/openwrt/sdk:-` and +The default `targets=all` selects the SDK tags actually available in GHCR for +each release, excluding architectures unsupported by OpenWrt's Rust package. +Release metadata supplies architectures, with release source definitions for +SDK-only targets omitted from `.targets.json`. Alternatively, supply a +comma-separated target list; unavailable or unsupported combinations are skipped +and reported, and an empty matrix fails. Images still use +`ghcr.io/openwrt/sdk:-` and publish to `lochnair/openwrt-sdk-rust:-` on Docker Hub. No `latest` tag or additional registry is introduced. @@ -24,12 +28,17 @@ triggers. `publish=false` is the safe validation default. Dispatch directory, and each successfully built and checked image is pushed under its original tag. Before any push, `ci/verify-cache.sh` uses the installed SDK rustc in fresh containers and requires writes to both tiers, Redis/S3 hits, and backfilling. +Each matrix job publishes independently after its own verification succeeds. +Its local SDK image is removed at the end, including after verification or push +failure and during non-publishing runs. Cleanup leaves the shared builder and +compilation caches intact. Also set `SCCACHE_S3_ACCESS_KEY_ID` and `SCCACHE_S3_SECRET_ACCESS_KEY` from the runner cache service in Gitea repository or organization secrets. These are needed for non-publishing builds too; Docker Hub credentials are only used for publishing. -Run a non-publishing dispatch for the default six combinations, then an +Run a non-publishing dispatch with +`targets=x86/64,mediatek/filogic,octeon/generic` for the six validation combinations, then an approved publishing dispatch. Check the pulled image's Rust/Cargo installation and downstream package build before disabling the old Jenkins job and deleting `Jenkinsfile`. Do not retire Jenkins before this acceptance run. No Jenkins @@ -88,8 +97,8 @@ simultaneous dispatches. It is limited to eight CPUs and 20 GiB RAM with no swap. Make and Rust bootstrap both receive eight jobs; the original `-j1 V=sc` diagnostic retry also reduces bootstrap/Ninja to one job. Rust build-directory/archive cleanup remains. BuildKit -layer storage has a 50 GiB GC budget. Matrix jobs also run one at a time and -stop scheduling on a failure, as Jenkins' sequential stage loop did. +layer storage has a 50 GiB GC budget. Matrix jobs also run one at a time, with +`fail-fast: false` so a failed target does not stop the remaining builds. OpenWrt integration was inspected in the SDK's pinned packages feed: diff --git a/ci/VALIDATION.md b/ci/VALIDATION.md index 7fa4023..e2be2ad 100644 --- a/ci/VALIDATION.md +++ b/ci/VALIDATION.md @@ -1,3 +1,53 @@ +# All-target matrix validation — 2026-10-13 + +The Gitea workflow now defaults to `targets=all`. Target discovery reads every +page of the public GHCR SDK tag list and selects each release's target tags, +using release directory names to exclude architecture aliases. Architecture +metadata comes from the release's `.targets.json`, with the matching OpenWrt +source tag supplying `ARCH` for SDK-only targets absent from that JSON. + +Live generation and registry manifest checks produced: + +| OpenWrt release | Available target SDK images | Rust-supported matrix targets | +| --- | ---: | ---: | +| 25.12.5 (stable) | 94 | 93 | +| 24.10.8 (oldstable) | 90 | 89 | +| Total | 184 | 182 | + +Both releases exclude `ixp4xx/generic` (`armeb`). The architecture allowlist +matches `RUST_ARCH_DEPENDS` at the released packages feed revisions: +[25.12.5](https://github.com/openwrt/packages/blob/5caa62e0bc9f7fb9b0c12a23267bceb7724214dd/lang/rust/rust-values.mk) +and [24.10.8](https://github.com/openwrt/packages/blob/23abaa6f3b0fdfd76b570031107e5718476ff0c8/lang/rust/rust-values.mk). +All four Malta subtargets remain included for both releases. The older release +also retains SDK-only `lantiq/ase`, `lantiq/xway_legacy`, `ramips/rt288x` and +`realtek/rtl931x`. + +Validation passed: + +- All 182 generated base-image tags returned HTTP 200 and a manifest digest + from GHCR. Matrix entries are unique; `GITHUB_OUTPUT` matches JSON stdout. +- The explicit `x86/64,mediatek/filogic,octeon/generic` subset produces six + jobs. Selecting `siflower/sf21` produces only its 25.12.5 job. +- Six offline regression tests cover per-release availability, unsupported + architectures, explicit selections and duplicates, invalid/empty selections, + architecture aliases, SDK-only architecture fallback, missing metadata and + registry pagination. The matrix job runs these tests before live discovery. +- YAML parsing, actionlint 1.7.12 (excluding only the existing Gitea + `builtin:checkout` syntax), ShellCheck and `git diff --check` pass. +- Workflow assertions confirm `fail-fast: false`, unchanged `max-parallel: 1`, + timeout, runner, build/verification commands and cache environment. Each job + pushes after its own successful verification when `publish=true`. +- Mock Docker execution checks publication and cleanup after a successful push, + a failed push and an absent image. Cleanup runs with `always()` and removes + only the job's SDK tag; it does not prune caches, volumes or other images. + +The Redis/S3 infrastructure, sccache configuration, eight-job compilation limit, +eight-CPU/20 GiB builder limits and 50 GiB BuildKit GC budget are unchanged. +This validation did not compile all SDKs or dispatch/publish images. Architecture +eligibility and available base images do not guarantee every Rust bootstrap will +succeed. Live workflow/publication acceptance remains pending; the legacy +Jenkinsfile is retained unchanged until cutover acceptance. + # Migration validation — 2026-10-10 Inspected the original Jenkinsfile, Dockerfile, downstream `sqm-autorate-rust` diff --git a/ci/matrix.py b/ci/matrix.py index 3754704..be4ca99 100644 --- a/ci/matrix.py +++ b/ci/matrix.py @@ -1,40 +1,108 @@ #!/usr/bin/env python3 -"""Resolve the same stable/oldstable x target matrix as Jenkins.""" +"""Resolve Rust-supported SDK tags for each stable/oldstable release.""" import json import os import re -import subprocess -import tempfile +import sys +import urllib.parse import urllib.request -def resolve(versions, targets): +# OpenWrt packages/lang/rust/rust-values.mk: RUST_ARCH_DEPENDS. +RUST_ARCHES = { + "aarch64", "arm", "i386", "loongarch64", "mips", "mips64", + "mips64el", "mipsel", "powerpc", "powerpc64", "riscv64", "x86_64", +} + + +def fetch(url): + with urllib.request.urlopen(url, timeout=60) as response: + return response.read().decode() + + +def sdk_tags(): + token = json.loads(fetch("https://ghcr.io/token?service=ghcr.io&scope=repository:openwrt/sdk:pull"))["token"] + url = "https://ghcr.io/v2/openwrt/sdk/tags/list?n=1000" + tags = set() + while url: + request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + with urllib.request.urlopen(request, timeout=60) as response: + tags.update(json.load(response)["tags"]) + link = response.headers.get("Link", "") + next_page = re.search(r'<([^>]+)>;\s*rel="next"', link) + url = urllib.parse.urljoin(url, next_page[1]) if next_page else None + return tags + + +def release_targets(version, tags): + base = f"https://downloads.openwrt.org/releases/{version}" + # Directory names distinguish target/subtarget tags from architecture aliases. + boards = re.findall(r'href="([a-z0-9_+-]+)/"', fetch(f"{base}/targets/")) + suffix = f"-{version}" + targets = sorted({f"{board}/{tag[len(board) + 1:-len(suffix)]}" + for board in boards for tag in tags + if tag.startswith(f"{board}-") and tag.endswith(suffix)}) + if not targets: + raise ValueError(f"No SDK images available for OpenWrt {version}") + architectures = json.loads(fetch(f"{base}/.targets.json")) + result = {} + for target in targets: + if not re.fullmatch(r"[a-z0-9_+-]+/[a-z0-9_+-]+", target): + raise ValueError(f"Invalid SDK target: {target!r}") + if target in architectures: + arch = architectures[target] + arch = arch if arch in RUST_ARCHES else arch.split("_", 1)[0] + else: + # SDK-only targets (e.g. Malta) have no profiles in .targets.json. + board, subtarget = target.split("/") + source = f"https://raw.githubusercontent.com/openwrt/openwrt/v{version}/target/linux" + definition = fetch(f"{source}/{board}/{subtarget}/target.mk") + match = re.search(r"^ARCH\s*:?=\s*([a-z0-9_]+)\s*$", definition, re.MULTILINE) + if not match: + definition = fetch(f"{source}/{board}/Makefile") + match = re.search(r"^ARCH\s*:?=\s*([a-z0-9_]+)\s*$", definition, re.MULTILINE) + if not match: + raise ValueError(f"Cannot determine architecture for {target} on {version}") + arch = match[1] + result[target] = arch + return result + + +def resolve(versions, targets, tags): versions = [versions[k] for k in ("stable_version", "oldstable_version") if versions.get(k)] if not versions: raise ValueError("OpenWrt returned no stable or oldstable version") targets = [t.strip() for t in targets.split(",") if t.strip()] if not targets: raise ValueError("No targets selected") + if "all" in targets and targets != ["all"]: + raise ValueError("Select all or a comma-separated target list") for target in targets: - if not re.fullmatch(r"[a-z0-9_+-]+/[a-z0-9_+-]+", target): + if target != "all" and not re.fullmatch(r"[a-z0-9_+-]+/[a-z0-9_+-]+", target): raise ValueError(f"Invalid target: {target!r}") for version in versions: if not re.fullmatch(r"[a-zA-Z0-9_.-]+", version): raise ValueError(f"Invalid version: {version!r}") - return {"include": [{"target": t.replace("/", "-"), "version": v} - for v in versions for t in dict.fromkeys(targets)]} + matrix = [] + for version in dict.fromkeys(versions): + available = release_targets(version, tags) + selected = available if targets == ["all"] else dict.fromkeys(targets) + supported = [t for t in selected if available.get(t) in RUST_ARCHES] + skipped = [t for t in selected if t not in supported] + print(f"OpenWrt {version}: {len(supported)} targets selected; " + f"{len(available)} SDK images available", file=sys.stderr) + if skipped: + print(f"Skipped unavailable or Rust-unsupported targets: {', '.join(skipped)}", file=sys.stderr) + matrix.extend({"target": t.replace("/", "-"), "version": version} for t in supported) + if not matrix: + raise ValueError("No available Rust-supported SDK targets selected") + return {"include": matrix} if __name__ == "__main__": - with urllib.request.urlopen("https://downloads.openwrt.org/.versions.json", timeout=60) as response: - versions = json.load(response) - targets = os.environ.get("TARGETS", "x86/64,mediatek/filogic,octeon/generic") - if targets.strip() == "all": - with tempfile.TemporaryDirectory(prefix="openwrt-targets-") as source: - subprocess.run(["git", "clone", "--depth", "1", "https://github.com/openwrt/openwrt.git", source], check=True) - output = subprocess.check_output(["perl", "scripts/dump-target-info.pl", "targets"], cwd=source, text=True) - targets = ",".join(line.split()[0] for line in output.splitlines() if line.strip()) - matrix = json.dumps(resolve(versions, targets), separators=(",", ":")) + versions = json.loads(fetch("https://downloads.openwrt.org/.versions.json")) + targets = os.environ.get("TARGETS", "all") + matrix = json.dumps(resolve(versions, targets, sdk_tags()), separators=(",", ":")) print(matrix) with open(os.environ["GITHUB_OUTPUT"], "a") as output: output.write(f"matrix={matrix}\n") diff --git a/ci/test_matrix.py b/ci/test_matrix.py new file mode 100644 index 0000000..a8c22de --- /dev/null +++ b/ci/test_matrix.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Offline checks for release-specific SDK selection and registry pagination.""" +import contextlib +import io +import json +import unittest +from unittest.mock import patch + +import matrix + + +class MatrixTests(unittest.TestCase): + versions = {"stable_version": "25.12.5", "oldstable_version": "24.10.8"} + + def resolve(self, targets): + releases = { + "25.12.5": {"x86/64": "x86_64", "siflower/sf21": "riscv64", "ixp4xx/generic": "armeb"}, + "24.10.8": {"x86/64": "x86_64", "malta/be64": "mips64", "arc/generic": "arc"}, + } + with patch.object(matrix, "release_targets", side_effect=lambda v, tags: releases[v]): + with contextlib.redirect_stderr(io.StringIO()): + return matrix.resolve(self.versions, targets, set())["include"] + + def test_all_uses_each_releases_available_supported_sdks(self): + self.assertEqual(self.resolve("all"), [ + {"target": "x86-64", "version": "25.12.5"}, + {"target": "siflower-sf21", "version": "25.12.5"}, + {"target": "x86-64", "version": "24.10.8"}, + {"target": "malta-be64", "version": "24.10.8"}, + ]) + + def test_explicit_targets_deduplicate_and_skip_unavailable_combinations(self): + self.assertEqual(self.resolve("siflower/sf21,x86/64,x86/64,ixp4xx/generic"), [ + {"target": "siflower-sf21", "version": "25.12.5"}, + {"target": "x86-64", "version": "25.12.5"}, + {"target": "x86-64", "version": "24.10.8"}, + ]) + + def test_invalid_or_empty_selection_fails(self): + for targets in ("", "../bad", "x86/64;echo", "all,x86/64", "ixp4xx/generic", "missing/generic"): + with self.subTest(targets=targets), self.assertRaises(ValueError): + self.resolve(targets) + for versions in ({}, {"stable_version": "../bad"}): + with self.assertRaises(ValueError): + matrix.resolve(versions, "all", set()) + + def test_discovery_excludes_aliases_and_other_releases_and_keeps_sdk_only_targets(self): + base = "https://downloads.openwrt.org/releases/25.12.5" + source = "https://raw.githubusercontent.com/openwrt/openwrt/v25.12.5/target/linux" + responses = { + f"{base}/targets/": 'x86ixp4xx' + 'maltaramips', + f"{base}/.targets.json": json.dumps({"x86/64": "x86_64", "ixp4xx/generic": "armeb_xscale"}), + f"{source}/malta/be64/target.mk": "ARCH:=mips64\n", + f"{source}/ramips/rt288x/target.mk": "SUBTARGET:=rt288x\n", + f"{source}/ramips/Makefile": "ARCH:=mipsel\n", + } + tags = {"x86-64-25.12.5", "ixp4xx-generic-25.12.5", "malta-be64-25.12.5", + "ramips-rt288x-25.12.5", "x86_64-25.12.5", "arm_cortex-a7-25.12.5", + "x86-generic-24.10.8", "x86-64-v25.12.5"} + with patch.object(matrix, "fetch", side_effect=lambda url: responses[url]): + self.assertEqual(matrix.release_targets("25.12.5", tags), { + "ixp4xx/generic": "armeb", "malta/be64": "mips64", + "ramips/rt288x": "mipsel", "x86/64": "x86_64", + }) + + def test_missing_images_or_unknown_architecture_fails(self): + with patch.object(matrix, "fetch", return_value='malta'): + with self.assertRaisesRegex(ValueError, "No SDK images"): + matrix.release_targets("25.12.5", set()) + def response(url): + if url.endswith("/targets/"): + return 'malta' + return "{}" if url.endswith(".json") else "SUBTARGET:=be\n" + with patch.object(matrix, "fetch", side_effect=response): + with self.assertRaisesRegex(ValueError, "Cannot determine architecture"): + matrix.release_targets("25.12.5", {"malta-be-25.12.5"}) + + def test_registry_pagination(self): + first = io.BytesIO(b'{"tags":["x86-64-25.12.5"]}') + first.headers = {"Link": '; rel="next"'} + second = io.BytesIO(b'{"tags":["malta-be-24.10.8"]}') + second.headers = {} + with patch.object(matrix, "fetch", return_value='{"token":"public-pull-token"}'): + with patch.object(matrix.urllib.request, "urlopen", side_effect=[first, second]) as opened: + self.assertEqual(matrix.sdk_tags(), {"x86-64-25.12.5", "malta-be-24.10.8"}) + self.assertEqual(opened.call_args_list[1].args[0].full_url, + "https://ghcr.io/v2/openwrt/sdk/tags/list?last=x86-64-25.12.5&n=1000") + + +if __name__ == "__main__": + unittest.main()