diff --git a/.github/ORDINARY_RUNNER_IMAGES.md b/.github/ORDINARY_RUNNER_IMAGES.md new file mode 100644 index 00000000000..4c15197e1e1 --- /dev/null +++ b/.github/ORDINARY_RUNNER_IMAGES.md @@ -0,0 +1,37 @@ +# Coexisting ordinary AMD64 runner images + +The selector retains the existing generic ordinary labels only for the exact +already-provisioned AMD64 contract `d272d01bcf3dfa620bbab1e9f31c3e1987862d33ec876bbad83c80f29de41a58`. +For any different AMD64 manifest (including a recipe-only change), branch pushes +and PRs that do not change that manifest require Linux/X64 plus: + +`platform-image-manifest--` + +This is an ordinary pool label, not a candidate label or an admission token. +Runtime `ci-image-contract verify` remains mandatory. PRs changing AMD64 +requirements still need the exact current-head published candidate and its +unchanged trust gates. Explicit ARM64 validation and its provisioning contract +remain unchanged; new AMD64 ordinary contracts deliberately use X64 capacity. + +## Safe rollout order + +1. Keep legacy runner registrations/images/labels available for older branches + and existing PR heads. Do not overwrite an active runner or change its image. +2. Obtain the exact published immutable new image from a successful trusted + publication. Verify manifest/recipe/provenance, confinement and applicable + compiler/KVM checks before creating separate capacity with fresh registration + and work volumes. Preserve the existing repository/group scope and resource + reserve/cleanup safeguards; do not copy Gateway credentials to the host. +3. Give that new capacity only the corresponding versioned kind label, never + `rust-ci`, `kotlin-ci`, `npm-pr`, or any `platform-image-pr-*` candidate label. + Do not label an image merely because its tool versions look similar. +4. Merge the selector repair normally before a new manifest becomes the branch + default. Verify actual candidate jobs and new ordinary capacity before merging + the recipe PR; source tests or a queued runner are not execution proof. +5. Observe real ordinary exact-contract jobs and preserve old capacity until all + remaining consuming branches/heads are migrated or no longer need it. Rollback + must preserve both contracts; never relabel an incompatible image to clear CI. + +This source change does not create runners or register/alter any label, image, +permission, workload, candidate allocator, or cleanup timer. Missing compatible +new capacity intentionally queues work rather than selecting an old image. diff --git a/.github/actions/s3-layer-cache-settings/action.yaml b/.github/actions/s3-layer-cache-settings/action.yaml index 770be990586..5aca3677773 100644 --- a/.github/actions/s3-layer-cache-settings/action.yaml +++ b/.github/actions/s3-layer-cache-settings/action.yaml @@ -104,4 +104,6 @@ runs: cacheToManifestNames.push('${{ inputs.name }}'); } - core.setOutput('cache_to', `${settingsString},mode=${{ inputs.mode }},name=${cacheToManifestNames.join(';')}`); + // Cache export is an optimization, not a build/publication gate. + // Preserve build and registry failures; only tolerate cache upload outages. + core.setOutput('cache_to', `${settingsString},mode=${{ inputs.mode }},name=${cacheToManifestNames.join(';')},ignore-error=true`); diff --git a/.github/scripts/runner-image.py b/.github/scripts/runner-image.py index 6f55c45b158..bc7ff445aa8 100644 --- a/.github/scripts/runner-image.py +++ b/.github/scripts/runner-image.py @@ -15,6 +15,8 @@ MANIFEST = ".github/runner-requirements.json" ARM64_MANIFEST = ".github/runner-requirements.arm64.json" REPO = "dashpay/platform" +# Only this already-provisioned AMD64 contract may use legacy generic labels. +LEGACY_AMD64_FINGERPRINT = "d272d01bcf3dfa620bbab1e9f31c3e1987862d33ec876bbad83c80f29de41a58" def require(condition, message): @@ -107,11 +109,7 @@ def export_environment(manifest, output): handle.write(f"{key}={value}\n") -def select(manifest, kind, output, wait_seconds, arch=None, validation=False): - require(arch in (None, "", "X64", "ARM64"), "Unsupported runner architecture") - require(not arch or kind == "rust", "Architecture selection is only supported for Rust") - require(not validation or (kind == "rust" and arch == "ARM64"), - "The validation-only pool is for explicitly selected ARM64 Rust jobs") +def ordinary_labels(manifest, kind, arch=None, validation=False): # Linux describes the runner process, not the physical host: ARM64 Linux # containers on Macs remain in this pool; native macOS stays for Swift. fallback = ["self-hosted"] + (["Linux"] if kind == "rust" else []) @@ -119,6 +117,22 @@ def select(manifest, kind, output, wait_seconds, arch=None, validation=False): fallback.append(arch) fallback.append("rust-ci-validation" if validation else {"rust": "rust-ci", "kotlin": "kotlin-ci", "npm": "npm-pr"}[kind]) + if arch != "ARM64" and fingerprint(manifest) != LEGACY_AMD64_FINGERPRINT: + # New AMD64 pools must not carry rust-ci/kotlin-ci/npm-pr: old branches + # still request those labels and must keep using their exact old image. + require(manifest["requirements"]["platform"] == "linux/amd64", + "Versioned ordinary pool requires an AMD64 manifest") + return ["self-hosted", "Linux", "X64", + f"platform-image-manifest-{fingerprint(manifest)}-{kind}"] + return fallback + + +def select(manifest, kind, output, wait_seconds, arch=None, validation=False): + require(arch in (None, "", "X64", "ARM64"), "Unsupported runner architecture") + require(not arch or kind == "rust", "Architecture selection is only supported for Rust") + require(not validation or (kind == "rust" and arch == "ARM64"), + "The validation-only pool is for explicitly selected ARM64 Rust jobs") + fallback = ordinary_labels(manifest, kind, arch, validation) event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) requested = event.get("pull_request") labels, changed = fallback, False @@ -131,7 +145,7 @@ def select(manifest, kind, output, wait_seconds, arch=None, validation=False): # Only validation capacity has the new ARM64 image before rollout. # Keep this PR's ordinary job on unchanged AMD64 capacity while its # separate ARM64 job proves the new manifest on the validation pool. - labels = fallback = ["self-hosted", "Linux", "X64", "rust-ci"] + labels = fallback = ordinary_labels(manifest, kind, arch="X64") if arch == "ARM64": # ARM64 is explicitly provisioned from a published immutable image. # The AMD64/KVM candidate publisher is not ARM64 validation. The @@ -173,6 +187,12 @@ def select(manifest, kind, output, wait_seconds, arch=None, validation=False): require(run["path"] == ".github/workflows/runner-image-candidate.yml" and run["event"] == "pull_request_target", "Unexpected candidate publisher") if run["conclusion"] == "success": + # Publication may finish during the retry sleep after + # this PR was updated/closed. Do not queue a stale label + # that the allocator must refuse to service. + current = api(f"pulls/{pr['number']}") + require(current["state"] == "open" and current["head"]["sha"] == head, + "PR changed before selecting its candidate") labels = ["self-hosted", "Linux", "X64", f"platform-image-pr-{pr['number']}-{head}-{candidate['description'][7:]}-{kind}"] break diff --git a/.github/scripts/tests/fixtures/legacy-runner-requirements.json b/.github/scripts/tests/fixtures/legacy-runner-requirements.json new file mode 100644 index 00000000000..2987919fc83 --- /dev/null +++ b/.github/scripts/tests/fixtures/legacy-runner-requirements.json @@ -0,0 +1,228 @@ +{ + "schema": 1, + "recipe_revision": "e49e8bc9977f5f961a76ba1d1f7673c72173679f", + "requirements": { + "schema": 2, + "contract_version": "1", + "platform": "linux/amd64", + "ubuntu_image": "ubuntu:24.04@sha256:496754492fb28b4d3049432f2ca787449331e23fb14f0dd3fffea86bf5a93eb4", + "apt_snapshot": "20260920T000000Z", + "rust_version": "1.98.1", + "rust_manifest_sha256": "a7c8774a5fd8441c997d94c029776cbc5eb111e9d72ab5d256fa69866644347e", + "artifacts": [ + { + "name": "runner", + "url": "https://github.com/actions/runner/releases/download/v2.337.0/actions-runner-linux-x64-2.337.0.tar.gz", + "sha256": "70920811a4f8ad4328818682bca5c6469c1c942fab52448868071d0063816613", + "format": "tar", + "destination": "/opt/actions-runner" + }, + { + "name": "cargo-llvm-cov", + "url": "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.9.1/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "b3f68e625481fed9b16444174f3fa5ebcdbde4a1878803a35eabe2dcefcdc41a", + "format": "tar", + "destination": "/opt/ci/bin/cargo-llvm-cov", + "binary": "cargo-llvm-cov" + }, + { + "name": "cargo-nextest", + "url": "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-0.9.144/cargo-nextest-0.9.144-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "8a4f726272b0a1c499bd87ca3978bfbb1a8c20bb08ccf075b9996e2081bd1e1e", + "format": "tar", + "destination": "/opt/ci/bin/cargo-nextest", + "binary": "cargo-nextest" + }, + { + "name": "cargo-machete", + "url": "https://github.com/bnjbvr/cargo-machete/releases/download/v0.9.2/cargo-machete-v0.9.2-x86_64-unknown-linux-musl.tar.gz", + "sha256": "48200087f54c55aabcd4db4af1e25742b49846c02a1b1bfa134711945b35b2e9", + "format": "tar", + "destination": "/opt/ci/bin/cargo-machete", + "binary": "cargo-machete" + }, + { + "name": "cargo-ndk", + "url": "https://github.com/bbqsrc/cargo-ndk/releases/download/v4.1.2/cargo-ndk-x86_64-unknown-linux-gnu-v4.1.2.tgz", + "sha256": "9451622c4567e8abb2c8005001855e32901c0b716ccdd3a173a4375fd03426e1", + "format": "tar", + "destination": "/opt/ci/bin/cargo-ndk", + "binary": "cargo-ndk" + }, + { + "name": "protoc", + "url": "https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip", + "sha256": "7ca037bfe5e5cabd4255ccd21dd265f79eb82d3c010117994f5dc81d2140ee88", + "format": "zip", + "destination": "/opt/protoc" + }, + { + "name": "rustup-init", + "url": "https://static.rust-lang.org/rustup/archive/1.28.2/x86_64-unknown-linux-gnu/rustup-init", + "sha256": "20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c", + "format": "file", + "destination": "/opt/ci/rustup-init", + "build_only": true + }, + { + "name": "platforms;android-35", + "url": "https://dl.google.com/android/repository/platform-35_r02.zip", + "upstream_sha1": "0bb560a90a7a2cbd0dd8348224d518b638fe7949", + "format": "zip", + "destination": "/opt/android-sdk/platforms/android-35", + "archive_root": "android-35", + "package_xml": "\n \n 35\n 13\n true\n \n \n \n 2\n \n Android SDK Platform 35\n \n ", + "sha256": "0988cacad01b38a18a47bac14a0695f246bc76c1b06c0eeb8eb0dc825ab0c8e0" + }, + { + "name": "ndk;28.1.13356709", + "url": "https://dl.google.com/android/repository/android-ndk-r28b-linux.zip", + "upstream_sha1": "f574d3165405bd59ffc5edaadac02689075a729f", + "format": "zip", + "destination": "/opt/android-sdk/ndk/28.1.13356709", + "archive_root": "android-ndk-r28b", + "package_xml": "\n \n \n 28\n 1\n 13356709\n \n NDK (Side by side) 28.1.13356709\n \n ", + "sha256": "e9f2759862cecfd48c20bbb7d8cfedbb020f4d91b5f78d9a2fc106f7db3c27ed" + }, + { + "name": "build-tools;35.0.0", + "url": "https://dl.google.com/android/repository/build-tools_r35_linux.zip", + "upstream_sha1": "2cfaa0bbb2336e9ec18ed3ecea84fa2e2af607bc", + "format": "zip", + "destination": "/opt/android-sdk/build-tools/35.0.0", + "archive_root": "android-15", + "package_xml": "\n \n \n 35\n 0\n 0\n \n Android SDK Build-Tools 35\n \n ", + "sha256": "bd3a4966912eb8b30ed0d00b0cda6b6543b949d5ffe00bea54c04c81e1561d88" + }, + { + "name": "cmdline-tools;19.0", + "url": "https://dl.google.com/android/repository/commandlinetools-linux-13114758_latest.zip", + "upstream_sha1": "5fdcc763663eefb86a5b8879697aa6088b041e70", + "format": "zip", + "destination": "/opt/android-sdk/cmdline-tools/19.0", + "archive_root": "cmdline-tools", + "package_xml": "\n \n \n 19\n 0\n \n Android SDK Command-line Tools\n \n ", + "sha256": "7ec965280a073311c339e571cd5de778b9975026cfcbe79f2b1cdcb1e15317ee" + }, + { + "name": "platform-tools", + "url": "https://dl.google.com/android/repository/platform-tools_r37.0.1-linux.zip", + "upstream_sha1": "477254aa5f903c15cf51001717bdf347fb6b53e0", + "format": "zip", + "destination": "/opt/android-sdk/platform-tools", + "archive_root": "platform-tools", + "package_xml": "\n \n \n 37\n 0\n 1\n \n Android SDK Platform-Tools\n \n ", + "sha256": "d230f13842f60f782a8645f9c813f8f845bf36089ea7289f28c48f17979313f1" + }, + { + "name": "emulator", + "url": "https://dl.google.com/android/repository/emulator-linux_x64-15917651.zip", + "upstream_sha1": "1b1f78891abf8ec268264356e1365c25519e8379", + "format": "zip", + "destination": "/opt/android-sdk/emulator", + "archive_root": "emulator", + "package_xml": "\n \n \n 37\n 1\n 11\n \n Android Emulator\n \n ", + "sha256": "95771e0ae431897b2a4bd2d97fa095f29a8b0624a7b216baf529f9306161c266" + }, + { + "name": "system-images;android-35;default;x86_64", + "url": "https://dl.google.com/android/repository/sys-img/android/x86_64-35_r02.zip", + "upstream_sha1": "2d857d170c0d1b827149565da34b3383e5306f7f", + "format": "zip", + "destination": "/opt/android-sdk/system-images/android-35/default/x86_64", + "archive_root": "x86_64", + "package_xml": "\n \n 35\n 13\n true\n \n default\n Default Android System Image\n \n x86_64\n \n \n 2\n \n Intel x86_64 Atom System Image\n \n \n \n 29\n 1\n 11\n \n \n \n \n ", + "sha256": "6dd7de33e63ef105cf2fabea6badda1dbe7665c96d8908e5f6e1407e63ff4556" + } + ], + "bootstrap_ca": { + "url": "https://snapshot.ubuntu.com/ubuntu/20260920T000000Z/pool/main/c/ca-certificates/ca-certificates_20240203_all.deb", + "sha256": "641de77d8f142cfd62a1a6f964ba67b20754d3337c480efb529d086075a06c9a", + "version": "20240203" + }, + "apt_packages": [ + "ca-certificates", + "curl", + "git", + "gh", + "jq", + "python3", + "unzip", + "zip", + "xz-utils", + "bzip2", + "gnupg", + "build-essential", + "clang", + "llvm", + "libsnappy-dev", + "cmake", + "libgmp-dev", + "libssl-dev", + "pkg-config", + "openjdk-17-jdk-headless", + "libicu74", + "libkrb5-3", + "zlib1g", + "libgcc-s1", + "libstdc++6", + "libcurl4t64", + "liblttng-ust1t64", + "libunwind8", + "libpulse0", + "libx11-xcb1", + "libnss3", + "libxcomposite1", + "libxcursor1", + "libxi6", + "libxrandr2", + "libxtst6", + "libasound2t64", + "libgl1", + "libegl1", + "libdbus-1-3", + "libxdamage1", + "libxfixes3" + ], + "java_major": 17, + "versions": { + "runner": "2.337.0", + "llvm_cov": "0.9.1", + "nextest": "0.9.144", + "machete": "0.9.2", + "cargo_ndk": "4.1.2", + "protoc": "32.0", + "rustup": "1.28.2" + }, + "android": { + "api": 35, + "build_tools": "35.0.0", + "ndk": "28.1.13356709", + "cmdline_tools": "19.0", + "system_image": "system-images;android-35;default;x86_64", + "abi": "x86_64" + }, + "client_codegen": { + "schema": 1, + "versions": { + "protobuf": "3.18.1", + "grpc": "1.46.3", + "grpc_java": "1.42.1" + }, + "sources": [ + { + "url": "https://github.com/protocolbuffers/protobuf/releases/download/v3.18.1/protobuf-cpp-3.18.1.tar.gz", + "sha256": "6ee35eda3f79e49608d2ace8d866313fdec539d8bb14c6c54e8d2a16fa4e6780" + }, + { + "url": "https://github.com/grpc/grpc/archive/refs/tags/v1.46.3.tar.gz", + "sha256": "d6cbf22cb5007af71b61c6be316a79397469c58c82a942552a62e708bce60964" + }, + { + "url": "https://github.com/grpc/grpc-java/archive/refs/tags/v1.42.1.tar.gz", + "sha256": "33775a1ad05974bbba6ff97801cd9b326485b21fab91b08a4e1bc53e500e6326" + } + ] + } + } +} diff --git a/.github/scripts/tests/test_candidate_controller_pin.py b/.github/scripts/tests/test_candidate_controller_pin.py new file mode 100644 index 00000000000..c24e468c192 --- /dev/null +++ b/.github/scripts/tests/test_candidate_controller_pin.py @@ -0,0 +1,24 @@ +"""Keep the trusted candidate caller on the reviewed exact-contract controller.""" +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[3] +REVIEWED_CONTROLLER = "1be03edb7f4f18548d525b17dfac6ee31db17c99" + + +class CandidateControllerPinTests(unittest.TestCase): + def test_should_bind_workflow_and_checkout_to_the_same_reviewed_commit(self): + """Both pins must adopt the reviewed merge, never mutable or mixed refs.""" + workflow = (ROOT / ".github/workflows/runner-image-candidate.yml").read_text() + workflow_refs = re.findall( + r"^\s+uses: dashpay/dash-selfhosted-image/" + r"\.github/workflows/platform-candidate\.yml@(\S+)\s*$", + workflow, re.MULTILINE, + ) + checkout_refs = re.findall( + r"^\s+control_revision: (\S+)\s*$", workflow, re.MULTILINE, + ) + self.assertEqual(workflow_refs, [REVIEWED_CONTROLLER]) + self.assertEqual(checkout_refs, [REVIEWED_CONTROLLER]) diff --git a/.github/scripts/tests/test_ordinary_image_routing.py b/.github/scripts/tests/test_ordinary_image_routing.py new file mode 100644 index 00000000000..d2749e4b57f --- /dev/null +++ b/.github/scripts/tests/test_ordinary_image_routing.py @@ -0,0 +1,85 @@ +"""Exact ordinary image pools must coexist with legacy development branches.""" +import copy +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[3] +spec = importlib.util.spec_from_file_location('ordinary_runner_image', ROOT / '.github/scripts/runner-image.py') +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class OrdinaryImageRoutingTests(unittest.TestCase): + def setUp(self): + self.legacy = runner.read_manifest(Path(__file__).parent / 'fixtures/legacy-runner-requirements.json') + self.new = copy.deepcopy(self.legacy) + self.new['recipe_revision'] = '1be03edb7f4f18548d525b17dfac6ee31db17c99' + self.assertEqual(runner.fingerprint(self.legacy), 'd272d01bcf3dfa620bbab1e9f31c3e1987862d33ec876bbad83c80f29de41a58') + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.event = Path(self.temp.name) / 'event.json' + self.output = Path(self.temp.name) / 'output' + + def select(self, manifest, kind='rust', event=None, files=None, arch=None, validation=False): + self.event.write_text(json.dumps(event or {})) + pr = {'number': 99, 'state': 'open', 'head': {'sha': 'a' * 40}, 'changed_files': len(files or [])} + def api(path): + if path == 'pulls/99': return pr + if path == 'pulls/99/files?per_page=100&page=1': return [{'filename': p} for p in files or []] + raise AssertionError('Unexpected API request: ' + path) + with patch.dict(os.environ, {'GITHUB_EVENT_PATH': str(self.event)}), patch.object(runner, 'api', side_effect=api): + runner.select(manifest, kind, self.output, 0, arch, validation) + result = dict(line.split('=', 1) for line in self.output.read_text().splitlines()) + return json.loads(result['labels']), result['image_changed'] + + def test_legacy_rust_kotlin_npm_labels_remain_unchanged(self): + for kind, labels in [('rust', ['self-hosted', 'Linux', 'rust-ci']), ('kotlin', ['self-hosted', 'kotlin-ci']), ('npm', ['self-hosted', 'npm-pr'])]: + with self.subTest(kind=kind): self.assertEqual(self.select(self.legacy, kind), (labels, 'false')) + + def test_new_recipe_never_uses_generic_legacy_capacity(self): + for kind in ('rust', 'kotlin', 'npm'): + with self.subTest(kind=kind): + labels, changed = self.select(self.new, kind) + self.assertEqual(labels, ['self-hosted', 'Linux', 'X64', f'platform-image-manifest-{runner.fingerprint(self.new)}-{kind}']) + self.assertEqual(changed, 'false') + self.assertTrue(set(labels).isdisjoint({'rust-ci', 'kotlin-ci', 'npm-pr', 'rust-ci-validation'})) + self.assertFalse(any(label.startswith('platform-image-pr-') for label in labels)) + + def test_changed_requirements_even_with_legacy_recipe_need_exact_pool(self): + manifest = copy.deepcopy(self.legacy) + manifest['requirements']['versions']['protoc'] = '32.1' + labels, _ = self.select(manifest) + self.assertEqual(labels[-1], 'platform-image-manifest-' + runner.fingerprint(manifest) + '-rust') + + def test_new_base_unchanged_application_pr_uses_versioned_capacity(self): + labels, changed = self.select(self.new, event={'pull_request': {'number': 99, 'head': {'sha': 'a' * 40}}}, files=['Cargo.lock']) + self.assertEqual(labels[-1], 'platform-image-manifest-' + runner.fingerprint(self.new) + '-rust') + self.assertEqual(changed, 'false') + + def test_arm64_change_cannot_reset_new_amd64_pool_to_generic(self): + labels, changed = self.select(self.new, event={'pull_request': {'number': 99, 'head': {'sha': 'a' * 40}}}, files=[runner.ARM64_MANIFEST]) + self.assertEqual(labels[-1], 'platform-image-manifest-' + runner.fingerprint(self.new) + '-rust') + self.assertEqual(changed, 'false') + + def test_explicit_arm64_pools_are_not_amd64_versioned_pools(self): + for validation, kind_label in [(False, 'rust-ci'), (True, 'rust-ci-validation')]: + labels, _ = self.select(self.new, arch='ARM64', validation=validation) + self.assertEqual(labels, ['self-hosted', 'Linux', 'ARM64', kind_label]) + + def test_explicit_x64_uses_exact_manifest_pool(self): + labels, _ = self.select(self.new, arch='X64') + self.assertEqual(labels[:3], ['self-hosted', 'Linux', 'X64']) + self.assertEqual(labels[-1], 'platform-image-manifest-' + runner.fingerprint(self.new) + '-rust') + + def test_full_manifest_label_is_stable_under_key_order_only(self): + reordered = dict(reversed(list(self.new.items()))) + self.assertEqual(self.select(self.new), self.select(reordered)) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/tests/test_runner_image.py b/.github/scripts/tests/test_runner_image.py index c811e406d35..660e057acc8 100644 --- a/.github/scripts/tests/test_runner_image.py +++ b/.github/scripts/tests/test_runner_image.py @@ -26,7 +26,8 @@ class SelectorTests(unittest.TestCase): def setUp(self): - self.manifest = runner.read_manifest(ROOT / runner.MANIFEST) + # Historical publisher fixtures bind the legacy contract, not a future branch default. + self.manifest = runner.read_manifest(Path(__file__).parent / "fixtures/legacy-runner-requirements.json") self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.event = Path(self.temp.name) / "event.json" @@ -233,7 +234,44 @@ def publish(_seconds): self.assertEqual(self.select(wait_seconds=60)["image_changed"], "true") sleep.assert_called_once_with(20) self.assertEqual(self.status_calls(), [STATUS_PATH, STATUS_PATH]) - self.assertEqual([call.args[0] for call in self.api.call_args_list].count("pulls/5151"), 2) + self.assertEqual([call.args[0] for call in self.api.call_args_list].count("pulls/5151"), 3) + + def test_should_reject_head_change_or_closure_during_candidate_retry(self): + good = FIXTURE["statuses"][0] + for kind in ("rust", "kotlin", "npm"): + for change in ("head", "closed"): + with self.subTest(kind=kind, change=change): + self.pr["state"] = "open" + self.pr["head"]["sha"] = HEAD + event = {"pull_request": copy.deepcopy(self.pr)} + self.responses[STATUS_PATH] = [dict(good, state="pending")] + + def publish_after_pr_changes(_seconds): + self.responses[STATUS_PATH] = [good] + if change == "head": + self.pr["head"]["sha"] = "e" * 40 + else: + self.pr["state"] = "closed" + + with patch.object(runner.time, "monotonic", return_value=0), \ + patch.object(runner.time, "sleep", side_effect=publish_after_pr_changes): + with self.assertRaisesRegex(ValueError, "PR changed before selecting"): + self.select(event, kind=kind, wait_seconds=60) + self.assertFalse(self.output.exists()) + + def test_should_fail_closed_if_final_candidate_head_check_is_unavailable(self): + original = self.api_response + + def fail_after_publisher_validation(path): + response = original(path) + if path == RUN_PATH: + self.responses["pulls/5151"] = URLError("final PR check unavailable") + return response + + with patch.object(self, "api_response", side_effect=fail_after_publisher_validation): + with self.assertRaisesRegex(URLError, "final PR check unavailable"): + self.select() + self.assertFalse(self.output.exists()) def test_environment_export_rejects_multiline_values_before_writing(self): self.manifest["requirements"]["versions"]["protoc"] = "32.0\nINJECTED=yes" @@ -311,12 +349,29 @@ def test_arm64_verify_uses_exact_contract_without_android_exports(self): self.assertFalse(any("ANDROID" in name or "NDK" in name for name in values)) def test_shared_rust_toolchain_does_not_drift_between_architectures(self): - amd = self.manifest["requirements"] + # This checks current contracts; historical publisher tests keep the frozen fixture. + amd = runner.read_manifest(ROOT / runner.MANIFEST)["requirements"] arm = runner.read_manifest(ROOT / runner.ARM64_MANIFEST)["requirements"] self.assertEqual(arm["versions"], {k: v for k, v in amd["versions"].items() if k != "cargo_ndk"}) for key in ("rust_version", "rust_manifest_sha256", "apt_snapshot", "java_major", "client_codegen"): self.assertEqual(amd[key], arm[key], key) + def test_should_detect_live_amd64_drift_despite_frozen_publisher_fixture(self): + manifests = {ROOT / name: runner.read_manifest(ROOT / name) + for name in (runner.MANIFEST, runner.ARM64_MANIFEST)} + manifests[ROOT / runner.MANIFEST]["requirements"]["java_major"] += 1 + with patch.object(runner, "read_manifest", side_effect=manifests.__getitem__): + with self.assertRaises(AssertionError): + self.test_shared_rust_toolchain_does_not_drift_between_architectures() + + def test_should_allow_matching_live_updates_despite_frozen_publisher_fixture(self): + manifests = {ROOT / name: runner.read_manifest(ROOT / name) + for name in (runner.MANIFEST, runner.ARM64_MANIFEST)} + for manifest in manifests.values(): + manifest["requirements"]["java_major"] += 1 + with patch.object(runner, "read_manifest", side_effect=manifests.__getitem__): + self.test_shared_rust_toolchain_does_not_drift_between_architectures() + def test_rejected_image_contract_stops_before_environment_export(self): with patch.dict(os.environ, {"RUNNER_OS": "Linux", "RUNNER_ARCH": "ARM64", "GITHUB_ENV": str(self.output)}), \ diff --git a/.github/scripts/tests/test_s3_layer_cache.py b/.github/scripts/tests/test_s3_layer_cache.py new file mode 100644 index 00000000000..d0c53939a5d --- /dev/null +++ b/.github/scripts/tests/test_s3_layer_cache.py @@ -0,0 +1,53 @@ +"""Execute the cache settings action with synthetic, credential-free inputs.""" +import json +from pathlib import Path +import subprocess +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[3] + + +class LayerCacheTests(unittest.TestCase): + def settings(self, shared=False, endpoint='https://cache.example.invalid'): + action = (ROOT / '.github/actions/s3-layer-cache-settings/action.yaml').read_text() + script = textwrap.dedent(action.split(' script: |\n', 1)[1]) + values = {'inputs.region': 'test-region', 'inputs.bucket': 'test-bucket', + 'inputs.prefix': 'cache-layers/linux/amd64/', 'inputs.endpoint': endpoint, + 'inputs.head_ref': 'refs/pull/42/merge', 'inputs.name': 'dashmate-helper', + 'github.sha': 'a' * 40, 'inputs.mode': 'max', + 'inputs.cache_to_name': str(shared).lower()} + for key, value in values.items(): + script = script.replace('${{ ' + key + ' }}', value) + self.assertNotIn('${{', script) + harness = 'const output = {}; const core = {setOutput: (k, v) => output[k] = v};\n' + result = subprocess.check_output(['node', '-e', harness + script + + '\nconsole.log(JSON.stringify(output));'], text=True) + return json.loads(result) + + def test_should_tolerate_only_export_errors_and_preserve_imports(self): + settings = self.settings() + self.assertTrue(settings['cache_to'].endswith(',ignore-error=true')) + self.assertEqual(settings['cache_to'].count('ignore-error='), 1) + imports = settings['cache_from'].splitlines() + self.assertEqual(len(imports), 3) + for item in imports: + self.assertNotIn('ignore-error', item) + self.assertNotIn('mode=', item) + self.assertIn('prefix=cache-layers/linux/amd64/', item) + self.assertIn('endpoint_url=https://cache.example.invalid', item) + expected = 'dashmate-helper_sha_' + 'a' * 40 + ';dashmate-helper_tag_refs-pull-42-merge' + self.assertIn('name=' + expected + ',ignore-error=true', settings['cache_to']) + + def test_should_keep_shared_manifest_writes_explicitly_opt_in(self): + self.assertNotIn(';dashmate-helper,', self.settings()['cache_to']) + self.assertIn(';dashmate-helper,ignore-error=true', self.settings(shared=True)['cache_to']) + self.assertNotIn('endpoint_url=', self.settings(endpoint='')['cache_to']) + + def test_should_not_ignore_build_or_registry_publication_failures(self): + action = (ROOT / '.github/actions/docker/action.yaml').read_text() + build = action.split(' - name: Build and push Docker image', 1)[1] + self.assertNotIn('continue-on-error', build) + self.assertNotIn('ignore-error', build) + self.assertIn('cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}', build) + self.assertIn('name-canonical=true,push=true', build) diff --git a/.github/workflows/runner-image-candidate.yml b/.github/workflows/runner-image-candidate.yml index 5e4708466fc..a93edb07c89 100644 --- a/.github/workflows/runner-image-candidate.yml +++ b/.github/workflows/runner-image-candidate.yml @@ -23,10 +23,10 @@ jobs: if: >- (github.event.action != 'closed' && !github.event.pull_request.draft) || (github.event.action == 'closed' && github.event.pull_request.merged) - uses: dashpay/dash-selfhosted-image/.github/workflows/platform-candidate.yml@7d901150bd3d0789d50c46f365b058f2f5f1f52d + uses: dashpay/dash-selfhosted-image/.github/workflows/platform-candidate.yml@1be03edb7f4f18548d525b17dfac6ee31db17c99 with: pull_request: ${{ github.event.pull_request.number }} - control_revision: 7d901150bd3d0789d50c46f365b058f2f5f1f52d + control_revision: 1be03edb7f4f18548d525b17dfac6ee31db17c99 mode: ${{ github.event.action == 'closed' && 'promote' || 'candidate' }} secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}