ci: backport reliability fixes to v5 without changing runner images

.github/ORDINARY_RUNNER_IMAGES.md

@@ -0,0 +1,37 @@
+# Coexisting ordinary AMD64 runner images
+
+The selector retains the existing generic ordinary labels only for the exact
+already-provisioned AMD64 contract `d272d01bcf3dfa620bbab1e9f31c3e1987862d33ec876bbad83c80f29de41a58`.
+For any different AMD64 manifest (including a recipe-only change), branch pushes
+and PRs that do not change that manifest require Linux/X64 plus:
+
+`platform-image-manifest-<canonical-full-manifest-sha256>-<rust|kotlin|npm>`
+
+This is an ordinary pool label, not a candidate label or an admission token.
+Runtime `ci-image-contract verify` remains mandatory. PRs changing AMD64
+requirements still need the exact current-head published candidate and its
+unchanged trust gates. Explicit ARM64 validation and its provisioning contract
+remain unchanged; new AMD64 ordinary contracts deliberately use X64 capacity.
+
+## Safe rollout order
+
+1. Keep legacy runner registrations/images/labels available for older branches
+   and existing PR heads. Do not overwrite an active runner or change its image.
+2. Obtain the exact published immutable new image from a successful trusted
+   publication. Verify manifest/recipe/provenance, confinement and applicable
+   compiler/KVM checks before creating separate capacity with fresh registration
+   and work volumes. Preserve the existing repository/group scope and resource
+   reserve/cleanup safeguards; do not copy Gateway credentials to the host.
+3. Give that new capacity only the corresponding versioned kind label, never
+   `rust-ci`, `kotlin-ci`, `npm-pr`, or any `platform-image-pr-*` candidate label.
+   Do not label an image merely because its tool versions look similar.
+4. Merge the selector repair normally before a new manifest becomes the branch
+   default. Verify actual candidate jobs and new ordinary capacity before merging
+   the recipe PR; source tests or a queued runner are not execution proof.
+5. Observe real ordinary exact-contract jobs and preserve old capacity until all
+   remaining consuming branches/heads are migrated or no longer need it. Rollback
+   must preserve both contracts; never relabel an incompatible image to clear CI.
+
+This source change does not create runners or register/alter any label, image,
+permission, workload, candidate allocator, or cleanup timer. Missing compatible
+new capacity intentionally queues work rather than selecting an old image.

.github/actions/s3-layer-cache-settings/action.yaml

@@ -104,4 +104,6 @@ runs:
             cacheToManifestNames.push('${{ inputs.name }}');
           }
 
-          core.setOutput('cache_to', `${settingsString},mode=${{ inputs.mode }},name=${cacheToManifestNames.join(';')}`);
+          // Cache export is an optimization, not a build/publication gate.
+          // Preserve build and registry failures; only tolerate cache upload outages.
+          core.setOutput('cache_to', `${settingsString},mode=${{ inputs.mode }},name=${cacheToManifestNames.join(';')},ignore-error=true`);

.github/scripts/runner-image.py

@@ -15,6 +15,8 @@
 MANIFEST = ".github/runner-requirements.json"
 ARM64_MANIFEST = ".github/runner-requirements.arm64.json"
 REPO = "dashpay/platform"
+# Only this already-provisioned AMD64 contract may use legacy generic labels.
+LEGACY_AMD64_FINGERPRINT = "d272d01bcf3dfa620bbab1e9f31c3e1987862d33ec876bbad83c80f29de41a58"
 
 
 def require(condition, message):
@@ -107,18 +109,30 @@ def export_environment(manifest, output):
             handle.write(f"{key}={value}\n")
 
 
-def select(manifest, kind, output, wait_seconds, arch=None, validation=False):
-    require(arch in (None, "", "X64", "ARM64"), "Unsupported runner architecture")
-    require(not arch or kind == "rust", "Architecture selection is only supported for Rust")
-    require(not validation or (kind == "rust" and arch == "ARM64"),
-            "The validation-only pool is for explicitly selected ARM64 Rust jobs")
+def ordinary_labels(manifest, kind, arch=None, validation=False):
     # Linux describes the runner process, not the physical host: ARM64 Linux
     # containers on Macs remain in this pool; native macOS stays for Swift.
     fallback = ["self-hosted"] + (["Linux"] if kind == "rust" else [])
     if arch:
         fallback.append(arch)
     fallback.append("rust-ci-validation" if validation else
                     {"rust": "rust-ci", "kotlin": "kotlin-ci", "npm": "npm-pr"}[kind])
+    if arch != "ARM64" and fingerprint(manifest) != LEGACY_AMD64_FINGERPRINT:
+        # New AMD64 pools must not carry rust-ci/kotlin-ci/npm-pr: old branches
+        # still request those labels and must keep using their exact old image.
+        require(manifest["requirements"]["platform"] == "linux/amd64",
+                "Versioned ordinary pool requires an AMD64 manifest")
+        return ["self-hosted", "Linux", "X64",
+                f"platform-image-manifest-{fingerprint(manifest)}-{kind}"]
+    return fallback
+
+
+def select(manifest, kind, output, wait_seconds, arch=None, validation=False):
+    require(arch in (None, "", "X64", "ARM64"), "Unsupported runner architecture")
+    require(not arch or kind == "rust", "Architecture selection is only supported for Rust")
+    require(not validation or (kind == "rust" and arch == "ARM64"),
+            "The validation-only pool is for explicitly selected ARM64 Rust jobs")
+    fallback = ordinary_labels(manifest, kind, arch, validation)
     event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
     requested = event.get("pull_request")
     labels, changed = fallback, False
@@ -131,7 +145,7 @@ def select(manifest, kind, output, wait_seconds, arch=None, validation=False):
             # Only validation capacity has the new ARM64 image before rollout.
             # Keep this PR's ordinary job on unchanged AMD64 capacity while its
             # separate ARM64 job proves the new manifest on the validation pool.
-            labels = fallback = ["self-hosted", "Linux", "X64", "rust-ci"]
+            labels = fallback = ordinary_labels(manifest, kind, arch="X64")
         if arch == "ARM64":
             # ARM64 is explicitly provisioned from a published immutable image.
             # The AMD64/KVM candidate publisher is not ARM64 validation. The
@@ -173,6 +187,12 @@ def select(manifest, kind, output, wait_seconds, arch=None, validation=False):
                     require(run["path"] == ".github/workflows/runner-image-candidate.yml"
                             and run["event"] == "pull_request_target", "Unexpected candidate publisher")
                     if run["conclusion"] == "success":
+                        # Publication may finish during the retry sleep after
+                        # this PR was updated/closed. Do not queue a stale label
+                        # that the allocator must refuse to service.
+                        current = api(f"pulls/{pr['number']}")
+                        require(current["state"] == "open" and current["head"]["sha"] == head,
+                                "PR changed before selecting its candidate")
                         labels = ["self-hosted", "Linux", "X64",
                                   f"platform-image-pr-{pr['number']}-{head}-{candidate['description'][7:]}-{kind}"]
                         break