From 61a65c35eafe1435e8cea02b5453651a0c8a7db3 Mon Sep 17 00:00:00 2001 From: Erik Jaaniso Date: Fri, 9 Oct 2026 11:15:17 +0300 Subject: [PATCH 1/2] ci: stop the image attestation from writing an unused storage record With push-to-registry the attest action also writes an org-level artifact metadata storage record, which needs a permission the image job does not have and came back "no artifacts found": two warnings on every release. The record is unused, provenance is the attestation itself, so skip it rather than widen the token. The comment that blamed download-artifact for the warnings was wrong and is gone. --- .github/workflows/release.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c403f89..a701193 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -519,10 +519,6 @@ jobs: attestations: write # write the image build-provenance attestation steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Both download-artifact steps in this workflow log two warnings per run: v8 also - # writes an "artifact metadata storage record", which wants `artifact-metadata: write`. - # That permission is deliberately not granted and the record is not used — provenance - # here comes from the attestation below. The download itself is unaffected. - name: Download the released x86_64 gnu (glibc) service binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -610,3 +606,10 @@ jobs: subject-name: ${{ steps.push.outputs.image }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true + # With push-to-registry the action also tries to write an org-level "artifact + # metadata storage record" (POST /orgs/{org}/artifacts/metadata/storage-record), + # which needs `artifact-metadata: write` and, for this image, came back "no + # artifacts found" — two warnings on every release. The record only feeds the + # org's linked-artifacts page and is not used; provenance is the attestation + # itself, so skip it rather than widen the token. + create-storage-record: false From fb1838af76327f65228d6b7d49f91a73f92d754c Mon Sep 17 00:00:00 2001 From: Erik Jaaniso Date: Fri, 9 Oct 2026 11:15:18 +0300 Subject: [PATCH 2/2] ci: pin what the harvest reads, not which userportal tag deploys it The two userportal extension tags fired on every userportal release, twice in three weeks, and both times nothing here had to change. Watch the facts instead: the deployed profile files by hash and the dcat fork's upstream base against the conformance requirements, resolved at whatever tag the userportal's own ckan/Dockerfile names. Replace the gdi-metadata README-phrase pin with content drift of the vendored shapes: `drift` takes a set filter and distinct exit codes, and the pins leg runs it for gdi-metadata, fatal under PINS_STRICT. A rebuilt base image is a note in the weekly job and fatal only under BASE_IMAGE_STRICT, which release and pins-strict set; image-scan already fails on a fixable CVE in the pinned base. One classify_fetch_failure beside curl_to decides gone, rate-limited or unreachable for every fetch site, so a 404 counts as drift everywhere. --- .github/workflows/scheduled.yml | 33 +- CONTRIBUTING.md | 34 +- Dockerfile | 4 +- conformance/README.md | 50 ++- conformance/check_ckanext.py | 24 +- conformance/requirements.txt | 33 +- scripts/ci-local.sh | 129 ++++--- scripts/tests/test_base_image_digest_pin.py | 43 ++- scripts/tests/test_gate_ordering.py | 140 ++++++- scripts/tests/test_userportal_profile_pins.py | 284 ++++++++++++++ scripts/tests/test_vendored_drift_filter.py | 196 ++++++++++ scripts/tests/test_vendored_pins.py | 56 ++- scripts/vendored.sh | 353 ++++++++++++------ 13 files changed, 1120 insertions(+), 259 deletions(-) create mode 100755 scripts/tests/test_userportal_profile_pins.py create mode 100755 scripts/tests/test_vendored_drift_filter.py diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 6d091c0..644f26d 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -224,12 +224,13 @@ jobs: # conformance/ga4gh-beacon-v2-default-model/, conformance/ga4gh-vrs-1.3/) from the commit # pinned in its VENDORED.md and asserts it is still byte-identical — turning the # "verbatim at commit X" claim from documentation into an enforced check. `check` also - # runs the `pins` assertions: the userportal's deployed ckanext-dcat/ckanext-fairdatapoint - # refs and gdi-metadata's declared HealthDCAT-AP release (so a federation move surfaces - # here as a ticket, not a silent harvest break). It is off the per-PR `ci` workflow - # because it needs network access and gates nothing on a PR. A red run means a vendored file - # was locally edited, a pin drifted, or an upstream deploy ref moved — re-vendor with - # `scripts/vendored.sh fetch` + bump VENDORED.md, update EXTERNAL_PINS, or revert the edit. + # runs the `pins` assertions: the userportal's deployed profile code, at the tags its own + # ckan/Dockerfile names, and the beacon-v2 release watch (so a consumer change surfaces + # here as a ticket, not a silent harvest break). It is off the per-PR `ci` workflow because + # it needs network access and gates nothing on a PR. A red run means a vendored file was + # locally edited, or upstream content or a pinned token changed — re-vendor with + # `scripts/vendored.sh fetch` + bump VENDORED.md, re-check and re-pin in scripts/vendored.sh, + # or revert the edit. # (The authored FDP shapes under conformance/shapes/fdp/ are not vendored — see # their PROVENANCE.md — so they are not byte-checked here.) GITHUB_TOKEN raises the # unauthenticated rate limit for the raw fetches. @@ -249,7 +250,9 @@ jobs: # `continue-on-error`, because upstream moving is news, not a defect in this tree. # The policy is to follow the federation in lockstep rather than chase it, so this # surfaces the change as exit 3, visible in the run, without failing a job that then - # gets muted. The `notify` step below still reports the run. + # gets muted. The `notify` step below still reports the run. The gdi-metadata shapes + # are the exception: the conformance gate validates against them, so the `pins` job + # below runs `drift gdi-metadata` fatally. - name: Report whether upstream has moved past our pinned commits continue-on-error: true env: @@ -495,13 +498,14 @@ jobs: key: gdi-corpus-${{ hashFiles('scripts/fetch-corpus.sh') }} - run: ./scripts/ci-local.sh corpus - # External pin freshness: the userportal deploy refs and gdi-metadata lineage this tree - # follows, plus the distroless runtime digest in the Dockerfile. Weekly, not per-PR, - # because it is a verdict about the world rather than about the commit: it goes red when - # the federation moves or the distroless base is rebuilt, neither of which any pull - # request caused, and a gate that reddens for reasons the author cannot fix is a gate - # people learn to ignore. The leg already downgrades an unreachable upstream to a warning - # rather than a failure. + # External pin freshness: the userportal's deployed profile code, the vendored gdi-metadata + # shapes against upstream, the beacon-v2 release and action tags, plus the distroless + # runtime digest in the Dockerfile. Weekly, not per-PR, because it is a verdict about the + # world rather than about the commit: it goes red when a consumer contract moves, which no + # pull request caused, and a gate that reddens for reasons the author cannot fix is a gate + # people learn to ignore. An unreachable upstream is a warning. A rebuilt distroless base + # is a note, not a failure: `image-scan` above already fails on a fixable CVE in the pinned + # one. pins: runs-on: ubuntu-26.04 steps: @@ -511,6 +515,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} # Fatal here, unlike a developer's working tree: this job exists to be the place # drift is noticed, and a warning in a weekly log nobody opens is not that. + # BASE_IMAGE_STRICT stays unset, per the job comment above. PINS_STRICT: "1" run: ./scripts/ci-local.sh pins diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e86e9b..365789a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -545,15 +545,18 @@ checks each pinned SHA against its comment upstream. The `pins` leg has no `ci.yml` job: it is a verdict about the world rather than about the commit, so it runs weekly in `scheduled.yml` as its own `pins` job (with `PINS_STRICT=1`), alongside `vendored-sync`. -`scripts/vendored.sh pins` asserts that the userportal's deployed CKAN-extension refs, the -gdi-metadata HealthDCAT-AP lineage, the GA4GH beacon-v2 latest release tag and the pinned -GitHub Action tags all still resolve to what this tree pins, and `ci-local.sh` itself -compares the Dockerfile's distroless runtime digest against upstream, through Docker. It -needs network, and unauthenticated GitHub API calls are rate-limited per IP, so export -`GITHUB_TOKEN` — sent to `api.github.com` only — to lift the budget. Real drift, -including a pinned path that 404s, warns in `pins` and fails `pins-strict`, which `release` -and the weekly job run. A rate-limited or unreachable host only warns, because that is a -verdict about your connection. +`scripts/vendored.sh pins` asserts that the userportal's deployed profile code (at the tags +its own `ckan/Dockerfile` names, so a userportal release that moves a tag without moving the +profile passes), the GA4GH beacon-v2 latest release tag and the pinned GitHub Action tags +still resolve to what this tree pins; `vendored.sh drift gdi-metadata` compares the vendored +shapes against upstream `main`; and `ci-local.sh` itself compares the Dockerfile's distroless +runtime digest against upstream, through Docker. It needs network, and unauthenticated +GitHub API calls are rate-limited per IP, so export `GITHUB_TOKEN` — sent to +`api.github.com` only — to lift the budget. Real drift, including a pinned path that 404s, +warns in `pins` and fails under `PINS_STRICT=1`, which `pins-strict`, `release` and the weekly +job set. A rebuilt base image is a note unless `BASE_IMAGE_STRICT=1`, which only `pins-strict` +and `release` set, since `image-scan` already fails on a fixable CVE in the pinned base. A +rate-limited or unreachable host only warns, because that is a verdict about your connection. ### Scheduled / on-demand jobs (`scheduled.yml`) @@ -586,9 +589,9 @@ inside the `release` meta-leg are marked. vendored conformance file is byte-identical to its pinned commit. Because that commit is an immutable SHA it stays green however far upstream moves, so `scripts/vendored.sh drift` answers the other question, comparing the same files against the upstream branch - named in each `VENDORED.md`. Exit 3 means upstream moved, which is news rather than a - defect, so the job runs it `continue-on-error` and it stays out of `all`. Local: both - need network. + named in each `VENDORED.md`. Exit 3 means upstream moved, which for Beacon and VRS is + news rather than a defect, so the job runs it `continue-on-error`; the `pins` leg runs + it for the gdi-metadata set alone, fatally. Local: both need network. - **`cross-native`** — macOS and Windows `gdi-dataset-tool` builds. No local leg: it needs those operating systems. - **`cross-linux-arm`** — aarch64-linux service build-verify (gnu and static musl, full @@ -609,9 +612,10 @@ inside the `release` meta-leg are marked. rather than per-PR because the leg range-downloads those slices first; the corpus is cached between runs. Local: `ci-local.sh corpus`. -- **`pins`** — external pin freshness: the userportal deploy refs and gdi-metadata lineage - this tree follows, plus the distroless runtime digest in the Dockerfile. Weekly because - it is a verdict about the world, not about the commit. Local: +- **`pins`** — external pin freshness: the userportal's deployed profile code, the vendored + gdi-metadata shapes against upstream, the beacon-v2 release and action tags, plus the + distroless runtime digest in the Dockerfile (a note there). Weekly because it is a + verdict about the world, not about the commit. Local: `PINS_STRICT=1 ci-local.sh pins` (the job sets that so drift is fatal there; a working tree only warns). - **`notify`** — opens or updates one tracking issue on a failed weekly run. No local diff --git a/Dockerfile b/Dockerfile index 68d75c9..cff2f5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,8 +119,8 @@ FROM ${BIN_SOURCE} AS binsrc # committing it. Re-resolve with # docker buildx imagetools inspect gcr.io/distroless/cc-debian13:nonroot # (or the registry manifest API) and bump tag+digest together. `ci-local.sh pins` -# checks this digest against the current upstream one on every run (WARN in `all`, -# fail under `PINS_STRICT=1`). +# checks this digest against the current upstream one on every run (a note in `all` and +# in the weekly job, fatal under `BASE_IMAGE_STRICT=1`, which `release` sets). FROM gcr.io/distroless/cc-debian13:nonroot@sha256:e792ab3d241a468a4fd7519ddbbebe66b49b5f365771716ea688ad40b6c6f1c2 # The service reads its config from GDI_CONFIG (default /etc/gdi-node-standalone/node.toml) diff --git a/conformance/README.md b/conformance/README.md index 116dc2e..c41622e 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -111,7 +111,8 @@ into `target/conformance/union.ttl`. It then runs: resolution. It overrides no predicate read, so the parent's predicate set is the deployed one. It ships in the harvester extension (`gdi-userportal-ckanext-fairdatapoint`), not in `ckanext-dcat`, so a harvester bump can - change the profile with the `ckanext-dcat` pin standing still; both refs are watched (see + change the profile with the `ckanext-dcat` pin standing still; the profile code of both + forks is watched by content, at the tags the userportal deploys (see [Version pins](#version-pins)). The check prefers the real `ckanext-dcat` `RDFParser` on the parent profile, since the @@ -200,13 +201,15 @@ You can also run a check by hand once a `union.ttl` exists: hash-locked `requirements.lock` (regenerate both together with `uv pip compile --universal --generate-hashes`); bump them in a reviewed change. `ckanext-dcat` is pinned to `2.4.4`, the upstream base of the GDI fork the userportal - deploys (`gdi-userportal-ckanext-dcat @ v2.5.0`, with harvester - `gdi-userportal-ckanext-fairdatapoint @ v1.7.2`, per gdi-userportal-ckan-docker). The - profile the deployment's harvest sources name is `fairdatapoint_dcat_ap`, supplied by the - harvester ref as a subclass of the fork's `euro_health_dcat_ap`, so the `ckanext-dcat` - pin alone would not see a profile change. That is why both refs are watched below. - `check_ckanext.py` runs an rdflib hand-mirror of the inherited `euro_health_dcat_ap` - reads, because the real `RDFParser` needs full CKAN and so never loads in this venv. + deploys (`gdi-userportal-ckanext-dcat`, with the harvester + `gdi-userportal-ckanext-fairdatapoint`, at the tags gdi-userportal-ckan-docker's + `ckan/Dockerfile` names). The profile the deployment's harvest sources name is + `fairdatapoint_dcat_ap`, supplied by the harvester as a subclass of the fork's + `euro_health_dcat_ap`, so the `ckanext-dcat` pin alone would not see a profile change. + That is why both profile files are watched below by content at the deployed tag, and the + fork's `pyproject.toml` version against this `2.4.4` pin. `check_ckanext.py` runs an + rdflib hand-mirror of the inherited `euro_health_dcat_ap` reads, because the real + `RDFParser` needs full CKAN and so never loads in this venv. - **Guards on all of the above.** Which gate reaches each one differs, and the difference matters, so it is stated per guard rather than claimed for the list. The offline integrity of these files is fully covered by `scripts/ci-local.sh all`; the two guards @@ -218,17 +221,20 @@ You can also run a check by hand once a `union.ttl` exists: `"required"` array from a Beacon schema makes everything downstream pass more easily. - `vendored.sh pins` (network, in the `pins` leg). Under `all`, both an unreachable network and real drift warn and pass: drift here means the federation moved, not that - this tree broke, so it is reported rather than fatal. `release` and `pins-strict` set - `PINS_STRICT=1` and fail closed on drift. Four watches (`EXTERNAL_PINS`): the - userportal's two deployed CKAN-extension refs, gdi-metadata's declared HealthDCAT-AP - release, and the latest GA4GH beacon-v2 release tag — the only watch on a standard - rather than on what the federation deploys. That last one reads the releases API, + this tree broke, so it is reported rather than fatal. The weekly job, `release` and + `pins-strict` set `PINS_STRICT=1` and fail closed on drift. Two watches: + `USERPORTAL_PROFILE_PINS`, the deployed profile files by hash and the dcat fork's + upstream base against this `requirements.txt`, resolved at the tags the userportal's + own `ckan/Dockerfile` names, so a userportal release that moves a tag but neither fact + passes; and `EXTERNAL_PINS`, the latest GA4GH beacon-v2 release tag — the only watch on + a standard rather than on what the federation deploys. That one reads the releases API, because a release is a tag and tags are in no file the repo contains: beacon-v2's own CHANGELOG tops out at `2.0.0` while the repo is tagged `v2.2.0`, so a file-based watch would be a dead guard. The same subcommand also runs `check_action_pins`, which is about this repo rather than the federation: it re-resolves every `uses: owner/repo@ # ` in the workflows and reports a SHA that no longer is - that tag. + that tag. The `pins` leg also runs `vendored.sh drift gdi-metadata` (below), with the + same warn-or-fail split. - `pip-audit` (in the `supply_chain` leg) — the advisory scan for the one corner `cargo deny` cannot see. It audits the `.lock` files, not the `.txt`: pip-audit resolves a requirements file, and the loose `.txt` can resolve a newer version than the lock pins @@ -245,11 +251,17 @@ You can also run a check by hand once a `union.ttl` exists: narrow — `verify` above already proves nothing in the tree was added, deleted or edited — leaving only the question of whether upstream still serves those bytes at that commit, which is a verdict about someone else's repository. - - `vendored.sh drift` (network) — the question `check` cannot answer: has upstream moved? - Same files, compared against the upstream branch (`**Branch:**` in each `VENDORED.md`). - Exit 3 means upstream moved, which is news rather than a defect, since this tree is - pinned. It is not in `ci-local.sh all`: it makes tens of network fetches, and a gate - that reddens because someone else committed is one people learn to ignore. + - `vendored.sh drift [set]` (network) — the question `check` cannot answer: has upstream + moved? Same files, compared against the upstream branch (`**Branch:**` in each + `VENDORED.md`). Exit 3 means upstream moved (a file removed upstream counts), 2 that a + file could not be fetched, 1 that this tree is broken (a vendored file missing locally, + or a `VENDORED.md` that does not parse). For + the Beacon and VRS sets that is news, not a defect: the node tracks a release. The + gdi-metadata shapes are the contract the conformance gate validates against, so the + `pins` leg runs `drift gdi-metadata` for them, a warning in `all` and fatal under + `PINS_STRICT=1`. The all-sets form is not in `ci-local.sh all`: it makes tens of + network fetches, and a gate that reddens because someone else committed is one people + learn to ignore. - Both are also scheduled weekly in `.github/workflows/scheduled.yml`, which runs on GitHub Actions (see [`CONTRIBUTING.md`](../CONTRIBUTING.md)). Two caveats. In a fork, scheduled workflows run only if the fork enables Actions, and GitHub disables a diff --git a/conformance/check_ckanext.py b/conformance/check_ckanext.py index f409d7c..f1a1565 100755 --- a/conformance/check_ckanext.py +++ b/conformance/check_ckanext.py @@ -15,8 +15,9 @@ ``ckanext-dcat``. It adds tag validation, ``tags_translated`` sanitising and label resolution on top of the parent's reads and overrides no predicate, so the parent's predicate set below is the deployed one. A harvester bump therefore changes this profile -without touching the ``ckanext-dcat`` pin, which is why both refs are watched by -``vendored.sh pins`` (see ``conformance/README.md``). +without touching the ``ckanext-dcat`` pin, which is why both forks' profile files are +watched by content, at the tags the userportal deploys, by ``vendored.sh pins`` (see +``conformance/README.md``). In practice the rdflib mirror (tier 2) is what runs: the real ``ckanext-dcat`` ``RDFParser`` needs full CKAN (``ckantoolkit`` imports ``ckan``), which the isolated @@ -177,21 +178,22 @@ def fallback_direct_parse(g: Graph) -> None: installed ``ckanext/dcat/profiles/euro_health_dcat_ap.py`` and ``euro_dcat_ap_3``). Pins: the deployed profile is ``fairdatapoint_dcat_ap`` from - ``gdi-userportal-ckanext-fairdatapoint @ v1.7.2``, a subclass that adds tag sanitising - and label resolution and overrides no predicate read, so this hand-mirror tracks its - parent in the GDI fork ``gdi-userportal-ckanext-dcat @ v2.5.0`` (what the userportal - deploys; ``requirements.txt`` pins the matching upstream base ``ckanext-dcat==2.4.4`` - for the optional real-parser tier). Both refs are watched by ``vendored.sh pins``, - because a harvester bump can change the profile with the ``ckanext-dcat`` pin standing - still. + ``gdi-userportal-ckanext-fairdatapoint``, a subclass that adds tag sanitising and label + resolution and overrides no predicate read, so this hand-mirror tracks its parent in the + GDI fork ``gdi-userportal-ckanext-dcat`` (what the userportal deploys; + ``requirements.txt`` pins the fork's upstream base ``ckanext-dcat==2.4.4`` for the + optional real-parser tier). ``vendored.sh pins`` watches both profile files by hash at + the tags the userportal's ``ckan/Dockerfile`` names, because a harvester bump can change + the profile with the ``ckanext-dcat`` pin standing still. The parent ``euro_health_dcat_ap`` profile reads ``healthdcatap:{numberOfRecords,healthCategory,hdab}`` (namespace ``http://healthdataportal.eu/ns/health#``) and ``dpv:hasLegalBasis`` (``https://w3id.org/dpv#``), plus the DCAT-AP core ``dct:title``, ``dct:identifier`` and ``dcat:accessURL`` — the subset asserted below, checked against the fork's - ``profiles/euro_health_dcat_ap.py`` at v2.5.0. Re-check this list when bumping, so the - best-effort tier cannot drift from the deployed profile. + ``profiles/euro_health_dcat_ap.py`` at the hash ``USERPORTAL_PROFILE_PINS`` pins. + Re-check this list when that hash drifts, so the best-effort tier cannot drift from the + deployed profile. """ catalogs = list(g.subjects(RDF.type, DCAT.Catalog)) _require( diff --git a/conformance/requirements.txt b/conformance/requirements.txt index 44d947e..8144ff7 100644 --- a/conformance/requirements.txt +++ b/conformance/requirements.txt @@ -22,24 +22,23 @@ # # - pyshacl / rdflib: the SHACL engine + RDF graph for check_fdp.py / check_dataset.py. # - ckanext-dcat: the consumer-compat profile parser for check_ckanext.py. The -# userportal deploys the GDI *fork* installed from git — -# `gdi-userportal-ckanext-dcat @ v2.5.0` (+ the harvester -# `gdi-userportal-ckanext-fairdatapoint @ v1.7.2`), per -# gdi-userportal-ckan-docker's `ckan/Dockerfile`. We pin the matching -# upstream base `2.4.4` here, not the git fork: the fork's `RDFParser` -# imports `ckan.plugins`, so the real-parser tier cannot load in this -# CKAN-less venv either way — check_ckanext.py always degrades to its -# best-effort direct rdflib consumer-parse (it says so in its output), -# so a git install would only add per-run network fragility for zero -# functional gain. pyshacl is the must-have gate; ckanext is nice-to-have. +# userportal deploys the GDI *fork* installed from git +# (`gdi-userportal-ckanext-dcat`, with the harvester +# `gdi-userportal-ckanext-fairdatapoint`), at the tags named in +# gdi-userportal-ckan-docker's `ckan/Dockerfile`. We pin the fork's +# upstream base here, not the git fork: the fork's `RDFParser` imports +# `ckan.plugins`, so the real-parser tier cannot load in this CKAN-less +# venv either way — check_ckanext.py always degrades to its best-effort +# direct rdflib consumer-parse (it says so in its output), so a git +# install would only add per-run network fragility for zero functional +# gain. pyshacl is the must-have gate; ckanext is nice-to-have. # PIN NOTE: check_ckanext.py's fallback hand-mirrors the predicate reads -# of the fork's `euro_health_dcat_ap` profile — VERIFIED against -# gdi-userportal-ckanext-dcat @ v2.5.0 (the mirrored predicates match -# upstream 2.4.4; the fork reads a few more). When bumping, re-verify -# that mirrored list against the fork's `profiles/euro_health_dcat_ap.py` -# (and `euro_dcat_ap_3.py`) so the best-effort check cannot silently -# validate stale predicates. The deployed fork refs are drift-watched by -# `scripts/vendored.sh pins`. +# of the fork's `euro_health_dcat_ap` profile. `scripts/vendored.sh pins` +# watches that file (and the harvester's subclass) by hash at the deployed +# tag, and the fork's `pyproject.toml` version against the pin below. When +# it drifts: re-verify the mirrored list against the fork's +# `profiles/euro_health_dcat_ap.py` (and `euro_dcat_ap_3.py`), re-pin the +# hash, and bump the version below if the base moved. pyshacl==0.40.1 rdflib==7.6.0 ckanext-dcat==2.4.4 diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index fc3d5d8..053198d 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -135,14 +135,16 @@ # machete cargo machete, the unused-dependency gate # pip-audit advisory scan of the conformance and crypt4gh Python lockfiles # sbom cargo-cyclonedx SBOM generation -# pins (network) the external upstream pins (userportal refs, gdi-metadata -# lineage), the GitHub action pins (`uses: owner/repo@sha # tag`) and -# the Dockerfile base-image digest still resolve to what is pinned -# here. The base-image half needs Docker and skips visibly without -# it. Drift and an unreachable host are both warnings, because neither -# is a defect in this tree; `pins-strict` makes only drift fatal. -# pins-strict the same check with drift fatal, which is what `release` runs. -# Being behind is a warning per change and a failure at a tag. +# pins (network) the upstream content this tree depends on still is what it +# was pinned against: the userportal's deployed profile code, the +# vendored gdi-metadata shapes, the beacon-v2 release, the GitHub action +# pins (`uses: owner/repo@sha # tag`) and the Dockerfile base-image +# digest (needs Docker; skips visibly without it). Drift and an +# unreachable host are both warnings here: neither is a defect in this +# tree. +# pins-strict the same check with drift fatal, which is what `release` runs +# (PINS_STRICT=1 BASE_IMAGE_STRICT=1). The weekly job sets only the +# former, so a rebuilt base image is a note there. # licenses cargo-about drift guard: THIRD-PARTY-LICENSES.md must match the # shipped dependency graph. In `release`. # reuse REUSE 3.3 licensing compliance: every file resolves to a copyright @@ -243,9 +245,11 @@ # GATE_LOCK the slot's lock file (default: # /tmp/gdi-node-standalone-gate-.lock). One file per machine, # shared by every checkout, because what they contend for is the CPU. -# PINS_STRICT set to 1 to make pin drift, external or base-image, fatal instead of -# a warning. It is what the `pins-strict` target sets and what -# `release` exports. +# PINS_STRICT set to 1 to make pin drift (userportal profile content, gdi-metadata +# shapes, external and action pins) fatal instead of a warning. Set by +# the weekly `pins` job, `pins-strict` and `release`. +# BASE_IMAGE_STRICT set to 1 to make a rebuilt base image fatal too; otherwise it is a +# note. Set by `pins-strict` and `release`, not by the weekly job. # SWEEP_MAXSIZE the target/ ceiling for `sweep`, with an explicit unit (default: # 20GB). A unitless value is rejected, because cargo-sweep would read # it as megabytes. @@ -1313,8 +1317,9 @@ _first_upstream_digest() { # exports the former), so a release cannot silently skip this the way `all` may. # # Returns 0 when clean, unreachable (offline/registry down — a verdict about the -# network, not the pin), or skipped; 1 only on confirmed drift, which `pins()` turns -# into WARN (`all`) or FAIL (`PINS_STRICT=1`, the same split as the external pins below. +# network, not the pin), or skipped; 1 on confirmed drift, which `_base_image_pin_gate` +# turns into a note or, under BASE_IMAGE_STRICT=1, a failure; 2 when the Dockerfiles +# disagree with each other about the pin, a defect in this tree that is fatal in every mode. _base_image_freshness() { local ref='gcr.io/distroless/cc-debian13:nonroot' skip_unless docker "base image digest freshness" "base image freshness NOT checked" || return 0 @@ -1328,7 +1333,7 @@ _base_image_freshness() { if [[ "$(printf '%s\n' "$pinned" | grep -c .)" -ne 1 ]]; then printf '%sthe Dockerfiles pin %s at DIFFERENT digests — bump them together:%s\n%s\n' \ "$C_ERR" "$ref" "$C_OFF" "$pinned" - return 1 + return 2 fi upstream="$(docker buildx imagetools inspect "$ref" 2>/dev/null | _first_upstream_digest)" if [[ -z "$upstream" ]]; then @@ -1339,16 +1344,25 @@ _base_image_freshness() { _base_image_digest_check "$pinned" "$upstream" } -# Turns `_base_image_freshness`'s clean/drift verdict into the same warn-in-`all`, -# fail-under-`PINS_STRICT=1` split the external pins below use. Under strict mode it returns -# 1 rather than dying, and `pins()` acts on that only after the external pins have reported: -# a `die` here ended the leg before they were fetched. +# Turns `_base_image_freshness`'s verdict into what `pins()` acts on, after the other halves +# have reported (a `die` here once ended the leg before the external pins were fetched): +# 0 clean, or upstream moved without BASE_IMAGE_STRICT=1: a note. The weekly job runs +# PINS_STRICT=1 alone; distroless rebuilding its tag is not the federation moving, and +# a fixable CVE in the pinned base already fails `image-scan`. +# 1 upstream moved under BASE_IMAGE_STRICT=1 (`release`, `pins-strict`): shipping on a +# stale base is the case that costs. +# 2 the Dockerfiles disagree about the pin: a defect in this tree, fatal in every mode. _base_image_pin_gate() { - _base_image_freshness && return 0 - if [[ "${PINS_STRICT:-0}" == "1" ]]; then + local frc=0 + _base_image_freshness || frc=$? + case "$frc" in + 0) return 0 ;; + 2) return 2 ;; + esac + if [[ "${BASE_IMAGE_STRICT:-0}" == "1" ]]; then return 1 fi - printf '%sWARNING: base image digest DRIFT — see above. Not fatal here — `release` and `pins-strict` fail closed on this.%s\n' "$C_ERR" "$C_OFF" + printf '%snote: a fresher gcr.io/distroless/cc-debian13:nonroot exists — see above. Not fatal here (`image-scan` covers the CVE case); fatal under BASE_IMAGE_STRICT=1. Bump tag+digest together when convenient.%s\n' "$C_DIM" "$C_OFF" return 0 } @@ -1363,39 +1377,63 @@ _base_image_pin_gate() { # and skipped: it is a verdict about the network, not about the pins. Drift is a verdict # about the federation rather than about this tree, so it does not fail `all` either. # -# Every pin in EXTERNAL_PINS points at a repository this project does not control: two CKAN -# extension refs, a lineage string in the metadata repository's README, and the beacon-v2 -# `releases/latest` tag. Any of them can move because someone else committed, and -# `all_legs()` is a bare loop under `set -e`, so a fatal verdict here would take every -# later leg with it, including everything that compiles or tests. Upstream moving is not a -# defect in this tree, and a gate that reddens for it is a gate people learn to ignore. +# Everything this leg watches lives in repositories this project does not control: the +# userportal's deployed profile code, the gdi-metadata shapes, the beacon-v2 release tag, +# the action tags. Any of them can move because someone else committed, and `all_legs()` +# is a bare loop under `set -e`, so a fatal verdict here would take every later leg with +# it. Upstream moving is not a defect in this tree, and a gate that reddens for it is a +# gate people learn to ignore. # -# The signal is relocated rather than weakened: drift is reported on every run, and stays -# fatal where being behind the federation costs something, in `pins-strict` on demand and -# in `release` before any artifact ships, which exports PINS_STRICT=1. +# The signal is relocated rather than weakened: drift is reported on every run, and is +# fatal where being behind costs something: the weekly `pins` job and `pins-strict` +# (PINS_STRICT=1), and `release` before any artifact ships. pins() { - step "external pins — userportal deploy refs + gdi-metadata lineage (network); base image digest freshness (Docker)" - # Both halves report before either verdict is acted on. Strict-mode drift is collected + step "pins — userportal profile content, gdi-metadata shapes, release + action pins (network); base image digest (Docker)" + # Every half reports before any verdict is acted on. Strict-mode drift is collected # and fails the leg once, at the end, naming every half that drifted. - local fatal=() rc=0 - _base_image_pin_gate || - fatal+=("base image digest drift — see above. Re-resolve with 'docker buildx imagetools inspect gcr.io/distroless/cc-debian13:nonroot' and bump tag+digest together in the Dockerfile.") + local fatal=() rc=0 brc=0 + _base_image_pin_gate || brc=$? + case "$brc" in + 0) ;; + 2) fatal+=("the Dockerfiles pin the base image at different digests — see above. Bump them together.") ;; + *) fatal+=("base image digest drift — see above. Re-resolve with 'docker buildx imagetools inspect gcr.io/distroless/cc-debian13:nonroot' and bump tag+digest together in the Dockerfile.") ;; + esac bash scripts/vendored.sh pins || rc=$? case "$rc" in 0) ;; 2) printf '%sWARNING: external pins could not be fetched (offline / upstream down) — SKIPPED, not verified. Re-run scripts/ci-local.sh pins when connected.%s\n' "$C_ERR" "$C_OFF" ;; *) if [[ "${PINS_STRICT:-0}" == "1" ]]; then - fatal+=("external pin drift — see above. Follow the federation in lockstep (bump EXTERNAL_PINS in scripts/vendored.sh + the matching note in conformance/requirements.txt); do not get ahead of it.") + fatal+=("external pin drift — see above. Follow the federation in lockstep (re-check what the drifted line names, then re-pin it in scripts/vendored.sh); do not get ahead of it.") else printf '%sWARNING: external pin DRIFT — see above. The federation moved; this tree did not break.\n' "$C_ERR" - printf ' Follow it in lockstep (bump EXTERNAL_PINS in scripts/vendored.sh + the matching note\n' - printf ' in conformance/requirements.txt); do not get ahead of it. Not fatal here — `release`\n' - printf ' and `pins-strict` fail closed on this.%s\n' "$C_OFF" + printf ' Follow it in lockstep (re-check what the drifted line names, then re-pin it in\n' + printf ' scripts/vendored.sh); do not get ahead of it. Not fatal here — `release` and\n' + printf ' `pins-strict` fail closed on this.%s\n' "$C_OFF" + fi + ;; + esac + # The gdi-metadata shapes against upstream main. Beacon and VRS are tracked by release, so + # their drift is news; these are the contract the conformance gate validates against, so + # drift is a finding. Exit 3 = moved, 2 = could not fetch, 1 = this tree or the call is + # broken. + local src=0 + bash scripts/vendored.sh drift gdi-metadata || src=$? + case "$src" in + 0) ;; + 2) printf '%sWARNING: gdi-metadata shapes could not be fetched (offline / upstream down) — SKIPPED, not verified. Re-run scripts/ci-local.sh pins when connected.%s\n' "$C_ERR" "$C_OFF" ;; + 3) + if [[ "${PINS_STRICT:-0}" == "1" ]]; then + fatal+=("gdi-metadata shapes drift — see above. Re-vendor with 'scripts/vendored.sh fetch gdi-metadata', bump its VENDORED.md, re-check the node's mapping.") + else + printf '%sWARNING: gdi-metadata shapes DRIFT — see above. The conformance gate validates against the\n' "$C_ERR" + printf ' vendored copy: re-vendor with `scripts/vendored.sh fetch gdi-metadata` and bump its\n' + printf ' VENDORED.md. Not fatal here — `release` and `pins-strict` fail closed on this.%s\n' "$C_OFF" fi ;; + *) fatal+=("scripts/vendored.sh drift gdi-metadata exited ${src} — a defect in this tree or call (a vendored file missing locally, an unparseable VENDORED.md, or the set filter), not a verdict about upstream.") ;; esac - [[ ${#fatal[@]} -eq 0 ]] || die "pin drift under PINS_STRICT=1:$(printf '\n - %s' "${fatal[@]}")" + [[ ${#fatal[@]} -eq 0 ]] || die "pin drift:$(printf '\n - %s' "${fatal[@]}")" } # --- Seams ------------------------------------------------------------------- @@ -1995,15 +2033,18 @@ release() { # # PINS_STRICT restores the other half: external pin drift is a warning in the per-change # gate, but shipping while behind the federation is the case that costs something, so it - # fails closed here. GATE_STRICT_LEGS closes the last gap: a leg that `all` lets skip on a + # fails closed here. BASE_IMAGE_STRICT does the same for a base image upstream has rebuilt + # since it was pinned, which the weekly job only notes. GATE_STRICT_LEGS closes the last + # gap: a leg that `all` lets skip on a # missing tool dies here instead, because a release gate that skipped the alert-rule tests # is not a release gate. - export GATE_FORCE=1 PINS_STRICT=1 GATE_STRICT_LEGS=1 + export GATE_FORCE=1 PINS_STRICT=1 BASE_IMAGE_STRICT=1 GATE_STRICT_LEGS=1 all; lint_docker; otel; fuzz_short; vendored_check; licenses; coverage; e2e; e2e_observability; cross_compile; cross_arm; image_provenance; image_scan; e2e_full } # The full vendored-integrity check: re-fetches every vendored file from its pinned -# upstream commit, asserts byte-identity, then checks the external pins. `vendored_files`, +# upstream commit, asserts byte-identity, then checks the userportal profile content and the +# external pins. `vendored_files`, # in `all`, only counts what is on disk; this is what proves the content still matches what # was reviewed. # @@ -2706,7 +2747,7 @@ main() { vendored-check) vendored_check ;; ruff) ruff ;; pins) pins ;; - pins-strict) PINS_STRICT=1 pins ;; + pins-strict) PINS_STRICT=1 BASE_IMAGE_STRICT=1 pins ;; seams) seams ;; conformance) conformance ;; crypt4gh) crypt4gh ;; diff --git a/scripts/tests/test_base_image_digest_pin.py b/scripts/tests/test_base_image_digest_pin.py index 2245ec0..6089127 100755 --- a/scripts/tests/test_base_image_digest_pin.py +++ b/scripts/tests/test_base_image_digest_pin.py @@ -2,9 +2,9 @@ """Guard: the `pins` leg's base-image digest comparison behaves on equal/unequal input. `ci-local.sh`'s `pins` leg compares the Dockerfile's pinned distroless digest against the -one `docker buildx imagetools inspect` resolves now, warning in `all` and failing under -`PINS_STRICT=1` for a release. Without it a base pin can fall arbitrarily far behind -upstream between releases with nothing executed noticing. +one `docker buildx imagetools inspect` resolves now, a note in `all` and in the weekly job +and a failure under `BASE_IMAGE_STRICT=1` for a release. Without it a base pin can fall +arbitrarily far behind upstream between releases with nothing executed noticing. The comparison (`_base_image_digest_check`) is split out from the Docker call around it (`_base_image_freshness`) so it can be tested without Docker or a network. This file @@ -66,6 +66,43 @@ def run_check(pinned: str, upstream: str) -> subprocess.CompletedProcess: ) +class BaseImageFreshnessReturnCodes(unittest.TestCase): + """`_base_image_freshness` tells "upstream moved" (1) from "this tree is inconsistent" (2). + + `_base_image_pin_gate` demotes the former to a note unless BASE_IMAGE_STRICT=1 and must + keep the latter fatal, so the two need distinct codes. Run with the Docker-touching + helpers stubbed: two different pins for the base are reported before the registry is + asked, so no `docker` is needed to reach that branch. + """ + + def test_two_different_pins_for_the_base_return_2_before_asking_the_registry(self): + body = extract_function("_base_image_freshness") + script = ( + "set -euo pipefail\n" + "C_DIM='' ; C_ERR='' ; C_OFF=''\n" + "skip_unless() { return 0; }\n" + "assert_nonempty() { :; }\n" + "_all_dockerfile_digest_pins() {\n" + " echo 'gcr.io/distroless/cc-debian13:nonroot sha256:" + "a" * 64 + "'\n" + " echo 'gcr.io/distroless/cc-debian13:nonroot sha256:" + "b" * 64 + "'\n" + "}\n" + "docker() { echo 'registry was asked' >&2; exit 99; }\n" + f"{body}\n" + "rc=0; _base_image_freshness || rc=$?\n" + 'echo "rc=$rc"\n' + ) + proc = subprocess.run( + ["bash", "-c", script], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + self.assertIn("rc=2", proc.stdout, proc.stdout + proc.stderr) + self.assertNotIn("registry was asked", proc.stderr) + self.assertIn("DIFFERENT digests", proc.stdout) + + class BaseImageDigestCheck(unittest.TestCase): def test_equal_digests_are_clean(self): proc = run_check(DIGEST_A, DIGEST_A) diff --git a/scripts/tests/test_gate_ordering.py b/scripts/tests/test_gate_ordering.py index dbd14d6..9d377d8 100755 --- a/scripts/tests/test_gate_ordering.py +++ b/scripts/tests/test_gate_ordering.py @@ -313,6 +313,15 @@ def test_release_makes_pin_drift_fatal_again(self): "behind the federation actually costs something, so it must fail closed", ) + def test_release_makes_base_image_drift_fatal_too(self): + # The weekly job takes a moved base image as a note; a release ships on it. + self.assertRegex( + self.release_body, + r"(?m)^\s*(export\s+[^\n]*)?\bBASE_IMAGE_STRICT=1\b", + "`release` no longer exports BASE_IMAGE_STRICT=1, so a tag can ship on a base " + "image upstream has since rebuilt", + ) + def test_release_forbids_skipped_legs(self): # `all` lets `promtool` skip visibly without Docker. The export is what makes # `record_skip` die instead, so a release gate cannot skip the alert-rule tests. @@ -356,14 +365,23 @@ def function_source(text: str, name: str) -> str: def run_pins( - *, base_drift: bool, external_rc: int, strict: bool + *, + base_drift: bool, + external_rc: int, + strict: bool, + shapes_rc: int = 0, + base_strict: bool = False, + base_rc: int | None = None, ) -> subprocess.CompletedProcess: - """Run the real `pins()` with its Docker half and its network half stubbed. - - `_base_image_freshness` returns 1 on drift and 0 when clean, as the real one does, and - `scripts/vendored.sh` exits 0 when clean, 1 on drift and 2 when unreachable. Each stub - announces itself on stdout, so a test can tell a half that ran from one that never - did. `pins`, `_base_image_pin_gate`, `step` and `die` are taken from ci-local.sh. + """Run the real `pins()` with its Docker half and its network halves stubbed. + + `_base_image_freshness` returns 1 on drift and 0 when clean, as the real one does + (`base_rc` overrides that, for its other codes), and `scripts/vendored.sh` exits 0 when + clean, 1 on drift and 2 when unreachable for `pins`, and `shapes_rc` for + `drift gdi-metadata` (0 clean, 1 tree broken, 2 unreachable, 3 upstream moved). Each + stub announces + itself on stdout, so a test can tell a half that ran from one that never did. `pins`, + `_base_image_pin_gate`, `step` and `die` are taken from ci-local.sh. """ text = CI_LOCAL.read_text(encoding="utf-8") script = "\n".join( @@ -376,19 +394,28 @@ def run_pins( ), ( f"_base_image_freshness() {{ echo 'stub: base image freshness ran'; " - f"return {1 if base_drift else 0}; }}" + f"return {base_rc if base_rc is not None else (1 if base_drift else 0)}; }}" ), "pins", ] ) - env = {k: v for k, v in os.environ.items() if k != "PINS_STRICT"} + env = { + k: v + for k, v in os.environ.items() + if k not in ("PINS_STRICT", "BASE_IMAGE_STRICT") + } if strict: env["PINS_STRICT"] = "1" + if base_strict: + env["BASE_IMAGE_STRICT"] = "1" with tempfile.TemporaryDirectory() as tmp: stub = pathlib.Path(tmp, "scripts", "vendored.sh") stub.parent.mkdir() stub.write_text( - f'echo "stub: vendored.sh $* ran"\nexit {external_rc}\n', encoding="utf-8" + 'echo "stub: vendored.sh $* ran"\n' + f'case "$1" in drift) exit {shapes_rc} ;; esac\n' + f"exit {external_rc}\n", + encoding="utf-8", ) return subprocess.run( ["bash", "-c", script], @@ -409,7 +436,7 @@ class PinDriftFatalityTest(unittest.TestCase): """ def test_strict_base_drift_does_not_hide_the_external_pins(self): - proc = run_pins(base_drift=True, external_rc=0, strict=True) + proc = run_pins(base_drift=True, external_rc=0, strict=True, base_strict=True) self.assertIn( "stub: vendored.sh pins ran", proc.stdout, @@ -419,7 +446,7 @@ def test_strict_base_drift_does_not_hide_the_external_pins(self): self.assertIn("base image digest drift", proc.stderr) def test_strict_drift_in_both_halves_names_both(self): - proc = run_pins(base_drift=True, external_rc=1, strict=True) + proc = run_pins(base_drift=True, external_rc=1, strict=True, base_strict=True) self.assertEqual(1, proc.returncode, proc.stderr) self.assertIn("base image digest drift", proc.stderr) self.assertIn("external pin drift", proc.stderr) @@ -433,7 +460,7 @@ def test_strict_external_drift_alone_is_fatal(self): def test_strict_base_drift_survives_unreachable_external_pins(self): # Unreachable is not fatal on its own, so its arm must not end the leg with a # success that swallows the base-image verdict held above it. - proc = run_pins(base_drift=True, external_rc=2, strict=True) + proc = run_pins(base_drift=True, external_rc=2, strict=True, base_strict=True) self.assertIn("could not be fetched", proc.stdout) self.assertEqual(1, proc.returncode, proc.stderr) self.assertIn("base image digest drift", proc.stderr) @@ -454,10 +481,95 @@ def test_drift_is_only_a_warning_without_strict_mode(self): # fatal verdict here would skip every leg after `pins` because an upstream moved. proc = run_pins(base_drift=True, external_rc=1, strict=False) self.assertEqual(0, proc.returncode, proc.stderr) - self.assertIn("WARNING: base image digest DRIFT", proc.stdout) + self.assertIn("a fresher", proc.stdout) self.assertIn("WARNING: external pin DRIFT", proc.stdout) +class BaseImageStrictnessTest(unittest.TestCase): + """A moved base image is a note under PINS_STRICT alone, fatal under BASE_IMAGE_STRICT. + + The weekly job sets only PINS_STRICT=1; `release` and `pins-strict` add the other. + Two Dockerfiles pinning the base at different digests is a defect in this tree, not + upstream news, and stays fatal in every mode. + """ + + def test_a_moved_base_under_pins_strict_alone_is_a_note(self): + proc = run_pins(base_drift=True, external_rc=0, strict=True) + self.assertEqual(0, proc.returncode, proc.stderr) + self.assertIn("a fresher", proc.stdout) + self.assertIn("BASE_IMAGE_STRICT", proc.stdout) + self.assertNotIn("base image digest drift", proc.stderr) + + def test_a_moved_base_is_fatal_under_base_image_strict(self): + proc = run_pins(base_drift=True, external_rc=0, strict=False, base_strict=True) + self.assertEqual(1, proc.returncode, proc.stderr) + self.assertIn("base image digest drift", proc.stderr) + self.assertIn("stub: vendored.sh pins ran", proc.stdout) + + def test_inconsistent_dockerfile_pins_are_fatal_in_every_mode(self): + for strict, base_strict in ((False, False), (True, False), (False, True)): + proc = run_pins( + base_drift=True, + external_rc=0, + strict=strict, + base_strict=base_strict, + base_rc=2, + ) + self.assertEqual(1, proc.returncode, (strict, base_strict, proc.stdout)) + self.assertIn("different digests", proc.stderr.lower()) + + +class PinsStrictTargetTest(unittest.TestCase): + def test_pins_strict_makes_both_halves_fatal(self): + text = CI_LOCAL.read_text(encoding="utf-8") + self.assertRegex( + text, + r"(?m)^\s*pins-strict\)\s+(?=.*\bPINS_STRICT=1\b)(?=.*\bBASE_IMAGE_STRICT=1\b).*\bpins\b", + "the `pins-strict` target must set PINS_STRICT=1 and BASE_IMAGE_STRICT=1, since " + "it is documented as the check `release` runs", + ) + + +class ShapeDriftFatalityTest(unittest.TestCase): + """The gdi-metadata shapes half of `pins()`: `vendored.sh drift gdi-metadata`. + + Fatal under PINS_STRICT like external pin drift, a warning otherwise. Exit 2 is no + network. Exit 1 is the set filter matching nothing, a defect in the leg's own call, + fatal in every mode. + """ + + def test_strict_shape_drift_is_fatal_and_named(self): + proc = run_pins(base_drift=False, external_rc=0, strict=True, shapes_rc=3) + self.assertEqual(1, proc.returncode, proc.stderr) + self.assertIn("gdi-metadata shapes", proc.stderr) + + def test_shape_drift_is_only_a_warning_without_strict_mode(self): + proc = run_pins(base_drift=False, external_rc=0, strict=False, shapes_rc=3) + self.assertEqual(0, proc.returncode, proc.stderr) + self.assertIn("WARNING: gdi-metadata shapes", proc.stdout) + + def test_unreachable_shapes_are_not_fatal_under_strict_mode(self): + proc = run_pins(base_drift=False, external_rc=0, strict=True, shapes_rc=2) + self.assertEqual(0, proc.returncode, proc.stderr) + self.assertIn("SKIPPED", proc.stdout) + + def test_a_set_filter_that_matches_nothing_is_fatal_in_every_mode(self): + for strict in (True, False): + proc = run_pins(base_drift=False, external_rc=0, strict=strict, shapes_rc=1) + self.assertEqual(1, proc.returncode, (strict, proc.stdout, proc.stderr)) + + def test_an_unexpected_drift_exit_still_names_the_other_findings(self): + # The leg promises one failure naming every half; a solo `die` here would eat the + # base-image and external verdicts collected above it. + proc = run_pins( + base_drift=True, external_rc=1, strict=True, base_strict=True, shapes_rc=1 + ) + self.assertEqual(1, proc.returncode, proc.stderr) + self.assertIn("base image digest drift", proc.stderr) + self.assertIn("external pin drift", proc.stderr) + self.assertIn("drift gdi-metadata exited 1", proc.stderr) + + class AnchoringTest(unittest.TestCase): def test_dev_reset_anchors_before_it_reads_the_working_directory(self): text = strip_comments(DEV_RESET.read_text(encoding="utf-8")) diff --git a/scripts/tests/test_userportal_profile_pins.py b/scripts/tests/test_userportal_profile_pins.py new file mode 100755 index 0000000..b24807b --- /dev/null +++ b/scripts/tests/test_userportal_profile_pins.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Guard: `vendored.sh pins` watches what the userportal's profile reads, not which tag it deploys. + +`check_ckanext.py` hand-mirrors the deployed harvest profile, and the conformance venv pins +the GDI dcat fork's upstream base. Both are facts about the forks at whatever tag the +userportal deploys, so `check_userportal_profiles` reads the tags from the userportal's own +`ckan/Dockerfile` and compares content at those tags: a `sha256=` entry by hash, a +`requirement=` entry against the version `conformance/requirements.txt` pins. + +Run against a stub `curl` on PATH that serves fixture files by URL. Exit codes follow +`check_pins`: 0 clean, 1 drift, 2 unreachable. +""" + +import hashlib +import os +import re +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +from _helpers import SCRIPTS, strip_comments + +VENDORED = SCRIPTS / "vendored.sh" +RAW = "https://raw.githubusercontent.com" + + +def _extract(name: str) -> str: + src = VENDORED.read_text(encoding="utf-8") + m = re.search(rf"^{re.escape(name)}\(\) \{{.*?^\}}", src, re.MULTILINE | re.DOTALL) + assert m, f"{name} is no longer defined in vendored.sh in an extractable form" + return m.group(0) + + +#: Serves `$STUB_FILES/`, 404 (curl exit 22) for anything else, and +#: curl exit 7 with no HTTP code when STUB_NET_DOWN=1. +STUB_CURL = """#!/usr/bin/env bash +url=""; out="" +while [ $# -gt 0 ]; do + case "$1" in + -o) out="$2"; shift ;; + https://*) url="$1" ;; + esac + shift +done +if [ "${STUB_NET_DOWN:-0}" = 1 ]; then printf '000 '; exit 7; fi +key="${url#https://raw.githubusercontent.com/}"; key="${key//\\//__}" +if [ -f "$STUB_FILES/$key" ]; then cp "$STUB_FILES/$key" "$out"; printf '200 41'; exit 0; fi +printf '404 41'; exit 22 +""" + +HEALTH_PROFILE = "class EuropeanHealthDCATAPProfile:\n reads = ['hdab']\n" +FDP_PROFILE = ( + "class FAIRDataPointDCATAPProfile(EuropeanHealthDCATAPProfile):\n pass\n" +) + + +def sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def dockerfile( + dcat_ref: str = "v9.9.9", + fdp_ref: str = "v1.0.0", + fdp: bool = True, + quoted: bool = False, +) -> str: + # The userportal installs forks in two shapes: `-e git+URL.git@ref#egg=name`, and the + # quoted `'name[extras] @ git+URL.git@ref'` it already uses for ckanext-scheming. + dcat = ( + f" 'ckanext-dcat[x] @ git+https://github.com/Org/fork-dcat.git@{dcat_ref}' \\" + if quoted + else f" -e git+https://github.com/Org/fork-dcat.git@{dcat_ref}#egg=ckanext-dcat \\" + ) + lines = ["RUN pip install \\", dcat] + if fdp: + lines.append( + f" -e git+https://github.com/Org/fork-fdp.git@{fdp_ref}#egg=ckanext-fairdatapoint \\" + ) + lines.append(" ckanext-dcat") + return "\n".join(lines) + "\n" + + +def pyproject(version: str) -> str: + # A dependency string carrying the pinned version too, so an unanchored match on the + # bare number would pass a moved [project] version. + return ( + "[project]\n" + 'name = "ckanext-dcat"\n' + f'version = "{version}"\n' + 'dependencies = ["ckanext-scheming>=2.4.4"]\n' + ) + + +class UserportalProfilePinsTest(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="gdi-userportal-pins-")) + self.addCleanup( + lambda: subprocess.run(["rm", "-rf", str(self.tmp)], check=False) + ) + stub_dir = self.tmp / "bin" + stub_dir.mkdir() + stub = stub_dir / "curl" + stub.write_text(STUB_CURL, encoding="utf-8") + stub.chmod(stub.stat().st_mode | stat.S_IXUSR) + self.files = self.tmp / "files" + self.files.mkdir() + (self.tmp / "conformance").mkdir() + (self.tmp / "conformance" / "requirements.txt").write_text( + "pyshacl==0.40.1\nckanext-dcat==2.4.4\nrdflib==7.6.0\n", encoding="utf-8" + ) + self.serve("Org/portal/main/ckan/Dockerfile", dockerfile()) + self.serve("Org/fork-dcat/v9.9.9/pyproject.toml", pyproject("2.4.4")) + self.serve( + "Org/fork-dcat/v9.9.9/ckanext/dcat/profiles/health.py", HEALTH_PROFILE + ) + self.serve("Org/fork-fdp/v1.0.0/ckanext/fairdatapoint/profiles.py", FDP_PROFILE) + + def serve(self, path: str, content: str) -> None: + (self.files / path.replace("/", "__")).write_text(content, encoding="utf-8") + + def run_check(self, net_down: bool = False) -> subprocess.CompletedProcess: + pins = [ + "Org/fork-dcat|pyproject.toml|requirement=ckanext-dcat|dcat base", + f"Org/fork-dcat|ckanext/dcat/profiles/health.py|sha256={sha256(HEALTH_PROFILE)}|health profile", + f"Org/fork-fdp|ckanext/fairdatapoint/profiles.py|sha256={sha256(FDP_PROFILE)}|fdp profile", + ] + script = "\n".join( + [ + "set -euo pipefail", + f"RAW={RAW!r}", + 'USERPORTAL_DOCKERFILE="Org/portal/main|ckan/Dockerfile"', + "USERPORTAL_PROFILE_PINS=(", + *(f" {p!r}" for p in pins), + ")", + _extract("curl_to"), + _extract("classify_fetch_failure"), + _extract("check_userportal_profiles"), + "rc=0; check_userportal_profiles || rc=$?", + 'echo "rc=$rc"', + ] + ) + env = { + **os.environ, + "PATH": f"{self.tmp / 'bin'}{os.pathsep}{os.environ['PATH']}", + "STUB_FILES": str(self.files), + "STUB_NET_DOWN": "1" if net_down else "0", + "GITHUB_TOKEN": "", + } + return subprocess.run( + ["bash", "-c", script], + cwd=self.tmp, + env=env, + capture_output=True, + text=True, + check=False, + ) + + def rc(self, proc: subprocess.CompletedProcess) -> int: + m = re.search(r"^rc=(\d+)$", proc.stdout, re.MULTILINE) + self.assertIsNotNone(m, f"no rc line:\n{proc.stdout}\n{proc.stderr}") + return int(m.group(1)) + + def test_matching_content_at_the_deployed_tags_passes(self): + proc = self.run_check() + self.assertEqual(0, self.rc(proc), proc.stdout + proc.stderr) + self.assertRegex( + proc.stdout, r"OK\s+health profile\s+\(Org/fork-dcat@v9\.9\.9\)" + ) + self.assertRegex(proc.stdout, r"OK\s+fdp profile\s+\(Org/fork-fdp@v1\.0\.0\)") + self.assertRegex(proc.stdout, r"OK\s+dcat base\s+\(Org/fork-dcat@v9\.9\.9") + + def test_a_new_tag_carrying_identical_content_is_not_drift(self): + # The reason this check exists: a userportal release that moves the tag but not + # the profile or the base must pass without anyone touching this tree. + self.serve("Org/portal/main/ckan/Dockerfile", dockerfile("v10.0.0", "v1.1.0")) + self.serve("Org/fork-dcat/v10.0.0/pyproject.toml", pyproject("2.4.4")) + self.serve( + "Org/fork-dcat/v10.0.0/ckanext/dcat/profiles/health.py", HEALTH_PROFILE + ) + self.serve("Org/fork-fdp/v1.1.0/ckanext/fairdatapoint/profiles.py", FDP_PROFILE) + proc = self.run_check() + self.assertEqual(0, self.rc(proc), proc.stdout + proc.stderr) + self.assertIn("@v10.0.0", proc.stdout) + + def test_the_quoted_install_form_resolves_the_tag_without_its_quote(self): + self.serve("Org/portal/main/ckan/Dockerfile", dockerfile(quoted=True)) + proc = self.run_check() + self.assertEqual(0, self.rc(proc), proc.stdout + proc.stderr) + self.assertRegex( + proc.stdout, r"OK\s+health profile\s+\(Org/fork-dcat@v9\.9\.9\)" + ) + + def test_changed_profile_content_at_the_deployed_tag_is_drift(self): + self.serve( + "Org/fork-dcat/v9.9.9/ckanext/dcat/profiles/health.py", + HEALTH_PROFILE + " reads.append('extra')\n", + ) + proc = self.run_check() + self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) + self.assertRegex( + proc.stdout, r"DRIFT\s+health profile\s+\(Org/fork-dcat@v9\.9\.9" + ) + self.assertRegex(proc.stdout, r"OK\s+fdp profile") + + def test_a_moved_upstream_base_version_is_drift(self): + # The expected version is the one in requirements.txt; nothing else states it. + self.serve("Org/fork-dcat/v9.9.9/pyproject.toml", pyproject("2.5.0")) + proc = self.run_check() + self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) + self.assertRegex(proc.stdout, r"DRIFT\s+dcat base\s+\(Org/fork-dcat@v9\.9\.9") + self.assertIn("2.5.0", proc.stdout) + self.assertIn("2.4.4", proc.stdout) + + def test_a_fork_no_longer_installed_from_the_dockerfile_is_drift(self): + self.serve("Org/portal/main/ckan/Dockerfile", dockerfile(fdp=False)) + proc = self.run_check() + self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) + self.assertRegex(proc.stdout, r"GONE\s+fdp profile\s+\(Org/fork-fdp") + self.assertRegex(proc.stdout, r"OK\s+health profile") + + def test_a_pinned_path_missing_at_the_deployed_tag_is_drift(self): + ( + self.files / "Org__fork-fdp__v1.0.0__ckanext__fairdatapoint__profiles.py" + ).unlink() + proc = self.run_check() + self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) + self.assertRegex(proc.stdout, r"GONE\s+fdp profile\s+\(HTTP 404") + + def test_a_missing_dockerfile_is_drift_not_unreachable(self): + # The userportal moving its Dockerfile is the deployment changing shape; read as + # a network failure, every later run would pass with a warning. + (self.files / "Org__portal__main__ckan__Dockerfile").unlink() + proc = self.run_check() + self.assertEqual(1, self.rc(proc), proc.stdout + proc.stderr) + self.assertRegex(proc.stdout, r"GONE\s+userportal ckan/Dockerfile") + + def test_no_network_is_unreachable_not_drift(self): + proc = self.run_check(net_down=True) + self.assertEqual(2, self.rc(proc), proc.stdout + proc.stderr) + self.assertIn("UNREACHABLE", proc.stdout) + self.assertNotIn("DRIFT", proc.stdout) + + +class ShippedPinsTest(unittest.TestCase): + def test_the_shipped_pins_are_well_formed(self): + src = VENDORED.read_text(encoding="utf-8") + m = re.search( + r"^USERPORTAL_PROFILE_PINS=\((.*?)^\)", src, re.MULTILINE | re.DOTALL + ) + assert m, "USERPORTAL_PROFILE_PINS is no longer an array literal in vendored.sh" + entries = re.findall(r'^\s*"([^"]+)"', m.group(1), re.MULTILINE) + reqs = (SCRIPTS.parent / "conformance" / "requirements.txt").read_text( + encoding="utf-8" + ) + self.assertTrue(entries) + for e in entries: + fields = e.split("|") + self.assertEqual(4, len(fields), e) + exp = fields[2] + self.assertRegex( + exp, r"^(sha256=[0-9a-f]{64}|requirement=[A-Za-z0-9_.-]+)$", e + ) + if exp.startswith("requirement="): + name = exp.split("=", 1)[1] + self.assertRegex( + reqs, + rf"(?m)^{re.escape(name)}==", + f"{name} is not pinned in conformance/requirements.txt", + ) + + def test_both_pin_subcommands_run_the_profile_check(self): + # Read what the script runs, not what its comments say. + src = strip_comments(VENDORED.read_text(encoding="utf-8")) + dispatch = src[src.index('case "$cmd" in') :] + for arm in ("check)", "pins)"): + body = dispatch[dispatch.index(arm) :] + body = body[: body.index(";;")] + self.assertIn("check_userportal_profiles", body, f"the `{arm}` arm") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_vendored_drift_filter.py b/scripts/tests/test_vendored_drift_filter.py new file mode 100755 index 0000000..e7aea01 --- /dev/null +++ b/scripts/tests/test_vendored_drift_filter.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Guard: `vendored.sh drift [set]` watches one set and tells drift from no network. + +The `pins` leg runs `drift gdi-metadata` fatally under PINS_STRICT, since the shapes are the +contract the conformance gate validates against, while Beacon and VRS drift stays advisory. +That needs a set filter and distinct exit codes: 0 clean, 2 fetch failure without drift, 3 +upstream moved (a file removed upstream counts), 1 this tree broken or a set name matching +nothing (a typo must not read as "unreachable" and pass as a warning). + +Run through the real script with a stub `curl` that serves the local vendored files back, +one of them altered, or nothing. +""" + +import os +import re +import shutil +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + +from _helpers import REPO_ROOT, SCRIPTS + +VENDORED = SCRIPTS / "vendored.sh" +SET_DIR = REPO_ROOT / "conformance" / "shapes" / "gdi-metadata" + +#: Serves `$STUB_LOCAL/` for `$STUB_PREFIX/`, altering `$STUB_DRIFT_FILE`, and +#: records every URL. STUB_NET_DOWN=1 is curl exit 7 with no HTTP code. +STUB_CURL = """#!/usr/bin/env bash +url=""; out="" +while [ $# -gt 0 ]; do + case "$1" in + -o) out="$2"; shift ;; + https://*) url="$1" ;; + esac + shift +done +printf '%s\\n' "$url" >> "$STUB_URLS" +if [ "${STUB_NET_DOWN:-0}" = 1 ]; then printf '000 '; exit 7; fi +case "$url" in + "$STUB_PREFIX"/*) + rel="${url#"$STUB_PREFIX"/}" + if [ -f "$STUB_LOCAL/$rel" ]; then + cp "$STUB_LOCAL/$rel" "$out" + [ "$rel" = "${STUB_DRIFT_FILE:-}" ] && printf '\\n# moved upstream\\n' >> "$out" + printf '200 41'; exit 0 + fi ;; +esac +printf '404 41'; exit 22 +""" + + +def md_field(key: str) -> str: + text = (SET_DIR / "VENDORED.md").read_text(encoding="utf-8") + m = re.search(rf"\*\*{key}:\*\*[^`]*`([^`]*)`", text) + assert m, f"{key} not in {SET_DIR}/VENDORED.md" + return m.group(1) + + +class DriftFilterTest(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="gdi-drift-")) + self.addCleanup( + lambda: subprocess.run(["rm", "-rf", str(self.tmp)], check=False) + ) + stub = self.tmp / "curl" + stub.write_text(STUB_CURL, encoding="utf-8") + stub.chmod(stub.stat().st_mode | stat.S_IXUSR) + self.urls = self.tmp / "urls" + self.urls.touch() + repo, path = md_field("Repository"), md_field("Path").rstrip("/") + self.prefix = ( + f"https://raw.githubusercontent.com/{repo}/{md_field('Branch')}/{path}" + ) + + def run_drift( + self, + *args: str, + drift_file: str = "", + net_down: bool = False, + local: Path = SET_DIR, + ): + env = { + **os.environ, + "PATH": f"{self.tmp}{os.pathsep}{os.environ['PATH']}", + "STUB_URLS": str(self.urls), + "STUB_PREFIX": self.prefix, + "STUB_LOCAL": str(local), + "STUB_DRIFT_FILE": drift_file, + "STUB_NET_DOWN": "1" if net_down else "0", + "GITHUB_TOKEN": "", + } + return subprocess.run( + ["bash", str(VENDORED), "drift", *args], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + def test_a_set_filter_fetches_only_that_set(self): + proc = self.run_drift("gdi-metadata") + self.assertEqual(0, proc.returncode, proc.stdout + proc.stderr) + urls = self.urls.read_text(encoding="utf-8").split() + self.assertTrue(urls) + self.assertTrue(all(u.startswith(self.prefix) for u in urls), urls) + self.assertNotIn("beacon", proc.stdout) + + def test_an_upstream_change_in_the_filtered_set_exits_3(self): + proc = self.run_drift("gdi-metadata", drift_file="Dataset.ttl") + self.assertEqual(3, proc.returncode, proc.stdout + proc.stderr) + self.assertRegex(proc.stdout, r"DRIFT\s+Dataset\.ttl") + + def test_no_network_exits_2_and_is_not_called_drift(self): + proc = self.run_drift("gdi-metadata", net_down=True) + self.assertEqual(2, proc.returncode, proc.stdout + proc.stderr) + self.assertIn("FETCH-FAIL", proc.stdout) + self.assertNotIn("DRIFT", proc.stdout) + + def test_a_set_name_matching_nothing_is_an_error_not_a_verdict(self): + proc = self.run_drift("no-such-set") + self.assertEqual(1, proc.returncode, proc.stdout + proc.stderr) + self.assertIn("no vendored set matches", proc.stderr) + self.assertEqual("", self.urls.read_text(encoding="utf-8")) + + def test_a_shape_removed_upstream_is_drift_not_a_fetch_failure(self): + # A 404 is upstream saying the file is gone, which is the strongest form of + # "moved"; read as a network failure it would pass the strict pins leg as SKIPPED. + upstream = self.tmp / "upstream" + shutil.copytree(SET_DIR, upstream) + (upstream / "Resource.ttl").unlink() + proc = self.run_drift("gdi-metadata", local=upstream) + self.assertEqual(3, proc.returncode, proc.stdout + proc.stderr) + self.assertRegex(proc.stdout, r"GONE\s+Resource\.ttl") + self.assertNotIn("verdict about the", proc.stderr) + + def run_drift_in_copy(self, mutate) -> subprocess.CompletedProcess: + """Run `drift gdi-metadata` from a copy of the tree with `mutate(set_dir)` applied. + + The script anchors on its own location, so a copy holding `scripts/vendored.sh` and + the one set is a complete tree for this subcommand. Upstream still serves the real + set, so whatever `mutate` did is a local fact, not drift. + """ + tree = self.tmp / "tree" + (tree / "scripts").mkdir(parents=True) + shutil.copy(VENDORED, tree / "scripts" / "vendored.sh") + copy = tree / "conformance" / "shapes" / "gdi-metadata" + shutil.copytree(SET_DIR, copy) + mutate(copy) + env = { + **os.environ, + "PATH": f"{self.tmp}{os.pathsep}{os.environ['PATH']}", + "STUB_URLS": str(self.urls), + "STUB_PREFIX": self.prefix, + "STUB_LOCAL": str(SET_DIR), + "STUB_DRIFT_FILE": "", + "STUB_NET_DOWN": "0", + "GITHUB_TOKEN": "", + } + return subprocess.run( + ["bash", str(tree / "scripts" / "vendored.sh"), "drift", "gdi-metadata"], + cwd=tree, + env=env, + capture_output=True, + text=True, + check=False, + ) + + def test_a_vendored_file_missing_locally_is_a_tree_defect_not_ok(self): + # Every surviving file matches upstream, so the counters alone say "clean"; the + # VENDORED.md file count is what knows a file is missing. + proc = self.run_drift_in_copy(lambda d: (d / "Resource.ttl").unlink()) + self.assertEqual(1, proc.returncode, proc.stdout + proc.stderr) + self.assertNotIn("OK: every vendored file", proc.stdout) + self.assertIn("FILE-COUNT MISMATCH", proc.stderr) + + def test_an_unparseable_vendored_md_exits_1_cleanly(self): + def blank_commit(d): + md = d / "VENDORED.md" + md.write_text( + re.sub( + r"(?m)^- \*\*Commit:\*\*.*$", "", md.read_text(encoding="utf-8") + ), + encoding="utf-8", + ) + + proc = self.run_drift_in_copy(blank_commit) + self.assertEqual(1, proc.returncode, proc.stdout + proc.stderr) + self.assertNotIn("unbound variable", proc.stderr) + self.assertIn("could not parse", proc.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_vendored_pins.py b/scripts/tests/test_vendored_pins.py index 478ff14..db5a31d 100755 --- a/scripts/tests/test_vendored_pins.py +++ b/scripts/tests/test_vendored_pins.py @@ -116,20 +116,52 @@ def test_the_token_goes_to_the_api_host_only(self): self.assertFalse([a for a in argv if a.startswith("Authorization")]) -class RateLimitIsReportedTest(unittest.TestCase): - def test_both_pin_checks_print_the_remedy_on_a_rate_limit(self): - # The leg-level half: the flag curl_to sets must reach a line naming the cause and - # GITHUB_TOKEN in both checks. A remedy printed by one and not the other sends the - # reader to the wrong function. - for fn in ("check_pins", "check_action_pins"): - body = _extract(fn) - self.assertRegex( - body, - r'CURL_RATELIMITED[^\n]*\n\s*echo "\s*RATE-LIMITED[^\n]*GITHUB_TOKEN', - f"{fn} does not print a RATE-LIMITED line naming GITHUB_TOKEN when " - "curl_to reports a spent budget", +class FetchFailuresAreClassifiedOnceTest(unittest.TestCase): + """One classifier decides gone vs rate-limited vs unreachable, and every fetch site uses it. + + Three hand-copied versions of the split let the fourth site omit it, which is how a + 404 on a vendored shape or on the userportal's Dockerfile read as "network" and passed + the strict leg. The classifier is where the flag `curl_to` sets becomes a line naming + the cause and, for a spent budget, GITHUB_TOKEN. + """ + + FETCH_SITES = ( + "check_pins", + "check_action_pins", + "check_userportal_profiles", + "process_set", + ) + + def test_the_classifier_names_the_rate_limit_remedy(self): + body = _extract("classify_fetch_failure") + self.assertRegex(body, r"RATE-LIMITED[^\n]*GITHUB_TOKEN") + + def test_every_fetch_site_uses_the_classifier_and_none_reimplements_it(self): + from _helpers import strip_comments + + for fn in self.FETCH_SITES: + body = strip_comments(_extract(fn)) + self.assertIn("classify_fetch_failure", body, f"{fn} does not classify") + self.assertNotIn( + "CURL_HTTP_CODE == 404", body, f"{fn} re-implements the 404 split" ) + def test_the_classifier_tells_gone_from_the_network(self): + script = ( + f"{_extract('classify_fetch_failure')}\n" + "CURL_HTTP_CODE=404 CURL_RATELIMITED=0 classify_fetch_failure 22; echo\n" + "CURL_HTTP_CODE=403 CURL_RATELIMITED=1 classify_fetch_failure 22; echo\n" + "CURL_HTTP_CODE=500 CURL_RATELIMITED=0 classify_fetch_failure 22; echo\n" + "CURL_HTTP_CODE=0 CURL_RATELIMITED=0 classify_fetch_failure 7; echo\n" + ) + proc = subprocess.run( + ["bash", "-c", script], capture_output=True, text=True, check=False + ) + kinds = [line.split()[0] for line in proc.stdout.splitlines() if line.strip()] + self.assertEqual( + ["GONE", "RATE-LIMITED", "UNREACHABLE", "UNREACHABLE"], kinds, proc.stdout + ) + if __name__ == "__main__": unittest.main() diff --git a/scripts/vendored.sh b/scripts/vendored.sh index 0d3c85a..958edeb 100755 --- a/scripts/vendored.sh +++ b/scripts/vendored.sh @@ -45,22 +45,25 @@ # commit in that set's VENDORED.md and the matching build constant # (api_version / gdi_metadata_version) by hand, per VENDORED.md, then # re-run `check`. -# drift Report whether upstream has moved. `check` cannot answer that, because -# it fetches each set at its immutable pinned commit and so stays green -# however far upstream advances. This compares the same files against -# the upstream branch (VENDORED.md `**Branch:**`). Exit 3 = upstream -# moved, which is news, not a defect. Network; not in `ci-local.sh all`. -# pins Assert each EXTERNAL_PINS entry below (the userportal's deployed -# CKAN-extension refs and gdi-metadata's declared HealthDCAT-AP release) -# still contains its expected token. `check` runs this too, so it needs -# no separate invocation there. +# drift [SET] Report whether upstream has moved: the same files compared against the +# upstream branch (VENDORED.md `**Branch:**`), which `check` cannot see +# because it fetches each set at its pinned commit. Every set, or SET +# alone (substring match, as `fetch`). Exit 3 = upstream moved (a file +# removed upstream counts), 2 = could not fetch, 1 = this tree is broken +# or SET matches nothing. News for Beacon and VRS, which the +# node tracks by release; a finding for the gdi-metadata shapes, which is +# why `ci-local.sh pins` runs `drift gdi-metadata`. Network. +# pins Assert the upstream content this repo depends on is still what it was +# pinned against: USERPORTAL_PROFILE_PINS (at the tags the userportal's +# ckan/Dockerfile names), EXTERNAL_PINS, and the workflows' action pins. +# `check` runs this too. # # Scope: `check` and `fetch` verify the fidelity of the files this repo does vendor. They # do not detect that upstream has added a file worth having, because each set vendors a # curated subset (beacon omits `examples*/`, for one), so picking up new files stays a # re-vendor decision rather than drift. `pins` covers the opposite case: upstream files -# this repo does not vendor but silently depends on (harvest/profile version, metadata -# lineage), asserting a substring rather than byte-equality. `pins` makes one +# this repo does not vendor but depends on, by hash where the content matters (the deployed +# profile code) and by substring where a token does (a release tag). `pins` makes one # `api.github.com` call for the beacon-v2 release watch and one per distinct Action pin, # against an unauthenticated budget of 60/h per IP. Set GITHUB_TOKEN to raise it; # the token is sent to api.github.com only (see curl_to). A rate-limited 403 is reported @@ -80,27 +83,41 @@ SETS=( "conformance/ga4gh-vrs-1.3" ) -# External upstream references this repo does not vendor byte-for-byte, but which the -# node's conformance and harvest story silently depends on and must not drift unnoticed: -# * the userportal's deployed CKAN-extension pins: the DCAT profile parser -# (`ckanext-dcat`) that the check_ckanext.py hand-mirror tracks, and the FDP harvester -# (`ckanext-fairdatapoint`) that crawls this node's FDP; -# * the gdi-metadata model's declared HealthDCAT-AP release ("lineage"). The node follows -# gdi-metadata and the userportal profile; its built-in [fairdp.publish] also adds what -# later HealthDCAT-AP releases ask for, where GDI's model has room for it. This fires -# when gdi-metadata moves off Release 5, the signal to re-vendor the shapes. -# Each entry asserts the remote file still contains an expected substring (a version tag -# or the declared release) rather than byte-diffing it: the upstream files change often -# for reasons that do not concern this repo, and only the pinned token matters. +# What the userportal's harvest reads from this node is decided by two files this repo does +# not vendor: the `fairdatapoint_dcat_ap` profile in the GDI fork of ckanext-fairdatapoint +# and its parent `euro_health_dcat_ap` in the GDI fork of ckanext-dcat, which +# `conformance/check_ckanext.py` hand-mirrors. The mirror holds while that code is what it +# was mirrored from; the conformance venv's `ckanext-dcat` pin holds while the dcat fork's +# upstream base is that version. Both are checked at whatever tag the userportal deploys, +# read from its own `ckan/Dockerfile`, so a userportal release that moves a tag without +# moving either passes. +USERPORTAL_DOCKERFILE="GenomicDataInfrastructure/gdi-userportal-ckan-docker/main|ckan/Dockerfile" +# Format: "|||