From 2bec25788b1a516207eb028f26a93c4db6206803 Mon Sep 17 00:00:00 2001 From: Kinflou Date: Thu, 12 Nov 2026 07:24:32 +0800 Subject: [PATCH 1/5] Build the images reproducibly: same inputs, same digest Drop the revision label and build-arg (the build provenance attestation records the commit), set SOURCE_DATE_EPOCH from DEBIAN_SNAPSHOT and export with rewrite-timestamp, remove logs and caches that vary between builds, and pin BuildKit. A pull request builds each image twice without a cache and fails if the manifest digests differ. A publish whose per-architecture manifests are the ones :latest already has pushes and attests nothing new. --- .github/workflows/build.yml | 141 ++++++++++++++++++++++++++++++------ README.md | 4 +- images/Dockerfile | 8 -- images/apt-install.sh | 11 ++- images/oci-diff.py | 100 +++++++++++++++++++++++++ images/oci-digest.sh | 14 ++++ images/rustup.sh | 6 +- images/source-date-epoch.sh | 14 ++++ images/versions.md | 35 +++++++++ 9 files changed, 301 insertions(+), 32 deletions(-) create mode 100755 images/oci-diff.py create mode 100755 images/oci-digest.sh create mode 100755 images/source-date-epoch.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d647f76..c8267dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,12 @@ name: build # build provenance attestation, and then run core's bench in the published # ci-bench as a real `container:` job. # +# The images are reproducible: the same inputs give the same digest (the +# "Reproducible builds" section of images/versions.md). A pull request proves it +# by building each image twice, without a cache, and comparing the digests. A +# publish whose digests are the ones :latest already has pushes nothing new and +# attests nothing: the consumers' Renovate sees no change. +# # There is no weekly rebuild: the apt packages are pinned to the date of # DEBIAN_SNAPSHOT in images/Dockerfile, so a rebuild would change nothing. @@ -29,12 +35,15 @@ env: CORE_SHA: f9d53265b82c86502439fea1759e256973865853 jobs: - # Build each image natively for each architecture, load it into the local - # daemon and run the smoke test: as root and as another uid, with no network. + # Build each image natively for each architecture, as an OCI archive with + # the timestamps rewritten to SOURCE_DATE_EPOCH (the date of DEBIAN_SNAPSHOT, + # images/source-date-epoch.sh), load it into the local daemon and run the + # smoke test: as root and as another uid, with no network. On a pull request, + # also build it twice more without any cache and require the same digest. build: name: build (${{ matrix.image }}, ${{ matrix.arch }}) runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} - timeout-minutes: 60 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -44,41 +53,84 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Set SOURCE_DATE_EPOCH + run: echo "SOURCE_DATE_EPOCH=$(sh images/source-date-epoch.sh)" >> "$GITHUB_ENV" - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + driver-opts: image=moby/buildkit:v0.34.2@sha256:c5f8862f52499773af85bc07695b2379ef08e8002352b0df2c4864c91d78dabe - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: images target: ${{ matrix.image }} platforms: linux/${{ matrix.arch }} - load: true - tags: ci-${{ matrix.image }}:test - build-args: REVISION=${{ github.sha }} + outputs: type=oci,name=ci-${{ matrix.image }}:test,dest=${{ runner.temp }}/image.tar,rewrite-timestamp=true + build-args: SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} provenance: false sbom: false cache-from: type=gha,scope=ci-${{ matrix.image }}-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=ci-${{ matrix.image }}-${{ matrix.arch }} + - name: Load the image + run: | + set -euo pipefail + docker load -i "$RUNNER_TEMP/image.tar" | tee "$RUNNER_TEMP/load.txt" + ref=$(sed -n 's/^Loaded image\(\( ID\)\?\): //p' "$RUNNER_TEMP/load.txt" | head -n 1) + docker tag "$ref" ci-${{ matrix.image }}:test + echo "manifest digest: $(bash images/oci-digest.sh "$RUNNER_TEMP/image.tar")" - name: Smoke test (root) run: docker run --rm --network none -v "$PWD/images/smoke.sh:/smoke.sh:ro" ci-${{ matrix.image }}:test bash /smoke.sh ${{ matrix.image }} - name: Smoke test (uid 1001) run: docker run --rm --network none --user 1001:1001 -e HOME=/tmp -v "$PWD/images/smoke.sh:/smoke.sh:ro" ci-${{ matrix.image }}:test bash /smoke.sh ${{ matrix.image }} - # The same image again as an OCI archive (from the build cache, so it is - # quick): the layers' sizes there are what a pull transfers. - - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + # The layers' sizes in the OCI archive are what a pull transfers. + - name: Image size + run: | + set -euo pipefail + compressed=$(bash images/oci-size.sh "$RUNNER_TEMP/image.tar") + uncompressed=$(docker image inspect ci-${{ matrix.image }}:test --format '{{.Size}}') + echo "ci-${{ matrix.image }} linux/${{ matrix.arch }}: $((compressed / 1000000)) MB compressed ($compressed bytes), $((uncompressed / 1000000)) MB uncompressed" | tee -a "$GITHUB_STEP_SUMMARY" + # The reproducibility check: the same image twice, each build with no + # cache (not the registry's, not the first build's), so every step runs + # again. The OCI manifest digest of the two exports must be the same. + - name: Build again, no cache (1) + if: github.event_name == 'pull_request' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: images target: ${{ matrix.image }} platforms: linux/${{ matrix.arch }} - outputs: type=oci,dest=${{ runner.temp }}/image.tar - build-args: REVISION=${{ github.sha }} + outputs: type=oci,dest=${{ runner.temp }}/repro-1.tar,rewrite-timestamp=true + build-args: SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} provenance: false sbom: false - cache-from: type=gha,scope=ci-${{ matrix.image }}-${{ matrix.arch }} - - name: Image size + no-cache: true + - name: Build again, no cache (2) + if: github.event_name == 'pull_request' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: images + target: ${{ matrix.image }} + platforms: linux/${{ matrix.arch }} + outputs: type=oci,dest=${{ runner.temp }}/repro-2.tar,rewrite-timestamp=true + build-args: SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} + provenance: false + sbom: false + no-cache: true + - name: Check the two builds are the same + if: github.event_name == 'pull_request' run: | set -euo pipefail - compressed=$(bash images/oci-size.sh "$RUNNER_TEMP/image.tar") - uncompressed=$(docker image inspect ci-${{ matrix.image }}:test --format '{{.Size}}') - echo "ci-${{ matrix.image }} linux/${{ matrix.arch }}: $((compressed / 1000000)) MB compressed ($compressed bytes), $((uncompressed / 1000000)) MB uncompressed" | tee -a "$GITHUB_STEP_SUMMARY" + one=$(bash images/oci-digest.sh "$RUNNER_TEMP/repro-1.tar") + two=$(bash images/oci-digest.sh "$RUNNER_TEMP/repro-2.tar") + cached=$(bash images/oci-digest.sh "$RUNNER_TEMP/image.tar") + echo "ci-${{ matrix.image }} linux/${{ matrix.arch }}: build 1 $one, build 2 $two (first build, with cache: $cached)" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$one" != "$two" ]; then + echo "::error::ci-${{ matrix.image }} linux/${{ matrix.arch }} is not reproducible: $one != $two" + python3 -I images/oci-diff.py "$RUNNER_TEMP/repro-1.tar" "$RUNNER_TEMP/repro-2.tar" | tee -a "$GITHUB_STEP_SUMMARY" + exit 1 + fi + if [ "$one" != "$cached" ]; then + echo "::warning::the cached build differs from the uncached ones: $cached != $one" + python3 -I images/oci-diff.py "$RUNNER_TEMP/image.tar" "$RUNNER_TEMP/repro-1.tar" | tee -a "$GITHUB_STEP_SUMMARY" + fi # core's instruction-count bench, in the image as `docker run` (the # published image runs it as a `container:` job, below). amd64 only: the # counts are per architecture, and this is a does-it-work check. @@ -98,7 +150,8 @@ jobs: -w /work ci-bench:test sh /bench-check.sh # Push each image and architecture by digest, then smoke-test exactly what - # was pushed. + # was pushed. The build is reproducible, so for an unchanged image this is + # the manifest the registry already holds: the push adds nothing. publish: name: publish (${{ matrix.image }}, ${{ matrix.arch }}) if: github.ref == 'refs/heads/main' @@ -119,7 +172,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Set SOURCE_DATE_EPOCH + run: echo "SOURCE_DATE_EPOCH=$(sh images/source-date-epoch.sh)" >> "$GITHUB_ENV" - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + driver-opts: image=moby/buildkit:v0.34.2@sha256:c5f8862f52499773af85bc07695b2379ef08e8002352b0df2c4864c91d78dabe - uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0 with: registry: ghcr.io @@ -131,12 +188,12 @@ jobs: context: images target: ${{ matrix.image }} platforms: linux/${{ matrix.arch }} - build-args: REVISION=${{ github.sha }} + build-args: SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} provenance: false sbom: false cache-from: type=gha,scope=ci-${{ matrix.image }}-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=ci-${{ matrix.image }}-${{ matrix.arch }} - outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true,rewrite-timestamp=true - name: Smoke test the pushed image env: DIGEST: ${{ steps.build.outputs.digest }} @@ -156,7 +213,10 @@ jobs: if-no-files-found: error retention-days: 1 - # Join the architectures of each image into one manifest, tag it, attest it. + # Join the architectures of each image into one manifest, tag it, attest it, + # unless :latest already has exactly these per-architecture manifests: then + # the image is unchanged, its index is only given the new tags (the same + # digest, nothing new is pushed) and it is not attested again. merge: name: merge and attest (${{ matrix.image }}) needs: publish @@ -180,6 +240,8 @@ jobs: pattern: digests-${{ matrix.image }}-* merge-multiple: true - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + driver-opts: image=moby/buildkit:v0.34.2@sha256:c5f8862f52499773af85bc07695b2379ef08e8002352b0df2c4864c91d78dabe - uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0 with: registry: ghcr.io @@ -193,13 +255,45 @@ jobs: type=sha,prefix=sha-,format=short type=sha,prefix=sha-,format=long type=raw,value=latest + # Is what was just built what :latest already is? Compare the + # per-architecture manifests, not the index: the index is derived from + # them, and a retag must not depend on how it was assembled. + - name: Compare with :latest + id: current + working-directory: ${{ runner.temp }}/digests + run: | + set -euo pipefail + built=$(printf 'sha256:%s\n' * | sort | tr '\n' ' ') + latest=$(docker buildx imagetools inspect "$IMAGE:latest" --raw 2> /dev/null \ + | jq -r '[.manifests[]? | select(.platform.os == "linux") | .digest] | sort | join(" ")' || true) + echo "built: $built" + echo "latest: ${latest:-none}" + if [ "${built% }" = "$latest" ]; then + digest=$(docker buildx imagetools inspect "$IMAGE:latest" --format '{{json .Manifest}}' | jq -r .digest) + echo "unchanged=true" >> "$GITHUB_OUTPUT" + echo "digest=$digest" >> "$GITHUB_OUTPUT" + else + echo "unchanged=false" >> "$GITHUB_OUTPUT" + fi - name: Create the manifest list + if: steps.current.outputs.unchanged != 'true' working-directory: ${{ runner.temp }}/digests run: | # shellcheck disable=SC2046 docker buildx imagetools create \ $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf "$IMAGE@sha256:%s " *) + # Unchanged: tag the index :latest already is, as it is. The digest does + # not change; the sha tags are what the bench job below and Renovate follow. + - name: Tag the unchanged image + if: steps.current.outputs.unchanged == 'true' + env: + DIGEST: ${{ steps.current.outputs.digest }} + run: | + # shellcheck disable=SC2046 + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + "$IMAGE@$DIGEST" - name: Resolve the digest id: digest run: | @@ -210,6 +304,10 @@ jobs: { echo "## ci-${{ matrix.image }}" echo + if [ "${{ steps.current.outputs.unchanged }}" = true ]; then + echo "Unchanged: the digest is the one :latest already had, nothing new was pushed or attested." + echo + fi echo "Digest: \`$digest\`" echo echo "Tags: $(jq -r '.tags | map("\`" + . + "\`") | join(", ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")" @@ -229,7 +327,8 @@ jobs: echo " image: $IMAGE@$digest" echo '```' } >> "$GITHUB_STEP_SUMMARY" - - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + - if: steps.current.outputs.unchanged != 'true' + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-name: ${{ env.IMAGE }} subject-digest: ${{ steps.digest.outputs.digest }} diff --git a/README.md b/README.md index aef20f7..05d27d8 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,9 @@ says why Debian. ## Use it Pin the image by digest, never by tag. The digest of each build is in its -[workflow run](../../actions/workflows/build.yml)'s summary. +[workflow run](../../actions/workflows/build.yml)'s summary. The builds are +reproducible ([`images/versions.md`](images/versions.md#reproducible-builds)): a +digest changes only when an input of the image does. ```yaml jobs: diff --git a/images/Dockerfile b/images/Dockerfile index 34d9700..e87f79b 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -188,9 +188,7 @@ RUN --mount=type=bind,source=arch.sh,target=/arch.sh \ COPY --from=dl-iai /out/ /usr/local/bin/ COPY --from=dl-gh /out/ /usr/local/bin/ ENV CI_RUST_STABLE=${RUST_STABLE} -ARG REVISION=unknown LABEL org.opencontainers.image.source="https://github.com/ComlineProject/ci-images" \ - org.opencontainers.image.revision="${REVISION}" \ org.opencontainers.image.description="ComlineProject CI: Rust stable, valgrind and iai-callgrind-runner for the benchmarks (see images/versions.md)" @@ -217,9 +215,7 @@ RUN --mount=type=bind,source=apt-install.sh,target=/apt-install.sh \ COPY --from=fuzz-cargo /out/bin/ /usr/local/bin/ COPY --from=dl-gh /out/ /usr/local/bin/ ENV CI_RUST_NIGHTLY=${RUST_NIGHTLY} -ARG REVISION=unknown LABEL org.opencontainers.image.source="https://github.com/ComlineProject/ci-images" \ - org.opencontainers.image.revision="${REVISION}" \ org.opencontainers.image.description="ComlineProject CI: Rust nightly, cargo-fuzz and python3 for the fuzz jobs (see images/versions.md)" @@ -239,9 +235,7 @@ RUN --mount=type=bind,source=arch.sh,target=/arch.sh \ sh /rustup.sh "$RUST_STABLE" stable -t wasm32-unknown-unknown -t wasm32-wasip1 COPY --from=dl-wasm /out/ /usr/local/bin/ ENV CI_RUST_STABLE=${RUST_STABLE} -ARG REVISION=unknown LABEL org.opencontainers.image.source="https://github.com/ComlineProject/ci-images" \ - org.opencontainers.image.revision="${REVISION}" \ org.opencontainers.image.description="ComlineProject CI: Rust stable with wasm32 (unknown and wasip1), wasm-bindgen, wasm-pack and binaryen (see images/versions.md)" @@ -269,7 +263,5 @@ ARG RUST_STABLE COPY --from=tools-cargo /out/bin/ /usr/local/bin/ COPY --from=dl-cue /out/ /usr/local/bin/ ENV CI_RUST_STABLE=${RUST_STABLE} -ARG REVISION=unknown LABEL org.opencontainers.image.source="https://github.com/ComlineProject/ci-images" \ - org.opencontainers.image.revision="${REVISION}" \ org.opencontainers.image.description="ComlineProject CI: Rust stable with musl, cue and cbindgen (see images/versions.md)" diff --git a/images/apt-install.sh b/images/apt-install.sh index 1836be9..e56634f 100755 --- a/images/apt-install.sh +++ b/images/apt-install.sh @@ -1,8 +1,17 @@ #!/bin/sh # apt-install.sh PACKAGE...: install packages from the sources of the image -# (the date-pinned snapshot) and leave no package lists behind. +# (the date-pinned snapshot) and leave behind nothing that differs from one +# build to the next: no package lists, no download cache, no logs (they hold +# the build's clock), no ldconfig or debconf cache, no compiled Python. set -eu export DEBIAN_FRONTEND=noninteractive apt-get update -o APT::Update::Error-Mode=any apt-get install -y --no-install-recommends "$@" rm -rf /var/lib/apt/lists/* +# Files only: the directories belong to packages. +find /var/cache/apt /var/log -type f -delete +rm -f /var/cache/ldconfig/aux-cache /var/cache/debconf/*-old +# Byte-compiled Python (python3's postinst writes it) records the time of the +# build; Python writes it again, where it can, when a module is imported. +find /usr /etc /var -name '*.pyc' -delete +find /usr /etc /var -name __pycache__ -type d -empty -delete diff --git a/images/oci-diff.py b/images/oci-diff.py new file mode 100755 index 0000000..568e26d --- /dev/null +++ b/images/oci-diff.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""oci-diff.py A.tar B.tar: say what differs between two OCI archives of one image. + +Prints the manifest and config differences, then for every layer whose digest +differs, each file that is only in one of the two, or differs in type, mode, +owner, mtime, link target or content hash (at most --max per layer). Meant for +the log of a failed reproducibility check; it does not need either image to be +loadable. Run it with `python3 -I`. +""" +import argparse +import hashlib +import json +import tarfile + + +def read_blob(tar, digest): + algo, _, hexa = digest.partition(":") + return tar.extractfile(f"blobs/{algo}/{hexa}").read() + + +def manifest_of(tar): + index = json.loads(tar.extractfile("index.json").read()) + desc = index["manifests"][0] + while True: + doc = json.loads(read_blob(tar, desc["digest"])) + if "manifests" not in doc: + return desc["digest"], doc + desc = doc["manifests"][0] + + +def layer_files(tar, digest): + """path -> (type, mode, uid, gid, mtime, linkname, sha256 of content)""" + algo, _, hexa = digest.partition(":") + files = {} + with tarfile.open(fileobj=tar.extractfile(f"blobs/{algo}/{hexa}"), mode="r|*") as layer: + for m in layer: + h = "" + if m.isreg(): + h = hashlib.sha256(layer.extractfile(m).read()).hexdigest() + files[m.name] = (chr(m.type[0]) if isinstance(m.type, bytes) else m.type, + oct(m.mode), m.uid, m.gid, int(m.mtime), m.linkname, h) + return files + + +def diff_json(a, b, path=""): + if isinstance(a, dict) and isinstance(b, dict): + for k in sorted(set(a) | set(b)): + yield from diff_json(a.get(k), b.get(k), f"{path}/{k}") + elif isinstance(a, list) and isinstance(b, list) and len(a) == len(b): + for i, (x, y) in enumerate(zip(a, b)): + yield from diff_json(x, y, f"{path}[{i}]") + elif a != b: + yield f"{path}: {a!r} != {b!r}" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("a") + ap.add_argument("b") + ap.add_argument("--max", type=int, default=60) + args = ap.parse_args() + ta, tb = tarfile.open(args.a), tarfile.open(args.b) + da, ma = manifest_of(ta) + db, mb = manifest_of(tb) + print(f"manifest A {da}\nmanifest B {db}") + if da == db: + print("identical") + return + print("\nconfig differences:") + ca = json.loads(read_blob(ta, ma["config"]["digest"])) + cb = json.loads(read_blob(tb, mb["config"]["digest"])) + for line in diff_json(ca, cb): + print(" " + line) + la, lb = ma["layers"], mb["layers"] + if len(la) != len(lb): + print(f"\nlayer count differs: {len(la)} != {len(lb)}") + for i, (x, y) in enumerate(zip(la, lb)): + if x["digest"] == y["digest"]: + continue + print(f"\nlayer {i}: {x['digest']} != {y['digest']} (sizes {x['size']}, {y['size']})") + fa, fb = layer_files(ta, x["digest"]), layer_files(tb, y["digest"]) + shown = 0 + for name in sorted(set(fa) | set(fb)): + if fa.get(name) == fb.get(name): + continue + shown += 1 + if shown > args.max: + print(" ...") + break + if name not in fa: + print(f" only in B: {name}") + elif name not in fb: + print(f" only in A: {name}") + else: + print(f" {name}\n A {fa[name]}\n B {fb[name]}") + if shown == 0: + print(" same files and metadata: the difference is in the layer's encoding (tar or gzip)") + + +main() diff --git a/images/oci-digest.sh b/images/oci-digest.sh new file mode 100755 index 0000000..2dcd272 --- /dev/null +++ b/images/oci-digest.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# oci-digest.sh IMAGE.tar: the digest of the (first) image manifest in an OCI +# archive: what a registry would hold the image under. It does not depend on +# the archive's own tar headers, only on the manifest, so two builds of the +# same image have the same digest exactly when their config and every layer are +# the same bytes. +set -euo pipefail +tarball=$1 +blob() { tar -xOf "$tarball" "blobs/${1%%:*}/${1#*:}"; } +digest=$(tar -xOf "$tarball" index.json | jq -r '.manifests[0].digest') +while jq -e '.manifests' <<< "$(blob "$digest")" > /dev/null; do + digest=$(blob "$digest" | jq -r '.manifests[0].digest') +done +echo "$digest" diff --git a/images/rustup.sh b/images/rustup.sh index b10db94..a8a1b74 100755 --- a/images/rustup.sh +++ b/images/rustup.sh @@ -4,7 +4,9 @@ # e.g. -t wasm32-unknown-unknown), make it the default, and link LINK- # to it, so `stable` / `nightly` name the pinned toolchain without a download. # Everything under RUSTUP_HOME and CARGO_HOME is made open to every uid, in -# this one step so it adds no second copy of the files to the image. +# this one step so it adds no second copy of the files to the image. What +# rustup and cargo downloaded (the archives, the registry index) is removed: +# it is not needed afterwards, and it is not the same from one run to the next. set -eu . /arch.sh toolchain=$1 link=$2 @@ -18,4 +20,6 @@ rustup set auto-self-update disable rustup toolchain install "$toolchain" --profile minimal "$@" rustup default "$toolchain" ln -s "${toolchain}-${gnu}" "$RUSTUP_HOME/toolchains/${link}-${gnu}" +rm -rf "$RUSTUP_HOME/downloads" "$RUSTUP_HOME/tmp" \ + "$CARGO_HOME/registry" "$CARGO_HOME/git" "$CARGO_HOME/.package-cache" "$CARGO_HOME/.package-cache-mutate" chmod -R a+rwX "$RUSTUP_HOME" "$CARGO_HOME" diff --git a/images/source-date-epoch.sh b/images/source-date-epoch.sh new file mode 100755 index 0000000..eaee50b --- /dev/null +++ b/images/source-date-epoch.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# source-date-epoch.sh [DOCKERFILE]: print the build timestamp of the images, +# as Unix seconds: the date of DEBIAN_SNAPSHOT in the Dockerfile. It is the +# value of SOURCE_DATE_EPOCH, and the only place the images' timestamps come +# from, so it moves when (and only when) the snapshot does. +set -eu +dockerfile=${1:-"$(dirname "$0")/Dockerfile"} +snapshot=$(sed -n 's/^ARG DEBIAN_SNAPSHOT=\([0-9TZ]*\)$/\1/p' "$dockerfile") +case $snapshot in + [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z) ;; + *) echo "no DEBIAN_SNAPSHOT (YYYYMMDDTHHMMSSZ) in $dockerfile" >&2; exit 1 ;; +esac +iso=$(printf '%s' "$snapshot" | sed 's/^\(....\)\(..\)\(..\)T\(..\)\(..\)\(..\)Z$/\1-\2-\3 \4:\5:\6/') +date -u -d "$iso" +%s diff --git a/images/versions.md b/images/versions.md index d02ca17..fc003e8 100644 --- a/images/versions.md +++ b/images/versions.md @@ -57,6 +57,41 @@ Not in any image: Node (GitHub mounts its own into a job container for actions; a job that runs `node` in a step uses `actions/setup-node`), curl and unzip (only a build stage has curl), `pkg-config`, xz-utils. +## Reproducible builds + +Building an unchanged image again gives the same digest, so a merge to `main` +that touches no input publishes no new digest and Renovate opens no bump pull +requests in the consuming repositories. A digest changes only when an input +does: + +- **Base image**: `BASE_IMAGE`, by digest. +- **apt packages**: `DEBIAN_SNAPSHOT`, one snapshot.debian.org date. +- **Downloaded files**: every version `ARG`, each with its SHA-256 hash per + architecture, checked before use. The Rust toolchains are exact versions + (`RUST_STABLE`, `RUST_NIGHTLY`); cargo-fuzz and cbindgen are `--locked`. +- **Timestamps**: `SOURCE_DATE_EPOCH` is the date of `DEBIAN_SNAPSHOT` + (`images/source-date-epoch.sh`), and every image is exported with BuildKit's + `rewrite-timestamp=true`, so no file or history entry carries the build's + clock. BuildKit itself is pinned by digest in `build.yml`, because the layers' + compression comes from it. +- **Labels**: static. There is no `org.opencontainers.image.revision`; the + commit a digest was built from is in its build provenance attestation. +- **Build leftovers**: what varies between two runs is not left in the image: + `/var/log`, `/var/cache/apt`, `/var/lib/apt/lists`, the ldconfig and debconf + caches, compiled Python (`*.pyc`), and what rustup and cargo downloaded + (`images/apt-install.sh`, `images/rustup.sh`). + +The `build` job of a pull request checks it: it builds each image twice, with no +cache, exports both as OCI archives and fails if their manifest digests differ, +printing the files that differ (`images/oci-diff.py`). On `main`, `publish` +pushes each architecture by digest (the same manifest as before is no new +push), and `merge and attest` leaves `:latest`, the index and the attestation +alone when the per-architecture manifests are the ones `:latest` already has, +and only adds the `sha-` tags to it. + +A change to any input above, to a script in `images/`, or to the Dockerfile, is a +new digest, as it should be. + ## Size From the first build (pull request #3, `Image size` step of each build job). The From 1d2427882fb3ee057488902fc68175980c64b5d0 Mon Sep 17 00:00:00 2001 From: Kinflou Date: Thu, 12 Nov 2026 07:38:08 +0800 Subject: [PATCH 2/5] build: smoke-test a type=docker build, export the OCI archive from its cache (docker load cannot read OCI) --- .github/workflows/build.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8267dc..630d356 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,30 +63,36 @@ jobs: context: images target: ${{ matrix.image }} platforms: linux/${{ matrix.arch }} - outputs: type=oci,name=ci-${{ matrix.image }}:test,dest=${{ runner.temp }}/image.tar,rewrite-timestamp=true + outputs: type=docker,name=ci-${{ matrix.image }}:test,rewrite-timestamp=true build-args: SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} provenance: false sbom: false cache-from: type=gha,scope=ci-${{ matrix.image }}-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=ci-${{ matrix.image }}-${{ matrix.arch }} - - name: Load the image - run: | - set -euo pipefail - docker load -i "$RUNNER_TEMP/image.tar" | tee "$RUNNER_TEMP/load.txt" - ref=$(sed -n 's/^Loaded image\(\( ID\)\?\): //p' "$RUNNER_TEMP/load.txt" | head -n 1) - docker tag "$ref" ci-${{ matrix.image }}:test - echo "manifest digest: $(bash images/oci-digest.sh "$RUNNER_TEMP/image.tar")" - name: Smoke test (root) run: docker run --rm --network none -v "$PWD/images/smoke.sh:/smoke.sh:ro" ci-${{ matrix.image }}:test bash /smoke.sh ${{ matrix.image }} - name: Smoke test (uid 1001) run: docker run --rm --network none --user 1001:1001 -e HOME=/tmp -v "$PWD/images/smoke.sh:/smoke.sh:ro" ci-${{ matrix.image }}:test bash /smoke.sh ${{ matrix.image }} - # The layers' sizes in the OCI archive are what a pull transfers. + # The same image again as an OCI archive (from the build cache, so it is + # quick): the layers' sizes there are what a pull transfers, and its + # manifest digest is what the registry will hold. + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: images + target: ${{ matrix.image }} + platforms: linux/${{ matrix.arch }} + outputs: type=oci,dest=${{ runner.temp }}/image.tar,rewrite-timestamp=true + build-args: SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} + provenance: false + sbom: false + cache-from: type=gha,scope=ci-${{ matrix.image }}-${{ matrix.arch }} - name: Image size run: | set -euo pipefail compressed=$(bash images/oci-size.sh "$RUNNER_TEMP/image.tar") uncompressed=$(docker image inspect ci-${{ matrix.image }}:test --format '{{.Size}}') echo "ci-${{ matrix.image }} linux/${{ matrix.arch }}: $((compressed / 1000000)) MB compressed ($compressed bytes), $((uncompressed / 1000000)) MB uncompressed" | tee -a "$GITHUB_STEP_SUMMARY" + echo "manifest digest: $(bash images/oci-digest.sh "$RUNNER_TEMP/image.tar")" # The reproducibility check: the same image twice, each build with no # cache (not the registry's, not the first build's), so every step runs # again. The OCI manifest digest of the two exports must be the same. From 5f5ae8f6e5b5a79539179bb9b8dbb50e8cb6da75 Mon Sep 17 00:00:00 2001 From: Kinflou Date: Thu, 12 Nov 2026 07:46:51 +0800 Subject: [PATCH 3/5] source-date-epoch: refuse a snapshot date in the future; oci-diff: count mtime-only differences --- images/oci-diff.py | 13 +++++++++++-- images/source-date-epoch.sh | 9 ++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/images/oci-diff.py b/images/oci-diff.py index 568e26d..a0d987f 100755 --- a/images/oci-diff.py +++ b/images/oci-diff.py @@ -3,7 +3,8 @@ Prints the manifest and config differences, then for every layer whose digest differs, each file that is only in one of the two, or differs in type, mode, -owner, mtime, link target or content hash (at most --max per layer). Meant for +owner, link target or content hash (at most --max per layer); files that +differ only in mtime are counted, with a few examples. Meant for the log of a failed reproducibility check; it does not need either image to be loadable. Run it with `python3 -I`. """ @@ -80,9 +81,13 @@ def main(): print(f"\nlayer {i}: {x['digest']} != {y['digest']} (sizes {x['size']}, {y['size']})") fa, fb = layer_files(ta, x["digest"]), layer_files(tb, y["digest"]) shown = 0 + mtime_only = [] for name in sorted(set(fa) | set(fb)): if fa.get(name) == fb.get(name): continue + if name in fa and name in fb and fa[name][:4] + fa[name][5:] == fb[name][:4] + fb[name][5:]: + mtime_only.append(name) + continue shown += 1 if shown > args.max: print(" ...") @@ -93,7 +98,11 @@ def main(): print(f" only in A: {name}") else: print(f" {name}\n A {fa[name]}\n B {fb[name]}") - if shown == 0: + if mtime_only: + print(f" {len(mtime_only)} files differ only in mtime, for example:") + for name in mtime_only[:5]: + print(f" {name}: {fa[name][4]} != {fb[name][4]}") + if shown == 0 and not mtime_only: print(" same files and metadata: the difference is in the layer's encoding (tar or gzip)") diff --git a/images/source-date-epoch.sh b/images/source-date-epoch.sh index eaee50b..3b33dca 100755 --- a/images/source-date-epoch.sh +++ b/images/source-date-epoch.sh @@ -11,4 +11,11 @@ case $snapshot in *) echo "no DEBIAN_SNAPSHOT (YYYYMMDDTHHMMSSZ) in $dockerfile" >&2; exit 1 ;; esac iso=$(printf '%s' "$snapshot" | sed 's/^\(....\)\(..\)\(..\)T\(..\)\(..\)\(..\)Z$/\1-\2-\3 \4:\5:\6/') -date -u -d "$iso" +%s +epoch=$(date -u -d "$iso" +%s) +# rewrite-timestamp only lowers mtimes: a file made before the epoch keeps the +# time it was made at, and the digest then depends on the clock. +if [ "$epoch" -gt "$(date -u +%s)" ]; then + echo "DEBIAN_SNAPSHOT $snapshot is in the future: builds would not be reproducible" >&2 + exit 1 +fi +echo "$epoch" From 341da5cb112f7a30dcb1938f7ede42a27b93379f Mon Sep 17 00:00:00 2001 From: Kinflou Date: Thu, 12 Nov 2026 08:08:46 +0800 Subject: [PATCH 4/5] rustup.sh: sort rustlib/components (its order varies per run); pin the Dockerfile frontend by digest --- images/Dockerfile | 2 +- images/rustup.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index e87f79b..f4892d5 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1 +# syntax=docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e # # The CI images of ComlineProject, one Dockerfile, one build target each: # diff --git a/images/rustup.sh b/images/rustup.sh index a8a1b74..fb0e8ed 100755 --- a/images/rustup.sh +++ b/images/rustup.sh @@ -20,6 +20,10 @@ rustup set auto-self-update disable rustup toolchain install "$toolchain" --profile minimal "$@" rustup default "$toolchain" ln -s "${toolchain}-${gnu}" "$RUSTUP_HOME/toolchains/${link}-${gnu}" +# rustup writes the list of the components it installed in an order that varies +# from run to run (it comes out of a hash map); a sorted list means the same. +LC_ALL=C sort -o "$RUSTUP_HOME/toolchains/${toolchain}-${gnu}/lib/rustlib/components" \ + "$RUSTUP_HOME/toolchains/${toolchain}-${gnu}/lib/rustlib/components" rm -rf "$RUSTUP_HOME/downloads" "$RUSTUP_HOME/tmp" \ "$CARGO_HOME/registry" "$CARGO_HOME/git" "$CARGO_HOME/.package-cache" "$CARGO_HOME/.package-cache-mutate" chmod -R a+rwX "$RUSTUP_HOME" "$CARGO_HOME" From 9480cc188b39cc59884cb1ab60ea7a2a5447708b Mon Sep 17 00:00:00 2001 From: Kinflou Date: Thu, 12 Nov 2026 08:24:07 +0800 Subject: [PATCH 5/5] versions.md: what the reproducibility check found --- images/versions.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/images/versions.md b/images/versions.md index fc003e8..60ad890 100644 --- a/images/versions.md +++ b/images/versions.md @@ -79,7 +79,13 @@ does: - **Build leftovers**: what varies between two runs is not left in the image: `/var/log`, `/var/cache/apt`, `/var/lib/apt/lists`, the ldconfig and debconf caches, compiled Python (`*.pyc`), and what rustup and cargo downloaded - (`images/apt-install.sh`, `images/rustup.sh`). + (`images/apt-install.sh`, `images/rustup.sh`). rustup's list of installed + components (`lib/rustlib/components`) is written in a different order each + run, so it is sorted. The `cargo install` binaries (cargo-fuzz, cbindgen) + came out identical on both architectures without path remapping, so none is + applied. +- **Tools of the build**: BuildKit (`driver-opts` in `build.yml`) and the + Dockerfile frontend (`# syntax=` line) are pinned by digest. The `build` job of a pull request checks it: it builds each image twice, with no cache, exports both as OCI archives and fails if their manifest digests differ, @@ -89,6 +95,11 @@ push), and `merge and attest` leaves `:latest`, the index and the attestation alone when the per-architecture manifests are the ones `:latest` already has, and only adds the `sha-` tags to it. +`DEBIAN_SNAPSHOT` must not be later than the time of the build: +`rewrite-timestamp` only lowers modification times, so files made earlier than +`SOURCE_DATE_EPOCH` would keep the build's clock. `images/source-date-epoch.sh` +fails when the date is in the future. + A change to any input above, to a script in `images/`, or to the Dockerfile, is a new digest, as it should be.