From 80682a10f304d35f249f0f8962ed3092b408c404 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 23:15:56 +0000 Subject: [PATCH] apt: reach every Debian and Ubuntu archive over https, through the mirror shim deb.debian.org, archive.ubuntu.com and old-releases.ubuntu.com were plain http, so on any walk without --run, and on every hop before the signed ones, anyone on the network path could serve the box any .deb. The archives are now host/path mirrors through tools/mirror.py, which the box reaches as plain http on 127.0.0.1 and which fetches over https: trusty's and etch's apt have no https of their own. The shim refetches a live archive's dists/ metadata each time, as it does rpm repodata, except by-hash files and the dated snapshot.debian.org tree. The VM tier's release-pocket proxy fetches from the shim as the host sees it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EQ9ijFk8JWP18n4iscLJXi Signed-off-by: Claude --- SPEC.md | 5 +++++ adapters/debian.toml | 5 +++-- adapters/ubuntu.toml | 11 +++++++++-- tools/mirror.py | 17 ++++++++++++----- tools/vm.py | 3 +-- 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/SPEC.md b/SPEC.md index 635e3e7..fe5aff1 100644 --- a/SPEC.md +++ b/SPEC.md @@ -514,6 +514,11 @@ off after the box's own fragments, and stops the hop if `apt-config dump` still `/etc/apt/apt.conf`, which apt reads last); an unsigned hop removes that file. Box builds still use `debootstrap --no-check-gpg`. +Transport: every Debian and Ubuntu archive is given as host/path, so the box fetches it as plain http from the mirror +shim on 127.0.0.1, and the shim fetches it from the archive over https (apt before 1.5 has no https of its own). That covers the unsigned hops too, where transport is the only +check. The shim refetches a live archive's `dists/` metadata on every request, as it does rpm `repodata/`, except +files named by their hash (`by-hash/`) and the dated snapshot.debian.org tree, which never change. + With `wend walk --run` every byte is checked against the run's pins anyway; signatures are what protects `--live` walks and CI replays. diff --git a/adapters/debian.toml b/adapters/debian.toml index 03f96f6..4036db8 100644 --- a/adapters/debian.toml +++ b/adapters/debian.toml @@ -27,7 +27,8 @@ writable = [ # archive.debian.org has lost files (some of etch's, lenny's, squeeze's and wheezy's packages answer 404 now). The # archived releases come from a dated snapshot.debian.org tree instead: its dists/*/Release files are byte for byte # archive.debian.org's, and it has every file they list. It answers with redirects, which etch's and lenny's apt -# can't follow, so it goes through tools/mirror.py, the shim EL uses (the same port: one shim serves both). +# can't follow, so it goes through tools/mirror.py, the shim EL uses (the same port: one shim serves both). The +# current mirror goes through it too, so the box reaches deb.debian.org over https (SPEC "Signed packages"). shim = "http://127.0.0.1:8085" probe = "snapshot.debian.org/archive/debian-archive/20261003T165950Z/debian/dists/etch/Release" @@ -35,7 +36,7 @@ probe = "snapshot.debian.org/archive/debian-archive/20261003T165950Z/debian/dist # Where each release's packages live today. Old ones move to the archive; the harness picks per release. A host/path # goes through the [mirror] shim. archive = "snapshot.debian.org/archive/debian-archive/20261003T165950Z/debian" -current = "http://deb.debian.org/debian" +current = "deb.debian.org/debian" current_from = "bullseye" # `signed`: the hop into the release checks the archive's signatures (SPEC "Signed packages"). Checked with gpgv # against the keyring the box has on the release before (debian-archive-keyring's last version there): from stretch's diff --git a/adapters/ubuntu.toml b/adapters/ubuntu.toml index b9ade9c..21bf2d8 100644 --- a/adapters/ubuntu.toml +++ b/adapters/ubuntu.toml @@ -20,12 +20,19 @@ writable = [ "/etc/mysql/mysql.conf.d/*.cnf", ] +[mirror] +# tools/mirror.py, the shim EL and Debian use (the same port: one shim serves all three). It reaches the archive over +# https for the box: trusty's apt has no https of its own (SPEC "Signed packages"). +shim = "http://127.0.0.1:8085" +probe = "archive.ubuntu.com/ubuntu/dists/trusty/Release" + [releases] # Ubuntu moves a release to old-releases.ubuntu.com some time after its support ends; every LTS from trusty on is # still on the main archive (ESM keeps them there). The release pocket only: it is frozen at release time, so a hop's # pinned Release file stays good. -updates and -security move, and a route that read them couldn't be replayed. -archive = "http://old-releases.ubuntu.com/ubuntu" -current = "http://archive.ubuntu.com/ubuntu" +# Both go through the [mirror] shim, which reaches them over https: trusty's apt 1.0 has no https of its own. +archive = "old-releases.ubuntu.com/ubuntu" +current = "archive.ubuntu.com/ubuntu" current_from = "trusty" # `signed`: the hop into the release checks the archive's signatures (SPEC "Signed packages"). Checked with gpgv # against the keyring the box has on the release before (ubuntu-keyring in its release pocket): every Release from diff --git a/tools/mirror.py b/tools/mirror.py index 67e0197..ba16b42 100644 --- a/tools/mirror.py +++ b/tools/mirror.py @@ -2,7 +2,9 @@ CentOS 5's yum (python 2.4, 2007's openssl) can't speak today's TLS, and vault.centos.org redirects http to https. Debian's archived releases come from a dated snapshot.debian.org tree, which answers with redirects that etch's and -lenny's apt (before 0.7.21) can't follow; this follows them. +lenny's apt (before 0.7.21) can't follow; this follows them. Every Debian and Ubuntu archive goes through it too, so +the box speaks plain http only to this shim and the archive is reached over https: apt before 1.5 has no https of its +own, and before the hops that check signatures, transport is all that stands between the box and a forged .deb. A chroot shares the host's network, so the box points at http://127.0.0.1:// and this fetches https:/// once and serves it from disk after that. Only the hosts below are reachable through it. Started by tools/sandbox.py when an adapter has a [mirror] table. Standard library only. @@ -24,7 +26,9 @@ import urllib.request ALLOWED = {"vault.centos.org", "repo.almalinux.org", "vault.almalinux.org", "archives.fedoraproject.org", - "dl.fedoraproject.org", "snapshot.debian.org"} + "dl.fedoraproject.org", "snapshot.debian.org", "deb.debian.org", "archive.ubuntu.com", + "old-releases.ubuntu.com"} +FROZEN = {"snapshot.debian.org"} # dated trees: a path's bytes never change, metadata included CACHE = sys.argv[2] if len(sys.argv) > 2 else "/var/tmp/wend/mirror" STORE = os.path.join(CACHE, ".sha256") # not reachable through the shim: a request's first segment must be a host LOG = os.path.join(CACHE, ".fetched.log") @@ -136,12 +140,15 @@ def _go(self, body): return self.send_error(403) if local.endswith("/"): local += ".index" - # Live mirrors change their metadata; package files never change, and neither do repodata files named by - # their own hash (refetching those every time meant 170MB per dnf run, which timed out under load). + # Live mirrors change their metadata (rpm repodata, apt dists/); package files never change, and neither do + # metadata files named by their own hash (refetching those every time meant 170MB per dnf run, which timed out + # under load), apt's by-hash files included. name = os.path.basename(local) head = name.split("-", 1)[0] hashed = len(head) >= 40 and all(c in "0123456789abcdef" for c in head) - volatile = ("/repodata/" in parts[1] and "vault" not in parts[0] and not hashed) or local.endswith(".index") + live = "vault" not in parts[0] and parts[0] not in FROZEN + volatile = (("/repodata/" in parts[1] or "/dists/" in parts[1]) and live and not hashed) or \ + local.endswith(".index") if not os.path.exists(local) or volatile: err = None for attempt in range(3): # transfers through a busy proxy get cut off now and then diff --git a/tools/vm.py b/tools/vm.py index 1c54ec0..7e880a2 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -588,11 +588,10 @@ def build(box_id): srv.httpd.shutdown() srv.httpd.server_close() else: # debian-installer: netboot from the start release's mirror, the preseed and post_install() in its initrd - mirror = backends.of(a).hop_vars(a, box_doc["release"])["mirror"] host_mirror = backends.Apt().hop_vars(load(f"adapters/{fam}.toml"), box_doc["release"])["mirror"] for f in ("linux", "initrd.gz"): # fetched on the host: a mirror through the shim is the host's 127.0.0.1 fetch(f"{host_mirror}/{inst['netboot']}/{f}", os.path.join(d, f)) - u = urllib.parse.urlsplit(mirror) + u = urllib.parse.urlsplit(host_mirror) # the proxy runs on the host, so it reaches the shim at 127.0.0.1 proxy = (PocketProxy(f"{u.scheme}://{u.netloc}", box_doc.get("vm", {}).get("pockets", [])) if inst.get("release_pocket_only") else None) extra = gzip.compress(newc({"preseed.cfg": preseed(box_doc, a, inst, proxy).encode(),