apt: reach every Debian and Ubuntu archive over https, through the mirror shim

SPEC.md

@@ -514,6 +514,11 @@ off after the box's own fragments, and stops the hop if `apt-config dump` still
 `/etc/apt/apt.conf`, which apt reads last); an unsigned hop removes that file. Box builds still use
 `debootstrap --no-check-gpg`.
 
+Transport: every Debian and Ubuntu archive is given as host/path, so the box fetches it as plain http from the mirror
+shim on 127.0.0.1, and the shim fetches it from the archive over https (apt before 1.5 has no https of its own). That covers the unsigned hops too, where transport is the only
+check. The shim refetches a live archive's `dists/` metadata on every request, as it does rpm `repodata/`, except
+files named by their hash (`by-hash/`) and the dated snapshot.debian.org tree, which never change.
+
 With `wend walk --run` every byte is checked against the run's pins anyway; signatures are what protects `--live`
 walks and CI replays.
 

adapters/debian.toml

@@ -27,15 +27,16 @@ writable = [
 # archive.debian.org has lost files (some of etch's, lenny's, squeeze's and wheezy's packages answer 404 now). The
 # archived releases come from a dated snapshot.debian.org tree instead: its dists/*/Release files are byte for byte
 # archive.debian.org's, and it has every file they list. It answers with redirects, which etch's and lenny's apt
-# can't follow, so it goes through tools/mirror.py, the shim EL uses (the same port: one shim serves both).
+# can't follow, so it goes through tools/mirror.py, the shim EL uses (the same port: one shim serves both). The
+# current mirror goes through it too, so the box reaches deb.debian.org over https (SPEC "Signed packages").
 shim = "http://127.0.0.1:8085"
 probe = "snapshot.debian.org/archive/debian-archive/20261003T165950Z/debian/dists/etch/Release"
 
 [releases]
 # Where each release's packages live today. Old ones move to the archive; the harness picks per release. A host/path
 # goes through the [mirror] shim.
 archive = "snapshot.debian.org/archive/debian-archive/20261003T165950Z/debian"
-current = "http://deb.debian.org/debian"
+current = "deb.debian.org/debian"
 current_from = "bullseye"
 # `signed`: the hop into the release checks the archive's signatures (SPEC "Signed packages"). Checked with gpgv
 # against the keyring the box has on the release before (debian-archive-keyring's last version there): from stretch's

adapters/ubuntu.toml

@@ -20,12 +20,19 @@ writable = [
   "/etc/mysql/mysql.conf.d/*.cnf",
 ]
 
+[mirror]
+# tools/mirror.py, the shim EL and Debian use (the same port: one shim serves all three). It reaches the archive over
+# https for the box: trusty's apt has no https of its own (SPEC "Signed packages").
+shim = "http://127.0.0.1:8085"
+probe = "archive.ubuntu.com/ubuntu/dists/trusty/Release"
+
 [releases]
 # Ubuntu moves a release to old-releases.ubuntu.com some time after its support ends; every LTS from trusty on is
 # still on the main archive (ESM keeps them there). The release pocket only: it is frozen at release time, so a hop's
 # pinned Release file stays good. -updates and -security move, and a route that read them couldn't be replayed.
-archive = "http://old-releases.ubuntu.com/ubuntu"
-current = "http://archive.ubuntu.com/ubuntu"
+# Both go through the [mirror] shim, which reaches them over https: trusty's apt 1.0 has no https of its own.
+archive = "old-releases.ubuntu.com/ubuntu"
+current = "archive.ubuntu.com/ubuntu"
 current_from = "trusty"
 # `signed`: the hop into the release checks the archive's signatures (SPEC "Signed packages"). Checked with gpgv
 # against the keyring the box has on the release before (ubuntu-keyring in its release pocket): every Release from

tools/mirror.py

@@ -2,7 +2,9 @@
 
 CentOS 5's yum (python 2.4, 2007's openssl) can't speak today's TLS, and vault.centos.org redirects http to https.
 Debian's archived releases come from a dated snapshot.debian.org tree, which answers with redirects that etch's and
-lenny's apt (before 0.7.21) can't follow; this follows them.
+lenny's apt (before 0.7.21) can't follow; this follows them. Every Debian and Ubuntu archive goes through it too, so
+the box speaks plain http only to this shim and the archive is reached over https: apt before 1.5 has no https of its
+own, and before the hops that check signatures, transport is all that stands between the box and a forged .deb.
 A chroot shares the host's network, so the box points at http://127.0.0.1:<port>/<host>/<path> and this fetches
 https://<host>/<path> once and serves it from disk after that. Only the hosts below are reachable through it.
 Started by tools/sandbox.py when an adapter has a [mirror] table. Standard library only.
@@ -24,7 +26,9 @@
 import urllib.request
 
 ALLOWED = {"vault.centos.org", "repo.almalinux.org", "vault.almalinux.org", "archives.fedoraproject.org",
-           "dl.fedoraproject.org", "snapshot.debian.org"}
+           "dl.fedoraproject.org", "snapshot.debian.org", "deb.debian.org", "archive.ubuntu.com",
+           "old-releases.ubuntu.com"}
+FROZEN = {"snapshot.debian.org"}  # dated trees: a path's bytes never change, metadata included
 CACHE = sys.argv[2] if len(sys.argv) > 2 else "/var/tmp/wend/mirror"
 STORE = os.path.join(CACHE, ".sha256")  # not reachable through the shim: a request's first segment must be a host
 LOG = os.path.join(CACHE, ".fetched.log")
@@ -136,12 +140,15 @@ def _go(self, body):
             return self.send_error(403)
         if local.endswith("/"):
             local += ".index"
-        # Live mirrors change their metadata; package files never change, and neither do repodata files named by
-        # their own hash (refetching those every time meant 170MB per dnf run, which timed out under load).
+        # Live mirrors change their metadata (rpm repodata, apt dists/); package files never change, and neither do
+        # metadata files named by their own hash (refetching those every time meant 170MB per dnf run, which timed out
+        # under load), apt's by-hash files included.
         name = os.path.basename(local)
         head = name.split("-", 1)[0]
         hashed = len(head) >= 40 and all(c in "0123456789abcdef" for c in head)
-        volatile = ("/repodata/" in parts[1] and "vault" not in parts[0] and not hashed) or local.endswith(".index")
+        live = "vault" not in parts[0] and parts[0] not in FROZEN
+        volatile = (("/repodata/" in parts[1] or "/dists/" in parts[1]) and live and not hashed) or \
+            local.endswith(".index")
         if not os.path.exists(local) or volatile:
             err = None
             for attempt in range(3):  # transfers through a busy proxy get cut off now and then

tools/vm.py

@@ -799,11 +799,10 @@ def install(box_id, box_doc, d):
             srv.httpd.shutdown()
             srv.httpd.server_close()
     else:  # debian-installer: netboot from the start release's mirror, the preseed and post_install() in its initrd
-        mirror = backends.of(a).hop_vars(a, box_doc["release"])["mirror"]
         host_mirror = backends.Apt().hop_vars(load(f"adapters/{fam}.toml"), box_doc["release"])["mirror"]
         for f in ("linux", "initrd.gz"):  # fetched on the host: a mirror through the shim is the host's 127.0.0.1
             fetch(f"{host_mirror}/{inst['netboot']}/{f}", os.path.join(d, f))
-        u = urllib.parse.urlsplit(mirror)
+        u = urllib.parse.urlsplit(host_mirror)  # the proxy runs on the host, so it reaches the shim at 127.0.0.1
         proxy = (PocketProxy(f"{u.scheme}://{u.netloc}", box_doc.get("vm", {}).get("pockets", []))
                  if inst.get("release_pocket_only") else None)
         extra = gzip.compress(newc({"preseed.cfg": preseed(box_doc, a, inst, proxy).encode(),