From 8ed097fc8e5cf573a52e344b4af668257be8fbd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 02:24:03 +0000 Subject: [PATCH] mirror shim: reuse a running shim only if it is this checkout's tools/mirror.py A shim left on the port by another checkout serves with its own allow-list: after #123, an older one answers 403 for deb.debian.org and Ubuntu's archives mid-route. The shim now names its source's sha256 at /.wend-shim, and ensure_mirror refuses any other (an older shim answers 403 there), saying to stop it with vm.py shim-stop. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EQ9ijFk8JWP18n4iscLJXi Signed-off-by: Claude --- SPEC.md | 4 +++- tools/mirror.py | 12 ++++++++++++ tools/sandbox.py | 24 +++++++++++++++++++----- tools/test_fetched.py | 25 +++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/SPEC.md b/SPEC.md index c2b6dcb..c091199 100644 --- a/SPEC.md +++ b/SPEC.md @@ -429,7 +429,9 @@ See `adapters/el.toml`. Notes that matter: `http://127.0.0.1://`, for a fixed list of hosts, and caching what it fetched. It also keeps every file it fetched under its sha256 (`.sha256/` in its cache, which no request can reach) and logs every file it serves (`.fetched.log`: ` sha256: `). A live repo's `repomd.xml` is refetched each - time, so the store is the only place an older one survives. `wend replay --out RUN.json` turns the lines the + time, so the store is the only place an older one survives. The shim answers `/.wend-shim` with its own source's sha256, and + the harness reuses a shim already on the port only if that matches its own `tools/mirror.py`: one from another + checkout would serve with its own allow-list, so the harness stops and says to `shim-stop` it. `wend replay --out RUN.json` turns the lines the replay added to the log into a download list beside the run, `RUN.fetched/`: `fetched.txt` (each distinct file, sorted) and `blobs/` (the bytes of every file in it that isn't a package, and of live EPEL's packages, which go when they're updated). The gate uploads it for the archive (`docs/archive.md`, "Capture"). diff --git a/tools/mirror.py b/tools/mirror.py index ba16b42..7aa70a8 100644 --- a/tools/mirror.py +++ b/tools/mirror.py @@ -28,6 +28,10 @@ ALLOWED = {"vault.centos.org", "repo.almalinux.org", "vault.almalinux.org", "archives.fedoraproject.org", "dl.fedoraproject.org", "snapshot.debian.org", "deb.debian.org", "archive.ubuntu.com", "old-releases.ubuntu.com"} +# The sha256 of this file, served at /.wend-shim: tools/sandbox.py reuses a shim already on the port only if it is this +# one (one from another checkout would answer with its own allow-list and rules). +with open(os.path.abspath(__file__), "rb") as _f: + VERSION = hashlib.sha256(_f.read()).hexdigest() FROZEN = {"snapshot.debian.org"} # dated trees: a path's bytes never change, metadata included CACHE = sys.argv[2] if len(sys.argv) > 2 else "/var/tmp/wend/mirror" STORE = os.path.join(CACHE, ".sha256") # not reachable through the shim: a request's first segment must be a host @@ -129,6 +133,14 @@ class H(http.server.BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" def _go(self, body): + if self.path == "/.wend-shim": + v = VERSION.encode() + self.send_response(200) + self.send_header("Content-Length", str(len(v))) + self.end_headers() + if body: + self.wfile.write(v) + return parts = self.path.lstrip("/").split("?")[0].split("/", 1) # Only /: no empty or dot segments, so the path can't leave the cache # (os.path.join would throw CACHE away for "/etc/shadow"; the shim runs as root). diff --git a/tools/sandbox.py b/tools/sandbox.py index 9d92906..347c17c 100644 --- a/tools/sandbox.py +++ b/tools/sandbox.py @@ -12,6 +12,7 @@ import subprocess import sys import time +import urllib.error import urllib.request import xml.etree.ElementTree as ET @@ -141,17 +142,30 @@ def ensure_mirror(adapter): return shim = adapter["mirror"]["shim"] probe = f"{shim}/{adapter['mirror']['probe']}" + mine = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mirror.py") + want = hashlib.sha256(open(mine, "rb").read()).hexdigest() for i in range(30): + try: # a shim from another checkout answers with its own allow-list: never reuse one that isn't this file + with urllib.request.urlopen(f"{shim}/.wend-shim", timeout=60) as r: + have = r.read().decode(errors="replace").strip() + except urllib.error.HTTPError: + have = "an older tools/mirror.py" # one from before /.wend-shim answers 403 + except Exception: + have = None # nothing listening yet + if have is not None and have != want: + raise RuntimeError(f"a mirror shim from another checkout ({have[:16]}) is listening on {shim}, not this " + f"checkout's tools/mirror.py ({want[:16]}); stop it (python3 tools/vm.py shim-stop PID) " + "and run again") try: + if have is None: + raise OSError("no shim yet") urllib.request.urlopen(probe, timeout=60).read() return except Exception: - if i == 0: + if i == 0 and have is None: os.makedirs(WORK, exist_ok=True) - subprocess.Popen([sys.executable, "-B", os.path.join(os.path.dirname(os.path.abspath(__file__)), - "mirror.py"), shim.rsplit(":", 1)[1], - f"{WORK}/mirror"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - start_new_session=True) + subprocess.Popen([sys.executable, "-B", mine, shim.rsplit(":", 1)[1], f"{WORK}/mirror"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) time.sleep(2) raise RuntimeError(f"the mirror shim at {shim} isn't answering {probe}") diff --git a/tools/test_fetched.py b/tools/test_fetched.py index 25c4d92..82dd72a 100644 --- a/tools/test_fetched.py +++ b/tools/test_fetched.py @@ -108,6 +108,31 @@ def get(rel, rng=None): check(not os.path.exists(f"{work}/again/blobs/{sha[md]}") and open(f"{work}/again/missing.txt").read().split() == [f"https://{md}"], "a store file that doesn't hash to its name is reported missing, never copied") + import http.server + import threading + import sandbox + mine = hashlib.sha256(open(f"{HERE}/mirror.py", "rb").read()).hexdigest() + check(get(".wend-shim").decode() == mine, "the shim names its own source's sha256 at /.wend-shim") + try: + sandbox.ensure_mirror({"mirror": {"shim": f"http://127.0.0.1:{port}", "probe": md}}) + check(True, "ensure_mirror reuses a running shim that is this checkout's tools/mirror.py") + except RuntimeError as e: + check(False, f"ensure_mirror refused this checkout's own shim: {e}") + + class Old(http.server.BaseHTTPRequestHandler): # a shim from before /.wend-shim: 403 for any unknown host + def do_GET(self): + self.send_error(403) + + def log_message(self, *a): + pass + old = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Old) + threading.Thread(target=old.serve_forever, daemon=True).start() + try: + sandbox.ensure_mirror({"mirror": {"shim": f"http://127.0.0.1:{old.server_address[1]}", "probe": md}}) + check(False, "ensure_mirror reused a shim from another checkout") + except RuntimeError as e: + check("another checkout" in str(e), "ensure_mirror refuses a shim from another checkout, naming shim-stop") + old.shutdown() finally: shim.terminate() shim.wait()