mirror shim: reuse a running shim only if it is this checkout's tools/mirror.py

SPEC.md

@@ -429,7 +429,9 @@ See `adapters/el.toml`. Notes that matter:
   `http://127.0.0.1:<port>/<host>/<path>`, for a fixed list of hosts, and caching what it fetched. It also keeps
   every file it fetched under its sha256 (`.sha256/<hex>` in its cache, which no request can reach) and logs every
   file it serves (`.fetched.log`: `<https url> sha256:<hex> <size>`). A live repo's `repomd.xml` is refetched each
-  time, so the store is the only place an older one survives. `wend replay --out RUN.json` turns the lines the
+  time, so the store is the only place an older one survives. The shim answers `/.wend-shim` with its own source's sha256, and
+  the harness reuses a shim already on the port only if that matches its own `tools/mirror.py`: one from another
+  checkout would serve with its own allow-list, so the harness stops and says to `shim-stop` it. `wend replay --out RUN.json` turns the lines the
   replay added to the log into a download list beside the run, `RUN.fetched/`: `fetched.txt` (each distinct file,
   sorted) and `blobs/<hex>` (the bytes of every file in it that isn't a package, and of live EPEL's packages, which
   go when they're updated). The gate uploads it for the archive (`docs/archive.md`, "Capture").

tools/mirror.py

@@ -28,6 +28,10 @@
 ALLOWED = {"vault.centos.org", "repo.almalinux.org", "vault.almalinux.org", "archives.fedoraproject.org",
            "dl.fedoraproject.org", "snapshot.debian.org", "deb.debian.org", "archive.ubuntu.com",
            "old-releases.ubuntu.com"}
+# The sha256 of this file, served at /.wend-shim: tools/sandbox.py reuses a shim already on the port only if it is this
+# one (one from another checkout would answer with its own allow-list and rules).
+with open(os.path.abspath(__file__), "rb") as _f:
+    VERSION = hashlib.sha256(_f.read()).hexdigest()
 FROZEN = {"snapshot.debian.org"}  # dated trees: a path's bytes never change, metadata included
 CACHE = sys.argv[2] if len(sys.argv) > 2 else "/var/tmp/wend/mirror"
 STORE = os.path.join(CACHE, ".sha256")  # not reachable through the shim: a request's first segment must be a host
@@ -129,6 +133,14 @@ class H(http.server.BaseHTTPRequestHandler):
     protocol_version = "HTTP/1.1"
 
     def _go(self, body):
+        if self.path == "/.wend-shim":
+            v = VERSION.encode()
+            self.send_response(200)
+            self.send_header("Content-Length", str(len(v)))
+            self.end_headers()
+            if body:
+                self.wfile.write(v)
+            return
         parts = self.path.lstrip("/").split("?")[0].split("/", 1)
         # Only <allowed host>/<relative path>: no empty or dot segments, so the path can't leave the cache
         # (os.path.join would throw CACHE away for "/etc/shadow"; the shim runs as root).

tools/sandbox.py

@@ -12,6 +12,7 @@
 import subprocess
 import sys
 import time
+import urllib.error
 import urllib.request
 import xml.etree.ElementTree as ET
 
@@ -141,17 +142,30 @@ def ensure_mirror(adapter):
         return
     shim = adapter["mirror"]["shim"]
     probe = f"{shim}/{adapter['mirror']['probe']}"
+    mine = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mirror.py")
+    want = hashlib.sha256(open(mine, "rb").read()).hexdigest()
     for i in range(30):
+        try:  # a shim from another checkout answers with its own allow-list: never reuse one that isn't this file
+            with urllib.request.urlopen(f"{shim}/.wend-shim", timeout=60) as r:
+                have = r.read().decode(errors="replace").strip()
+        except urllib.error.HTTPError:
+            have = "an older tools/mirror.py"  # one from before /.wend-shim answers 403
+        except Exception:
+            have = None  # nothing listening yet
+        if have is not None and have != want:
+            raise RuntimeError(f"a mirror shim from another checkout ({have[:16]}) is listening on {shim}, not this "
+                               f"checkout's tools/mirror.py ({want[:16]}); stop it (python3 tools/vm.py shim-stop PID) "
+                               "and run again")
         try:
+            if have is None:
+                raise OSError("no shim yet")
             urllib.request.urlopen(probe, timeout=60).read()
             return
         except Exception:
-            if i == 0:
+            if i == 0 and have is None:
                 os.makedirs(WORK, exist_ok=True)
-                subprocess.Popen([sys.executable, "-B", os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                                                     "mirror.py"), shim.rsplit(":", 1)[1],
-                                  f"{WORK}/mirror"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
-                                 start_new_session=True)
+                subprocess.Popen([sys.executable, "-B", mine, shim.rsplit(":", 1)[1], f"{WORK}/mirror"],
+                                 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True)
             time.sleep(2)
     raise RuntimeError(f"the mirror shim at {shim} isn't answering {probe}")
 

tools/test_fetched.py

@@ -108,6 +108,31 @@ def get(rel, rng=None):
     check(not os.path.exists(f"{work}/again/blobs/{sha[md]}") and
           open(f"{work}/again/missing.txt").read().split() == [f"https://{md}"],
           "a store file that doesn't hash to its name is reported missing, never copied")
+    import http.server
+    import threading
+    import sandbox
+    mine = hashlib.sha256(open(f"{HERE}/mirror.py", "rb").read()).hexdigest()
+    check(get(".wend-shim").decode() == mine, "the shim names its own source's sha256 at /.wend-shim")
+    try:
+        sandbox.ensure_mirror({"mirror": {"shim": f"http://127.0.0.1:{port}", "probe": md}})
+        check(True, "ensure_mirror reuses a running shim that is this checkout's tools/mirror.py")
+    except RuntimeError as e:
+        check(False, f"ensure_mirror refused this checkout's own shim: {e}")
+
+    class Old(http.server.BaseHTTPRequestHandler):  # a shim from before /.wend-shim: 403 for any unknown host
+        def do_GET(self):
+            self.send_error(403)
+
+        def log_message(self, *a):
+            pass
+    old = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Old)
+    threading.Thread(target=old.serve_forever, daemon=True).start()
+    try:
+        sandbox.ensure_mirror({"mirror": {"shim": f"http://127.0.0.1:{old.server_address[1]}", "probe": md}})
+        check(False, "ensure_mirror reused a shim from another checkout")
+    except RuntimeError as e:
+        check("another checkout" in str(e), "ensure_mirror refuses a shim from another checkout, naming shim-stop")
+    old.shutdown()
 finally:
     shim.terminate()
     shim.wait()