ledger(H62): pre-register k3d-gpu against Intel's EAS default (EAS v1.2.0-rc.1 with EAT 2.0), with its runner

.github/workflows/ci.yml

@@ -268,6 +268,7 @@ jobs:
       - run: python3 bench/h54gpu/test_h54gpu.py
       - run: python3 bench/h57d/test_h57d.py
       - run: python3 bench/h59/test_h59.py
+      - run: python3 bench/h62/test_h62.py
       - run: python3 bench/admission/test_ctl.py
       - run: python3 bench/admission/test_analyze.py
       - run: bash bench/admission/test_run.sh

bench/eatflow/crescent/serving.py

@@ -7,7 +7,12 @@
 Slot mode (H56, SERVING_SLOT=i with SERVING_CPUS = the slice): only slot i's containers
 are this hold's, so a sibling hold's cluster and load count as foreign; each row also
 carries own_busy_pct / other_busy_pct (/proc/stat over the slice and the rest of the
-online CPUs) and own_mhz (mean scaling_cur_freq over the slice)."""
+online CPUs) and own_mhz (mean scaling_cur_freq over the slice).
+
+VM mode (H62, SERVING_VMS = comma-separated container names): those containers run this
+campaign's KVM guests, so each row records their CPU-s/s as vm_cpu and leaves them out of cpu
+and serving_cpu. Their names carry no k3dgpu prefix, so another row's hold counts them as
+foreign serving load."""
 import json
 import os
 import signal
@@ -40,6 +45,19 @@ def _slot():
 
 
 SLOT = _slot()
+
+
+def vms_from(value):
+    """SERVING_VMS: the container names of this campaign's guests (H62), or ()."""
+    return tuple(v.strip() for v in value.split(",") if v.strip())
+
+
+def split_vms(cpu, vms):
+    """(serving cpu, vm cpu): the readings of the containers named in vms move to the second."""
+    return ({n: v for n, v in cpu.items() if n not in vms}, {n: v for n, v in cpu.items() if n in vms})
+
+
+VMS = vms_from(os.environ.get("SERVING_VMS", ""))
 stop = False
 
 
@@ -113,8 +131,11 @@ def main(out):
                    for n in cs if cur.get(n) is not None and prev.get(n) is not None}
             with open("/proc/loadavg") as la:
                 load1 = float(la.read().split()[0])
+            cpu, vm_cpu = split_vms(cpu, VMS)
             row = {"ts": round(t1, 3), "seconds": round(t1 - t0, 3), "load1": load1, "cpu": cpu,
                    "serving_cpu": round(sum(cpu.values()), 3), "aiperf_procs": aiperf()}
+            if VMS:
+                row["vm_cpu"] = vm_cpu
             if SLOT is not None:
                 st1 = load._read("/proc/stat")
                 row.update(slot=SLOT, **slot_fields(st, st1, own))

bench/eatflow/crescent/test_crescent.py

@@ -55,6 +55,34 @@ def test_slot_fields(self):
             shutil.rmtree(tmp)
 
 
+class ServingVmTest(unittest.TestCase):
+    """H62 (EXPERIMENTS.md "H62 pre-registration"): the two KVM guests are this campaign's arms.
+    SERVING_VMS names their containers; their CPU is recorded as vm_cpu and kept out of the
+    serving covariate. Without it, every row reads as before."""
+
+    CPU = {"qwen38-a-hr": 1.25, "qwen38-b-hr": 1.0, "h62-vm-a": 3.5, "h62-vm-b": 2.0}
+
+    def test_vms_split_out_of_the_serving_covariate(self):
+        serving_cpu, vm_cpu = serving.split_vms(self.CPU, ("h62-vm-a", "h62-vm-b"))
+        self.assertEqual(serving_cpu, {"qwen38-a-hr": 1.25, "qwen38-b-hr": 1.0})
+        self.assertEqual(vm_cpu, {"h62-vm-a": 3.5, "h62-vm-b": 2.0})
+
+    def test_without_vms_nothing_moves(self):
+        self.assertEqual(serving.split_vms(self.CPU, ()), (self.CPU, {}))
+        self.assertEqual(serving.vms_from(""), ())
+        self.assertEqual(serving.vms_from("h62-vm-a, h62-vm-b,"), ("h62-vm-a", "h62-vm-b"))
+
+    def test_a_vm_missing_from_the_reading_is_not_invented(self):
+        serving_cpu, vm_cpu = serving.split_vms({"qwen38-a-hr": 1.0}, ("h62-vm-a",))
+        self.assertEqual((serving_cpu, vm_cpu), ({"qwen38-a-hr": 1.0}, {}))
+
+    def test_the_guests_are_not_this_campaigns_by_name(self):
+        # Another row's hold counts the guests as foreign serving load: their names carry no
+        # k3dgpu prefix, and hold.sh's cleanup (k3dgpu-e-*, k3dgpu-a-*) never removes them.
+        for name in ("h62-vm-a", "h62-vm-b"):
+            self.assertFalse(serving.is_mine(name), name)
+
+
 class LockwaitTest(unittest.TestCase):
     def run_log(self, text, waits=()):
         import lockwait

bench/eatflow/live.sh

@@ -43,6 +43,7 @@
 #                   solo: ARM = H53, one arm (LIVE_SOLO) per hold on the Arc Pro B70s (bench/eatflow/solo.py);
 #                   pairs: ARM = H47, H49, H47b, H49b, H50 or H50o, both arms live in cluster k3dgpu-e-<arm>, one hold
 #                   of interleaved pairs into LIVE_PAIRS_DIR (LIVE_HOLD = this hold's number);
+#                   ARM = H62: both servers in KVM guests, driven by LIVE_REMOTE_A and LIVE_REMOTE_B (bench/h62);
 #                   code: H43 (amendment W1), the coding agent on C12 with `k3d-gpu sandbox up`, one
 #                   block per level in LIVE_CODE_LEVELS (default "typical heavy heavy typical") into
 #                   LIVE_CODE_DIR/h43-<arm>-<level> (shared by an arm's holds, so a level's halves resume);
@@ -170,6 +171,7 @@ t0=$(date +%s)
 el() { echo $(( $(date +%s) - t0 )); }
 stamp() { echo "== $1 $(date +%T) (+$(el)s)" | tee -a "${OUT}/live.log"; }
 pids=()
+REMOTE=()   # H62: the remote arms' control scripts (their port-forwards stop with the hold)
 # DevUI bearer token for this hold only (the agent and the harness read DEVUI_AUTH_TOKEN).
 DEVUI_AUTH_TOKEN=$(python3 -c "import secrets; print(secrets.token_hex(16))")
 export DEVUI_AUTH_TOKEN
@@ -180,6 +182,7 @@ cleanup() {
     for p in "${pids[@]}"; do kill "${p}" 2>/dev/null; done
     [ -f "${OUT}/mcp.pid" ] && kill "$(cat "${OUT}/mcp.pid")" 2>/dev/null
     for f in "${OUT}"/agent-*.pid; do [ -f "${f}" ] && kill "$(cat "${f}")" 2>/dev/null; done
+    for r in "${REMOTE[@]}"; do "${r}" unforward >/dev/null 2>&1; done
     docker rm -f "${PFX}-litellm" "${PFX}-litellm-a" "${PFX}-litellm-b" "${PFX}-${lc}-srv" "${PFX}-rapl" \
         "${PFX}-redis" >/dev/null 2>&1
     k3d cluster delete "${K3D_GPU_CLUSTER}" >/dev/null 2>&1
@@ -449,14 +452,60 @@ if [ "${MODE}" = pairs ]; then
     # What to deploy: the family's arms, or the row's own B arm (H50o: B3).
     DESIGN=${FAMILY}
     [ "${ARM}" = H50o ] && DESIGN=H50o
+    [ "${ARM}" = H62 ] && DESIGN=H62
     [ "${ARM}" = H50 ] && KV_GIB=${LIVE_EAT_KV_GIB:-40}   # EAT's shipped KV cache (40 GiB)
     mkdir -p "${PD}"
     PD=$(cd "${PD}" && pwd)
     case ${DESIGN} in
         H47) A=A1 B=A4 ;;
         H49) A=A1 B=B1; export K3D_GPU_NODE_LABELS=ei-infra-eligible=true,ei-inference-eligible=true ;;
         H50o) A=A1 B=B3 ;;
+        H62) A=A1 B=B5 ;;
     esac
+    if [ "${DESIGN}" = H62 ]; then
+        # H62 ("H62 pre-registration"): each server runs in its own KVM guest on this host
+        # (bench/h62); nothing is deployed here. LIVE_REMOTE_A and LIVE_REMOTE_B name each arm's
+        # control script (bench/h62/arm.sh <arm>): restart its serving pod, record it, forward
+        # its Service to a host port and name its pod cgroup for the idle-arm check. From the
+        # LiteLLM gateways on, the hold is H50's. The local arms below run unchanged in the else
+        # branch.
+        RA=${LIVE_REMOTE_A:?H62 needs LIVE_REMOTE_A} RB=${LIVE_REMOTE_B:?H62 needs LIVE_REMOTE_B}
+        REMOTE=("${RA}" "${RB}")
+        stamp "restart both servers"
+        left=$(( CAP - $(el) - 900 ))     # ready with at least 900 s of the hold left
+        "${RA}" restart "${left}" > "${OUT}/restart-a.log" 2>&1 &
+        ra=$!
+        "${RB}" restart "${left}" > "${OUT}/restart-b.log" 2>&1 &
+        rb=$!
+        wait "${ra}"; xa=$?
+        wait "${rb}"; xb=$?
+        if [ "${xa}" != 0 ] || [ "${xb}" != 0 ]; then
+            "${RA}" logs "${OUT}/remote-a" > /dev/null 2>&1
+            "${RB}" logs "${OUT}/remote-b" > /dev/null 2>&1
+            if [ "${xa}" = 3 ] || [ "${xb}" = 3 ]; then
+                echo "NOT DELIVERABLE: not ready with 900 s of the hold left (restart A ${xa}, B ${xb})" | tee -a "${OUT}/live.log"
+                exit 3
+            fi
+            echo "restart failed (A ${xa}, B ${xb})" | tee -a "${OUT}/live.log"
+            exit 1
+        fi
+        stamp ready
+        "${RA}" records "${OUT}/remote-a" > "${OUT}/records-a.log" 2>&1 || { echo "A records failed"; exit 1; }
+        "${RB}" records "${OUT}/remote-b" > "${OUT}/records-b.log" 2>&1 || { echo "B records failed"; exit 1; }
+        "${RA}" forward >> "${OUT}/live.log" 2>&1 || { echo "A forward failed"; exit 1; }
+        "${RB}" forward >> "${OUT}/live.log" 2>&1 || { echo "B forward failed"; exit 1; }
+        P_SRV=$("${RA}" port) BPORT=$("${RB}" port) API_B=/v1
+        cg_a=$("${RA}" cgroup) || { echo "no cgroup for the A pod"; exit 1; }
+        cg_b=$("${RB}" cgroup) || { echo "no cgroup for the B pod"; exit 1; }
+        echo "pod cgroups: ${A} ${cg_a}; ${B} ${cg_b}" | tee -a "${OUT}/live.log"
+        python3 "${HERE}/pairs.py" usage "${cg_a}" "${cg_b}" >> "${OUT}/live.log" 2>&1 || { echo "pod cpu.stat unreadable"; exit 1; }
+        wait_http "http://127.0.0.1:${P_SRV}/v1/models" 120 || { echo "A server not answering"; exit 3; }
+        curl -fsS "http://127.0.0.1:${P_SRV}/v1/models" > "${OUT}/models-a.json"
+        wait_http "http://127.0.0.1:${BPORT}${API_B}/models" 300 || { echo "B server not answering"; exit 3; }
+        curl -fsS "http://127.0.0.1:${BPORT}${API_B}/models" > "${OUT}/models-b.json"
+        SERVED_A=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['data'][0]['id'])" "${OUT}/models-a.json")
+        SERVED_B=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['data'][0]['id'])" "${OUT}/models-b.json")
+    else
     # The B arm's server (every design but H47's one pod): namespace, Deployment, Service,
     # service port and OpenAI path.
     B_NS=default B_DEP=${B_REL} B_SVC=${B_REL}-service B_SPORT=80 B_API=/v1
@@ -590,6 +639,7 @@ if [ "${MODE}" = pairs ]; then
         curl -fsS "http://127.0.0.1:${BPORT}${API_B}/models" > "${OUT}/models-b.json"
         SERVED_B=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['data'][0]['id'])" "${OUT}/models-b.json")
     fi
+    fi
     litellm() {  # litellm x served port llm_port api_path
         printf 'model_list:\n  - model_name: %s\n    litellm_params:\n      model: openai/%s\n      api_base: http://127.0.0.1:%s%s\n      api_key: not-needed\n      timeout: 600\n' \
             "${ALIAS}" "$2" "$3" "$5" > "${OUT}/litellm-$1.yaml"

bench/eatflow/pairs.py

@@ -17,12 +17,16 @@
 pairs' pre-windows. bg_cpu, package W and throttle during the attempt are reported, never
 gated.
 
-Load rule L3 (H50, H50o, H53; EXPERIMENTS.md "Crescent amendment A3"): the same
+Load rule L3 (H50, H50o, H53, H62; EXPERIMENTS.md "Crescent amendment A3"): the same
 pre-windows, no match filter. Every complete pair enters the statistic; a balance gate (a
 sign-flip test of arm on each pre covariate, p < 0.01 gates) replaces it; a two-regressor
 adjustment and L2's matched subset are reported, never gated. The arm order of pair j is drawn
 from the registered seed (order), and each pairs directory carries one registration.json
 (register). Python 3 standard library only.
+
+A pod cgroup is a local cgroup v2 directory, or "exec:<command line>" for a pod on another
+machine (H62, EXPERIMENTS.md "H62 pre-registration": each server runs in its own KVM guest):
+the command prints that pod's cpu.stat, and the idle-arm check reads it like a local one.
 """
 import glob
 import json
@@ -103,9 +107,30 @@ def total_jiffies(stat_text, cpus):
     return tot
 
 
+EXEC = "exec:"              # a pod cgroup read through a command (H62: a pod in a KVM guest)
+EXEC_TIMEOUT_S = 15.0
+
+
+def _exec_text(cmdline):
+    """stdout of a command line (shlex-split, no shell), or "" when it fails or times out."""
+    argv = shlex.split(cmdline)
+    if not argv:
+        return ""
+    try:
+        p = subprocess.run(argv, capture_output=True, text=True, timeout=EXEC_TIMEOUT_S)
+    except (OSError, subprocess.SubprocessError):
+        return ""
+    return p.stdout if p.returncode == 0 else ""
+
+
 def cgroup_usage_usec(path):
-    """cpu.stat usage_usec of a cgroup v2 directory (None when unreadable)."""
-    for line in _read(os.path.join(path, "cpu.stat")).splitlines():
+    """cpu.stat usage_usec of a cgroup v2 directory, or of "exec:<command line>", a command that
+    prints a pod's cpu.stat on another machine (None when unreadable)."""
+    if path.startswith(EXEC):
+        text = _exec_text(path[len(EXEC):])
+    else:
+        text = _read(os.path.join(path, "cpu.stat"))
+    for line in text.splitlines():
         k, _, v = line.partition(" ")
         if k == "usage_usec" and v.strip().isdigit():
             return int(v)
@@ -512,13 +537,14 @@ def boot_share(d, pred, boot=stats.BOOT, seed=stats.SEED, stat=None):
 
 # Rule L3 rows: the seed of their arm order, fixed in the pre-registration (amendment A3).
 ORDER_SEEDS = {"H50": "H50-crescent-2026-10-20", "H50o": "H50o-crescent-2026-10-20",
-               "H53": "H53-crescent-2026-10-20"}
+               "H53": "H53-crescent-2026-10-20", "H62": "H62-crescent-2026-10-29"}
 
 # Hypothesis id -> (design, load rule). H47b and H49b are H47 and H49 under rule L2; H50 is
 # H49b's design on the Xeon 6741P, H50o the same with arm B3 (EXPERIMENTS.md "H50-H52 pre-registration");
-# H53 the H49 statistic over solo-hold block pairs on the Arc Pro B70s (solo.py; block-level verdict, "H53 pre-registration").
+# H53 the H49 statistic over solo-hold block pairs on the Arc Pro B70s (solo.py; block-level verdict, "H53 pre-registration");
+# H62 H50's design with arm B5, Intel's EAS default, each server in a KVM guest ("H62 pre-registration").
 DESIGNS = {"H47": ("H47", "L1"), "H49": ("H49", "L1"), "H47b": ("H47", "L2"), "H49b": ("H49", "L2"),
-           "H50": ("H49", "L3"), "H50o": ("H49", "L3"), "H53": ("H49", "L3")}
+           "H50": ("H49", "L3"), "H50o": ("H49", "L3"), "H53": ("H49", "L3"), "H62": ("H49", "L3")}
 
 
 # The ledger a rule-L3 row must be pre-registered in before any of its holds runs.
@@ -814,13 +840,20 @@ def analyze(attempts, pairs, hyp, a, b, holds=None, registration=None, solo=Fals
 if __name__ == "__main__":
     # `pairs.py design HYP`: "<design> <rule>" (live.sh reads the rule from here, DESIGNS only).
     # `pairs.py arm HYP N`: the arm solo hold N must run (solo-hold.sh checks it before the lock).
+    # `pairs.py usage SPEC...`: each pod cgroup's usage_usec, read once; exit 1 if one is unreadable
+    # (live.sh checks the H62 arms' exec: specs with it before a hold).
     try:
         if len(sys.argv) == 3 and sys.argv[1] == "design":
             print(*design(sys.argv[2]))
         elif len(sys.argv) == 4 and sys.argv[1] == "arm":
             design(sys.argv[2])
             print(expected_arm(sys.argv[2], int(sys.argv[3])))
+        elif len(sys.argv) >= 3 and sys.argv[1] == "usage":
+            got = [(spec, cgroup_usage_usec(spec)) for spec in sys.argv[2:]]
+            for spec, v in got:
+                print(f"usage_usec {v} {spec}")
+            sys.exit(1 if any(v is None for _, v in got) else 0)
         else:
-            raise SystemExit("usage: pairs.py design HYP | pairs.py arm HYP N")
+            raise SystemExit("usage: pairs.py design HYP | pairs.py arm HYP N | pairs.py usage SPEC...")
     except (ValueError, KeyError) as e:
         raise SystemExit(f"pairs.py: {e!r}")