bench(h59): the H59-family runner (amendment R1); H57d amendment R2, Od renders in the hold's cluster

.github/workflows/ci.yml

@@ -269,6 +269,9 @@ jobs:
       - run: python3 bench/h54gpu/test_h54gpu.py
       - run: python3 bench/h57d/test_h57d.py
       - run: python3 bench/h59/test_h59.py
+      # The H59-family runner (amendment R1): renders through the launcher against the fit
+      # fixtures, arm.sh against a local stub server, its records scored by h59.py.
+      - run: python3 bench/h59/test_run.py
       - run: python3 bench/admission/test_ctl.py
       - run: python3 bench/admission/test_analyze.py
       - run: bash bench/admission/test_run.sh

bench/h57d/arm.sh

@@ -49,7 +49,11 @@ stamp "start ${MODE} (${left} s of the hold left)"
 M=${OUT}/deploy.yaml
 case ${DEPLOY} in
     k3d-gpu)
-        K3D_GPU_AGENT_DEVICE=CPU K3D_GPU_AGENT_MODEL=$(field model) K3D_GPU_AGENT_MODEL_NAME=${MODEL} K3D_GPU_AGENT_DRAFT=off \
+        # K3D_GPU_CLUSTER: `agent manifest` reads the node's hybrid label and allocatable CPU
+        # through context k3d-${K3D_GPU_CLUSTER}; unset, that is k3d-gpu, which the hold's
+        # kubeconfig does not hold, and Od renders with no P-core label and 1 CPU (H57d
+        # amendment R2).
+        K3D_GPU_CLUSTER=${CLUSTER} K3D_GPU_AGENT_DEVICE=CPU K3D_GPU_AGENT_MODEL=$(field model) K3D_GPU_AGENT_MODEL_NAME=${MODEL} K3D_GPU_AGENT_DRAFT=off \
             K3D_GPU_AUTOQUANT=off "${REPO}/scripts/k3d-gpu" agent manifest > "${OUT}/agent.yaml" 2> "${OUT}/agent.err" \
             && ${PY} "${REPO}/bench/apb/stage.py" ovms "${OUT}/agent.yaml" "${OUT}/staged.yaml" \
             && ${PY} "${D}/h57d.py" od-ro "${OUT}/staged.yaml" "${M}" ;;

bench/h57d/test_h57d.py

@@ -9,6 +9,7 @@
 import subprocess
 import sys
 import tempfile
+import time
 import unittest
 
 HERE = os.path.dirname(os.path.abspath(__file__))
@@ -644,5 +645,74 @@ def test_engine_flags_block(self):
         self.assertIn("  llamacpp: " + h57d.LLAMACPP, lines)
 
 
+ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
+FIX = os.path.join(ROOT, "tests", "fixtures", "fit")
+# kubectl stand-in: `k3d-gpu` names --context on every call; a hold's kubeconfig holds only the
+# hold's cluster, so any other context reads "context was not found". apply fails, which stops
+# arm.sh right after the render.
+KUBECTL = r'''#!/usr/bin/env python3
+import os, sys
+a = sys.argv[1:]
+ctx = None
+while a and a[0] == "--context":
+    ctx, a = a[1], a[2:]
+if ctx is not None and ctx != "k3d-" + os.environ["STUB_CLUSTER"]:
+    sys.exit(f"error: context was not found for specified context: {ctx}")
+s = " ".join(a)
+if s == "get nodes -l k3d-gpu.io/hybrid-cpu=true -o name":
+    print("node/k3d-x-server-0")
+elif "allocatable" in s:
+    print("cpu 32")
+elif a[:1] == ["apply"]:
+    sys.exit("apply refused by the test")
+'''
+
+
+@unittest.skipUnless(HAVE_YAML, "PyYAML")
+class OdRendersInTheHoldsCluster(unittest.TestCase):
+    """H57d amendment R2: arm.sh renders Od with K3D_GPU_CLUSTER set to the hold's cluster, so
+    `agent manifest` reads the hold's hybrid node: the P-core label and the 16 P-core CPUs (H57dp's
+    deviation was a render through context k3d-gpu: no label, 1 CPU)."""
+
+    def test_od_render_has_the_pcore_balloon(self):
+        with tempfile.TemporaryDirectory() as d:
+            b = os.path.join(d, "bin")
+            os.makedirs(b)
+            with open(os.path.join(b, "kubectl"), "w") as f:
+                f.write(KUBECTL)
+            os.chmod(os.path.join(b, "kubectl"), 0o755)
+            cache = os.path.join(d, "cache")
+            os.makedirs(cache)
+            for n in os.listdir(os.path.join(FIX, "models")):
+                src = os.path.join(FIX, "models", n)
+                os.symlink(src, os.path.join(cache, f"{n}@main"))
+            sysd = os.path.join(d, "root", "sys", "devices")
+            for name, cpus in (("cpu_core", "0-15"), ("cpu_atom", "16-31")):
+                os.makedirs(os.path.join(sysd, name))
+                with open(os.path.join(sysd, name, "cpus"), "w") as f:
+                    f.write(cpus + "\n")
+            open(os.path.join(d, "kubeconfig"), "w").close()
+            env = {"PATH": b + ":/usr/bin:/bin", "HOME": d, "REPO": ROOT, "CLUSTER": "k3dgpu-d-h57d-1",
+                   "STUB_CLUSTER": "k3dgpu-d-h57d-1", "KUBECONFIG": os.path.join(d, "kubeconfig"),
+                   "DEADLINE": str(int(time.time()) + 5000), "MODE": "perf",
+                   "K3D_GPU_SYSROOT": os.path.join(d, "root"), "K3D_GPU_FIT_CACHE": cache,
+                   "K3D_GPU_FIT_HOST": os.path.join(FIX, "hosts", "desktop-i9-nogpu.json"),
+                   "K3D_GPU_HF_FIXTURES": os.path.join(ROOT, "tests", "fixtures", "hf"),
+                   "XDG_STATE_HOME": os.path.join(d, "state"), "XDG_CACHE_HOME": os.path.join(d, "xdg")}
+            out = os.path.join(d, "Od")
+            r = subprocess.run(["bash", os.path.join(HERE, "arm.sh"), "Od", out], env=env,
+                               capture_output=True, text=True, timeout=300)
+            with open(os.path.join(out, "status.json")) as f:
+                self.assertIn("apply failed", json.load(f)["reason"], r.stdout + r.stderr)
+            with open(os.path.join(out, "agent.err")) as f:
+                err = f.read()
+            with open(os.path.join(out, "deploy.yaml")) as f:
+                docs = [x for x in yaml.safe_load_all(f) if x]
+        dep = [x for x in docs if x["kind"] == "Deployment"][0]
+        tpl = dep["spec"]["template"]
+        self.assertEqual(tpl["metadata"]["labels"].get("k3d-gpu.io/cores"), "performance", err)
+        self.assertEqual(str(tpl["spec"]["containers"][0]["resources"]["requests"]["cpu"]), "16", err)
+
+
 if __name__ == "__main__":
     unittest.main()

bench/h59/arm.sh

@@ -0,0 +1,203 @@
+#!/usr/bin/env bash
+# arm.sh ARM OUT: one H59-family arm in the hold's cluster (EXPERIMENTS.md "H59 pre-registration"
+# and "H59 amendment R1"): render it with `k3d-gpu agent manifest --model` in the hold's cluster,
+# stage and check it (run.py render), wait for the model list and a 16-token chat, record the
+# server's image, command line, CPUs and fit, run the hold's phases (MODE=perf: 60 s idle, 3
+# warmup requests, N=1, N=4, N=16 on crescent, TTFT, prefix reuse; MODE=quality: GSM8K items
+# 0-49), then delete the arm.
+#   REPO  this repository;  G  the campaign dir (models under G/models)
+#   CLUSTER  the hold's cluster;  KUBECONFIG  its scratch kubeconfig;  H59_PROFILE  the host
+#   DEADLINE  epoch second the hold's arms must end by;  MODE  perf | quality
+#   HOLDNUM  the hold's number (the prefix phase's seed);  SERVING  crescent's serving.jsonl
+#   H59_PORT  local port (default random);  H59_IDLE_S  idle window (default 60)
+# status.json: measured | not_deliverable | not_reached | failed | smoke, with the reason.
+set -uo pipefail
+ARM=$1 OUT=$2
+mkdir -p "${OUT}"
+D=${REPO:?}/bench/h59
+H=${REPO}/bench/h54
+PY=python3
+RUN=${D}/run.py
+LOAD=${REPO}/bench/llm_load.py
+ENERGY=${REPO}/bench/energy.sh
+BOOT=900
+IDLE=${H59_IDLE_S:-60}
+now() { date +%s; }
+stamp() { echo "$(date -u +%FT%TZ) ${ARM} $*" | tee -a "${OUT}/arm.log"; }
+status() { ${PY} -c 'import json,sys; json.dump({"status": sys.argv[1], "reason": sys.argv[2], "at": sys.argv[3]}, open(sys.argv[4], "w"))' \
+    "$1" "$2" "$(date -u +%FT%TZ)" "${OUT}/status.json"; stamp "status $1: $2"; }
+field() { ${PY} "${RUN}" field "${ARM}" "$1"; }
+pfield() { ${PY} "${RUN}" profile "${H59_PROFILE:?}" | sed -n "s/^$1=//p"; }
+NODE=k3d-${CLUSTER:?}-server-0
+NSP=k3d-gpu-agent SVC=$(field svc) SPORT=$(field port) API=$(field api)
+PHASES=$(pfield phases) RULE=$(pfield load_rule)
+MODEL=gemma
+PORT=${H59_PORT:-$(( 19000 + RANDOM % 1000 ))}
+pf=""
+pf_loop() {  # kubectl port-forward can drop on a long phase: respawn it
+    trap 'kill "${k}" 2>/dev/null; exit 0' TERM
+    while :; do
+        kubectl -n "${NSP}" port-forward "svc/${SVC}" "${PORT}:${SPORT}" >> "${OUT}/pf.log" 2>&1 &
+        k=$!
+        wait "${k}"
+        echo "port-forward exited $(date -u +%FT%TZ)" >> "${OUT}/pf.log"
+        sleep 1
+    done
+}
+cleanup_pf() { [ -z "${pf}" ] || kill "${pf}" 2>/dev/null; pf=""; }
+trap cleanup_pf EXIT
+
+need=900
+[ "${MODE:?}" = quality ] && need=600
+left=$(( DEADLINE - $(now) ))
+if [ "${left}" -lt "${need}" ]; then status not_reached "${left} s of the hold left (< ${need})"; exit 0; fi
+stamp "start ${MODE} (${left} s of the hold left)"
+
+# ---- render, stage, check (run.py render): agent.yaml, deploy.yaml, check.json, fit.json --------
+M=${OUT}/deploy.yaml
+if ! why=$(${PY} "${RUN}" render "${ARM}" "${OUT}" 2>&1); then
+    status failed "$(tail -1 <<< "${why}")"
+    exit 1
+fi
+kubectl apply -f "${M}" > "${OUT}/apply.log" 2>&1 || { status failed "apply failed: $(tail -1 "${OUT}/apply.log")"; exit 1; }
+
+# ---- readiness: the model list and a 16-token chat ------------------------------------------
+t_apply=$(now)
+limit=$(( BOOT < DEADLINE - t_apply - 300 ? BOOT : DEADLINE - t_apply - 300 ))
+base="http://127.0.0.1:${PORT}${API}"
+ready=""
+while [ $(( $(now) - t_apply )) -lt "${limit}" ]; do
+    if [ -z "${pf}" ] || ! kill -0 "${pf}" 2>/dev/null; then
+        pf=""
+        if kubectl -n "${NSP}" get pods -l "app=${SVC}" -o jsonpath='{.items[*].status.phase}' 2>/dev/null | grep -q Running; then
+            pf_loop &
+            pf=$!
+            sleep 3
+        fi
+    fi
+    if [ -n "${pf}" ] && curl -fsS --max-time 10 "${base}/models" > "${OUT}/models.json" 2>/dev/null \
+       && curl -fsS --max-time 120 "${base}/chat/completions" -H 'Content-Type: application/json' \
+            -d "{\"model\":\"${MODEL}\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Say hello.\"}]}" \
+            > "${OUT}/hello.json" 2>/dev/null; then
+        ready=1
+        break
+    fi
+    sleep 10
+done
+t_ready=$(now)
+echo "{\"apply\": ${t_apply}, \"ready\": ${t_ready}, \"boot_s\": $(( t_ready - t_apply )), \"limit_s\": ${limit}}" > "${OUT}/boot.json"
+kubectl get pods -A -o wide > "${OUT}/pods.txt" 2>&1
+pod=$(kubectl -n "${NSP}" get pods -l "app=${SVC}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+ctr=$(kubectl -n "${NSP}" get pod "${pod}" -o jsonpath='{.spec.containers[0].name}' 2>/dev/null)
+# cpu_usec: the k3d node container's cgroup CPU time, or nothing when it cannot be read.
+cpu_usec() {
+    local id
+    id=$(docker inspect -f '{{.Id}}' "${NODE}" 2>/dev/null) || return 1
+    awk '/^usage_usec/{print $2}' "/sys/fs/cgroup/system.slice/docker-${id}.scope/cpu.stat" 2>/dev/null
+}
+teardown() {
+    cleanup_pf
+    kubectl -n "${NSP}" logs "${pod}" --all-containers > "${OUT}/server.log" 2>&1
+    kubectl -n "${NSP}" describe pod "${pod}" > "${OUT}/describe.txt" 2>&1
+    kubectl delete -f "${M}" --wait=true --timeout=300s > "${OUT}/delete.log" 2>&1
+    # OVMS's compiled-model cache leaves with the arm, so no later boot reads it (H54's teardown).
+    [ "${ARM}" != O ] || docker exec "${NODE}" sh -c 'rm -rf /var/apb-models/.ov_cache/*' >> "${OUT}/delete.log" 2>&1
+    # The node's CPU reads idle (< 1 CPU-s/s over 10 s) before the next arm, at most 180 s; a
+    # cgroup that cannot be read ends the wait (recorded).
+    local a b
+    for _ in $(seq 18); do
+        a=$(cpu_usec) && [ -n "${a}" ] || { echo "idle wait: node cgroup not readable" >> "${OUT}/delete.log"; break; }
+        sleep 10
+        b=$(cpu_usec) && [ -n "${b}" ] || break
+        [ $(( (b - a) / 10000000 )) -lt 1 ] && break
+    done
+}
+if [ -z "${ready}" ]; then
+    if [ "${limit}" -lt "${BOOT}" ]; then status not_reached "not ready within the ${limit} s the hold left (boot limit ${BOOT} s)"
+    else status not_deliverable "no model list and 16-token chat within ${BOOT} s"; fi
+    teardown
+    exit 0
+fi
+stamp "ready after $(( t_ready - t_apply )) s"
+kubectl -n "${NSP}" get pod "${pod}" -o jsonpath='{.spec.containers[0].image} {.status.containerStatuses[0].imageID}' > "${OUT}/image.txt"
+kubectl -n "${NSP}" get pod "${pod}" -o json > "${OUT}/pod.json"
+# The server's command line, the CPU set it got (the P-core balloon on a hybrid host), the ISA it
+# sees and its threads' affinity.
+kubectl -n "${NSP}" exec "${pod}" -c "${ctr}" -- sh -c \
+    'echo "cmdline $(tr "\0" " " < /proc/1/cmdline)"; echo "cpuset $(cat /sys/fs/cgroup/cpuset.cpus.effective)"; grep Cpus_allowed_list /proc/1/status; echo "avx512f $(grep -c avx512f /proc/cpuinfo) amx $(grep -c amx_tile /proc/cpuinfo) avx_vnni $(grep -c avx_vnni /proc/cpuinfo) nproc $(nproc)"; echo "server threads by affinity:"; for t in /proc/1/task/*; do grep -s Cpus_allowed_list "${t}/status" | cut -f2; done | sort | uniq -c' > "${OUT}/cpus.txt" 2>&1
+# O: the graph.pbtxt the --pull init container wrote from this hold's arguments.
+[ "${ARM}" != O ] || kubectl -n "${NSP}" exec "${pod}" -c "${ctr}" -- cat "/models/OpenVINO/gemma-4-26b-a4b-it-int4-ov/graph.pbtxt" > "${OUT}/graph.pbtxt" 2>&1
+
+if [ -n "${SMOKE:-}" ]; then
+    # Wiring smoke (discarded, disclosed): one streamed 16-token request with ignore_eos, one chat
+    # with chat_template_kwargs and the prefix phase's first agent, then the arm is deleted.
+    ${PY} "${LOAD}" --base-url "${base}" --model "${MODEL}" --requests 1 --concurrency 1 --warmup 0 --max-tokens 16 \
+        --ignore-eos --unique-prompts --out "${OUT}/smoke-load.json" >> "${OUT}/arm.log" 2>&1
+    curl -sS --max-time 300 "${base}/chat/completions" -H 'Content-Type: application/json' \
+        -d "{\"model\":\"${MODEL}\",\"max_tokens\":16,\"chat_template_kwargs\":{\"enable_thinking\":false},\"messages\":[{\"role\":\"user\",\"content\":\"Say hello.\"}]}" \
+        > "${OUT}/smoke-kwargs.json" 2>&1
+    status smoke "ready; wiring checks in smoke-*.json"
+    teardown
+    exit 0
+fi
+
+# ---- phases ----------------------------------------------------------------------------------
+W0="" W1=""
+idle() {  # idle [SUFFIX]: the 60 s idle window (RAPL); on crescent also the void rule's CPU window
+    local t0
+    t0=$(date +%s.%N)
+    bash "${ENERGY}" --zone intel-rapl:0 --interval 5 --json "${OUT}/energy-idle${1:+-$1}.json" -- sleep "${IDLE}" 2>> "${OUT}/arm.log"
+    W0=${t0} W1=$(date +%s.%N)
+    echo "{\"phase\": \"idle${1:+-$1}\", \"t0\": ${W0}, \"t1\": ${W1}}" >> "${OUT}/phases.jsonl"
+}
+load() {  # load NAME REQUESTS CONCURRENCY MAX_TOKENS [extra llm_load args]
+    local name=$1 n=$2 c=$3 mt=$4
+    shift 4
+    bash "${ENERGY}" --zone intel-rapl:0 --interval 5 --json "${OUT}/energy-${name}.json" -- \
+        ${PY} "${LOAD}" --base-url "${base}" --model "${MODEL}" --requests "${n}" --concurrency "${c}" --warmup 0 \
+        --max-tokens "${mt}" --temperature 0 --unique-prompts --timeout 1800 --label "H59 ${ARM} ${name}" \
+        --out "${OUT}/load-${name}.json" "$@" 2>> "${OUT}/arm.log"
+}
+phase() {  # phase NAME REQUESTS CONCURRENCY MAX_TOKENS [extra]: run and record; on crescent (H54's
+    # rule, amendment R4) a void phase is re-run once after a fresh idle window
+    local name=$1 t0 t1 v try
+    for try in 1 2; do
+        t0=$(date +%s.%N)
+        load "$@"
+        t1=$(date +%s.%N)
+        if [ "${RULE}" != h54-void ]; then
+            echo "{\"phase\": \"${name}\", \"t0\": ${t0}, \"t1\": ${t1}}" >> "${OUT}/phases.jsonl"
+            return
+        fi
+        v=$(${PY} "${H}/h54.py" void "${SERVING:?}" "${t0}" "${t1}" "${W0}" "${W1}")
+        echo "{\"phase\": \"${name}\", \"try\": ${try}, \"t0\": ${t0}, \"t1\": ${t1}, \"w0\": ${W0}, \"w1\": ${W1}, \"check\": ${v}}" >> "${OUT}/phases.jsonl"
+        case ${v} in *'"void": true'*) [ "${try}" = 1 ] && { mv "${OUT}/load-${name}.json" "${OUT}/load-${name}-void1.json"; idle "${name}"; continue; } ;; esac
+        return
+    done
+}
+has() { [[ ",${PHASES}," == *",$1,"* ]]; }
+if [ "${MODE}" = perf ]; then
+    idle
+    load warmup 3 1 256 --ignore-eos
+    phase n1 8 1 256 --ignore-eos
+    phase n4 16 4 256 --ignore-eos
+    if has n16; then phase n16 32 16 256 --ignore-eos; fi
+    phase ttft 4 1 16 --prompt "$(cat "${H}/ttft-prompt.txt")"
+    t0=$(date +%s.%N)
+    ${PY} "${RUN}" prefix "${base}" "${MODEL}" "$(( 5900 + ${HOLDNUM:-0} ))" "${OUT}/prefix.json" >> "${OUT}/arm.log" 2>&1
+    echo "{\"phase\": \"prefix\", \"t0\": ${t0}, \"t1\": $(date +%s.%N)}" >> "${OUT}/phases.jsonl"
+    # Measured when the N=4 phase completed a request (h59.py scores the completed ones); a server
+    # that stopped answering after readiness is failed (H54 amendment R3).
+    if ${PY} -c 'import json,sys; s=json.load(open(sys.argv[1]))["phases"][0]["samples"]; sys.exit(not any(x.get("ok") for x in s))' \
+        "${OUT}/load-n4.json" 2>/dev/null; then status measured "perf phases recorded"
+    else status failed "server stopped answering: the N=4 phase completed no request"; fi
+else
+    t0=$(date +%s.%N)
+    ${PY} "${H}/h54.py" gsm8k "${base}" "${MODEL}" "${H}/gsm8k-test-0-199.jsonl" "${OUT}/quality.json" 0-49 >> "${OUT}/arm.log" 2>&1
+    echo "{\"phase\": \"quality\", \"t0\": ${t0}, \"t1\": $(date +%s.%N)}" >> "${OUT}/phases.jsonl"
+    if ${PY} -c 'import json,sys; q=json.load(open(sys.argv[1])); sys.exit(not (q.get("n") and q.get("errors", 0) < q["n"]))' \
+        "${OUT}/quality.json" 2>/dev/null; then status measured "quality recorded"
+    else status failed "quality not recorded, or every item errored"; fi
+fi
+teardown
+exit 0

bench/h59/hold-run.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# hold-run.sh R ARM...: inside hold.sh's lock and timeout: create the hold's cluster with the
+# host profile's vendors, node image and kubelet thresholds, start the P-core load sampler on a
+# hybrid host (H57d's, which the desktop and wildct load gate reads), run each arm (arm.sh) until
+# the profile's arms_end_s after the hold started. CLUSTER, KUBECONFIG, REPO, G, MODE,
+# H59_PROFILE, H59_IMAGE (and SERVING on crescent) come from hold.sh; hold.sh deletes the cluster.
+set -uo pipefail
+R=$1
+shift
+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+t0=$(date +%s)
+pfield() { python3 "${HERE}/run.py" profile "${H59_PROFILE:?}" | sed -n "s/^$1=//p"; }
+# The k3d shim in ${G}/bin adds the campaign's model directory at /var/apb-models in the node.
+K3D_GPU_VENDORS=$(pfield vendors) K3D_GPU_IMAGE=${H59_IMAGE:?} K3D_GPU_K3S_ARGS=$(pfield k3s_args) \
+    K3D_GPU_CLUSTER=${CLUSTER:?} timeout 300 "${REPO:?}/scripts/k3d-gpu" up > "${R}/up.log" 2>&1 \
+    || { echo "up failed"; tail -20 "${R}/up.log"; exit 1; }
+kubectl get node -o json > "${R}/node.json"
+docker exec "k3d-${CLUSTER}-server-0" ls /var/apb-models > /dev/null || { echo "models volume missing in the node"; exit 1; }
+if [ "$(pfield sampler)" = pcore ]; then
+    python3 "${REPO}/bench/h57d/h57d.py" sample "${R}/load.jsonl" "k3d-${CLUSTER}-server-0" &
+    sp=$!
+    trap 'kill "${sp}" 2>/dev/null' EXIT
+fi
+export DEADLINE=$(( t0 + $(pfield arms_end_s) ))
+for arm in "$@"; do
+    bash "${HERE}/arm.sh" "${arm}" "${R}/${arm}"
+done