batch: land #182, #236, #195, #224, #237, #226, #239, #230, #210, #233, #196, #235, #207, #213, #228, #227

.github/workflows/ci.yml

@@ -207,6 +207,7 @@ jobs:
       - run: python3 tests/ledger_drift_check.py
       - run: tests/pe_premise_check_test.sh
       - run: tests/pe_premise_check.sh
+      - run: python3 tests/model_cards_license_test.py
 
   bench-tests:
     name: Bench harness tests
@@ -225,6 +226,9 @@ jobs:
       # tests/fit_test.py (the fit arithmetic on fixture hosts and models, the guard
       # under simulated PSI) and the launcher's fit, agent manifest and agent guard.
       - run: tests/fit_test.sh
+      # The llama.cpp agent backend the fit picks (GGUF download, one-time GGUF conversion,
+      # MTP on request) and `up` leaving busy Intel cards out of the device plugin.
+      - run: tests/llamacpp_test.sh
 
   tune-tests:
     name: k3d-gpu tune tests
@@ -248,10 +252,13 @@ jobs:
       - run: python3 bench/test_agent_trace.py
       - run: python3 bench/eatflow/crescent/test_holds.py
       - run: tests/hold_own_test.sh
+      - run: tests/wildct_hold_test.sh
+      - run: python3 bench/wildct/test_wildct.py
       - run: python3 bench/load/test_load.py
       - run: python3 bench/eatflow/test_agentbench.py
       - run: python3 bench/eatflow/test_pairs.py
       - run: python3 bench/eatflow/test_solo.py
+      - run: tests/xhost_tools_test.sh
       - run: python3 bench/eatflow/test_h4345.py
       - run: python3 bench/models/test_export_qat_ov.py
       - run: python3 bench/models/test_qat_grid.py

Makefile

@@ -32,6 +32,7 @@ test:
 	tests/quickstart_test.sh
 	tests/quickstart_adversarial_test.sh
 	tests/fit_test.sh
+	tests/llamacpp_test.sh
 	python3 tests/fuzz/props.py
 
 # Shell scripts: *.sh plus any file whose shebang runs sh or bash.

README.md

@@ -573,7 +573,7 @@ name in place of `gpu`):
 ```bash
 docker exec k3d-gpu-server-0 touch /var/lib/rancher/k3s/server/manifests/nri-balloons.yaml.skip
 helm upgrade --install nri-resource-policy-balloons nri-resource-policy-balloons \
-  --repo https://containers.github.io/nri-plugins --version v0.14.0 \
+  --repo https://containers.github.io/nri-plugins --version v0.15.0 \
   --namespace kube-system -f values.yaml --force   # Helm 4: --force-conflicts
 ```
 
@@ -713,7 +713,7 @@ documents GPU support for NVIDIA only.
 ### Node Feature Discovery
 
 Every cluster also runs [Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery)
-v0.19.0 with Intel's
+v0.20.1 with Intel's
 [NodeFeatureRules](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/v0.37.1/deployments/nfd/overlays/node-feature-rules)
 v0.37.1, baked into the image (namespace `node-feature-discovery`). About 30 s
 after `up`, nodes carry the host's CPU, kernel and PCI features. On an
@@ -881,7 +881,7 @@ is deleted.
 | `K3D_GPU_AGENT_TIMEOUT` | `1800` | seconds `agent up` waits for download + load |
 | `K3D_GPU_AGENT_LOG_LEVEL` | `INFO` | `DEBUG` also logs the compiled model's execution device (and every request) |
 | `K3D_GPU_AGENT_IMAGE` | `openvino/model_server@sha256:a928…fc58` (2026.4.1-gpu) | server image |
-| `K3D_GPU_AGENT_BACKEND` | `ovms` (`vllm` on a Gaudi) | `ovms`, `vllm` or `llm-scaler` (see below) |
+| `K3D_GPU_AGENT_BACKEND` | `ovms` (`vllm` on a Gaudi; with `--model M`, the engine the fit picks, and on the CPU llama.cpp where an EXPERIMENTS.md H59-family row is SUPPORTED for the CPU class) | `ovms`, `vllm`, `llm-scaler` or `llama.cpp` (see below and [docs/fit.md](docs/fit.md)) |
 
 #### Any Hugging Face model: `agent up --model`
 
@@ -940,7 +940,7 @@ metadata. Metadata is cached for a day under `~/.cache/k3d-gpu/hf`. Without
 | `K3D_GPU_AGENT_DRAFT` | the optimizer's choice | `off` disables speculative decoding, `mtp` forces an MTP head or drafter |
 | `K3D_GPU_MODEL_OPT` | `auto` | `off` serves `--model` exactly as given |
 | `K3D_GPU_HF_FIXTURES` | unset | directory of saved metadata bundles to read instead of Hugging Face |
-| `K3D_GPU_MODEL_OPT_LEDGER` | the checkout's `EXPERIMENTS.md`, else the packaged `console/ledger-rules.json` | ledger read for promotions and for the per-CPU-class Fast Draft and autoquant defaults; packages ship `console/ledger-rules.json`, the H27/H34 rows generated from `EXPERIMENTS.md` (CI fails when it drifts) |
+| `K3D_GPU_MODEL_OPT_LEDGER` | the checkout's `EXPERIMENTS.md`, else the packaged `console/ledger-rules.json` | ledger read for promotions and for the per-CPU-class defaults: OVMS Fast Draft (H34), autoquant (H27), llama.cpp as the CPU engine (H59) and its QAT MTP drafter (H57dm, H57wm); packages ship `console/ledger-rules.json`, those rows generated from `EXPERIMENTS.md` (CI fails when it drifts) |
 
 #### vLLM backends for discrete Arc (Battlemage)
 
@@ -949,7 +949,7 @@ OpenAI API at `/v1` from `svc/vllm`:
 
 | Backend | Image (pinned by digest) | Notes |
 |---------|--------------------------|-------|
-| `vllm` | `vllm/vllm-openai-xpu:v0.31.0` | upstream vLLM for Intel GPUs; validated on Arc Pro B-series |
+| `vllm` | `vllm/vllm-openai-xpu:v0.32.0` | upstream vLLM for Intel GPUs (Data Center GPU and Arc, per [vLLM's XPU guide](https://github.com/vllm-project/vllm/blob/v0.32.0/docs/getting_started/installation/gpu.xpu.inc.md)); v0.32.0 not yet run here on an Arc B-series card |
 | `llm-scaler` | `intel/llm-scaler-vllm:1.5` (vLLM 0.26.0-b2) | Intel's build for Arc B60/B70 |
 
 ```bash
@@ -1200,7 +1200,7 @@ and submits through execlists, so a GuC fault there warns instead. A failed HuC
 only warns: it runs media decode and encode, not inference. Without root the GuC is reported as not checked. The refused GPU's
 facts/v1 target reads `usable: false` with the `reason` and `fix`; `plan` then
 falls back to the next green cell on a usable target (OVMS on the CPU, or
-llama.cpp on Vulkan, pinned as `ghcr.io/ggml-org/llama.cpp:server-vulkan-v0.5.0`
+llama.cpp on Vulkan, pinned as `ghcr.io/ggml-org/llama.cpp:server-vulkan-b11960` (release v0.8.0)
 by digest) and lists the fix. With nothing left to pick the decision is `fix`,
 not an engine; a Gaudi without its driver plans the same way. `agent up` follows
 the same verdict: `auto` uses OVMS on the CPU and says why, and
@@ -1485,7 +1485,7 @@ not switch the Intel GPU path.
 ### Agent sandboxes on E-cores
 
 `k3d-gpu sandbox up` installs [kubernetes-sigs/agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox)
-v1.0.7 (controller image pinned by digest) and a `python-ecores` SandboxTemplate
+v1.0.8 (controller image pinned by digest) and a `python-ecores` SandboxTemplate
 plus a one-replica warm pool in namespace `agent-sandbox`. Sandbox pods carry
 `k3d-gpu.io/cores: efficient` and request one CPU, so on a hybrid host untrusted
 agent tool execution runs on E-cores, apart from the P-cores that serve inference.
@@ -1506,7 +1506,7 @@ EOF
 ```
 
 The sandbox serves `POST /execute {"command": "..."}` on port 8888
-(`registry.k8s.io/agent-sandbox/python-runtime-sandbox:v1.0.7`, pinned by digest).
+(`registry.k8s.io/agent-sandbox/python-runtime-sandbox:v1.0.8`, pinned by digest).
 
 What works on k3d, and what does not:
 

bench/apb/arm.sh

@@ -65,14 +65,19 @@ case ${ARM} in
         chart=${APB_EAT_CHART:?set APB_EAT_CHART to EAT core/helm-charts/vllm}
         python3 "${HERE}/stage.py" values "${chart}/xeon-values.yaml" "${REPO}/integrations/eat/vllm-values.yaml" 6 "${OUT}/values.yaml"
         # EAT's catalog entry cpu-qwen3-4b (core/playbooks/deploy-inference-models.yml), ingress off.
+        # The image is the one H8 ran as arm A3, pinned here so that a later tag in
+        # integrations/eat/vllm-values.yaml does not change what this arm reproduces.
         helm template vllm-qwen-3-4b-cpu "${chart}" --values "${OUT}/values.yaml" \
+            --set image.tag="v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1" \
             --set LLM_MODEL_ID=Qwen/Qwen3-4B-Instruct-2507 --set global.monitoring=false \
             --set svcmonitor.enabled=false --set ingress.enabled=false --set route.enabled=false > "${OUT}/eat.yaml"
         python3 "${HERE}/stage.py" vllm "${OUT}/eat.yaml" "${OUT}/deploy.yaml" Qwen3-4B-Instruct-2507 32768
         stage Qwen3-4B-Instruct-2507 Qwen3-4B-Instruct-2507 1001
         NS=default DEP=vllm-qwen-3-4b-cpu EP=default/vllm-qwen-3-4b-cpu-service ;;
     vllm-cuda)
         gpu=$(nvidia-smi --query-gpu=name,uuid --format=csv,noheader | awk -F', ' '/RTX 4000/ {print $2}')
+        # vLLM CUDA v0.31.0 (agent.lock vllm_cuda), the image H8 ran as arm A4, kept so H8 reproduces.
+        # No pending row runs this arm; a new CUDA arm pins vLLM's newest release.
         image=vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b
         sed -e "s|@VLLM_IMAGE@|${image}|" -e "s|@GPU_UUID@|${gpu}|" "${HERE}/vllm-cuda.yaml" > "${OUT}/cuda.yaml"
         python3 "${HERE}/stage.py" cuda "${OUT}/cuda.yaml" "${OUT}/deploy.yaml" Qwen3-4B-AWQ

bench/apb/h7.sh

@@ -54,7 +54,10 @@ K3D_GPU_AGENT_DEVICE=CPU K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3-4B-fp16-ov K3D_GPU_A
     "${K}" agent manifest > "${OUT}/agent.yaml" 2> "${OUT}/agent.err" || { cat "${OUT}/agent.err"; exit 1; }
 python3 "${HERE}/stage.py" ovms "${OUT}/agent.yaml" "${OUT}/ovms.yaml"
 python3 "${HERE}/stage.py" values "${CHART}/xeon-values.yaml" "${REPO}/integrations/eat/vllm-values.yaml" 6 "${OUT}/values.yaml"
+# B's image is the one H7 ran, pinned here so that a later tag in integrations/eat/vllm-values.yaml
+# does not change what this script reproduces.
 helm template vllm-qwen-3-4b-cpu "${CHART}" --values "${OUT}/values.yaml" --set LLM_MODEL_ID=Qwen/Qwen3-4B --set cpu=16 \
+    --set image.tag="v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1" \
     --set global.monitoring=false --set svcmonitor.enabled=false --set ingress.enabled=false --set route.enabled=false > "${OUT}/eat.yaml"
 python3 "${HERE}/stage.py" vllm "${OUT}/eat.yaml" "${OUT}/vllm.yaml" Qwen3-4B 32768
 stage Qwen3-4B-fp16-ov OpenVINO/Qwen3-4B-fp16-ov

bench/apb/h7x.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""h7x.py RUN_DIR: score an H7x run (EXPERIMENTS.md H7x) from bench/apb/h7x.sh's records.
+
+Reads b<i>-<E>-{decode,prefill,conc4}.json (bench/llm_load.py) and the matching
+b<i>-<E>-<w>-energy.json and b<i>-<E>-idle-energy.json (bench/energy.sh). E is A (OVMS fp16),
+B (vLLM CPU bf16, EAT chart defaults) or C (B without --enforce-eager).
+
+The blocks are those listed in restarts.txt ("b<i> <E> <restarts before> <restarts after>", one
+line per block whose workloads ran) plus any with a decode record. A block is void when its decode
+phase has fewer than 5 ok requests (none when it left no decode record), any request stopped short
+of 256 output tokens or has no decode rate, or the engine's pod restarted during the block. With
+>= 2 void blocks, or fewer than 4 valid blocks (a hold cut short by its cap), in arm A or B the
+run is INCONCLUSIVE.
+
+Decision (pre-registered in H7x): R = median A decode tok/s / median B decode tok/s over every
+request of the valid blocks; gap(x) = max(x, 1/x) - 1. Blocks pair in run order: the k-th A
+block with the k-th B block, void blocks included in the count (ABBAABBAAB gives 5 pairs); a pair
+with a void block is dropped. r_k = ratio of their block medians.
+  SUPPORTED     gap(R) < 0.15, gap(R_bw) < 0.15 and gap(r_k) < 0.15 for every pair
+  REFUTED       gap(R) >= 0.15 and every r_k on R's side of 1 with gap(r_k) >= 0.15
+  INCONCLUSIVE  otherwise
+R_bw (effective weight bandwidth, weight bytes x decode tok/s) uses the bytes passed in.
+Reported, never gating: C/B decode, prefill TTFT and conc4 per arm, and the H2 energy method
+(idle W from the 20 s model-loaded idle; incremental J = E - idle W x t) per token.
+Stdlib only.
+"""
+import glob
+import json
+import os
+import re
+import statistics as st
+import sys
+
+A_BYTES, B_BYTES = 8_044_936_594, 8_044_982_000   # H7's weights: fp16 IR .bin, bf16 safetensors
+
+
+def gap(x):
+    return max(x, 1 / x) - 1
+
+
+def load(path):
+    try:
+        return json.load(open(path))
+    except (OSError, ValueError):
+        return None
+
+
+def restarts(run):
+    """{block: (engine, restarted)} for every block whose workloads ran: h7x.sh writes
+    "b<i> <E> <restarts before> <restarts after>" to restarts.txt after each block's workloads."""
+    out = {}
+    try:
+        with open(os.path.join(run, "restarts.txt")) as f:
+            for line in f:
+                p = line.split()
+                if len(p) >= 2 and re.fullmatch(r"b\d+", p[0]) and p[1] in ("A", "B", "C"):
+                    out[int(p[0][1:])] = (p[1], len(p) == 4 and p[2] != p[3])
+    except OSError:
+        pass
+    return out
+
+
+def blocks(run):
+    """Blocks in run order: every block in restarts.txt or with a decode record. A block that ran
+    but left no decode record (its energy.sh failed before llm_load) is void, and keeps its place
+    in the run order."""
+    out, rs = [], restarts(run)
+    ran = {i: e for i, (e, _) in rs.items()}
+    for p in glob.glob(os.path.join(run, "b*-*-decode.json")):
+        m = re.match(r"b(\d+)-([ABC])-decode\.json$", os.path.basename(p))
+        if m:
+            ran[int(m.group(1))] = m.group(2)
+    for i, e in ran.items():
+        rec = {"block": i, "engine": e}
+        for w in ("decode", "prefill", "conc4"):
+            d = load(os.path.join(run, f"b{i}-{e}-{w}.json"))
+            ph = d["phases"][0] if d and d.get("phases") else None
+            rec[w] = ph
+            rec[w + "_energy"] = load(os.path.join(run, f"b{i}-{e}-{w}-energy.json"))
+        rec["idle"] = load(os.path.join(run, f"b{i}-{e}-idle-energy.json"))
+        dec = [s for s in (rec["decode"] or {}).get("samples", []) if s.get("ok")]
+        rec["decode_tps"] = [s["decode_tokens_per_s"] for s in dec if s.get("decode_tokens_per_s")]
+        rec["restarted"] = rs.get(i, (e, False))[1]
+        rec["void"] = (len(dec) < 5 or len(rec["decode_tps"]) < len(dec) or rec["restarted"]
+                       or any(s.get("output_tokens") != 256 for s in dec))
+        out.append(rec)
+    return sorted(out, key=lambda r: r["block"])
+
+
+def incremental_j_per_token(rec, w):
+    e, idle, ph = rec.get(w + "_energy"), rec.get("idle"), rec.get(w)
+    if not (e and idle and ph and ph.get("output_tokens_total") and idle.get("seconds")):
+        return None
+    p_idle = idle["joules"] / idle["seconds"]
+    return (e["joules"] - p_idle * e["seconds"]) / ph["output_tokens_total"]
+
+
+def score(recs):
+    arms = {e: [r for r in recs if r["engine"] == e and not r["void"]] for e in "ABC"}
+    void = {e: sum(1 for r in recs if r["engine"] == e and r["void"]) for e in "ABC"}
+    res = {"blocks": [{k: r[k] for k in ("block", "engine", "void", "decode_tps")} for r in recs], "void": void}
+    if void["A"] >= 2 or void["B"] >= 2 or len(arms["A"]) < 4 or len(arms["B"]) < 4:
+        return dict(res, verdict="INCONCLUSIVE", why=">= 2 void blocks, or < 4 valid blocks, in A or B")
+    med = {e: st.median(t for r in arms[e] for t in r["decode_tps"]) for e in "ABC" if arms[e]}
+    R = med["A"] / med["B"]
+    R_bw = (A_BYTES * med["A"]) / (B_BYTES * med["B"])
+    run_order = {e: [r for r in recs if r["engine"] == e] for e in "AB"}
+    pairs = [st.median(a["decode_tps"]) / st.median(b["decode_tps"])
+             for a, b in zip(run_order["A"], run_order["B"]) if not (a["void"] or b["void"])]
+    side = all((r > 1) == (R > 1) for r in pairs)
+    if gap(R) < 0.15 and gap(R_bw) < 0.15 and all(gap(r) < 0.15 for r in pairs):
+        v = "SUPPORTED"
+    elif gap(R) >= 0.15 and side and all(gap(r) >= 0.15 for r in pairs):
+        v = "REFUTED"
+    else:
+        v = "INCONCLUSIVE"
+
+    def reported(e):
+        rs = arms.get(e) or []
+        ttft = [s["ttft_s"] for r in rs for s in (r["prefill"] or {}).get("samples", []) if s.get("ok") and s.get("ttft_s")]
+        c4 = [r["conc4"]["aggregate_tokens_per_s"] for r in rs if r["conc4"] and r["conc4"].get("aggregate_tokens_per_s")]
+        jt = [x for x in (incremental_j_per_token(r, "decode") for r in rs) if x is not None]
+        return {"decode_tps_median": med.get(e), "prefill_ttft_median_s": st.median(ttft) if ttft else None,
+                "conc4_tps_median": st.median(c4) if c4 else None,
+                "decode_incremental_j_per_token_median": st.median(jt) if jt else None, "blocks": len(rs)}
+
+    return dict(res, verdict=v, R=round(R, 4), gap=round(gap(R), 4), R_bandwidth=round(R_bw, 4),
+                pair_ratios=[round(r, 4) for r in pairs],
+                C_over_B=round(med["C"] / med["B"], 4) if "C" in med else None,
+                arms={e: reported(e) for e in "ABC"})
+
+
+if __name__ == "__main__":
+    json.dump(score(blocks(sys.argv[1])), sys.stdout, indent=1)
+    print()