From c68ced447de1ea5ad2d53783922c0a56c2694c39 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 04:01:00 -0700 Subject: [PATCH 1/3] bench: record why the held arm pins stay below the newest release Three bench pins are older than their project's newest release on purpose. Each now says why next to the pin: - h54.py's llama.cpp b11749 feeds H54q, H57dp/H57de and H57wa/wm/wb, which pair with or replicate b11749 records. Moving it to v0.8.0 (b11960) needs a dated amendment for each of those rows first. - live.sh's A3/A3ref and apb/arm.sh's vllm-cuda arm run vLLM CUDA v0.31.0, the image the done rows ran. No pending row runs them, and the pin is kept so those rows reproduce. Comments only; nothing a run reads changes. --- bench/apb/arm.sh | 2 ++ bench/eatflow/live.sh | 2 ++ bench/h54/h54.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/bench/apb/arm.sh b/bench/apb/arm.sh index 4f1aac1f..5939faf6 100755 --- a/bench/apb/arm.sh +++ b/bench/apb/arm.sh @@ -73,6 +73,8 @@ case ${ARM} in NS=default DEP=vllm-qwen-3-4b-cpu EP=default/vllm-qwen-3-4b-cpu-service ;; vllm-cuda) gpu=$(nvidia-smi --query-gpu=name,uuid --format=csv,noheader | awk -F', ' '/RTX 4000/ {print $2}') + # vLLM CUDA v0.31.0 (agent.lock vllm_cuda), the image H8 ran as arm A4, kept so H8 reproduces. + # No pending row runs this arm; a new CUDA arm pins vLLM's newest release. image=vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b sed -e "s|@VLLM_IMAGE@|${image}|" -e "s|@GPU_UUID@|${gpu}|" "${HERE}/vllm-cuda.yaml" > "${OUT}/cuda.yaml" python3 "${HERE}/stage.py" cuda "${OUT}/cuda.yaml" "${OUT}/deploy.yaml" Qwen3-4B-AWQ diff --git a/bench/eatflow/live.sh b/bench/eatflow/live.sh index 00e8599a..ccda0b70 100755 --- a/bench/eatflow/live.sh +++ b/bench/eatflow/live.sh @@ -658,6 +658,8 @@ case ${ARM} in [ "${ARM}" = A3ref ] && w=Qwen3-4B-Instruct-2507 cluster_up gpu=$(nvidia-smi --query-gpu=name,uuid --format=csv,noheader | awk -F', ' '/RTX 4000/ {print $2}') + # vLLM CUDA v0.31.0 (agent.lock vllm_cuda), the image the done desktop rows' A3 and A3ref + # ran, kept so they reproduce. No pending row runs A3; a new CUDA arm pins vLLM's newest. image=vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b sed -e "s|@VLLM_IMAGE@|${image}|" -e "s|@GPU_UUID@|${gpu}|" \ -e "s|--served-model-name=Qwen/Qwen3-4B-AWQ|--served-model-name=${SERVED}|" \ diff --git a/bench/h54/h54.py b/bench/h54/h54.py index ee0ad1b1..3ee89b51 100644 --- a/bench/h54/h54.py +++ b/bench/h54/h54.py @@ -45,6 +45,9 @@ IMG = { "vllm024": "vllm/vllm-openai-cpu:v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1", "vllm032": "vllm/vllm-openai-cpu:v0.32.0@sha256:ffaba3989b9b96d95a576143c872a4110ce9800f5199b6a36eb27bdff1cbbe05", + # Held at b11749 while rows depend on it: H54q pairs Lg with H54a's b11749 records (amendment + # Q1), H57dp/H57de reuse H57d's b11749 quality records, and H57wa/wm/wb replicate H57d. A newer + # llama.cpp (v0.8.0 is b11960) needs a dated amendment for each of those rows first. "llamacpp": "ghcr.io/ggml-org/llama.cpp:server-b11749@sha256:9fbaa06d7f4449e2ae0d02b40e0431f7c212dcf231c9e3a7777a6afc2cc8f312", "sglang": "lmsysorg/sglang:v0.5.22-xeon@sha256:4e5c413ed9a31f38f74e18aa9b645d65a0b0a023d42f63edb4e2b412ea1a230f", "ovms": "openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58", From bb537ea1c271a0d9bd061878a32b349da0f02195 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 04:06:45 -0700 Subject: [PATCH 2/3] bench(apb): h7.sh and arm.sh pin the vLLM CPU image H7 and H8 ran Both scripts took B's image tag from the shared overlay integrations/eat/vllm-values.yaml, which still names v0.24.0. EAT's latest release ships v0.28.0, and that overlay is due to move with it. When it does, H7's and H8's scripts would start rendering a different engine from the one their run headers record. Each script now sets the tag those rows ran (v0.24.0 by index digest) on the helm command line, after the overlay. Rendered from EAT 0d3438d's chart, the output is byte-identical to before, and it stays on v0.24.0 when the overlay's tag is moved to v0.28.0. Test: bench_test.sh checks both scripts carry the pin; it failed before this change. --- bench/apb/arm.sh | 3 +++ bench/apb/h7.sh | 3 +++ tests/bench_test.sh | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/bench/apb/arm.sh b/bench/apb/arm.sh index 5939faf6..ea510911 100755 --- a/bench/apb/arm.sh +++ b/bench/apb/arm.sh @@ -65,7 +65,10 @@ case ${ARM} in chart=${APB_EAT_CHART:?set APB_EAT_CHART to EAT core/helm-charts/vllm} python3 "${HERE}/stage.py" values "${chart}/xeon-values.yaml" "${REPO}/integrations/eat/vllm-values.yaml" 6 "${OUT}/values.yaml" # EAT's catalog entry cpu-qwen3-4b (core/playbooks/deploy-inference-models.yml), ingress off. + # The image is the one H8 ran as arm A3, pinned here so that a later tag in + # integrations/eat/vllm-values.yaml does not change what this arm reproduces. helm template vllm-qwen-3-4b-cpu "${chart}" --values "${OUT}/values.yaml" \ + --set image.tag="v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1" \ --set LLM_MODEL_ID=Qwen/Qwen3-4B-Instruct-2507 --set global.monitoring=false \ --set svcmonitor.enabled=false --set ingress.enabled=false --set route.enabled=false > "${OUT}/eat.yaml" python3 "${HERE}/stage.py" vllm "${OUT}/eat.yaml" "${OUT}/deploy.yaml" Qwen3-4B-Instruct-2507 32768 diff --git a/bench/apb/h7.sh b/bench/apb/h7.sh index 817009e9..3e64a7d1 100755 --- a/bench/apb/h7.sh +++ b/bench/apb/h7.sh @@ -54,7 +54,10 @@ K3D_GPU_AGENT_DEVICE=CPU K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3-4B-fp16-ov K3D_GPU_A "${K}" agent manifest > "${OUT}/agent.yaml" 2> "${OUT}/agent.err" || { cat "${OUT}/agent.err"; exit 1; } python3 "${HERE}/stage.py" ovms "${OUT}/agent.yaml" "${OUT}/ovms.yaml" python3 "${HERE}/stage.py" values "${CHART}/xeon-values.yaml" "${REPO}/integrations/eat/vllm-values.yaml" 6 "${OUT}/values.yaml" +# B's image is the one H7 ran, pinned here so that a later tag in integrations/eat/vllm-values.yaml +# does not change what this script reproduces. helm template vllm-qwen-3-4b-cpu "${CHART}" --values "${OUT}/values.yaml" --set LLM_MODEL_ID=Qwen/Qwen3-4B --set cpu=16 \ + --set image.tag="v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1" \ --set global.monitoring=false --set svcmonitor.enabled=false --set ingress.enabled=false --set route.enabled=false > "${OUT}/eat.yaml" python3 "${HERE}/stage.py" vllm "${OUT}/eat.yaml" "${OUT}/vllm.yaml" Qwen3-4B 32768 stage Qwen3-4B-fp16-ov OpenVINO/Qwen3-4B-fp16-ov diff --git a/tests/bench_test.sh b/tests/bench_test.sh index c9e1c61d..71b265a8 100755 --- a/tests/bench_test.sh +++ b/tests/bench_test.sh @@ -359,6 +359,12 @@ PY if python3 -c 'import yaml' 2>/dev/null; then check "thermal-resource probe: arms B/T/C request what H23 pre-registered" h23_arms fi +# H7 and H8 ran EAT's vLLM CPU arm on v0.24.0 (their run headers). Their scripts pin that image +# themselves, so a newer tag in the shared overlay integrations/eat/vllm-values.yaml does not +# change what they reproduce. +check "apb (H7, H8): h7.sh and arm.sh eat-vllm-cpu pin the vLLM CPU image those rows ran" \ + sh -c 'for f in "$@"; do grep -q -- "--set image.tag=\"v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1\"" "$f" || { echo "$f"; exit 1; }; done' \ + _ "${ROOT}/bench/apb/h7.sh" "${ROOT}/bench/apb/arm.sh" if command -v cc >/dev/null; then check "research: stream_probe.c compiles" \ cc -fsyntax-only -fopenmp "${ROOT}/research/agents-per-box-ceiling/stream_probe.c" From c7d5719684b7c2b1a4ec3346856a15957e101a5b Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 04:37:58 -0700 Subject: [PATCH 3/3] bench(h54): the b11749 hold names only the rows that run llama.cpp The reason recorded next to h54.py's llama.cpp pin listed H57wb among the rows that depend on b11749. H57wb compares k3d-gpu auto (OVMS) with EAT stock vLLM CPU and runs no llama.cpp arm. The rows that do take this pin are H54q (Lg, paired with H54a's b11749 records under amendment Q1) and, through bench/h57d/h57d.py, H57dp, H57de, H57wa and H57wm. Comment only; nothing a run reads changes. --- bench/h54/h54.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bench/h54/h54.py b/bench/h54/h54.py index 3ee89b51..9c083a37 100644 --- a/bench/h54/h54.py +++ b/bench/h54/h54.py @@ -46,8 +46,9 @@ "vllm024": "vllm/vllm-openai-cpu:v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1", "vllm032": "vllm/vllm-openai-cpu:v0.32.0@sha256:ffaba3989b9b96d95a576143c872a4110ce9800f5199b6a36eb27bdff1cbbe05", # Held at b11749 while rows depend on it: H54q pairs Lg with H54a's b11749 records (amendment - # Q1), H57dp/H57de reuse H57d's b11749 quality records, and H57wa/wm/wb replicate H57d. A newer - # llama.cpp (v0.8.0 is b11960) needs a dated amendment for each of those rows first. + # Q1), and bench/h57d/h57d.py runs its llama.cpp arms on this pin: H57dp/H57de reuse H57d's + # b11749 quality records, and H57wa/H57wm replicate H57d's arms on wildct. A newer llama.cpp + # (v0.8.0 is b11960) needs a dated amendment for each of those rows first. "llamacpp": "ghcr.io/ggml-org/llama.cpp:server-b11749@sha256:9fbaa06d7f4449e2ae0d02b40e0431f7c212dcf231c9e3a7777a6afc2cc8f312", "sglang": "lmsysorg/sglang:v0.5.22-xeon@sha256:4e5c413ed9a31f38f74e18aa9b645d65a0b0a023d42f63edb4e2b412ea1a230f", "ovms": "openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58",