diff --git a/bench/apb/arm.sh b/bench/apb/arm.sh index 4f1aac1f..ea510911 100755 --- a/bench/apb/arm.sh +++ b/bench/apb/arm.sh @@ -65,7 +65,10 @@ case ${ARM} in chart=${APB_EAT_CHART:?set APB_EAT_CHART to EAT core/helm-charts/vllm} python3 "${HERE}/stage.py" values "${chart}/xeon-values.yaml" "${REPO}/integrations/eat/vllm-values.yaml" 6 "${OUT}/values.yaml" # EAT's catalog entry cpu-qwen3-4b (core/playbooks/deploy-inference-models.yml), ingress off. + # The image is the one H8 ran as arm A3, pinned here so that a later tag in + # integrations/eat/vllm-values.yaml does not change what this arm reproduces. helm template vllm-qwen-3-4b-cpu "${chart}" --values "${OUT}/values.yaml" \ + --set image.tag="v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1" \ --set LLM_MODEL_ID=Qwen/Qwen3-4B-Instruct-2507 --set global.monitoring=false \ --set svcmonitor.enabled=false --set ingress.enabled=false --set route.enabled=false > "${OUT}/eat.yaml" python3 "${HERE}/stage.py" vllm "${OUT}/eat.yaml" "${OUT}/deploy.yaml" Qwen3-4B-Instruct-2507 32768 @@ -73,6 +76,8 @@ case ${ARM} in NS=default DEP=vllm-qwen-3-4b-cpu EP=default/vllm-qwen-3-4b-cpu-service ;; vllm-cuda) gpu=$(nvidia-smi --query-gpu=name,uuid --format=csv,noheader | awk -F', ' '/RTX 4000/ {print $2}') + # vLLM CUDA v0.31.0 (agent.lock vllm_cuda), the image H8 ran as arm A4, kept so H8 reproduces. + # No pending row runs this arm; a new CUDA arm pins vLLM's newest release. image=vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b sed -e "s|@VLLM_IMAGE@|${image}|" -e "s|@GPU_UUID@|${gpu}|" "${HERE}/vllm-cuda.yaml" > "${OUT}/cuda.yaml" python3 "${HERE}/stage.py" cuda "${OUT}/cuda.yaml" "${OUT}/deploy.yaml" Qwen3-4B-AWQ diff --git a/bench/apb/h7.sh b/bench/apb/h7.sh index 817009e9..3e64a7d1 100755 --- a/bench/apb/h7.sh +++ b/bench/apb/h7.sh @@ -54,7 +54,10 @@ K3D_GPU_AGENT_DEVICE=CPU K3D_GPU_AGENT_MODEL=OpenVINO/Qwen3-4B-fp16-ov K3D_GPU_A "${K}" agent manifest > "${OUT}/agent.yaml" 2> "${OUT}/agent.err" || { cat "${OUT}/agent.err"; exit 1; } python3 "${HERE}/stage.py" ovms "${OUT}/agent.yaml" "${OUT}/ovms.yaml" python3 "${HERE}/stage.py" values "${CHART}/xeon-values.yaml" "${REPO}/integrations/eat/vllm-values.yaml" 6 "${OUT}/values.yaml" +# B's image is the one H7 ran, pinned here so that a later tag in integrations/eat/vllm-values.yaml +# does not change what this script reproduces. helm template vllm-qwen-3-4b-cpu "${CHART}" --values "${OUT}/values.yaml" --set LLM_MODEL_ID=Qwen/Qwen3-4B --set cpu=16 \ + --set image.tag="v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1" \ --set global.monitoring=false --set svcmonitor.enabled=false --set ingress.enabled=false --set route.enabled=false > "${OUT}/eat.yaml" python3 "${HERE}/stage.py" vllm "${OUT}/eat.yaml" "${OUT}/vllm.yaml" Qwen3-4B 32768 stage Qwen3-4B-fp16-ov OpenVINO/Qwen3-4B-fp16-ov diff --git a/bench/eatflow/live.sh b/bench/eatflow/live.sh index 00e8599a..ccda0b70 100755 --- a/bench/eatflow/live.sh +++ b/bench/eatflow/live.sh @@ -658,6 +658,8 @@ case ${ARM} in [ "${ARM}" = A3ref ] && w=Qwen3-4B-Instruct-2507 cluster_up gpu=$(nvidia-smi --query-gpu=name,uuid --format=csv,noheader | awk -F', ' '/RTX 4000/ {print $2}') + # vLLM CUDA v0.31.0 (agent.lock vllm_cuda), the image the done desktop rows' A3 and A3ref + # ran, kept so they reproduce. No pending row runs A3; a new CUDA arm pins vLLM's newest. image=vllm/vllm-openai@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b sed -e "s|@VLLM_IMAGE@|${image}|" -e "s|@GPU_UUID@|${gpu}|" \ -e "s|--served-model-name=Qwen/Qwen3-4B-AWQ|--served-model-name=${SERVED}|" \ diff --git a/bench/h54/h54.py b/bench/h54/h54.py index ee0ad1b1..9c083a37 100644 --- a/bench/h54/h54.py +++ b/bench/h54/h54.py @@ -45,6 +45,10 @@ IMG = { "vllm024": "vllm/vllm-openai-cpu:v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1", "vllm032": "vllm/vllm-openai-cpu:v0.32.0@sha256:ffaba3989b9b96d95a576143c872a4110ce9800f5199b6a36eb27bdff1cbbe05", + # Held at b11749 while rows depend on it: H54q pairs Lg with H54a's b11749 records (amendment + # Q1), and bench/h57d/h57d.py runs its llama.cpp arms on this pin: H57dp/H57de reuse H57d's + # b11749 quality records, and H57wa/H57wm replicate H57d's arms on wildct. A newer llama.cpp + # (v0.8.0 is b11960) needs a dated amendment for each of those rows first. "llamacpp": "ghcr.io/ggml-org/llama.cpp:server-b11749@sha256:9fbaa06d7f4449e2ae0d02b40e0431f7c212dcf231c9e3a7777a6afc2cc8f312", "sglang": "lmsysorg/sglang:v0.5.22-xeon@sha256:4e5c413ed9a31f38f74e18aa9b645d65a0b0a023d42f63edb4e2b412ea1a230f", "ovms": "openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58", diff --git a/tests/bench_test.sh b/tests/bench_test.sh index c9e1c61d..71b265a8 100755 --- a/tests/bench_test.sh +++ b/tests/bench_test.sh @@ -359,6 +359,12 @@ PY if python3 -c 'import yaml' 2>/dev/null; then check "thermal-resource probe: arms B/T/C request what H23 pre-registered" h23_arms fi +# H7 and H8 ran EAT's vLLM CPU arm on v0.24.0 (their run headers). Their scripts pin that image +# themselves, so a newer tag in the shared overlay integrations/eat/vllm-values.yaml does not +# change what they reproduce. +check "apb (H7, H8): h7.sh and arm.sh eat-vllm-cpu pin the vLLM CPU image those rows ran" \ + sh -c 'for f in "$@"; do grep -q -- "--set image.tag=\"v0.24.0@sha256:b0e6f8604413a8bfcca8836bb4b57b47f1aa653957e07f8903ece531309981a1\"" "$f" || { echo "$f"; exit 1; }; done' \ + _ "${ROOT}/bench/apb/h7.sh" "${ROOT}/bench/apb/arm.sh" if command -v cc >/dev/null; then check "research: stream_probe.c compiles" \ cc -fsyntax-only -fopenmp "${ROOT}/research/agents-per-box-ceiling/stream_probe.c"