ledger(H53): INCONCLUSIVE, B2 not deliverable on k3d-gpu; amendment R1 and the run's records

bench/eatflow/live.sh

@@ -280,6 +280,19 @@ stage() {  # stage <local dir> <path under /var/apb-models> <uid>
         docker exec "${n}" chown -R "$3:$3" /var/apb-models
     done
 }
+# stage_path <local dir> <path under /var/apb-models> <uid>: copy into every node, chown that
+# path only. Top level, so the solo (H53) and pairs paths both have it (H53 amendment R1).
+stage_path() {
+    local n
+    for n in $(nodes); do
+        docker exec "${n}" mkdir -p "/var/apb-models/$2" /var/apb-models/.ov_cache
+        docker cp -L "${M}/$1/." "${n}:/var/apb-models/$2/"
+        docker exec "${n}" rm -rf "/var/apb-models/$2/.cache" "/var/apb-models/$2/.git"
+        docker exec "${n}" chown -R "$3:$3" "/var/apb-models/$2"
+        docker exec "${n}" chown 5000:5000 /var/apb-models/.ov_cache
+        docker exec "${n}" chmod 1777 /var/apb-models
+    done
+}
 # cpusets: H56's cpuset.txt, "<role> <name> <cpus>" lines that crescent/h56.py checks.
 cpusets() {
     local n c
@@ -316,6 +329,8 @@ if [ "${MODE}" = solo ]; then
     # AG (k3d-gpu's auto device and draft: OVMS on one card) or B2 (EAT PR #21's vLLM XPU path,
     # its xpu-qwen36-27b profile on these weights, LIVE_B2_TP cards). LIVE_GPU_FREE: the PCI
     # addresses of the cards the hold may use; every other card is held by a placeholder pod.
+    # LIVE_PAIRS_GUARD: the task-start guard, s (unset: agentbench solo's own; a wiring smoke
+    # sets it past the hold, so only the warm-up runs; H53 amendment R1).
     PD=${LIVE_PAIRS_DIR:?} HOLD=${LIVE_HOLD:?} SOLO=${LIVE_SOLO:?LIVE_SOLO is AG or B2}
     read -r FAMILY RULE <<< "$(python3 "${HERE}/pairs.py" design "${ARM}")"
     [ "${ARM}" = H53 ] && [ "${RULE:-}" = L3 ] || { echo "solo: ARM must be H53 (a rule-L3 row)"; exit 2; }
@@ -428,7 +443,8 @@ if [ "${MODE}" = solo ]; then
     ${AB} solo --hyp "${ARM}" --hold "${HOLD}" --arm "${SOLO}" --out "${PD}" --deadline "${deadline}" --tasks B24 \
         --agent http://127.0.0.1:8090 --upstream "http://127.0.0.1:${P_LLM}" --proxy-port "${P_PROXY}" \
         --restart "${OUT}/agent.sh" --pod "${cg}" --mcp-url "http://127.0.0.1:${P_MCP}/mcp" --reset "${OUT}/reset-mcp.sh" \
-        --sampling "${SAMPLING}" --baseline "${PD}/baseline-hold${HOLD}.jsonl" --level "${LEVEL}" 2>&1 | tee -a "${OUT}/live.log"
+        --sampling "${SAMPLING}" --baseline "${PD}/baseline-hold${HOLD}.jsonl" --level "${LEVEL}" \
+        ${LIVE_PAIRS_GUARD:+--guard-s "${LIVE_PAIRS_GUARD}"} 2>&1 | tee -a "${OUT}/live.log"
     stamp finished
     exit 0
 fi
@@ -462,17 +478,6 @@ if [ "${MODE}" = pairs ]; then
     B_NS=default B_DEP=${B_REL} B_SVC=${B_REL}-service B_SPORT=80 B_API=/v1
     [ "${DESIGN}" = H50o ] && B_DEP=ovms-model-server B_SVC=ovms-model-server B_SPORT=8080 B_API=/v3
     DRAFT_DIR=OpenVINO-Qwen3-0.6B-int4-ov     # where OVMS --pull puts the draft, inside the target dir
-    stage_path() {  # stage_path <local dir> <path under /var/apb-models> <uid>: chown that path only
-        local n
-        for n in $(nodes); do
-            docker exec "${n}" mkdir -p "/var/apb-models/$2" /var/apb-models/.ov_cache
-            docker cp -L "${M}/$1/." "${n}:/var/apb-models/$2/"
-            docker exec "${n}" rm -rf "/var/apb-models/$2/.cache" "/var/apb-models/$2/.git"
-            docker exec "${n}" chown -R "$3:$3" "/var/apb-models/$2"
-            docker exec "${n}" chown 5000:5000 /var/apb-models/.ov_cache
-            docker exec "${n}" chmod 1777 /var/apb-models
-        done
-    }
     cluster_up
     stage_path "${TGT_DIR}" "${TGT}" 5000
     [ "${A_DRAFT}" = off ] || stage_path Qwen3-0.6B-int4-ov "${TGT}/${DRAFT_DIR}" 5000

bench/eatflow/results/2026-10-20/H53/prehold/fetch.log

@@ -0,0 +1,69 @@
+lock-taken 2026-10-29T21:44:37Z
+6d5cab69b902d2d4b83d95af218c1ea069d0d9fc
+Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
+Error while downloading from https://huggingface.co/Qwen/Qwen3.8-27B/resolve/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/model-00002-of-00018.safetensors: peer closed connection without sending complete message body (received 1344965215 bytes, expected 3043080328)
+Trying to resume download...
+✓ Downloaded
+  path: /home/devcloud/k3dgpu-bench/models/Qwen3.8-27B
+Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
+✓ Downloaded
+  path: /home/devcloud/k3dgpu-bench/models/Qwen3.8-27B-int4-ov
+ok  size 1570 .gitattributes
+ok  size 11544 LICENSE
+ok  size 65012 README.md
+ok  size 8952 chat_template.jinja
+ok  size 4312 config.json
+ok  size 238 crc32.txt
+ok  size 202 generation_config.json
+ok  size 3353259 merges.txt
+ok  ba0ce20aae489ad196733da5064bcdf159a1fe84f53336648196e1ebb7751b1c model-00001-of-00018.safetensors
+ok  06a148c01bfbe3faa14a5f184a7ff29a706f7ae1c8b2705d2058e26d17a001fb model-00002-of-00018.safetensors
+ok  2e1bf62cbcd406eaa64b60d10353e1f0ef4039d0976e56f05cabe953454f9968 model-00003-of-00018.safetensors
+ok  511e34063187882659753c4d93f3859f93c019fd438d8813071921c81d9a3f1a model-00004-of-00018.safetensors
+ok  635cb53446dc74f219740fc59e18b774f877b803b9722e289ca62575a6efa701 model-00005-of-00018.safetensors
+ok  0bc5214fac607f0e6cc92eec3789d4b8559410ef9fce66621ba8158e8410dae0 model-00006-of-00018.safetensors
+ok  80b0c49033e9a0d5762562aa12f4acdb7f54da586f3d0110f28c48d91cf07892 model-00007-of-00018.safetensors
+ok  7192c5b66185d3592927daabee1cc19e6f6e0ce75988ee20e824b624765fda79 model-00008-of-00018.safetensors
+ok  af3c48cc37af44f3db6ae0579baf019180d48d9c527caa0a1f03ff85813a56d8 model-00009-of-00018.safetensors
+ok  163490a76f3bea3a40855b7efc04ce6d27afaf1a34f0bbde495b9491f76457c9 model-00010-of-00018.safetensors
+ok  5f3ae1b948aeee39da77aec558e8236cd65fe4d7cb7686a76bb007acc563c6d8 model-00011-of-00018.safetensors
+ok  a3de1c7114677a8f5ac5c4892c90e8238ea5c1e2038c80e757dfc87c3902ca55 model-00012-of-00018.safetensors
+ok  06ab79a41f74c9c5cb734816feb0c7fc364104b227165ee7391231e1155aa02a model-00013-of-00018.safetensors
+ok  4138ed94603065ba884bbcadedb04d7718bb40117e85e6f5c6fc5b9c05b7a85b model-00014-of-00018.safetensors
+ok  69224e27b9de4e7dbf6fc936c6eaae08447bda3b80a6c31a871ab451173afd22 model-00015-of-00018.safetensors
+ok  73cb9a1089fb6155cb648609478d6633be8a5c7d9ca5a05bc8925ce8a553cefe model-00016-of-00018.safetensors
+ok  beb51f01056142ac4984bd800507b0dd0fd18de57f8e9ef6ea41d1a3598983a8 model-00017-of-00018.safetensors
+ok  1d3479509e21494658f9b64d317f5ea8e55c4025d28c702d6c4d0b356ce8ea06 model-00018-of-00018.safetensors
+ok  size 112216 model.safetensors.index.json
+ok  size 390 preprocessor_config.json
+ok  0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3 tokenizer.json
+ok  size 17928 tokenizer_config.json
+ok  size 385 video_preprocessor_config.json
+ok  size 6722759 vocab.json
+ok  size 1570 .gitattributes
+ok  size 8321 README.md
+ok  size 8952 chat_template.jinja
+ok  size 3679 config.json
+ok  size 213 generation_config.json
+ok  size 1790 openvino_config.json
+ok  48fc2c6a7a1056994982911eb9695826acef933c3d742364c56d16317a0b2a79 openvino_detokenizer.bin
+ok  size 16771 openvino_detokenizer.xml
+ok  95aac72ac7286707c5e0fd9b120ea7f22c93f3ae17e7f1263442587d1d887d50 openvino_language_model.bin
+ok  size 9736672 openvino_language_model.xml
+ok  bc8ef47f252ad5a72be11e48ba6a23da50a67d1cef03396a7d902ca9aa1d5bc8 openvino_mtp_model.bin
+ok  size 210992 openvino_mtp_model.xml
+ok  e2099d0d83dcdfb00c378b51b50a52d28116aee71eb4f81cef0fe19ae37a085f openvino_text_embeddings_model.bin
+ok  size 5920 openvino_text_embeddings_model.xml
+ok  ca1ff37f580b558f0119b57824240adc92624d6d2ef6dd96ca2a24f4f1368c05 openvino_tokenizer.bin
+ok  size 35358 openvino_tokenizer.xml
+ok  3792b6946c1f475aaac5d6f3960c7b912f612104e4f39ad68643b707370ed8e7 openvino_vision_embeddings_merger_model.bin
+ok  size 1330718 openvino_vision_embeddings_merger_model.xml
+ok  5a8661b052abd07563d0f7148934d1fb77d5a842647151304287beb5dcee5541 openvino_vision_embeddings_model.bin
+ok  size 8610 openvino_vision_embeddings_model.xml
+ok  82ccc97be0351cd82ebe1717fd7fa9adf72a44982a7ced8e8a845d869325d9e5 openvino_vision_embeddings_pos_model.bin
+ok  size 5887 openvino_vision_embeddings_pos_model.xml
+ok  size 390 preprocessor_config.json
+ok  size 1300 processor_config.json
+ok  87a7830d63fcf43bf241c3c5242e96e62dd3fdc29224ca26fed8ea333db72de4 tokenizer.json
+ok  size 1098 tokenizer_config.json
+rc=0 2026-10-29T21:55:52Z

bench/eatflow/results/2026-10-20/H53/prehold/frontdoor-post.txt

@@ -0,0 +1,2 @@
+2026-10-30T00:37:52Z :30000 pass=1 before=2 after=3 content=OK
+2026-10-30T00:38:03Z :30001 pass=1 before=3 after=4 content=OK

bench/eatflow/results/2026-10-20/H53/prehold/frontdoor-pre.txt

@@ -0,0 +1,2 @@
+2026-10-30T00:14:45Z :30000 pass=1 before=161 after=162 content=OK
+2026-10-30T00:14:45Z :30001 pass=1 before=57 after=58 content=OK

bench/eatflow/results/2026-10-20/H53/prehold/inspect-qwen38-a-hr.after.key.json

@@ -0,0 +1 @@
+{"args": ["/entry.sh"], "binds": ["/home/devcloud/launchers/hicache-mha.py:/opt/venv/lib/python3.12/site-packages/sglang/srt/mem_cache/pool_host/mha.py:ro", "/home/devcloud/launchers/empty-pycache:/opt/venv/lib/python3.12/site-packages/sglang/srt/mem_cache/pool_host/__pycache__:ro", "/dev/dri:/dev/dri", "/home/devcloud/launchers/qwen38-pair-entry-hicache.sh:/entry.sh:ro", "/home/devcloud/hicache-shared:/hicache", "/home/devcloud/launchers/hicache-common.py:/opt/venv/lib/python3.12/site-packages/sglang/srt/mem_cache/pool_host/common.py:ro", "/home/devcloud/launchers/hicache-mph.py:/opt/venv/lib/python3.12/site-packages/sglang/srt/mem_cache/memory_pool_host.py:ro", "/home/devcloud/launchers/empty-memcache-pyc:/opt/venv/lib/python3.12/site-packages/sglang/srt/mem_cache/__pycache__:ro", "/home/devcloud/models/Qwen3.8-27B-AWQ-INT4:/model:ro"], "devices": [{"CgroupPermissions": "rwm", "PathInContainer": "/dev/dri", "PathOnHost": "/dev/dri"}], "env": ["CCL_CONFIGURATION=cpu_gpu_dpcpp", "CCL_CONFIGURATION_PATH=", "CCL_ROOT=/opt/intel/oneapi/ccl/2022.1", "CLASSPATH=/opt/intel/oneapi/mpi/2021.18/share/java/mpi.jar", "CMAKE_PREFIX_PATH=/opt/intel/oneapi/tbb/2023.1/env/..:/opt/intel/oneapi/pti/1.0/lib/cmake/pti:/opt/intel/oneapi/mkl/2026.1/lib/cmake:/opt/intel/oneapi/dpl/2022.13/lib/cmake/oneDPL:/opt/intel/oneapi/compiler/2026.1:/opt/intel/oneapi/ccl/2022.1/lib/cmake/oneCCL", "CMPLR_ROOT=/opt/intel/oneapi/compiler/2026.1", "CPATH=/opt/intel/oneapi/umf/1.1/include:/opt/intel/oneapi/dev-utilities/2026.0/include:/opt/intel/oneapi/ccl/2022.1/include", "CPLUS_INCLUDE_PATH=/opt/intel/oneapi/umf/1.1/include:/opt/intel/oneapi/tbb/2023.1/env/../include:/opt/intel/oneapi/pti/1.0/include:/opt/intel/oneapi/mpi/2021.18/include:/opt/intel/oneapi/mkl/2026.1/include:/opt/intel/oneapi/dpl/2022.13/include", "CTX=8192", "C_INCLUDE_PATH=/opt/intel/oneapi/umf/1.1/include:/opt/intel/oneapi/tbb/2023.1/env/../include:/opt/intel/oneapi/pti/1.0/include:/opt/intel/oneapi/mpi/2021.18/include:/opt/intel/oneapi/mkl/2026.1/include", "DEBIAN_FRONTEND=noninteractive", "DIAGUTIL_PATH=/opt/intel/oneapi/compiler/2026.1/etc/compiler/sys_check/sys_check.sh", "DPLROOT=/opt/intel/oneapi/dpl/2022.13", "DPL_ROOT=/opt/intel/oneapi/dpl/2022.13", "FI_PROVIDER_PATH=/opt/intel/oneapi/mpi/2021.18/opt/mpi/libfabric/lib/prov:/usr/lib/x86_64-linux-gnu/libfabric", "GDB_INFO=/opt/intel/oneapi/debugger/2026.1/share/info/", "INFOPATH=/opt/intel/oneapi/debugger/2026.1/share/info", "INTEL_PYTHONHOME=/opt/intel/oneapi/debugger/2026.1/opt/debugger", "I_MPI_ROOT=/opt/intel/oneapi/mpi/2021.18", "KV_PORT=5557", "LANG=C.UTF-8", "LD_LIBRARY_PATH=/opt/intel/oneapi/tcm/1.5/lib:/opt/intel/oneapi/umf/1.1/lib:/opt/intel/oneapi/tcm/1.5/env/../lib:/opt/intel/oneapi/tbb/2023.1/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/pti/1.0/lib:/opt/intel/oneapi/mpi/2021.18/opt/mpi/libfabric/lib:/opt/intel/oneapi/mpi/2021.18/lib:/opt/intel/oneapi/mkl/2026.1/lib:/opt/intel/oneapi/debugger/2026.1/opt/debugger/lib:/opt/intel/oneapi/compiler/2026.1/opt/compiler/lib:/opt/intel/oneapi/compiler/2026.1/lib:/opt/intel/oneapi/ccl/2022.1/lib/", "LIBRARY_PATH=/opt/intel/oneapi/tcm/1.5/lib:/opt/intel/oneapi/umf/1.1/lib:/opt/intel/oneapi/tbb/2023.1/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/pti/1.0/lib:/opt/intel/oneapi/mpi/2021.18/lib:/opt/intel/oneapi/mkl/2026.1/lib:/opt/intel/oneapi/compiler/2026.1/lib:/opt/intel/oneapi/ccl/2022.1/lib/", "MANPATH=/opt/intel/oneapi/mpi/2021.18/share/man:/opt/intel/oneapi/debugger/2026.1/share/man:/opt/intel/oneapi/compiler/2026.1/share/man:", "MKLROOT=/opt/intel/oneapi/mkl/2026.1", "NLSPATH=/opt/intel/oneapi/compiler/2026.1/lib/compiler/locale/%l_%t/%N", "OCL_ICD_FILENAMES=/opt/intel/oneapi/compiler/2026.1/lib/libintelocl.so", "ONEAPI_DEVICE_SELECTOR=level_zero:gpu", "ONEAPI_ROOT=/opt/intel/oneapi", "PATH=/opt/venv/bin:/root/.local/bin:/opt/intel/oneapi/mpi/2021.18/bin:/opt/intel/oneapi/mkl/2026.1/bin:/opt/intel/oneapi/dev-utilities/2026.0/bin:/opt/intel/oneapi/debugger/2026.1/opt/debugger/bin:/opt/intel/oneapi/compiler/2026.1/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "PKG_CONFIG_PATH=/opt/intel/oneapi/tbb/2023.1/env/../lib/pkgconfig:/opt/intel/oneapi/mpi/2021.18/lib/pkgconfig:/opt/intel/oneapi/mkl/2026.1/lib/pkgconfig:/opt/intel/oneapi/dpl/2022.13/lib/pkgconfig:/opt/intel/oneapi/compiler/2026.1/lib/pkgconfig:/opt/intel/oneapi/ccl/2022.1/lib/pkgconfig/", "PORT=30000", "Pti_DIR=/opt/intel/oneapi/pti/1.0/lib/cmake/pti", "SETVARS_COMPLETED=1", "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1", "SGLANG_CHUNKED_PREFILL=4096", "SGLANG_CONTEXT_LEN=262144", "SGLANG_DFLASH_DISABLE_DRAFT_GRAPH=0", "SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR=/hicache", "SGLANG_HTTP_PORT=30000", "SGLANG_MAX_RUNNING=64", "SGLANG_MEM_FRACTION=0.85", "SGLANG_MODEL=/model", "SGLANG_MOE_CONFIG_DIR=/opt/xpu/moe-tuned", "SGLANG_PAGE_SIZE=64", "SGLANG_TP_SIZE=2", "TBBROOT=/opt/intel/oneapi/tbb/2023.1/env/..", "TCMROOT=/opt/intel/oneapi/tcm/1.5/env/..", "TCM_ROOT=/opt/intel/oneapi/tcm/1.5", "TP_SIZE=2", "UMF_ROOT=/opt/intel/oneapi/umf/1.1", "UV_PYTHON_INSTALL_DIR=/opt/uv/python", "VIRTUAL_ENV=/opt/venv", "XPU_DENSE_INT8=1", "XPU_DENSE_INT8_MAX_M=8", "XPU_GDN_DECODE_BV=16", "XPU_KERNELS=topk,gemv", "XPU_MAMBA_PAGED=1", "XPU_MOE_ALIGN_SKIP=1", "XPU_MOE_BUILD=/opt/xpu/build-gemv", "XPU_REPO=/opt/xpu", "XPU_SYMM_AR=1", "XPU_SYMM_AR_MAX_BYTES=131072", "XPU_TOPK_BUILD=/opt/xpu/build", "XPU_UNROLL_SPEC_VERIFY=1", "ZE_AFFINITY_MASK=0,1"], "image": "sha256:feb7b9130eff2fa26dfa09ab1a8b9a8423db013ad7d18f804f8b55a57cb2c175", "network": "bridge", "path": "bash", "ports": {"30000/tcp": [{"HostIp": "", "HostPort": "30000"}], "5557/tcp": [{"HostIp": "", "HostPort": "5557"}]}, "restart": {"MaximumRetryCount": 0, "Name": "no"}}