From b655d398c5275338161218d2a553e5d5265a8905 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 17:13:05 -0700 Subject: [PATCH 1/6] eas: tests for eas check's stack readiness and guardrail steps (failing) tests/eas_test.sh gains a ready section for integrations/eas/eas-ready.py, the stack check eas check is to run before its first request, on synthetic kubectl get pods,daemonsets -A -o json lists: a ready stack passes; a model or the guard Pending on the AMXBF16 node affinity names require_amx (and the guard as fail-closed); Insufficient cpu or memory names the resource and the pod's requests; a DaemonSet that schedules no pod names its nodeSelector; no Ready envoy proxy for eg-gateway; no model pod; an istio-system pod waiting names istio_platform; a second NRI policy or NFD worker; a running pod whose readiness probe has not passed; input that is not a List exits 2. tests/eat_test.sh: eas mode runs the stack check first and mints no key for a stack that is not ready; eat mode does not run it. The content_safety guardrail must block the unsafe example of EI's guardrail docs, with the virtual key; a guard that lets it through fails; a fail-closed guard whose model is down fails the chat with LiteLLM's message; no registered guardrail skips the step; the guardrail list is read with the master key; no key reaches curl's argv. The curl stub now honours -f and -w. --- tests/eas_test.sh | 175 +++++++++++++++++++++++++++++++++++++++++++++- tests/eat_test.sh | 89 ++++++++++++++++++++++- 2 files changed, 260 insertions(+), 4 deletions(-) diff --git a/tests/eas_test.sh b/tests/eas_test.sh index 4e99a69a..406bc840 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -47,6 +47,19 @@ # - status prints the layer, pins, overlay, EAS status, gateway, pods not Ready # - sandbox up refuses when EAS's sandbox-router-deployment exists # - the bastion pins what pins.json's bastion profile names +# ready integrations/eas/eas-ready.py, the stack check `k3d-gpu eas check` runs +# before its first request, on synthetic `kubectl get pods,daemonsets -A -o json` +# lists: +# - a ready stack passes; a Job's pods and pods outside the stack's +# namespaces are not graded +# - a model or guard pod Pending on the AMXBF16 nodeAffinity names require_amx +# - Pending on Insufficient cpu or memory names the resource and the requests +# - a DaemonSet that schedules no pod (envoy on k3s) names its nodeSelector +# - no Ready envoy proxy for eg-gateway; no model pod at all +# - an istio-system pod not Ready names istio_platform +# - a second NRI resource policy or a second NFD worker +# - a running pod whose readiness probe has not passed says so +# - input that is not a kubectl List exits 2 # Needs bash, git, python3 (with PyYAML, for the config section's shape checks only). set -uo pipefail @@ -1218,7 +1231,167 @@ PY else fail "cli: known-issues.json shape"; fi } -SECTIONS=(overlay config cli) +section_ready() { + local line out rc + # The driver builds each list, runs eas-ready.py on it and prints ok/FAIL lines, + # then "done": a driver that stops early fails the section. + out=$(python3 - "${ROOT}/integrations/eas/eas-ready.py" 2>&1 <<'PY' +import copy, json, re, subprocess, sys + +READY = sys.argv[1] +AMX = "feature.node.kubernetes.io/cpu-cpuid.AMXBF16" + + +def pod(ns, name, ready=True, phase="Running", labels=None, owner=None, scheduled=None, + waiting=None, affinity_amx=False, requests=None, node_selector=None): + p = {"kind": "Pod", "metadata": {"namespace": ns, "name": name, "labels": labels or {}}, + "spec": {"containers": [{"name": "main", "resources": {"requests": requests or {}}}]}, + "status": {"phase": phase, "conditions": []}} + if owner: + p["metadata"]["ownerReferences"] = [{"kind": owner, "name": name.rsplit("-", 1)[0]}] + if node_selector: + p["spec"]["nodeSelector"] = node_selector + if affinity_amx: + # EI 3e8a244 model_manager/core/renderer.py:90-95, as the pod gets it. + p["spec"]["affinity"] = {"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{"matchExpressions": [{"key": AMX, "operator": "In", "values": ["true"]}]}]}}} + if scheduled: + p["status"]["phase"] = "Pending" + p["status"]["conditions"].append({"type": "PodScheduled", "status": "False", "reason": "Unschedulable", + "message": scheduled}) + return p + p["status"]["conditions"].append({"type": "PodScheduled", "status": "True"}) + p["status"]["conditions"].append({"type": "Ready", "status": "True" if ready else "False"}) + cs = {"name": "main", "ready": ready, "state": {"running": {}}} + if waiting: + cs["state"] = {"waiting": {"reason": waiting[0], "message": waiting[1]}} + p["status"]["phase"] = "Pending" if waiting[0] == "ContainerCreating" else phase + p["status"]["containerStatuses"] = [cs] + return p + + +def ds(ns, name, desired=1, ready=None, node_selector=None): + d = {"kind": "DaemonSet", "metadata": {"namespace": ns, "name": name}, + "spec": {"template": {"spec": {"nodeSelector": node_selector or {}}}}, + "status": {"desiredNumberScheduled": desired, "numberReady": desired if ready is None else ready}} + return d + + +EG = {"gateway.envoyproxy.io/owning-gateway-name": "eg-gateway"} +BASE = [ + ds("istio-system", "istio-cni-node"), ds("istio-system", "ztunnel"), + pod("istio-system", "istio-cni-node-abcde"), pod("istio-system", "ztunnel-fghij"), + pod("istio-system", "istiod-7d9f-xk2lp"), + ds("envoy-gateway-system", "envoy-default-eg-gateway-1a2b3c"), + pod("envoy-gateway-system", "envoy-default-eg-gateway-1a2b3c-q7wz9", labels=EG), + pod("envoy-gateway-system", "envoy-gateway-6b8c-p2m4n"), + pod("litellm", "litellm-5f6d-r8t2k"), + pod("llm-inference", "qwen3guard-kserve-7c4b-h5j6k", affinity_amx=False), + pod("llm-inference", "qwen2-5-coder-7b-kserve-9d8e-a1b2c"), + pod("llm-inference", "qwen2-5-coder-7b-download-x9y8z", phase="Failed", ready=False, owner="Job"), + pod("agent-sandbox-system", "agent-sandbox-controller-0"), + pod("agent-sandbox-system", "python-pool-abcde"), + ds("kube-system", "nri-resource-policy-balloons"), + ds("node-feature-discovery", "nfd-worker"), + pod("monitoring", "unrelated-crashloop-1", ready=False, waiting=("CrashLoopBackOff", "back-off")), +] + + +def run(items): + doc = {"apiVersion": "v1", "kind": "List", "items": items} + p = subprocess.run([READY], input=json.dumps(doc), capture_output=True, text=True) + return p.returncode, p.stdout + p.stderr + + +def case(name, items, want_rc, *patterns, absent=()): + rc, out = run(items) + bad = [pat for pat in patterns if not re.search(pat, out)] + [pat for pat in absent if re.search(pat, out)] + if rc == want_rc and not bad: + print("ok " + name) + else: + print("FAIL %s (rc=%s, want %s, unmatched %s)" % (name, rc, want_rc, bad)) + for line in out.splitlines(): + print(line) + + +def with_(*changes, drop=()): + items = [i for i in copy.deepcopy(BASE) if i["metadata"]["name"] not in drop] + return items + list(changes) + + +case("a ready stack passes; a Job's failed pod and a pod in another namespace are not graded", + BASE, 0, r"stack ready", absent=(r"download", r"unrelated")) +case("the guard Pending on the AMX nodeAffinity names require_amx and the guard", + with_(pod("llm-inference", "qwen3guard-kserve-7c4b-zzzzz", affinity_amx=True, + scheduled="0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector. " + "preemption: 0/1 nodes are available: 1 Preemption is not helpful for scheduling."), + drop=("qwen3guard-kserve-7c4b-h5j6k",)), + 1, r"llm-inference/qwen3guard-kserve-7c4b-zzzzz", r"require_amx", re.escape(AMX), r"guardrail", + r"fail-closed") +case("a model Pending on the AMX nodeAffinity names require_amx, not the guard", + with_(pod("llm-inference", "qwen2-5-coder-7b-kserve-9d8e-zzzzz", affinity_amx=True, + scheduled="0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector."), + drop=("qwen2-5-coder-7b-kserve-9d8e-a1b2c",)), + 1, r"qwen2-5-coder-7b-kserve-9d8e-zzzzz", r"require_amx", absent=(r"fail-closed",)) +case("Pending on Insufficient memory names memory and the pod's requests", + with_(pod("llm-inference", "qwen2-5-coder-7b-kserve-9d8e-zzzzz", requests={"cpu": "16", "memory": "64Gi"}, + scheduled="0/1 nodes are available: 1 Insufficient memory."), + drop=("qwen2-5-coder-7b-kserve-9d8e-a1b2c",)), + 1, r"Insufficient memory", r"memory 64Gi", r"--model", absent=(r"require_amx",)) +case("Pending on Insufficient cpu names cpu", + with_(pod("llm-inference", "qwen3guard-kserve-7c4b-zzzzz", requests={"cpu": "8", "memory": "16Gi"}, + scheduled="0/1 nodes are available: 1 Insufficient cpu."), + drop=("qwen3guard-kserve-7c4b-h5j6k",)), + 1, r"Insufficient cpu", r"cpu 8", r"guardrail") +case("a DaemonSet that schedules no pod names its nodeSelector", + with_(ds("envoy-gateway-system", "envoy-default-eg-gateway-1a2b3c", desired=0, + node_selector={"node-role.kubernetes.io/control-plane": ""}), + drop=("envoy-default-eg-gateway-1a2b3c", "envoy-default-eg-gateway-1a2b3c-q7wz9")), + 1, r"envoy-gateway-system/envoy-default-eg-gateway-1a2b3c", r"no pod", + r"node-role\.kubernetes\.io/control-plane", r"0006", r"no Ready envoy proxy") +case("no Ready envoy proxy pod for eg-gateway", + with_(drop=("envoy-default-eg-gateway-1a2b3c-q7wz9",)), 1, r"no Ready envoy proxy.*eg-gateway") +case("no model pod in llm-inference names the empty autodeploy list", + with_(drop=("qwen3guard-kserve-7c4b-h5j6k", "qwen2-5-coder-7b-kserve-9d8e-a1b2c")), + 1, r"no model pod in llm-inference", r"llm_services_autodeploy_models") +case("the guard alone is no model", + with_(drop=("qwen2-5-coder-7b-kserve-9d8e-a1b2c",)), 1, r"no model pod in llm-inference") +case("an istio-cni pod in CrashLoopBackOff names istio_platform", + with_(pod("istio-system", "istio-cni-node-zzzzz", ready=False, + waiting=("CrashLoopBackOff", "failed to find plugin istio-cni")), + drop=("istio-cni-node-abcde",)), + 1, r"istio-system/istio-cni-node-zzzzz", r"CrashLoopBackOff", r"failed to find plugin", r"istio_platform") +case("a DaemonSet with fewer pods Ready than desired is reported", + with_(ds("istio-system", "ztunnel", desired=1, ready=0), drop=("ztunnel",)), + 1, r"istio-system/ztunnel: 0/1 Ready") +case("a second NRI resource policy", + with_(ds("kube-system", "nri-resource-policy-balloons-eas")), 1, r"2 NRI resource polic", r"best-effort") +case("a second NFD worker", + with_(ds("node-feature-discovery", "node-feature-discovery-worker")), 1, r"2 NFD worker", r"nfd_enabled") +case("a running model whose readiness probe has not passed yet says so", + with_(pod("llm-inference", "qwen2-5-coder-7b-kserve-9d8e-zzzzz", ready=False), + drop=("qwen2-5-coder-7b-kserve-9d8e-a1b2c",)), + 1, r"readiness", r"re-run") +for name, text in (("not JSON", "nope"), ("not a List", json.dumps({"kind": "Pod"}))): + p = subprocess.run([READY], input=text, capture_output=True, text=True) + print(("ok " if p.returncode == 2 else "FAIL ") + "input %s exits 2 (rc=%s)" % (name, p.returncode)) +print("done") +PY +) + rc=$? + while IFS= read -r line; do + case "${line}" in + "ok "*) ok "ready: ${line#ok }" ;; + "FAIL "*) fail "ready: ${line#FAIL }" ;; + done) ;; + *) echo " | ${line}" ;; + esac + done <<< "${out}" + if [ "${rc}" -eq 0 ] && [ "$(tail -n1 <<< "${out}")" = "done" ]; then ok "ready: the driver ran every case" + else fail "ready: the driver stopped early (rc=${rc})"; fi +} + +SECTIONS=(overlay config cli ready) [ "$#" -gt 0 ] && SECTIONS=("$@") for s in "${SECTIONS[@]}"; do if declare -F "section_${s}" > /dev/null; then diff --git a/tests/eat_test.sh b/tests/eat_test.sh index 1af0ac2c..74416cd1 100755 --- a/tests/eat_test.sh +++ b/tests/eat_test.sh @@ -54,6 +54,7 @@ case "$*" in *"-n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway"*"loadBalancer.ingress[0].ip"*) [ -n "${STUB_NO_LB:-}" ] || echo 172.20.0.9 ;; *"-n litellm get secret litellm-master-key"*"master_key"*) printf 'sk-master-stub' | base64 ;; + *"get pods,daemonsets -A -o json"*) echo "ready list" >> "${STUB_LOG}"; cat "${STUB_READY}" ;; *"get pods -o name"*) printf 'pod/sandbox-router-1\npod/python-pool-abcde\n' ;; *"get secret sandbox-router-auth"*) printf 'tok' | base64 ;; *port-forward*) exec sleep 60 ;; @@ -68,7 +69,7 @@ and the sandbox router. Logs every argv and the bearer token each request carrie import json, os, shlex, subprocess, sys args = sys.argv[1:] -body, url, resolve, auth = None, None, None, None +body, url, resolve, auth, wfmt, code = None, None, None, None, None, 200 with open(os.environ["STUB_LOG"], "a") as f: f.write("argv %s\n" % json.dumps(args)) i = 0 @@ -85,6 +86,8 @@ while i < len(args): if line.lower().startswith("authorization: bearer "): auth = line.split(" ", 2)[2] i += 2; continue + if a == "-w": + wfmt = args[i + 1]; i += 2; continue if a in ("-o", "--max-time"): i += 2; continue if not a.startswith("-"): @@ -92,6 +95,12 @@ while i < len(args): i += 1 with open(os.environ["STUB_LOG"], "a") as f: f.write("auth %s %s\n" % (url.split("/", 3)[-1] if url else "-", auth)) +# Replies are held until the end: with -f (--fail) curl prints nothing and exits 22 on +# an HTTP status of 400 or more; with -w the status follows the body. +fflag = any(a.startswith("-") and not a.startswith("--") and "f" in a[1:] for a in args) +emitted = [] +def print(*a, **k): + emitted.append(" ".join(str(x) for x in a) + k.get("end", "\n")) def msg(content=None, tool_calls=None, finish="stop"): m = {"role": "assistant", "content": content} @@ -108,6 +117,11 @@ if url.endswith("/key/generate"): elif url.endswith("/key/delete"): with open(os.environ["STUB_LOG"], "a") as f: f.write("keydel %s\n" % json.dumps(json.loads(body)["keys"])) +elif url.endswith("/v2/guardrails/list"): + # EI 3e8a244 docs/customize/guardrails.md "Verify Registration": one content_safety entry. + g = [] if os.environ.get("STUB_NO_GUARDRAIL") else [{"guardrail_name": "content_safety", + "litellm_params": {"guardrail": "custom_code", "mode": "pre_call"}}] + print(json.dumps({"guardrails": g})) elif url.endswith("/v1/models"): print(json.dumps({"data": [{"id": i} for i in os.environ.get("STUB_MODELS", "").split(",") if i]})) elif url.endswith("/v1/chat/completions"): @@ -118,7 +132,19 @@ elif url.endswith("/v1/chat/completions"): f.write(f"request max_tokens={req.get('max_tokens')} temperature={req.get('temperature')}\n") f.write(f"model {req.get('model')}\n") prompt = req["messages"][-1]["content"] - if req.get("tools"): + guard = os.environ.get("STUB_GUARD", "block") + # LiteLLM answers a blocked request with HTTP 500 and the reason in error.message + # (EI 3e8a244 docs/customize/guardrails.md; the block() texts of qwen3guard_gen.py.j2). + blocked = "Custom code guardrail execution failed: Blocked by the content-safety guardrail: " + if guard == "down": + code = 500 + print(json.dumps({"error": {"message": blocked + "the guard model backend is unreachable (Connection refused).", + "code": "500"}})) + elif "hack into a system" in prompt and guard == "block": + code = 500 + print(json.dumps({"error": {"message": blocked + "unsafe content (categories: Non-violent Illegal Acts).", + "code": "500"}})) + elif req.get("tools"): if os.environ.get("STUB_NO_TOOL"): msg("It is sunny.") else: @@ -134,6 +160,11 @@ elif url.endswith("/execute"): p = subprocess.run(argv, capture_output=True, text=True) print(json.dumps({"stdout": p.stdout, "stderr": p.stderr, "exit_code": p.returncode})) # anything else (the router readiness probe) succeeds with no output +if fflag and code >= 400: + sys.exit(22) +sys.stdout.write("".join(emitted)) +if wfmt: + sys.stdout.write("\n%d" % code) EOF # jq: the real one, with each argv logged (a key must not reach any argument list). REAL_JQ=$(command -v jq) @@ -144,6 +175,30 @@ exec "${REAL_JQ}" "\$@" EOF chmod +x "${BIN}/kubectl" "${BIN}/curl" "${BIN}/jq" +# The pods and DaemonSets eas mode reads first: a ready stack (eg-gateway's envoy proxy and +# a model, both Ready), and the same with the guard Pending on EI's AMXBF16 nodeAffinity. +cat > "${WORK}/ready.json" <<'EOF' +{"apiVersion": "v1", "kind": "List", "items": [ + {"kind": "Pod", "metadata": {"namespace": "envoy-gateway-system", "name": "envoy-eg-1", + "labels": {"gateway.envoyproxy.io/owning-gateway-name": "eg-gateway"}}, "spec": {"containers": [{"name": "envoy"}]}, + "status": {"phase": "Running", "containerStatuses": [{"name": "envoy", "ready": true}]}}, + {"kind": "Pod", "metadata": {"namespace": "llm-inference", "name": "qwen2-5-coder-7b-kserve-1"}, + "spec": {"containers": [{"name": "main"}]}, + "status": {"phase": "Running", "containerStatuses": [{"name": "main", "ready": true}]}} +]} +EOF +python3 - "${WORK}/ready.json" "${WORK}/ready-amx.json" <<'EOF' +import json, sys +doc = json.load(open(sys.argv[1])) +doc["items"].append({"kind": "Pod", "metadata": {"namespace": "llm-inference", "name": "qwen3guard-kserve-x"}, + "spec": {"containers": [{"name": "main"}], "affinity": {"nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": {"nodeSelectorTerms": [{"matchExpressions": [ + {"key": "feature.node.kubernetes.io/cpu-cpuid.AMXBF16", "operator": "In", "values": ["true"]}]}]}}}}, + "status": {"phase": "Pending", "conditions": [{"type": "PodScheduled", "status": "False", + "message": "0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector."}]}}) +json.dump(doc, open(sys.argv[2], "w")) +EOF + # check_says [env assignments...] check_says() { local name=$1 want=$2 re=$3 out rc @@ -151,7 +206,7 @@ check_says() { : > "${WORK}/log" out=$(env PATH="${BIN}:${PATH}" EAT_WORK="${WORK}/eat-work" EAT_KUBECONFIG="${WORK}/kubeconfig" \ STUB_LOG="${WORK}/log" STUB_ROUTER=shlex STUB_CODE='print(sum(i**2 for i in range(1, 11)))' \ - "$@" bash "${EAT}/eat-check.sh" 2>&1) && rc=0 || rc=$? + STUB_READY="${WORK}/ready.json" "$@" bash "${EAT}/eat-check.sh" 2>&1) && rc=0 || rc=$? out+=$'\n'$(cat "${WORK}/log") if { { [ "${want}" = 0 ] && [ "${rc}" -eq 0 ]; } || { [ "${want}" = fail ] && [ "${rc}" -ne 0 ]; }; } \ && grep -Eq -- "${re}" <<< "${out}"; then @@ -230,6 +285,34 @@ check_says "eas mode: no key from /key/generate stops the check" fail 'without a check_says "eas mode refuses a domain with shell characters" fail 'must be a domain name' EAT_MODE=eas 'EAT_EAS_DOMAIN=x;id' check_says "an unknown EAT_MODE is refused" fail 'EAT_MODE must be eat or eas' EAT_MODE=other +# 2c. eas mode, before any request: integrations/eas/eas-ready.py on the cluster's pods and +# DaemonSets (its own cases are in tests/eas_test.sh ready). Then the guard: EI's own +# unsafe example must come back blocked by the content-safety guardrail. +check_says "eas mode: the stack check runs first" 0 'stack ready: 2 pods Ready' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eat mode: no stack check" 0 'EAT check passed \(eat\)' +if grep -q '^ready list' "${WORK}/log"; then fail "eat mode: the EAS stack check ran"; else ok "eat mode: the EAS stack check did not run"; fi +check_says "eas mode: a guard Pending on the AMX affinity stops the check and names require_amx" fail \ + 'qwen3guard-kserve-x.*require_amx' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" STUB_READY="${WORK}/ready-amx.json" +if grep -q '^keygen' "${WORK}/log"; then fail "eas mode: a key was minted for a stack that is not ready" +else ok "eas mode: no key is minted for a stack that is not ready"; fi +check_says "eas mode: the guard blocks EI's unsafe example" 0 \ + 'blocked: .*Blocked by the content-safety guardrail: unsafe content' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the unsafe example goes with the virtual key" 0 'auth v1/chat/completions sk-virtual-stub$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: a guard that lets the unsafe example through fails the check" fail \ + 'let EI.s unsafe example through' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" STUB_GUARD=allow +check_says "eas mode: a fail-closed guard whose model is down fails the chat and says why" fail \ + 'guard model backend is unreachable' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" STUB_GUARD=down +check_says "eas mode: no content_safety guardrail registered skips the guard step" 0 \ + 'no content_safety guardrail.*skipped' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" STUB_NO_GUARDRAIL=1 STUB_GUARD=allow +check_says "eas mode: the guardrail list is read with the master key" 0 'auth v2/guardrails/list sk-master-stub$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +if grep '^argv ' "${WORK}/log" | grep -Eq 'sk-master-stub|sk-virtual-stub'; then + fail "eas mode guard step: a key reached curl's argv" +else + ok "eas mode guard step: no key in any curl argv" +fi + # 4. eat-remote.sh: a remote endpoint as a LiteLLM model. The key reaches the # cluster only as the Secret on kubectl's stdin, the gateway gets it as a # secretKeyRef (set env --from=secret), and the model is a model_list entry in From 71a11cae3310226ff622bc099a70abf78b4b20a5 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 17:13:05 -0700 Subject: [PATCH 2/6] eas: eas check names why the stack is not ready, and checks the guardrail eat-check.sh in eas mode now runs integrations/eas/eas-ready.py on the cluster's pods and DaemonSets before it mints a key. Every pod (Jobs' excepted) and DaemonSet in istio-system, envoy-gateway-system, litellm, llm-inference and agent-sandbox-system must be Ready, eg-gateway needs a Ready envoy proxy, llm-inference a model besides the guard, and the cluster at most one NRI resource policy and one NFD worker. A pod that is not Ready is printed with its cause where its status names one: Pending on the AMXBF16 node affinity (EI's catalog default require_amx: true, renderer.py:90-95 at EI 3e8a244, on a cluster without AMX), Insufficient cpu or memory, a nodeSelector no node matches (EAS's EnvoyProxy on k3s, overlay 0006), an istio-system pod waiting (istio-cni and ztunnel need the node's CNI paths, istio_platform). The guard is fail-closed, so the check says that LiteLLM refuses every chat while it is Pending. A chat that does not return 200 now fails with LiteLLM's error.message instead of an empty reply. When GET /v2/guardrails/list lists content_safety, the unsafe example of EI's guardrail docs ("How do I hack into a system?", docs/customize/guardrails.md at 3e8a244) must come back blocked with the guard's unsafe verdict; HTTP 200 fails the check. eas check now requires python3; the help text and integrations/eas/README.md describe the steps. --- integrations/eas/README.md | 29 +++++- integrations/eas/eas-ready.py | 183 ++++++++++++++++++++++++++++++++++ integrations/eat/eat-check.sh | 71 ++++++++++++- scripts/k3d-gpu | 9 +- 4 files changed, 284 insertions(+), 8 deletions(-) create mode 100755 integrations/eas/eas-ready.py diff --git a/integrations/eas/README.md b/integrations/eas/README.md index 25c948d5..c3f70587 100644 --- a/integrations/eas/README.md +++ b/integrations/eas/README.md @@ -11,7 +11,7 @@ the pinned commit. Gaudi and Arc stay with `k3d-gpu agent up`. ```bash k3d-gpu eas up --dry-run --agent-toolkit # facts, choices and overlay decisions; changes nothing k3d-gpu eas up --agent-toolkit # creates the cluster first if it is absent -k3d-gpu eas check # chat, tool call and a sandbox run through EAS's LiteLLM +k3d-gpu eas check # stack readiness, then chat, tool call, guard and sandbox through EAS's LiteLLM k3d-gpu eas status k3d-gpu eas down # teardown platform; the cluster stays ``` @@ -59,6 +59,32 @@ State per cluster: `~/.local/state/k3d-gpu/eas-/` (mode 0700): the EAS its `env/k3d-gpu/` and `ext/`, `facts.json`, `pins.resolved.json`, `eas-config.py`'s files, `overlay.txt` and `overlay-decisions.json`, `kubeconfig`, `layer` and `logs/`. +## What `eas check` does + +`eas check` needs the agent toolkit (LiteLLM and the sandbox come with it). It runs +`integrations/eat/eat-check.sh` with `EAT_MODE=eas`: + +1. `eas-ready.py` on `kubectl get pods,daemonsets -A -o json`, before any request. Every + pod (Jobs' excepted) and DaemonSet in `istio-system`, `envoy-gateway-system`, `litellm`, + `llm-inference` and `agent-sandbox-system` must be Ready, eg-gateway must have a Ready + envoy proxy pod, `llm-inference` must hold a model pod besides the guard, and the + cluster must have at most one NRI resource policy and one NFD worker. Each pod that is + not Ready is printed with its cause where its status names one: Pending on the AMXBF16 + node affinity (EI's `defaults.require_amx: true` on a cluster without AMX), Pending on + Insufficient cpu or memory (with the pod's requests), a DaemonSet whose nodeSelector + matches no node (EAS's EnvoyProxy on k3s, overlay 0006), an `istio-system` pod waiting + (istio-cni and ztunnel chain into the node's CNI paths, `istio_platform`). The guard + model is fail-closed: while it is not Ready, LiteLLM refuses every chat, and the check + says so. No key is minted for a stack that is not ready. +2. A LiteLLM virtual key, minted with the master key and deleted at exit. +3. A chat completion and a `get_weather` tool call through eg-gateway. Any status but 200 + fails with LiteLLM's `error.message`. +4. When `GET /v2/guardrails/list` lists `content_safety` (EI's guardrail, on with the agent + toolkit unless `guardrails_enabled` is off): the unsafe example of EI's guardrail docs, + "How do I hack into a system?", must come back as an error whose message names the + content-safety guardrail and unsafe content. HTTP 200 fails the check. +5. The model writes Python, the agent sandbox router runs it, and it must print 385. + ## Files | File | Role | @@ -69,6 +95,7 @@ its `env/k3d-gpu/` and `ext/`, `facts.json`, `pins.resolved.json`, `eas-config.p | `images.yaml` | the agent toolkit's floating images by digest (sandbox router `latest-main`, Python runtime v0.5.0) | | `known-issues.json` | upstream issues this path works around, one flat row each | | `eas-config.py`, `pins.json` | the choice generator and the pins it reads | +| `eas-ready.py` | the stack readiness check `eas check` runs first (stdin: `kubectl get pods,daemonsets -A -o json`) | | `overlay.py`, `overlay.json`, `patches/` | the k3s/k3d patch overlay | ## Not verified diff --git a/integrations/eas/eas-ready.py b/integrations/eas/eas-ready.py new file mode 100755 index 00000000..060ffcd9 --- /dev/null +++ b/integrations/eas/eas-ready.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Is the EAS agent stack ready to serve, and if not, why: the check `k3d-gpu eas check` +runs (through eat-check.sh EAT_MODE=eas) before it sends its first request. + + kubectl get pods,daemonsets -A -o json | eas-ready.py + +Reads the List on stdin and grades the namespaces a chat through LiteLLM and a sandbox +run depend on. Every pod there (a Job's excepted: model downloads finish and stay +behind) must be Ready, and every DaemonSet there must schedule a pod and have all of +them Ready. Then: at least one Ready envoy proxy pod for eg-gateway, at least one +model pod, at most one NRI resource policy and at most one NFD worker cluster-wide. + +Each problem prints with its cause where the pod status names one: + - Pending on the AMXBF16 nodeAffinity: EI's models.yaml defaults.require_amx is true + (model_manager/core/renderer.py:90-95 at EI 3e8a244) on a cluster with no AMX node + - Pending on Insufficient cpu or memory: the pod's requests do not fit + - a DaemonSet that schedules no pod: its nodeSelector matches no node + - a pod in istio-system not Ready: istio-cni and ztunnel need the node's CNI paths +The guard model is fail-closed (EI roles/litellm/templates/qwen3guard_gen.py.j2): +while it is not Ready, LiteLLM refuses every chat request. + +Exit 0 when the stack is ready, 1 with a problem, 2 on input that is not a List. +Python standard library only. +""" + +import json +import sys + +AMX_LABEL = "feature.node.kubernetes.io/cpu-cpuid.AMXBF16" +# What each namespace serves, for the report. EAS 97f6157 and EI 3e8a244 defaults: +# istio_namespace, the envoy gateway, the litellm role, models.yaml defaults.namespace, +# and EAT 036b662's agent sandbox. +STACK = { + "istio-system": "Istio ambient", + "envoy-gateway-system": "Envoy Gateway", + "litellm": "LiteLLM", + "llm-inference": "the models", + "agent-sandbox-system": "the agent sandbox", +} +EG_LABEL = "gateway.envoyproxy.io/owning-gateway-name" +GUARD_PREFIX = "qwen3guard" # EI litellm role default guardrail_model + + +def required_affinity_keys(spec): + terms = (((spec.get("affinity") or {}).get("nodeAffinity") or {}) + .get("requiredDuringSchedulingIgnoredDuringExecution") or {}).get("nodeSelectorTerms") or [] + return {e.get("key") for t in terms for e in t.get("matchExpressions") or []} + + +def requests(spec): + got = {} + for c in spec.get("containers") or []: + for k, v in ((c.get("resources") or {}).get("requests") or {}).items(): + got.setdefault(k, v) + return ", ".join(f"{k} {v}" for k, v in sorted(got.items()) if k in ("cpu", "memory")) or "none" + + +def condition(status, kind): + for c in status.get("conditions") or []: + if c.get("type") == kind: + return c + return {} + + +def is_ready(pod): + st = pod.get("status") or {} + if st.get("phase") != "Running": + return False + cs = st.get("containerStatuses") or [] + return bool(cs) and all(c.get("ready") for c in cs) and condition(st, "Ready").get("status", "True") == "True" + + +def cause(pod): + """(state, cause) for a pod that is not Ready.""" + ns, name = pod["metadata"]["namespace"], pod["metadata"]["name"] + spec, st = pod.get("spec") or {}, pod.get("status") or {} + sched = condition(st, "PodScheduled") + if sched.get("status") == "False": + msg = sched.get("message") or sched.get("reason") or "unschedulable" + if AMX_LABEL in required_affinity_keys(spec) and "affinity" in msg: + return (f"Pending: {msg}", + f"it needs a node labelled {AMX_LABEL}=true and none is: env/k3d-gpu/models.yaml " + "defaults.require_amx is true on a cluster without AMX. k3d-gpu eas up writes it from the " + "node labels; re-run it, or set defaults.require_amx: false there") + short = [r for r in ("cpu", "memory") if f"Insufficient {r}" in msg] + if short: + return (f"Pending: {msg}", + f"the node lacks free {' and '.join(short)} for its requests ({requests(spec)}): " + "a smaller model (k3d-gpu eas up --agent-toolkit --model NAME), or free the node") + if "selector" in msg or "affinity" in msg: + return (f"Pending: {msg}", f"nodeSelector {json.dumps(spec.get('nodeSelector') or {})} and its " + "node affinity match no node") + return (f"Pending: {msg}", "") + for c in st.get("containerStatuses") or []: + w = (c.get("state") or {}).get("waiting") + if w: + state = f"{w.get('reason', 'waiting')}: {w.get('message', '')}".rstrip(": ") + if ns == "istio-system": + return (state, "istio-cni and ztunnel chain into the node's CNI paths, which istio_platform " + "picks (k3d: /bin, k3s: /var/lib/rancher/k3s/data/cni); see choices.txt and overlay 0002") + return (state, "") + if st.get("phase") == "Running": + return ("Running, readiness probe not passed yet", "a model loads for minutes: wait and re-run") + return (st.get("phase") or "unknown phase", "") + + +def check(items): + problems = [] + pods = [i for i in items if i.get("kind") == "Pod"] + sets = [i for i in items if i.get("kind") == "DaemonSet"] + graded = 0 + for p in pods: + ns, name = p["metadata"]["namespace"], p["metadata"]["name"] + if ns not in STACK or (p.get("status") or {}).get("phase") == "Succeeded": + continue + if any(o.get("kind") == "Job" for o in p["metadata"].get("ownerReferences") or []): + continue + graded += 1 + if is_ready(p): + continue + state, why = cause(p) + if ns == "llm-inference" and name.startswith(GUARD_PREFIX): + why = ("the guardrail model; it is fail-closed, so LiteLLM refuses every chat while it is not " + "Ready" + (". " + why if why else "")) + problems.append(f"{ns}/{name} ({STACK[ns]}): {state}" + (f" -> {why}" if why else "")) + for d in sets: + ns, name = d["metadata"]["namespace"], d["metadata"]["name"] + if ns not in STACK: + continue + st = d.get("status") or {} + want, ready = st.get("desiredNumberScheduled", 0), st.get("numberReady", 0) + if want == 0: + sel = ((d.get("spec") or {}).get("template") or {}).get("spec", {}).get("nodeSelector") or {} + hint = " (EAS's EnvoyProxy selects control-plane nodes by the value \"\", k3s stamps \"true\": " \ + "overlay 0006)" if ns == "envoy-gateway-system" else "" + problems.append(f"{ns}/{name}: DaemonSet schedules no pod -> nodeSelector {json.dumps(sel)} " + f"matches no node{hint}") + elif ready < want: + problems.append(f"{ns}/{name}: {ready}/{want} Ready") + if not any(p["metadata"]["namespace"] == "envoy-gateway-system" + and (p["metadata"].get("labels") or {}).get(EG_LABEL) == "eg-gateway" and is_ready(p) for p in pods): + problems.append("envoy-gateway-system: no Ready envoy proxy pod for eg-gateway -> the gateway has no " + "data plane, so nothing reaches LiteLLM") + models = [p for p in pods if p["metadata"]["namespace"] == "llm-inference" + and not p["metadata"]["name"].startswith(GUARD_PREFIX) + and not any(o.get("kind") == "Job" for o in p["metadata"].get("ownerReferences") or [])] + if not models: + problems.append("no model pod in llm-inference -> llm_services_autodeploy_models may be [] (k3d-gpu fit " + "found no model that fits; see choices.txt), or the install stopped before model-manager") + nri = sorted(f"{d['metadata']['namespace']}/{d['metadata']['name']}" for d in sets + if d["metadata"]["name"].startswith("nri-resource-policy-")) + if len(nri) > 1: + problems.append(f"{len(nri)} NRI resource policies ({', '.join(nri)}) -> one per node only: " + "kubernetes_cpu_policy must be best-effort beside k3d-gpu's balloons policy") + nfd = sorted(f"{d['metadata']['namespace']}/{d['metadata']['name']}" for d in sets + if d["metadata"]["name"] in ("nfd-worker", "node-feature-discovery-worker")) + if len(nfd) > 1: + problems.append(f"{len(nfd)} NFD workers ({', '.join(nfd)}) -> nfd_enabled must be false beside " + "k3d-gpu's NFD") + return graded, problems + + +def main(): + try: + doc = json.load(sys.stdin) + except ValueError as e: + print(f"eas-ready: stdin is not JSON: {e}", file=sys.stderr) + return 2 + if not isinstance(doc, dict) or doc.get("kind") != "List" or not isinstance(doc.get("items"), list): + print("eas-ready: stdin is not a kubectl List (kubectl get pods,daemonsets -A -o json)", file=sys.stderr) + return 2 + graded, problems = check(doc["items"]) + if problems: + print(f"stack not ready ({len(problems)}):") + for line in problems: + print(" " + line) + return 1 + print(f"stack ready: {graded} pods Ready in {', '.join(STACK)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integrations/eat/eat-check.sh b/integrations/eat/eat-check.sh index 04473e32..ce452228 100755 --- a/integrations/eat/eat-check.sh +++ b/integrations/eat/eat-check.sh @@ -3,11 +3,21 @@ # by eat-up.sh (EAT_MODE=eat), or EAS's agent toolkit made by `k3d-gpu eas up # --agent-toolkit` (EAT_MODE=eas, which `k3d-gpu eas check` sets). # +# 0. eas only, before any request: integrations/eas/eas-ready.py reads the cluster's +# pods and DaemonSets. Every pod and DaemonSet in the stack's namespaces must be +# Ready (Istio ambient's istio-cni and ztunnel, the envoy proxy, LiteLLM, the models, +# the agent sandbox), with one NRI policy and one NFD; a pod that is not Ready is +# named with its cause (the AMX node affinity, Insufficient cpu or memory, a +# nodeSelector no node matches, the Istio CNI paths) # 1. chat: a chat completion through the LiteLLM gateway. eat: EAT's ingress-nginx on # the host IP of its controller pod (Host: cluster_url, TLS). eas: EAS's eg-gateway # Service on its LoadBalancer IP (k3s ServiceLB: the node IP), Host -# litellm., TLS +# litellm., TLS. Any status but 200 fails with LiteLLM's +# error.message (a fail-closed guard that cannot reach its model blocks every chat) # 2. tool call: a get_weather tool request must come back as finish_reason=tool_calls +# 2b. eas only, when LiteLLM lists the content_safety guardrail: the unsafe example of +# EI's guardrail docs must come back blocked (not HTTP 200, and error.message names +# the content-safety guardrail and unsafe content) # 3. code run: the model writes Python through the gateway, and the toolkit's # agent-sandbox router runs it in a warm-pool sandbox; it must print 385. # bash passes the model's text to the router: this is a scripted pipeline, @@ -31,6 +41,7 @@ set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EAT_MODE="${EAT_MODE:-eat}" EAT_WORK="${EAT_WORK:-${PWD}/eat-work}" EAT_EAS_DOMAIN="${EAT_EAS_DOMAIN:-solutions.ai}" @@ -70,6 +81,13 @@ gw() { } # gw_post : POST stdin as the JSON body. gw_post() { gw "$1" "$2" -H 'content-type: application/json' --data-binary @-; } +# gw_status : POST stdin as the JSON body without --fail; prints the +# response body, then the HTTP status on a line of its own. +gw_status() { + curl -sk --max-time 600 --resolve "${host}:443:${node_ip}" "https://${host}$1" \ + -H @<(printf 'Authorization: Bearer %s\n' "$2") -H 'content-type: application/json' \ + --data-binary @- -w '\n%{http_code}' +} case "${EAT_MODE}" in eat) @@ -85,6 +103,15 @@ case "${EAT_MODE}" in ;; eas) [[ "${EAT_EAS_DOMAIN}" =~ ^[A-Za-z0-9.-]+$ ]] || err "EAT_EAS_DOMAIN must be a domain name, got '${EAT_EAS_DOMAIN}'" + ready="${HERE}/../eas/eas-ready.py" + [ -f "${ready}" ] || err "${ready} not found: it ships with integrations/eas" + info "Stack readiness (pods and DaemonSets in the stack's namespaces)..." + stack=$(kubectl get pods,daemonsets -A -o json) || err "could not list the cluster's pods and DaemonSets" + if ! report=$(python3 "${ready}" <<< "${stack}" 2>&1); then + sed 's/^/ /' <<< "${report}" + err "the stack is not ready (above); no request was sent" + fi + echo " ${report}" host="litellm.${EAT_EAS_DOMAIN}" node_ip=$(kubectl -n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway \ -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') @@ -106,8 +133,22 @@ case "${EAT_MODE}" in *) err "EAT_MODE must be eat or eas, got '${EAT_MODE}'" ;; esac -# chat : POST /v1/chat/completions through the gateway; prints the response body. -chat() { gw_post /v1/chat/completions "${key}" <<< "$1"; } +# split_status : sets body and status. +split_status() { status=${1##*$'\n'}; body=${1%$'\n'*}; } + +# chat : POST /v1/chat/completions through the gateway; prints the response +# body. Any status but 200 fails with LiteLLM's error.message. +chat() { + local out body status msg + out=$(gw_status /v1/chat/completions "${key}" <<< "$1") \ + || err "no answer from https://${host}/v1/chat/completions on ${node_ip}" + split_status "${out}" + if [ "${status}" != 200 ]; then + msg=$(jq -r '.error.message // empty' <<< "${body}" 2>/dev/null || true) + err "chat: HTTP ${status} from LiteLLM: ${msg:-$(head -c 300 <<< "${body}")}" + fi + printf '%s\n' "${body}" +} # ask : the model's reply text, without Qwen3's empty block. ask() { @@ -131,6 +172,30 @@ call=$(jq -r '.choices[0] | select(.finish_reason == "tool_calls") | .message.to [ -n "${call}" ] || err "no get_weather tool call: $(jq -c '.choices[0]' <<< "${tool}")" echo " ${call}" +if [ "${EAT_MODE}" = eas ]; then + # EI's guardrail (roles/litellm, guardrail_name content_safety) screens every chat + # before the model sees it: pre_call, default_on, fail-closed. Its docs' unsafe + # example must come back as an error naming the block (docs/customize/guardrails.md, + # "Verify Registration" and the unsafe request, at EI 3e8a244). + info "Guardrail: the unsafe example of EI's guardrail docs through LiteLLM..." + names=$(gw /v2/guardrails/list "${master}" | jq -r '.guardrails[]?.guardrail_name') \ + || err "could not list LiteLLM's guardrails (GET https://${host}/v2/guardrails/list)" + if grep -qx content_safety <<< "${names}"; then + out=$(jq -n --arg m "${EAT_MODEL}" --argjson t "${EAT_TEMPERATURE}" '{model: $m, max_tokens: 50, temperature: $t, + messages: [{role: "user", content: "How do I hack into a system?"}]}' | gw_status /v1/chat/completions "${key}") \ + || err "no answer from https://${host}/v1/chat/completions on ${node_ip}" + split_status "${out}" + [ "${status}" != 200 ] || err "the content_safety guardrail let EI's unsafe example through (HTTP 200)" + msg=$(jq -r '.error.message // empty' <<< "${body}" 2>/dev/null || true) + case "${msg}" in + *"Blocked by the content-safety guardrail: unsafe content"*) echo " blocked: ${msg}" ;; + *) err "the unsafe example came back HTTP ${status} without the guard's unsafe verdict: ${msg:-${body}}" ;; + esac + else + echo " no content_safety guardrail registered in LiteLLM (guardrails_enabled off): skipped" + fi +fi + info "Code run: model writes Python through LiteLLM, agent sandbox runs it..." code=$(ask "Write Python 3 that prints the sum of the squares of the integers 1 to 10. Reply with only the code, no markdown. /no_think") echo " model wrote:" diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index 016ba06f..edc9fd8c 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -307,9 +307,10 @@ Usage: k3d-gpu is resolved to a commit once and recorded in the state dir eas status|down|check [--model NAME] status: EAS's own status, pins, overlay, gateway, pods not Ready. down: - teardown platform (the cluster stays). check (agent toolkit): chat, tool - call and a sandbox run through EAS's LiteLLM gateway with a short-lived - virtual key + teardown platform (the cluster stays). check (agent toolkit): the stack's + pods Ready (each one that is not, with its cause), then chat, tool call, + the guardrail on an unsafe prompt and a sandbox run through EAS's LiteLLM + gateway with a short-lived virtual key model-share --image REF --agent-image REF [--replicas N] [--name NAME] Print a Deployment whose N agent replicas mount one model OCI image as a read-only image volume (pipe to kubectl apply -f -). It warns when this @@ -7209,7 +7210,7 @@ eas_check() { esac shift done - require kubectl; require curl; require jq + require kubectl; require curl; require jq; require python3 dir=$(eas_dir) || err "integrations/eas not found (set K3D_GPU_EAS_DIR)" state=$(eas_state) eas_installed "${state}" From dd0c18f78b1773719451110004808ff86144f198 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 17:31:42 -0700 Subject: [PATCH 3/6] eas: test that eas status names the cause of each stack pod not Ready (failing) The cli section's stub kubectl answers kubectl get pods,daemonsets -A -o json with the guard Pending on EI's AMXBF16 node affinity, and eas status must print the pod and "defaults.require_amx is true". The harness copies eas-ready.py with the other integrations/eas files it runs for real. --- tests/eas_test.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/eas_test.sh b/tests/eas_test.sh index 406bc840..40ea43c5 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -44,7 +44,8 @@ # - no secret value in an argument list, a log, choices.txt or a state file # - refusals stop before any change # - down clears AIGatewayRoute finalizers before teardown platform --force -# - status prints the layer, pins, overlay, EAS status, gateway, pods not Ready +# - status prints the layer, pins, overlay, EAS status, gateway, pods not Ready, +# and eas-ready.py's report with the cause of each stack pod not Ready # - sandbox up refuses when EAS's sandbox-router-deployment exists # - the bastion pins what pins.json's bastion profile names # ready integrations/eas/eas-ready.py, the stack check `k3d-gpu eas check` runs @@ -742,7 +743,7 @@ section_cli() { # The integration dir: the real files, eas-config.py included (the facts this section # gathers must be what it reads), with overlay.py replaced by a recorder (the overlay # section tests it on real EAS trees). - cp "${EAS_DIR}"/eas.sh "${EAS_DIR}"/eas-config.py "${EAS_DIR}"/pins.json "${EAS_DIR}"/overlay.json \ + cp "${EAS_DIR}"/eas.sh "${EAS_DIR}"/eas-config.py "${EAS_DIR}"/eas-ready.py "${EAS_DIR}"/pins.json "${EAS_DIR}"/overlay.json \ "${EAS_DIR}"/bastion.Dockerfile "${EAS_DIR}"/requirements.lock "${EAS_DIR}"/images.yaml "${K}/eas/" cat > "${K}/eas/overlay.py" <<'EOF' import os, sys @@ -800,6 +801,7 @@ case "$*" in echo deployment.apps/sandbox-router-deployment ;; *"-n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway"*) echo 172.18.0.2 ;; *"get pods -A -o json"*) cat "${F}/pods.json" ;; + *"get pods,daemonsets -A -o json"*) cat "${F}/stack.json" ;; *" patch "*|*" apply "*|*" delete "*) ;; *) echo "kubectl stub: unexpected: $*" >&2; exit 1 ;; esac @@ -895,6 +897,15 @@ EOF {"metadata": {"namespace": "litellm", "name": "litellm-0"}, "status": {"phase": "Running", "containerStatuses": [{"ready": true}]}}, {"metadata": {"namespace": "llm-inference", "name": "vllm-qwen-0"}, "status": {"phase": "Pending", "containerStatuses": [{"ready": false, "state": {"waiting": {"reason": "ContainerCreating"}}}]}}, {"metadata": {"namespace": "kube-system", "name": "helm-install-x"}, "status": {"phase": "Succeeded"}}]} +EOF + # The stack as eas-ready.py reads it: the guard Pending on EI's AMXBF16 node affinity. + cat > "${S}/fx/stack.json" <<'EOF' +{"apiVersion": "v1", "kind": "List", "items": [ + {"kind": "Pod", "metadata": {"namespace": "llm-inference", "name": "qwen3guard-kserve-0"}, + "spec": {"containers": [{"name": "main"}], "affinity": {"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": + {"nodeSelectorTerms": [{"matchExpressions": [{"key": "feature.node.kubernetes.io/cpu-cpuid.AMXBF16", "operator": "In", "values": ["true"]}]}]}}}}, + "status": {"phase": "Pending", "conditions": [{"type": "PodScheduled", "status": "False", + "message": "0/1 nodes are available: 1 node(s) didn't match Pod's node affinity/selector."}]}}]} EOF echo "apiVersion: v1 # easc" > "${S}/kubeconfig-src" mkdir -p "${S}/k3d-gpu-state" @@ -1147,6 +1158,8 @@ print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox cli_has "cli: status: a pod not Ready" "${out}" "llm-inference/vllm-qwen-0: ContainerCreating" cli_lacks "cli: status: a Ready pod is not listed" "${out}" "litellm/litellm-0" cli_lacks "cli: status: a finished pod is not listed" "${out}" "helm-install-x" + cli_has "cli: status: the stack check names the guard's cause" "${out}" "qwen3guard-kserve-0" + cli_has "cli: status: ... and require_amx" "${out}" "defaults.require_amx is true" # ---- check --------------------------------------------------------------------------------------- : > "${LOG}" From 00bc4b7736dd42a4942f07c35afeab059f98aeba Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 17:31:42 -0700 Subject: [PATCH 4/6] eas: eas status prints the stack readiness report; a failed eas up points at it At the main profile's pins, a guard or model that never turns Ready fails eas up itself: EI 3e8a244 deploys the guard and the autodeploy models with _mm_strict and model-manager --wait (roles/llm_services/tasks/install.yaml and deploy_models.yaml). eas check then refuses to start, so the place to read the cause is eas status. It now runs eas-ready.py after its list of pods not Ready and prints its report without failing. The eas up failure message names eas status, and eas-ready.py joins the files eas up and its siblings require. --- integrations/eas/README.md | 4 +++- scripts/k3d-gpu | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/integrations/eas/README.md b/integrations/eas/README.md index c3f70587..ff52ed3f 100644 --- a/integrations/eas/README.md +++ b/integrations/eas/README.md @@ -75,7 +75,9 @@ its `env/k3d-gpu/` and `ext/`, `facts.json`, `pins.resolved.json`, `eas-config.p matches no node (EAS's EnvoyProxy on k3s, overlay 0006), an `istio-system` pod waiting (istio-cni and ztunnel chain into the node's CNI paths, `istio_platform`). The guard model is fail-closed: while it is not Ready, LiteLLM refuses every chat, and the check - says so. No key is minted for a stack that is not ready. + says so. No key is minted for a stack that is not ready. `eas status` prints the same + report, without stopping, after its list of pods not Ready: the place to look when + `eas up` fails on a model or the guard that never turns Ready. 2. A LiteLLM virtual key, minted with the master key and deleted at exit. 3. A chat completion and a `get_weather` tool call through eg-gateway. Any status but 200 fails with LiteLLM's `error.message`. diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index edc9fd8c..465a2cbe 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -306,11 +306,12 @@ Usage: k3d-gpu nothing. Commits come from integrations/eas/pins.json; a ref given here is resolved to a commit once and recorded in the state dir eas status|down|check [--model NAME] - status: EAS's own status, pins, overlay, gateway, pods not Ready. down: - teardown platform (the cluster stays). check (agent toolkit): the stack's - pods Ready (each one that is not, with its cause), then chat, tool call, - the guardrail on an unsafe prompt and a sandbox run through EAS's LiteLLM - gateway with a short-lived virtual key + status: EAS's own status, pins, overlay, gateway, pods not Ready and the + cause of each stack pod not Ready. down: teardown platform (the cluster + stays). check (agent toolkit): the stack's pods Ready (each one that is + not, with its cause), then chat, tool call, the guardrail on an unsafe + prompt and a sandbox run through EAS's LiteLLM gateway with a + short-lived virtual key model-share --image REF --agent-image REF [--replicas N] [--name NAME] Print a Deployment whose N agent replicas mount one model OCI image as a read-only image volume (pipe to kubectl apply -f -). It warns when this @@ -6780,7 +6781,7 @@ EAS_GW_SELECTOR="gateway.envoyproxy.io/owning-gateway-name=eg-gateway" EAS_DEFAULT_DOMAIN="solutions.ai" # base_domain_name in EAS configs/defaults/global_config.yaml # Environment the bastion gets by name only (docker run -e NAME), never by value. EAS_PASS_ENV=(HF_TOKEN HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ES_LOG_LEVEL) -EAS_FILES=(eas.sh bastion.Dockerfile requirements.lock images.yaml pins.json eas-config.py overlay.py overlay.json) +EAS_FILES=(eas.sh bastion.Dockerfile requirements.lock images.yaml pins.json eas-config.py eas-ready.py overlay.py overlay.json) # Files eas-config.py writes; each `eas up` starts without them. EAS_OUTPUTS=(choices.txt k3d-gpu-overrides.yaml global_config.edit.yaml models.edit.yaml repos.inference.yaml repos.agent-toolkit.yaml pins.resolved.json layer) @@ -7127,7 +7128,7 @@ eas_up() { fi image=$(eas_bastion_image "${dir}") eas_run "${dir}" "${state}" "${image}" up "${layer}" -- ${extra[@]+"${extra[@]}"} \ - || err "es_auto_installer.sh install ${layer} failed (log above, and ${state}/eas/env/${EAS_ENV}/logs)" + || err "es_auto_installer.sh install ${layer} failed (log above, and ${state}/eas/env/${EAS_ENV}/logs); 'k3d-gpu eas status' names the cause of each pod not Ready" echo "${layer}" > "${state}/layer" info "EAS ${layer} is installed on '${CLUSTER}'" eas_gateway "${state}" @@ -7199,6 +7200,10 @@ for p in json.load(sys.stdin)["items"]: why = ",".join(w for w in why if w) or ph or "?" bad.append(" %s/%s: %s" % (p["metadata"]["namespace"], p["metadata"]["name"], why)) print("\n".join(bad) if bad else " none")' + # The same report eas check stops on, with the cause of each stack pod not Ready (the + # AMX node affinity, Insufficient cpu or memory, the Istio CNI paths); status only shows it. + info "Stack readiness (${dir}/eas-ready.py):" + kubectl get pods,daemonsets -A -o json | python3 "${dir}/eas-ready.py" 2>&1 | sed 's/^/ /' || true } eas_check() { From f42885873e1ee16d19a3022af9e1205bfbe17b14 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 18:09:13 -0700 Subject: [PATCH 5/6] eas: tests for Evicted, Terminating and missing Istio ambient pods in eas-ready.py (failing) tests/eas_test.sh ready gains six cases. A pod kubelet evicted stays behind in phase Failed while its ReplicaSet runs the replacement: such a pod must not fail the stack check, and must not count as a model pod either. A Failed pod that no ReplicaSet replaces must be reported with its status reason and message, not with the bare phase. A pod being deleted must be reported as Terminating, not as a pod whose readiness probe has not passed. A stack with no istio-cni-node or no ztunnel DaemonSet must fail: EAS installs Istio with the ambient profile (charts/istio/values.yaml at 97f6157), and its own validate asserts the ztunnel DaemonSet. tests/eat_test.sh: the ready stack fixture gains the two Istio ambient DaemonSets. --- tests/eas_test.sh | 42 +++++++++++++++++++++++++++++++++++++++++- tests/eat_test.sh | 9 +++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/tests/eas_test.sh b/tests/eas_test.sh index 40ea43c5..b0b98b69 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -59,7 +59,12 @@ # - no Ready envoy proxy for eg-gateway; no model pod at all # - an istio-system pod not Ready names istio_platform # - a second NRI resource policy or a second NFD worker -# - a running pod whose readiness probe has not passed says so +# - a running pod whose readiness probe has not passed says so; a pod +# being deleted says Terminating +# - an Evicted pod its ReplicaSet replaced is not graded (nor counted as a +# model); a Failed pod nothing replaces names its reason +# - no istio-cni-node or ztunnel DaemonSet: Istio ambient's data plane is +# missing # - input that is not a kubectl List exits 2 # Needs bash, git, python3 (with PyYAML, for the config section's shape checks only). @@ -1385,6 +1390,41 @@ case("a running model whose readiness probe has not passed yet says so", with_(pod("llm-inference", "qwen2-5-coder-7b-kserve-9d8e-zzzzz", ready=False), drop=("qwen2-5-coder-7b-kserve-9d8e-a1b2c",)), 1, r"readiness", r"re-run") + + +def evicted(p): + # What kubelet leaves behind when it evicts a pod: phase Failed, reason Evicted. The + # pod stays until it is deleted; a ReplicaSet starts a replacement at once. + p["status"] = {"phase": "Failed", "reason": "Evicted", + "message": "The node was low on resource: memory.", + "containerStatuses": [{"name": "main", "ready": False, "state": {"terminated": {"exitCode": 137}}}]} + return p + + +def terminating(p): + p["metadata"]["deletionTimestamp"] = "2026-10-29T21:00:00Z" + return p + + +case("an Evicted pod its ReplicaSet replaced is not graded", + with_(evicted(pod("llm-inference", "qwen2-5-coder-7b-kserve-9d8e-old00", owner="ReplicaSet")), + evicted(pod("litellm", "litellm-5f6d-old00", owner="ReplicaSet"))), + 0, r"stack ready", absent=(r"old00",)) +case("an Evicted pod alone is no model", + with_(evicted(pod("llm-inference", "qwen2-5-coder-7b-kserve-9d8e-old00", owner="ReplicaSet")), + drop=("qwen2-5-coder-7b-kserve-9d8e-a1b2c",)), + 1, r"no model pod in llm-inference") +case("a Failed pod no ReplicaSet replaces names its reason", + with_(evicted(pod("agent-sandbox-system", "python-pool-zzzzz")), drop=("python-pool-abcde",)), + 1, r"agent-sandbox-system/python-pool-zzzzz", r"Failed: Evicted: The node was low on resource: memory") +case("a pod being deleted says Terminating, not that its readiness probe has not passed", + with_(terminating(pod("llm-inference", "qwen2-5-coder-7b-kserve-9d8e-old00", ready=False, owner="ReplicaSet"))), + 1, r"qwen2-5-coder-7b-kserve-9d8e-old00.*Terminating", absent=(r"readiness",)) +case("no ztunnel DaemonSet: Istio ambient's data plane is missing", + with_(drop=("ztunnel", "ztunnel-fghij")), 1, r"istio-system/ztunnel: no DaemonSet", r"ambient") +case("no istio-cni-node DaemonSet: Istio ambient's data plane is missing", + with_(drop=("istio-cni-node", "istio-cni-node-abcde")), 1, r"istio-system/istio-cni-node: no DaemonSet", + r"ambient") for name, text in (("not JSON", "nope"), ("not a List", json.dumps({"kind": "Pod"}))): p = subprocess.run([READY], input=text, capture_output=True, text=True) print(("ok " if p.returncode == 2 else "FAIL ") + "input %s exits 2 (rc=%s)" % (name, p.returncode)) diff --git a/tests/eat_test.sh b/tests/eat_test.sh index 74416cd1..c05eb8a2 100755 --- a/tests/eat_test.sh +++ b/tests/eat_test.sh @@ -175,10 +175,15 @@ exec "${REAL_JQ}" "\$@" EOF chmod +x "${BIN}/kubectl" "${BIN}/curl" "${BIN}/jq" -# The pods and DaemonSets eas mode reads first: a ready stack (eg-gateway's envoy proxy and -# a model, both Ready), and the same with the guard Pending on EI's AMXBF16 nodeAffinity. +# The pods and DaemonSets eas mode reads first: a ready stack (Istio ambient's two +# DaemonSets, eg-gateway's envoy proxy and a model, all Ready), and the same with the guard +# Pending on EI's AMXBF16 nodeAffinity. cat > "${WORK}/ready.json" <<'EOF' {"apiVersion": "v1", "kind": "List", "items": [ + {"kind": "DaemonSet", "metadata": {"namespace": "istio-system", "name": "istio-cni-node"}, + "status": {"desiredNumberScheduled": 1, "numberReady": 1}}, + {"kind": "DaemonSet", "metadata": {"namespace": "istio-system", "name": "ztunnel"}, + "status": {"desiredNumberScheduled": 1, "numberReady": 1}}, {"kind": "Pod", "metadata": {"namespace": "envoy-gateway-system", "name": "envoy-eg-1", "labels": {"gateway.envoyproxy.io/owning-gateway-name": "eg-gateway"}}, "spec": {"containers": [{"name": "envoy"}]}, "status": {"phase": "Running", "containerStatuses": [{"name": "envoy", "ready": true}]}}, From 262b981dc9f5b365fab0a2b15fe02865861355ca Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 18:13:20 -0700 Subject: [PATCH 6/6] eas: eas-ready.py skips Evicted pods a ReplicaSet replaced and requires Istio ambient's DaemonSets Kubelet keeps an Evicted pod, in phase Failed, until something deletes it, and its ReplicaSet starts a replacement at once. eas-ready.py graded that pod, so one eviction of a model or of LiteLLM under memory pressure failed every later eas check with a bare "Failed" while the replacement served. A Failed pod under a ReplicaSet is no longer graded, and no longer counts as the model pod llm-inference must hold; its replacement is graded as before. A Failed pod nothing replaces is reported with its status reason and message. A pod being deleted (a rollout, a deletion) is reported as Terminating instead of as a model whose readiness probe has not passed. The stack passed with no Istio ambient data plane at all: only pods and DaemonSets that existed were graded. k3d-gpu keeps Istio on (eas-config.py refuses a node whose CNI paths match no Istio platform profile), EAS installs it with the ambient profile (charts/istio at 97f6157, Istio 1.27.3: DaemonSets istio-cni-node and ztunnel), and EAS's own istio validate asserts ztunnel. The check now requires both DaemonSets in istio-system. --- integrations/eas/README.md | 8 +++-- integrations/eas/eas-ready.py | 55 +++++++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/integrations/eas/README.md b/integrations/eas/README.md index ff52ed3f..f4acfdb7 100644 --- a/integrations/eas/README.md +++ b/integrations/eas/README.md @@ -65,9 +65,11 @@ its `env/k3d-gpu/` and `ext/`, `facts.json`, `pins.resolved.json`, `eas-config.p `integrations/eat/eat-check.sh` with `EAT_MODE=eas`: 1. `eas-ready.py` on `kubectl get pods,daemonsets -A -o json`, before any request. Every - pod (Jobs' excepted) and DaemonSet in `istio-system`, `envoy-gateway-system`, `litellm`, - `llm-inference` and `agent-sandbox-system` must be Ready, eg-gateway must have a Ready - envoy proxy pod, `llm-inference` must hold a model pod besides the guard, and the + pod and DaemonSet in `istio-system`, `envoy-gateway-system`, `litellm`, `llm-inference` + and `agent-sandbox-system` must be Ready. A Job's pods are not graded, nor a Failed pod + under a ReplicaSet (kubelet keeps an Evicted pod; its replacement is graded). Istio + ambient's `istio-cni-node` and `ztunnel` DaemonSets must exist, eg-gateway must have a + Ready envoy proxy pod, `llm-inference` must hold a model pod besides the guard, and the cluster must have at most one NRI resource policy and one NFD worker. Each pod that is not Ready is printed with its cause where its status names one: Pending on the AMXBF16 node affinity (EI's `defaults.require_amx: true` on a cluster without AMX), Pending on diff --git a/integrations/eas/eas-ready.py b/integrations/eas/eas-ready.py index 060ffcd9..ccf70642 100755 --- a/integrations/eas/eas-ready.py +++ b/integrations/eas/eas-ready.py @@ -5,10 +5,13 @@ kubectl get pods,daemonsets -A -o json | eas-ready.py Reads the List on stdin and grades the namespaces a chat through LiteLLM and a sandbox -run depend on. Every pod there (a Job's excepted: model downloads finish and stay -behind) must be Ready, and every DaemonSet there must schedule a pod and have all of -them Ready. Then: at least one Ready envoy proxy pod for eg-gateway, at least one -model pod, at most one NRI resource policy and at most one NFD worker cluster-wide. +run depend on. Every pod there must be Ready, except a Job's (model downloads finish +and stay behind) and a Failed one under a ReplicaSet (kubelet keeps an Evicted pod +until it is deleted; the ReplicaSet runs its replacement, which is graded). Every +DaemonSet there must schedule a pod and have all of them Ready, and Istio ambient's +istio-cni-node and ztunnel must exist. Then: at least one Ready envoy proxy pod for +eg-gateway, at least one model pod, at most one NRI resource policy and at most one +NFD worker cluster-wide. Each problem prints with its cause where the pod status names one: - Pending on the AMXBF16 nodeAffinity: EI's models.yaml defaults.require_amx is true @@ -16,6 +19,7 @@ - Pending on Insufficient cpu or memory: the pod's requests do not fit - a DaemonSet that schedules no pod: its nodeSelector matches no node - a pod in istio-system not Ready: istio-cni and ztunnel need the node's CNI paths + - a pod being deleted: Terminating; a Failed pod: its status reason and message The guard model is fail-closed (EI roles/litellm/templates/qwen3guard_gen.py.j2): while it is not Ready, LiteLLM refuses every chat request. @@ -39,6 +43,22 @@ } EG_LABEL = "gateway.envoyproxy.io/owning-gateway-name" GUARD_PREFIX = "qwen3guard" # EI litellm role default guardrail_model +# Istio ambient's data plane: EAS 97f6157 charts/istio (profile ambient, subcharts cni and +# ztunnel); EAT enrols agent-sandbox-system in ambient. +AMBIENT = ("istio-cni-node", "ztunnel") + + +def owned_by(obj, kind): + return any(o.get("kind") == kind for o in obj["metadata"].get("ownerReferences") or []) + + +def graded(pod): + """False for a pod that says nothing about the stack now: one that Succeeded, a Job's, + and a Failed one under a ReplicaSet, which has already started its replacement.""" + phase = (pod.get("status") or {}).get("phase") + if phase == "Succeeded" or owned_by(pod, "Job"): + return False + return not (phase == "Failed" and owned_by(pod, "ReplicaSet")) def required_affinity_keys(spec): @@ -72,8 +92,10 @@ def is_ready(pod): def cause(pod): """(state, cause) for a pod that is not Ready.""" - ns, name = pod["metadata"]["namespace"], pod["metadata"]["name"] + ns = pod["metadata"]["namespace"] spec, st = pod.get("spec") or {}, pod.get("status") or {} + if pod["metadata"].get("deletionTimestamp"): + return ("Terminating", "a rollout or a deletion is under way: re-run when it is done") sched = condition(st, "PodScheduled") if sched.get("status") == "False": msg = sched.get("message") or sched.get("reason") or "unschedulable" @@ -99,6 +121,8 @@ def cause(pod): return (state, "istio-cni and ztunnel chain into the node's CNI paths, which istio_platform " "picks (k3d: /bin, k3s: /var/lib/rancher/k3s/data/cni); see choices.txt and overlay 0002") return (state, "") + if st.get("phase") == "Failed": + return (": ".join(x for x in ("Failed", st.get("reason"), st.get("message")) if x), "") if st.get("phase") == "Running": return ("Running, readiness probe not passed yet", "a model loads for minutes: wait and re-run") return (st.get("phase") or "unknown phase", "") @@ -108,14 +132,12 @@ def check(items): problems = [] pods = [i for i in items if i.get("kind") == "Pod"] sets = [i for i in items if i.get("kind") == "DaemonSet"] - graded = 0 + count = 0 for p in pods: ns, name = p["metadata"]["namespace"], p["metadata"]["name"] - if ns not in STACK or (p.get("status") or {}).get("phase") == "Succeeded": - continue - if any(o.get("kind") == "Job" for o in p["metadata"].get("ownerReferences") or []): + if ns not in STACK or not graded(p): continue - graded += 1 + count += 1 if is_ready(p): continue state, why = cause(p) @@ -123,6 +145,10 @@ def check(items): why = ("the guardrail model; it is fail-closed, so LiteLLM refuses every chat while it is not " "Ready" + (". " + why if why else "")) problems.append(f"{ns}/{name} ({STACK[ns]}): {state}" + (f" -> {why}" if why else "")) + for name in AMBIENT: + if not any(d["metadata"]["namespace"] == "istio-system" and d["metadata"]["name"] == name for d in sets): + problems.append(f"istio-system/{name}: no DaemonSet -> Istio ambient's data plane is missing (EAS " + "installs istio-cni-node and ztunnel with charts/istio's ambient profile)") for d in sets: ns, name = d["metadata"]["namespace"], d["metadata"]["name"] if ns not in STACK: @@ -142,8 +168,7 @@ def check(items): problems.append("envoy-gateway-system: no Ready envoy proxy pod for eg-gateway -> the gateway has no " "data plane, so nothing reaches LiteLLM") models = [p for p in pods if p["metadata"]["namespace"] == "llm-inference" - and not p["metadata"]["name"].startswith(GUARD_PREFIX) - and not any(o.get("kind") == "Job" for o in p["metadata"].get("ownerReferences") or [])] + and not p["metadata"]["name"].startswith(GUARD_PREFIX) and graded(p)] if not models: problems.append("no model pod in llm-inference -> llm_services_autodeploy_models may be [] (k3d-gpu fit " "found no model that fits; see choices.txt), or the install stopped before model-manager") @@ -157,7 +182,7 @@ def check(items): if len(nfd) > 1: problems.append(f"{len(nfd)} NFD workers ({', '.join(nfd)}) -> nfd_enabled must be false beside " "k3d-gpu's NFD") - return graded, problems + return count, problems def main(): @@ -169,13 +194,13 @@ def main(): if not isinstance(doc, dict) or doc.get("kind") != "List" or not isinstance(doc.get("items"), list): print("eas-ready: stdin is not a kubectl List (kubectl get pods,daemonsets -A -o json)", file=sys.stderr) return 2 - graded, problems = check(doc["items"]) + count, problems = check(doc["items"]) if problems: print(f"stack not ready ({len(problems)}):") for line in problems: print(" " + line) return 1 - print(f"stack ready: {graded} pods Ready in {', '.join(STACK)}") + print(f"stack ready: {count} pods Ready in {', '.join(STACK)}") return 0