From 608551537ee41f93ad5cad4af3afcd11b57cce1b Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 14:38:19 -0700 Subject: [PATCH 01/19] eas: tests for k3d-gpu eas up, down, status and check (failing) tests/eas_test.sh gains a cli section. It runs the launcher's eas functions and eas.sh against stub docker, kubectl, k3d, git, yq and k3d-gpu fit, with the real eas-config.py, so the facts the launcher gathers are the facts the generator reads, and a recording overlay.py. It covers the dry run (no docker or kubectl change), cluster creation, the bastion's docker run, the order of eas.sh's steps, ref resolution, secrets kept off argv and out of logs, refusals, AIGatewayRoute finalizers before teardown (EAS issue #71), status, the sandbox up refusal, and the bastion's pins against pins.json. tests/eat_test.sh covers eat-check.sh's EAS mode: the eg-gateway address, Host litellm., a virtual key minted from litellm-master-key and deleted after, and no key on curl's argument list in either mode. tests/packaging_test.sh and tests/image_pins_test.sh cover integrations/eas in the packages and in images.lock. --- tests/eas_test.sh | 519 ++++++++++++++++++++++++++++++++++++++- tests/eat_test.sh | 89 ++++++- tests/image_pins_test.sh | 22 ++ tests/packaging_test.sh | 46 ++++ 4 files changed, 669 insertions(+), 7 deletions(-) diff --git a/tests/eas_test.sh b/tests/eas_test.sh index bd0f04af..57a13a26 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -29,6 +29,24 @@ # role looks for, --explain prints permalinks, every source it can cite is # a permalink at a full commit SHA, and pins.json has the k3d-gpu.eas-pins/v1 # shape. +# cli `k3d-gpu eas up|down|status|check`, eas.sh (the in-bastion driver) and the +# bastion, against stub docker, kubectl, k3d, git, yq and `k3d-gpu fit`, with the +# real eas-config.py (so the facts it gathers are the facts the generator reads) +# and a recording overlay.py: +# - up --dry-run prints the choices and the overlay decisions and makes no +# docker or kubectl change +# - up creates the cluster with `k3d-gpu up` when it is absent +# - the bastion run: host network, root with the state handed back to this +# uid:gid, state dir rw, kubeconfig ro, secrets by name only +# - eas.sh order: clone at the SHA, overlay, manifest pins, init, the edits, +# install with the overrides last; never configure +# - refs: a profile and a SHA go as given, any other ref through git ls-remote +# - no secret value in an argument list, a log, choices.txt or a state file +# - refusals stop before any change +# - down clears AIGatewayRoute finalizers before teardown platform --force +# - status prints the layer, pins, overlay, EAS status, gateway, pods not Ready +# - sandbox up refuses when EAS's sandbox-router-deployment exists +# - the bastion pins what pins.json's bastion profile names # Needs bash, git, python3 (with PyYAML, for the config section's shape checks only). set -uo pipefail @@ -688,7 +706,506 @@ PY fi } -SECTIONS=(overlay config) +section_cli() { + local B="${WORK}/cli/bin" K="${WORK}/cli/k" S="${WORK}/cli/s" LOG="${WORK}/cli/log" + local EAS_DIR="${ROOT}/integrations/eas" LAUNCHER="${ROOT}/scripts/k3d-gpu" + local SECRET="hf_SECRET_7f3a9c" EAS_SHA="97f6157d6a72f7173c698c474d4e51d2c79f1fd6" + local EI_SHA="3e8a24486b6ee812adf2bf452f8ba5751c16c8bd" EAT_SHA="036b66234801e9bd950afd90830821859780d668" + local STATE="${S}/k3d-gpu-state/eas-easc" + mkdir -p "${B}" "${K}/eas" "${K}/eat" "${S}/fx" "${S}/home" "${S}/host/proc/sys/fs/inotify" + + # Helpers named apart from the other sections' (has there takes the needle first). + cli_eq() { if [ "$2" = "$3" ]; then ok "$1"; else fail "$1: got '$2', want '$3'"; fi; } + cli_has() { if grep -Fq -- "$3" <<< "$2"; then ok "$1"; else fail "$1: missing '$3' in: $(head -c 700 <<< "$2")"; fi; } + cli_lacks() { if grep -Fq -- "$3" <<< "$2"; then fail "$1: unexpected '$3'"; else ok "$1"; fi; } + # cli_before : occurs, and before . + cli_before() { + local a b + a=$(grep -nF -- "$3" <<< "$2" | head -1 | cut -d: -f1) + b=$(grep -nF -- "$4" <<< "$2" | head -1 | cut -d: -f1) + if [ -n "${a}" ] && [ -n "${b}" ] && [ "${a}" -lt "${b}" ]; then ok "$1"; else fail "$1: '$3' (line ${a:-none}) not before '$4' (line ${b:-none})"; fi + } + + # The integration dir: the real files, eas-config.py included (the facts this section + # gathers must be what it reads), with overlay.py replaced by a recorder (the overlay + # section tests it on real EAS trees). + cp "${EAS_DIR}"/eas.sh "${EAS_DIR}"/eas-config.py "${EAS_DIR}"/pins.json "${EAS_DIR}"/overlay.json \ + "${EAS_DIR}"/bastion.Dockerfile "${EAS_DIR}"/requirements.lock "${EAS_DIR}"/images.yaml "${K}/eas/" + cat > "${K}/eas/overlay.py" <<'EOF' +import os, sys +a = sys.argv[1:] +with open(os.environ["STUB_LOG"], "a") as f: + f.write("overlay %s\n" % " ".join(a)) +if os.environ.get("STUB_OVERLAY_STOP"): + print("STOP: the overlay does not fit eas@97f6157; regenerate the rows above that say STOP. Nothing was applied.", file=sys.stderr) + sys.exit(3) +print("# overlay at eas@97f6157 (HEAD), kubernetes_cpu_policy=%s, nodes=%s" % (a[a.index("--policy") + 1], a[a.index("--nodes") + 1])) +print("0002 istio global.platform -> apply [stub]\n0006 envoy control-plane affinity -> apply (variant a) [stub]") +EOF + cat > "${K}/eat/eat-check.sh" <<'EOF' +echo "eat-check mode=${EAT_MODE} kubeconfig=${EAT_KUBECONFIG} domain=${EAT_EAS_DOMAIN} model=${EAT_MODEL}" >> "${STUB_LOG}" +echo "EAT check passed (${EAT_MODE})." +EOF + + # ---- stubs: every call is one line in ${STUB_LOG} ------------------------------------- + cat > "${B}/docker" <<'EOF' +#!/usr/bin/env bash +echo "docker $*" >> "${STUB_LOG}" +case "$*" in + "info "*) echo '["name=seccomp,profile=builtin","name=cgroupns"]' ;; + "exec k3d-easc-server-0 test -S /var/run/nri/nri.sock") exit 0 ;; + "exec k3d-easc-server-0 crictl info") echo '{"config": {"cni": {"binDirs": ["/usr/bin"], "confDir": "/var/lib/rancher/k3s/agent/etc/cni/net.d"}}}' ;; + "image inspect "*) [ -e "${STUB_STATE}/image-present" ] ;; + "build "*) touch "${STUB_STATE}/image-present" ;; + "run "*) echo "bastion: ran" ;; + *) echo "docker stub: unexpected: $*" >&2; exit 1 ;; +esac +EOF + cat > "${B}/k3d" <<'EOF' +#!/usr/bin/env bash +echo "k3d $*" >> "${STUB_LOG}" +case "$*" in + "cluster get "*) [ -e "${STUB_STATE}/cluster" ] ;; + "kubeconfig get "*) cat "${STUB_STATE}/kubeconfig-src" ;; +esac +EOF + cat > "${B}/kubectl" <<'EOF' +#!/usr/bin/env bash +echo "kubectl $*" >> "${STUB_LOG}" +F="${STUB_STATE}/fx" +case "$*" in + *"config view --minify --flatten"*) echo "apiVersion: v1 # minified for ${STUB_CLUSTER}" ;; + *"get nodes -o json"*) cat "${F}/nodes.json" ;; + *"get nodes"*) echo "node/k3d-easc-server-0" ;; + *"get daemonsets -A -o json"*) cat "${F}/daemonsets.json" ;; + *"get namespaces -o json"*) cat "${F}/namespaces.json" ;; + *"get crd sandboxes.agents.x-k8s.io -o name"*) echo 'Error from server (NotFound)' >&2; exit 1 ;; + *"version -o json"*) echo '{"serverVersion": {"gitVersion": "v1.37.1+k3s1"}}' ;; + *"get aigatewayroutes.aigateway.envoyproxy.io -A"*) printf 'llm-inference aigw-qwen2-5-coder-7b\nllm-inference aigw-qwen3guard\n' ;; + *"-n agent-sandbox-system get deployment sandbox-router-deployment -o name"*) + [ -n "${STUB_EAS_SANDBOX:-}" ] || { echo 'Error from server (NotFound)' >&2; exit 1; } + echo deployment.apps/sandbox-router-deployment ;; + *"-n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway"*) echo 172.18.0.2 ;; + *"get pods -A -o json"*) cat "${F}/pods.json" ;; + *" patch "*|*" apply "*|*" delete "*) ;; + *) echo "kubectl stub: unexpected: $*" >&2; exit 1 ;; +esac +EOF + cat > "${B}/git" <<'EOF' +#!/usr/bin/env bash +echo "git $*" >> "${STUB_LOG}" +if [ "$1" = ls-remote ]; then + case "$4" in + release-1.1) printf '170eb3abb1956de0787727f21e356aef2b13ff24\trefs/heads/release-1.1\n' ;; + v9.9.9) printf '1111111111111111111111111111111111111111\trefs/tags/v9.9.9\n2222222222222222222222222222222222222222\trefs/tags/v9.9.9^{}\n' ;; + esac + exit 0 +fi +if [ "$1" = clone ]; then + # A checkout whose installer records its argv; init seeds env/ as the real one does. + d=$4 + mkdir -p "${d}/.git" "${d}/configs/repos" + : > "${d}/configs/repos/repos.agent-toolkit.yaml" + : > "${d}/configs/repos/repos.inference.yaml" + cat > "${d}/es_auto_installer.sh" <<'EOS' +#!/usr/bin/env bash +echo "installer $*" >> "${STUB_LOG}" +if [ "$1" = init ]; then + mkdir -p env/k3d-gpu && : > env/k3d-gpu/global_config.yaml && : > env/k3d-gpu/models.yaml && : > env/k3d-gpu/.solutions.yaml +fi +EOS + chmod +x "${d}/es_auto_installer.sh" +fi +exit 0 +EOF + # yq: records its argv and the strenv() values; answers the row listings eas.sh reads + # from eas-config.py's files (one `key: "value"` per line, as it writes them). + cat > "${B}/yq" <<'EOF' +#!/usr/bin/env python3 +import os, re, sys +a = sys.argv[1:] +env = " ".join("%s=%s" % (k, os.environ[k]) for k in ("L", "U", "R", "F", "N") if k in os.environ and "strenv(%s)" % k in " ".join(a)) +with open(os.environ["STUB_LOG"], "a") as f: + f.write("yq %s%s\n" % (" ".join(a), " [" + env + "]" if env else "")) +if "-r" in a: + text = open(a[-1]).read() + if "@tsv" in a[a.index("-r") + 1]: + for m in re.finditer(r'- layer: "([^"]*)"\n\s+url: "([^"]*)"\n\s+rev: "([^"]*)"', text): + print("\t".join(m.groups())) + elif ".models[].name" in a: + for m in re.finditer(r'^ - name: "([^"]*)"', text, re.M): + print(m.group(1)) +EOF + cat > "${B}/self" <<'EOF' +#!/usr/bin/env bash +echo "self $*" >> "${STUB_LOG}" +case "$1" in + up) touch "${STUB_STATE}/cluster" ;; + fit) + case "$3" in + Qwen/Qwen2.5-Coder-14B-Instruct) echo '{"schema": "k3d-gpu.fit/v1", "fits": false, "summary": "fit: does not fit this host", "flags": []}'; exit 3 ;; + meta-llama/Llama-3.1-8B-Instruct) exit 2 ;; + *) printf '{"schema": "k3d-gpu.fit/v1", "fits": true, "summary": "fit: %s on CPU via vllm", "flags": ["VLLM_CPU_KVCACHE_SPACE=12"]}\n' "$3" ;; + esac ;; +esac +EOF + chmod +x "${B}"/* + + # ---- fixtures: a k3d-gpu cluster on a 32-CPU, 125 GiB hybrid desktop -------------------- + cat > "${S}/fx/nodes.json" <<'EOF' +{"items": [{"metadata": {"name": "k3d-easc-server-0", + "labels": {"feature.node.kubernetes.io/cpu-cpuid.AVXVNNI": "true", "k3d-gpu.io/hybrid-cpu": "true", + "node-role.kubernetes.io/control-plane": "true"}, + "annotations": {"k3s.io/node-args": "[\"server\", \"--disable\", \"traefik\", \"--tls-san\", \"0.0.0.0\"]"}}, + "status": {"allocatable": {"cpu": "32", "memory": "131419028Ki", "gpu.intel.com/xe": "1", "pods": "110"}}}]} +EOF + cat > "${S}/fx/daemonsets.json" <<'EOF' +{"items": [ + {"metadata": {"namespace": "kube-system", "name": "nri-resource-policy-balloons", + "labels": {"app.kubernetes.io/managed-by": "Helm"}, + "annotations": {"meta.helm.sh/release-name": "nri-resource-policy-balloons", + "objectset.rio.cattle.io/owner-gvk": "k3s.cattle.io/v1, Kind=Addon", "objectset.rio.cattle.io/owner-name": "nri-balloons"}}}, + {"metadata": {"namespace": "node-feature-discovery", "name": "nfd-worker", "labels": {"app": "nfd"}, + "annotations": {"objectset.rio.cattle.io/owner-gvk": "k3s.cattle.io/v1, Kind=Addon", "objectset.rio.cattle.io/owner-name": "node-feature-discovery"}}}]} +EOF + echo '{"items": [{"metadata": {"name": "default"}}, {"metadata": {"name": "kube-system"}}, {"metadata": {"name": "node-feature-discovery"}}]}' > "${S}/fx/namespaces.json" + cp "${S}/fx/namespaces.json" "${S}/fx/namespaces.ok" + cat > "${S}/fx/pods.json" <<'EOF' +{"items": [ + {"metadata": {"namespace": "litellm", "name": "litellm-0"}, "status": {"phase": "Running", "containerStatuses": [{"ready": true}]}}, + {"metadata": {"namespace": "llm-inference", "name": "vllm-qwen-0"}, "status": {"phase": "Pending", "containerStatuses": [{"ready": false, "state": {"waiting": {"reason": "ContainerCreating"}}}]}}, + {"metadata": {"namespace": "kube-system", "name": "helm-install-x"}, "status": {"phase": "Succeeded"}}]} +EOF + echo "apiVersion: v1 # easc" > "${S}/kubeconfig-src" + mkdir -p "${S}/k3d-gpu-state" + cp "${S}/kubeconfig-src" "${S}/k3d-gpu-state/kubeconfig-easc" + touch "${S}/cluster" + printf 'MemTotal: 131500000 kB\nMemAvailable: 120000000 kB\n' > "${S}/host/proc/meminfo" + echo 128 > "${S}/host/proc/sys/fs/inotify/max_user_instances" + echo 1048576 > "${S}/host/proc/sys/fs/inotify/max_user_watches" + + # cli [args...]: a launcher function in a subshell, the stubs first on PATH, the + # launcher's own `up` and `fit` replaced by the self stub. + cli() { + (export PATH="${B}:${PATH}" STUB_LOG="${LOG}" STUB_STATE="${S}" STUB_CLUSTER=easc HOME="${S}/home" \ + K3D_GPU_STATE_DIR="${S}/k3d-gpu-state" K3D_GPU_CLUSTER=easc K3D_GPU_EAS_DIR="${K}/eas" \ + K3D_GPU_SHARE="${ROOT}/share" + unset KUBECONFIG K3D_GPU_KUBECONFIG K3D_GPU_CONTEXT + # shellcheck disable=SC1090 + source <(sed '/^case "\${1:-help}" in$/,$d' "${LAUNCHER}") + # shellcheck disable=SC2034 + SYSROOT="${S}/host" + qs_self() { echo "${B}/self"; } + "$@") 2>&1 + } + local out rc log + + # ---- up --dry-run ------------------------------------------------------------------------- + : > "${LOG}" + out=$(cli cmd_eas up --agent-toolkit --dry-run); rc=$? + log=$(cat "${LOG}") + cli_eq "cli: dry run exits 0" "${rc}" 0 + cli_has "cli: dry run prints eas-config's choices" "${out}" "ServiceLB on (no --disable servicelb in k3s.io/node-args) -> metallb_enabled: false" + cli_has "cli: the facts carry the baked balloons policy to eas-config" "${out}" "-> kubernetes_cpu_policy: best-effort (one NRI policy per node)" + cli_has "cli: the facts carry the node's CNI bin_dirs to eas-config" "${out}" 'containerd CNI bin_dirs ["/usr/bin"] -> istio_platform: k3d' + cli_has "cli: the facts carry the fit documents to eas-config" "${out}" "qwen2-5-coder-7b needs 99 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-7b]" + cli_has "cli: the inotify warning reaches the user" "${out}" "WARN: fs.inotify.max_user_instances 128 < 8192" + cli_has "cli: dry run prints the overlay decisions" "${out}" "0006 envoy control-plane affinity -> apply (variant a)" + cli_has "cli: the overlay gets eas-config's cpu policy and the node count" "${log}" \ + "overlay --eas ${STATE}/eas --policy best-effort --nodes 1 --json ${STATE}/overlay-decisions.json" + cli_lacks "cli: dry run: the overlay is not applied" "${log}" "--apply" + cli_has "cli: dry run says nothing was applied" "${out}" "Dry run: nothing applied" + if grep -E '^docker ' <<< "${log}" | grep -Ev '^docker (info --format|exec k3d-easc-server-0 (test -S /var/run/nri/nri.sock|crictl info)$)' | grep -q .; then + fail "cli: dry run: a docker call that is not a read: $(grep -E '^docker ' <<< "${log}" | grep -Ev '^docker (info|exec)' | head -1)" + else + ok "cli: dry run: every docker call is a read (info, exec test, exec crictl info)" + fi + if grep -E '^kubectl ' <<< "${log}" | grep -Eqv ' (get|version|config view) '; then + fail "cli: dry run: a kubectl call that is not a read: $(grep -E '^kubectl ' <<< "${log}" | grep -Ev ' (get|version|config view) ' | head -1)" + else + ok "cli: dry run: every kubectl call is a read" + fi + cli_lacks "cli: dry run: no installer call" "${log}" "installer " + cli_eq "cli: state dir is 0700" "$(stat -c %a "${STATE}")" 700 + cli_eq "cli: the bastion's kubeconfig is 0600" "$(stat -c %a "${STATE}/kubeconfig")" 600 + cli_has "cli: the bastion's kubeconfig is this cluster's context alone" "${log}" "kubectl --context k3d-easc config view --minify --flatten" + cli_has "cli: eas-config writes its five files" "$(ls "${STATE}")" "repos.agent-toolkit.yaml" + local facts + facts=$(python3 -c 'import json, sys +f = json.load(open(sys.argv[1])) +print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox"], f["docker_rootless"], sorted(f["host"]), + sorted(f["nodes"][0]), sorted(f["daemonsets"][0]), f["namespaces"], f["fit"]["meta-llama/Llama-3.1-8B-Instruct"], + f["fit"]["Qwen/Qwen2.5-Coder-14B-Instruct"]["fits"])' "${STATE}/facts.json" 2>&1) + cli_eq "cli: facts.json has the k3d-gpu.eas-facts/v1 shape eas-config.py documents" "${facts}" \ + "['agent_sandbox', 'cluster', 'containerd_cni_bin_dirs', 'daemonsets', 'docker_rootless', 'fit', 'host', 'namespaces', 'nodes', 'nri_socket', 'schema', 'server_version'] True ['/usr/bin'] None False ['inotify_max_user_instances', 'inotify_max_user_watches', 'mem_available_kib'] ['allocatable', 'annotations', 'labels', 'name'] ['annotations', 'labels', 'name', 'namespace'] ['default', 'kube-system', 'node-feature-discovery'] None False" + cli_has "cli: fit runs per eas-config candidate, vLLM on the CPU" "${log}" "self fit --model Qwen/Qwen2.5-Coder-7B-Instruct --engine vllm --device cpu --json" + cli_has "cli: pins.resolved.json records eas-config's commits" "$(cat "${STATE}/pins.resolved.json")" "\"sha\": \"${EAS_SHA}\"" + + # ---- refs ----------------------------------------------------------------------------------- + : > "${LOG}" + out=$(cli cmd_eas up --dry-run --eas-ref v1.2.0-rc.1); rc=$? + cli_eq "cli: --eas-ref v1.2.0-rc.1 (a profile) runs" "${rc}" 0 + cli_has "cli: a profile goes to eas-config as given" "${out}" "# k3d-gpu eas-config: profile v1.2.0-rc.1, layer inference" + cli_has "cli: the rc profile's EAS commit drives the overlay" "${out}" "enterprise-ai-solutions@5c27fb3" + cli_lacks "cli: a profile needs no ls-remote" "$(cat "${LOG}")" "git ls-remote" + : > "${LOG}" + out=$(cli cmd_eas up --dry-run --agent-toolkit --ei-ref release-1.1 --eat-ref v9.9.9); rc=$? + cli_eq "cli: refs resolve" "${rc}" 0 + cli_has "cli: a branch resolves through git ls-remote" "$(cat "${LOG}")" "git ls-remote -- https://github.com/intel/enterprise-inference release-1.1 release-1.1^{}" + cli_has "cli: the resolved commit is printed with its ref" "${out}" "ei: intel/enterprise-inference@170eb3a (release-1.1, git ls-remote" + cli_has "cli: eas-config gets the commit, not the ref" "${out}" "# ei intel/enterprise-inference 170eb3abb1956de0787727f21e356aef2b13ff24" + cli_has "cli: an annotated tag resolves to its commit" "$(cat "${STATE}/repos.agent-toolkit.yaml")" 'rev: "2222222222222222222222222222222222222222"' + : > "${LOG}" + out=$(cli cmd_eas up --dry-run --eas-ref 0123456789abcdef0123456789abcdef01234567); rc=$? + cli_has "cli: a full SHA is taken as given" "${out}" "eas: intel/enterprise-ai-solutions@0123456 (as given)" + cli_lacks "cli: a full SHA needs no ls-remote" "$(cat "${LOG}")" "git ls-remote" + out=$(cli cmd_eas up --dry-run --ei-ref no-such-branch); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'has no branch or tag no-such-branch' <<< "${out}"; then ok "cli: an unknown ref stops"; else fail "cli: an unknown ref: rc=${rc} ${out}"; fi + + # ---- refusals --------------------------------------------------------------------------------- + # cli_refused : non-zero, the message, and no change. + cli_refused() { + local name=$1 msg=$2 o r + shift 2 + : > "${LOG}" + o=$(cli cmd_eas "$@"); r=$? + if [ "${r}" -ne 0 ] && grep -qF -- "${msg}" <<< "${o}" \ + && ! grep -Eq '^docker (run|build)|^kubectl .* (patch|apply|delete|create) |^installer ' "${LOG}"; then + ok "cli: refused before any change: ${name}" + else + fail "cli: refused before any change: ${name} (rc=${r}): $(head -c 600 <<< "${o}") | $(grep -E '^docker (run|build)|^installer' "${LOG}" | head -2)" + fi + } + cli_refused "--layer erag (eas-config's reason)" "chatqna_intel_gpu is EXPERIMENTAL" up --layer erag + cli_refused "--layer erag reads as a refusal" "refused (above); nothing was changed" up --layer erag + cli_refused "an unknown layer" "--layer must be inference, agent-toolkit or erag" up --layer platform + cli_refused "an unknown option" "unknown eas up option: --nope" up --nope + cli_refused "a ref that is an option" "not a ref or model name: '-x'" up --eas-ref -x + cli_refused "a ref with shell characters" "not a ref or model name" up --ei-ref 'main;id' + cli_refused "no subcommand" "usage: k3d-gpu eas up" "" + cli_refused "down with an option" "eas down takes no options" down --force + echo '{"items": [{"metadata": {"name": "genai-gateway"}}, {"metadata": {"name": "ingress-nginx"}}]}' > "${S}/fx/namespaces.json" + cli_refused "EAT 1.x present (eas-config refuses)" "both stacks would own LiteLLM" up --agent-toolkit + cp "${S}/fx/namespaces.ok" "${S}/fx/namespaces.json" + STUB_OVERLAY_STOP=1 cli_refused "an overlay STOP" "the overlay stops at this EAS commit" up --dry-run + rm "${S}/cluster" + cli_refused "a dry run on an absent cluster" "a dry run reads a running cluster" up --dry-run + touch "${S}/cluster" + mv "${K}/eas/overlay.py" "${K}/eas/overlay.py.off" + cli_refused "an incomplete integration dir" "overlay.py missing" up + mv "${K}/eas/overlay.py.off" "${K}/eas/overlay.py" + + # ---- up: the cluster, the bastion and its run ------------------------------------------------ + rm -f "${S}/cluster" "${S}/image-present" + : > "${LOG}" + out=$(HF_TOKEN="${SECRET}" HTTPS_PROXY=http://proxy.test:3128 cli cmd_eas up --agent-toolkit -- -e '{"llm_services_autodeploy_models": []}'); rc=$? + log=$(cat "${LOG}") + cli_eq "cli: up exits 0" "${rc}" 0 + cli_has "cli: up creates the absent cluster with 'k3d-gpu up'" "${log}" "self up" + cli_before "cli: the cluster comes before the facts" "${log}" "self up" "get nodes -o json" + local tag runline + tag=$(cat "${EAS_DIR}/bastion.Dockerfile" "${EAS_DIR}/requirements.lock" | sha256sum | cut -c1-16) + cli_has "cli: the bastion is built when absent, tagged by Dockerfile + lock" "${log}" \ + "docker build -q -t k3d-gpu-eas-bastion:${tag} -f ${K}/eas/bastion.Dockerfile ${K}/eas" + runline=$(grep '^docker run' <<< "${log}") + cli_has "cli: bastion: host network" "${runline}" "--network host" + cli_has "cli: bastion: root (ensure_sudo), the state handed back to this uid:gid" "${runline}" "--user 0:0" + cli_has "cli: bastion: EAS_OWNER is this uid:gid" "${runline}" "-e EAS_OWNER=$(id -u):$(id -g)" + cli_has "cli: bastion: state dir rw" "${runline}" "-v ${STATE}:/eas-state " + cli_has "cli: bastion: kubeconfig ro" "${runline}" "-v ${STATE}/kubeconfig:/kube/config:ro" + cli_has "cli: bastion: integrations/ ro, as overlay.py's patch lookup needs" "${runline}" "-v ${K}:/k3d-gpu/integrations:ro" + cli_has "cli: bastion: the EAS commit eas-config resolved" "${runline}" "-e EAS_SHA=${EAS_SHA}" + cli_has "cli: bastion: eas-config's cpu policy and the node count for the overlay" "${runline}" "-e EAS_CPU_POLICY=best-effort -e EAS_NODES=1" + cli_has "cli: bastion: HF_TOKEN by name only" "${runline}" "-e HF_TOKEN " + cli_has "cli: bastion: a proxy by name only" "${runline}" "-e HTTPS_PROXY " + cli_has "cli: bastion: eas.sh up with the layer and the extra args" "${runline}" \ + "k3d-gpu-eas-bastion:${tag} bash /k3d-gpu/integrations/eas/eas.sh up agent-toolkit -- -e {\"llm_services_autodeploy_models\": []}" + cli_has "cli: up prints the gateway with its --resolve line" "${out}" "curl -k --resolve litellm.solutions.ai:443:172.18.0.2 https://litellm.solutions.ai/v1/models" + local leak + leak=$(grep -rlF -- "${SECRET}" "${S}" "${LOG}" 2>/dev/null; grep -F -- "${SECRET}" <<< "${out}") + cli_eq "cli: no secret value in argv, logs, choices, state files or output" "${leak}" "" + cli_has "cli: the image pins go to the state dir" "$(cat "${STATE}/k3d-gpu-images.yaml" 2>/dev/null)" "agent_sandbox_router_image:" + cli_eq "cli: the installed layer is recorded" "$(cat "${STATE}/layer" 2>/dev/null)" agent-toolkit + : > "${LOG}" + out=$(cli cmd_eas up --agent-toolkit); rc=$? + cli_lacks "cli: a present cluster is not recreated" "$(cat "${LOG}")" "self up" + cli_lacks "cli: a present bastion is reused, not rebuilt" "$(cat "${LOG}")" "docker build" + : > "${LOG}" + out=$(cli cmd_eas up); rc=$? + cli_eq "cli: up of the inference layer exits 0" "${rc}" 0 + cli_lacks "cli: the inference layer has no LiteLLM line" "${out}" "litellm." + cli_has "cli: the inference layer prints the gateway" "${out}" "gateway: eg-gateway at 172.18.0.2" + mkdir -p "${STATE}/eas/env/k3d-gpu" + out=$(cli cmd_eas check); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'run .k3d-gpu eas up --agent-toolkit. first' <<< "${out}"; then + ok "cli: check refuses without the agent toolkit (no LiteLLM)" + else + fail "cli: check without the agent toolkit: rc=${rc} ${out}" + fi + cli cmd_eas up --agent-toolkit >/dev/null + + # ---- eas.sh in the bastion: its steps, in order ------------------------------------------------ + local ES="${WORK}/cli/es" + mkdir -p "${ES}" + command cp -f "${STATE}"/k3d-gpu-overrides.yaml "${STATE}"/global_config.edit.yaml "${STATE}"/models.edit.yaml \ + "${STATE}"/repos.agent-toolkit.yaml "${ES}/" + : > "${LOG}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit -- -e x=1 2>&1); rc=$? + log=$(cat "${LOG}") + cli_eq "cli: eas.sh up exits 0" "${rc}" 0 + cli_has "cli: eas.sh clones EAS" "${log}" "git clone -q https://github.com/intel/enterprise-ai-solutions ${ES}/eas" + cli_has "cli: eas.sh checks out the SHA, clean" "${log}" "git -C ${ES}/eas checkout -q --force --detach ${EAS_SHA}" + cli_before "cli: clone, then checkout" "${log}" "git clone" "checkout -q --force --detach" + cli_before "cli: checkout, then clean" "${log}" "checkout -q --force --detach" "git -C ${ES}/eas clean -q -fd" + cli_before "cli: checkout, then the overlay, applied" "${log}" "checkout -q --force --detach" \ + "overlay --eas ${ES}/eas --policy best-effort --nodes 1 --json ${ES}/overlay-decisions.json --apply" + cli_has "cli: overlay.txt holds the decisions" "$(cat "${ES}/overlay.txt")" "0002 istio global.platform -> apply" + cli_before "cli: the overlay, then the manifest pins" "${log}" "overlay --eas" \ + "yq -i (.repos[] | select(.layer == strenv(L) and .url == strenv(U)) | .rev) = strenv(R) ${ES}/eas/configs/repos/repos.agent-toolkit.yaml [L=inference U=https://github.com/intel/enterprise-inference R=${EI_SHA}]" + cli_has "cli: the EAT repo is pinned too" "${log}" "[L=agent-toolkit U=https://github.com/intel/enterprise-agent-toolkit R=${EAT_SHA}]" + cli_before "cli: the manifest pins, then init" "${log}" "strenv(R) ${ES}/eas/configs/repos" "installer init" + cli_has "cli: init --env k3d-gpu" "${log}" "installer init agent-toolkit --env k3d-gpu" + cli_before "cli: init, then global_config.edit.yaml" "${log}" "installer init" \ + "yq -i . *= load(strenv(F)) ${ES}/eas/env/k3d-gpu/global_config.yaml [F=${ES}/global_config.edit.yaml]" + cli_before "cli: init, then models.edit.yaml defaults" "${log}" "installer init" \ + "yq -i .defaults *= load(strenv(F)).defaults ${ES}/eas/env/k3d-gpu/models.yaml [F=${ES}/models.edit.yaml]" + cli_has "cli: the picked model's catalog entry is edited by name" "${log}" "[F=${ES}/models.edit.yaml N=qwen2-5-coder-7b]" + cli_before "cli: the edits, then install" "${log}" "load(strenv(F)).defaults" "installer install" + cli_has "cli: install with the image pins, then the overrides, then the extra args" "${log}" \ + "installer install agent-toolkit --env k3d-gpu -- -e @${ES}/k3d-gpu-images.yaml -e @${ES}/k3d-gpu-overrides.yaml -e x=1" + cli_lacks "cli: configure is never called" "${log}" "installer configure" + cli_eq "cli: .venv links to the baked venv" "$(readlink "${ES}/eas/.venv")" "${WORK}/cli/venv" + : > "${LOG}" + PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit >/dev/null 2>&1 + cli_has "cli: a second up re-inits with --upgrade (moved pins)" "$(cat "${LOG}")" "installer init agent-toolkit --env k3d-gpu --upgrade" + cli_lacks "cli: a second up does not clone again" "$(cat "${LOG}")" "git clone" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA=main bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'EAS_SHA must be a full commit SHA' <<< "${out}"; then ok "cli: eas.sh refuses a floating EAS ref"; else fail "cli: eas.sh floating ref: rc=${rc} ${out}"; fi + sed -i 's/rev: "3e8a/rev: "main-3e8a/' "${ES}/repos.agent-toolkit.yaml" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'is not a commit SHA' <<< "${out}"; then ok "cli: eas.sh refuses a manifest rev that is not a SHA"; else fail "cli: eas.sh manifest rev: rc=${rc} ${out}"; fi + : > "${LOG}" + PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" down >/dev/null 2>&1 + cli_has "cli: eas.sh down: teardown platform, forced (no terminal)" "$(cat "${LOG}")" "installer teardown platform --env k3d-gpu --force" + : > "${LOG}" + PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" status >/dev/null 2>&1 + cli_has "cli: eas.sh status" "$(cat "${LOG}")" "installer status --env k3d-gpu" + + # ---- down ---------------------------------------------------------------------------------------- + mkdir -p "${STATE}/eas/env/k3d-gpu" + : > "${LOG}" + out=$(cli cmd_eas down); rc=$? + log=$(cat "${LOG}") + cli_eq "cli: down exits 0" "${rc}" 0 + cli_has "cli: down clears each AIGatewayRoute's finalizers (EAS #71)" "${log}" \ + 'kubectl --context k3d-easc -n llm-inference patch aigatewayroute aigw-qwen2-5-coder-7b --type=merge -p {"metadata":{"finalizers":[]}}' + cli_has "cli: down clears the guard model's route too" "${log}" "patch aigatewayroute aigw-qwen3guard" + cli_before "cli: finalizers cleared before teardown" "${log}" "patch aigatewayroute aigw-qwen3guard" "eas.sh down" + cli_has "cli: down keeps the cluster" "${out}" "the cluster stays" + cli_lacks "cli: down does not delete the cluster" "${log}" "cluster delete" + + # ---- status -------------------------------------------------------------------------------------- + echo agent-toolkit > "${STATE}/layer" + command cp -f "${ES}/overlay.txt" "${STATE}/overlay.txt" # what eas.sh up writes in the bastion + : > "${LOG}" + out=$(cli cmd_eas status); rc=$? + cli_eq "cli: status exits 0" "${rc}" 0 + cli_has "cli: status: the layer" "${out}" "Layer: agent-toolkit" + cli_has "cli: status: pins" "${out}" "eas: intel/enterprise-ai-solutions@97f6157 (profile main)" + cli_has "cli: status: overlay decisions" "${out}" "0006 envoy control-plane affinity -> apply (variant a)" + cli_has "cli: status: EAS's own status" "$(cat "${LOG}")" "eas.sh status" + cli_has "cli: status: the gateway with its --resolve line" "${out}" "--resolve litellm.solutions.ai:443:172.18.0.2" + cli_has "cli: status: a pod not Ready" "${out}" "llm-inference/vllm-qwen-0: ContainerCreating" + cli_lacks "cli: status: a Ready pod is not listed" "${out}" "litellm/litellm-0" + cli_lacks "cli: status: a finished pod is not listed" "${out}" "helm-install-x" + + # ---- check --------------------------------------------------------------------------------------- + : > "${LOG}" + out=$(cli cmd_eas check --model qwen2-5-coder-7b); rc=$? + cli_eq "cli: check exits 0" "${rc}" 0 + cli_has "cli: check runs eat-check.sh in eas mode on this cluster's kubeconfig and domain" "$(cat "${LOG}")" \ + "eat-check mode=eas kubeconfig=${STATE}/kubeconfig domain=solutions.ai model=qwen2-5-coder-7b" + + # ---- nothing installed --------------------------------------------------------------------------- + rm -rf "${STATE}/eas" + cli_refused "down with nothing installed" "nothing installed by 'k3d-gpu eas up'" down + cli_refused "status with nothing installed" "nothing installed by 'k3d-gpu eas up'" status + + # ---- sandbox up next to EAS's agent sandbox -------------------------------------------------------- + : > "${LOG}" + out=$(STUB_EAS_SANDBOX=1 cli sandbox_up); rc=$? + if [ "${rc}" -ne 0 ] && grep -q "belongs to EAS's agent toolkit" <<< "${out}" && ! grep -q ' apply ' "${LOG}"; then + ok "cli: sandbox up refuses when sandbox-router-deployment exists, before any apply" + else + fail "cli: sandbox up next to EAS: rc=${rc} ${out}" + fi + + # ---- the launcher's dispatcher and help ------------------------------------------------------------ + out=$(K3D_GPU_STATE_DIR="${S}/k3d-gpu-state" HOME="${S}/home" "${LAUNCHER}" eas 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'usage: k3d-gpu eas up' <<< "${out}"; then ok "cli: dispatcher: 'eas' reaches cmd_eas"; else fail "cli: dispatcher: rc=${rc} ${out}"; fi + cli_has "cli: help documents eas up" "$("${LAUNCHER}" help)" "eas up [--agent-toolkit] [--model NAME]" + + # ---- the bastion: its pins are pins.json's bastion profile ----------------------------------------- + local k3s kc want got + k3s=$(sed -n 's/^ARG K3S_TAG="\{0,1\}v\([0-9]*\.[0-9]*\)\..*/\1/p' "${ROOT}/Dockerfile") + kc=$(sed -n 's/^ARG KUBECTL_VERSION=v\([0-9]*\.[0-9]*\)\..*/\1/p' "${EAS_DIR}/bastion.Dockerfile") + if [ -n "${k3s}" ] && [ "${kc}" = "${k3s}" ]; then ok "cli: bastion kubectl minor ${kc} is the K3S_TAG minor"; else fail "cli: bastion kubectl minor '${kc}' vs K3S_TAG minor '${k3s}'"; fi + want=$(python3 - "${EAS_DIR}/pins.json" <<'PY' +import json, sys +rows = {r["name"]: r for r in json.load(open(sys.argv[1]))["pins"] if r["profile"] == "bastion"} +print("FROM ubuntu:%s@sha256:%s kubectl %s helm %s yq %s kubernetes.core %s" + % (rows["ubuntu"]["ref"], rows["ubuntu"]["sha"], rows["kubectl"]["ref"], rows["helm"]["ref"], rows["yq"]["ref"], + rows["kubernetes.core"]["ref"])) +PY +) + got="$(grep -m1 '^FROM ' "${EAS_DIR}/bastion.Dockerfile") kubectl $(sed -n 's/^ARG KUBECTL_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") helm $(sed -n 's/^ARG HELM_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") yq $(sed -n 's/^ARG YQ_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") kubernetes.core $(sed -n 's/^ARG KUBERNETES_CORE_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile")" + cli_eq "cli: bastion.Dockerfile pins what pins.json's bastion profile names" "${got}" "${want}" + cli_eq "cli: bastion FROM is the EAT bastion's digest" \ + "$(grep -m1 '^FROM ' "${EAS_DIR}/bastion.Dockerfile")" "$(grep -m1 '^FROM ' "${ROOT}/integrations/eat/bastion.Dockerfile")" + want=$(python3 -c 'import json, sys +print(next(r["sha"] for r in json.load(open(sys.argv[1]))["pins"] if r["profile"] == "main" and r["name"] == "eas"))' "${EAS_DIR}/pins.json") + cli_eq "cli: requirements.lock is for the pinned EAS commit" "$(head -1 "${EAS_DIR}/requirements.lock")" "# enterprise-ai-solutions@${want} requirements.txt" + cli_eq "cli: requirements.in names the same commit" "$(head -1 "${EAS_DIR}/requirements.in")" "$(head -1 "${EAS_DIR}/requirements.lock")" + if python3 - "${EAS_DIR}/pins.json" "${EAS_DIR}/requirements.lock" <<'PY' +import json, re, sys +lock = open(sys.argv[2]).read() +for r in json.load(open(sys.argv[1]))["pins"]: + if r["profile"] == "bastion" and r["repo"].startswith("pypi.org/project/"): + name = r["repo"].rsplit("/", 1)[1] + block = re.search(r"(?ms)^%s==%s \\\n(.*?)(?=^\S)" % (re.escape(name), re.escape(r["ref"])), lock + "\nEND") + assert block and "--hash=sha256:" + r["sha"] in block.group(1), (name, r["ref"]) +PY + then ok "cli: requirements.lock pins each pins.json Python package at its version, with its wheel's sha256" + else fail "cli: requirements.lock and pins.json's bastion Python rows disagree"; fi + if grep -E '^[A-Za-z0-9_.-]+==' "${EAS_DIR}/requirements.lock" | grep -qv -- ' \\$'; then + fail "cli: requirements.lock has a pin without hashes" + else + ok "cli: every pin in requirements.lock carries hashes" + fi + if grep -E '^ARG [A-Z_]*_(SHA256|VERSION)=' "${EAS_DIR}/bastion.Dockerfile" | grep -Eqv '=(v?[0-9.]+|[0-9a-f]{64})$'; then + fail "cli: bastion.Dockerfile has a version or sha256 ARG that is not exact" + else + ok "cli: bastion.Dockerfile pins every download by version and sha256" + fi + cli_eq "cli: every bastion download is checked" "$(grep -c 'sha256sum -c -' "${EAS_DIR}/bastion.Dockerfile")" 4 + while read -r mode _ _ path; do + if [ "${mode}" = 100755 ]; then ok "cli: executable in git: ${path}"; else fail "cli: not executable in git (${mode}): ${path}"; fi + done < <(git -C "${ROOT}" ls-files -s -- 'integrations/eas/*.sh') + if python3 - "${EAS_DIR}/known-issues.json" <<'PY' +import json, sys +rows = json.load(open(sys.argv[1]))["issues"] +keys = {"id", "upstream", "symptom", "workaround", "source", "checked"} +assert rows and all(set(r) == keys and all(isinstance(v, str) and v for v in r.values()) for r in rows), rows +assert len({r["id"] for r in rows}) == len(rows) +PY + then ok "cli: known-issues.json rows are flat {id, upstream, symptom, workaround, source, checked}" + else fail "cli: known-issues.json shape"; fi +} + +SECTIONS=(overlay config cli) [ "$#" -gt 0 ] && SECTIONS=("$@") for s in "${SECTIONS[@]}"; do if declare -F "section_${s}" > /dev/null; then diff --git a/tests/eat_test.sh b/tests/eat_test.sh index 177b6b33..fe181a26 100755 --- a/tests/eat_test.sh +++ b/tests/eat_test.sh @@ -9,6 +9,12 @@ # both for a router that shlex.splits the command (agent-sandbox v0.5.0 # python-runtime-sandbox) and for one that runs it under /bin/sh -c # - the check fails when the sandbox prints anything but 385 +# - no key reaches curl's argument list (headers and bodies go as files / stdin) +# 2b. eat-check.sh EAT_MODE=eas (k3d-gpu eas check) against stubs: the gateway is +# eg-gateway's LoadBalancer IP with Host litellm.; the requests carry a +# LiteLLM virtual key minted from Secret litellm/litellm-master-key and deleted at +# exit (pass or fail); the model defaults to the first chat model /v1/models lists; +# the sandbox step is the same as in eat mode. # 3. eat-up.sh refuses deploy_openshell=on, which the pinned EAT tag cannot deploy. # 5. The GenAI Gateway installs from copies of its charts with exact dependency # versions and every image name:tag@sha256 (gateway-charts.py, brownfield.sh). @@ -45,6 +51,9 @@ cat > "${BIN}/kubectl" <<'EOF' case "$*" in *"get nodes"*) echo 172.20.0.2 ;; # first labelled node: not where ingress runs *"-n ingress-nginx get pods"*"app.kubernetes.io/component=controller"*"status.phase=Running"*) echo 172.20.0.3 ;; + *"-n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway"*"loadBalancer.ingress[0].ip"*) + [ -n "${STUB_NO_LB:-}" ] || echo 172.20.0.9 ;; + *"-n litellm get secret litellm-master-key"*"master_key"*) printf 'sk-master-stub' | base64 ;; *"get pods -o name"*) printf 'pod/sandbox-router-1\npod/python-pool-abcde\n' ;; *"get secret sandbox-router-auth"*) printf 'tok' | base64 ;; *port-forward*) exec sleep 60 ;; @@ -54,23 +63,35 @@ EOF cat > "${BIN}/curl" <<'EOF' #!/usr/bin/env python3 -"""curl stub: answers the LiteLLM gateway and the sandbox router.""" +"""curl stub: answers the LiteLLM gateway (chat, /key/generate, /key/delete, /v1/models) +and the sandbox router. Logs every argv and the bearer token each request carries.""" import json, os, shlex, subprocess, sys args = sys.argv[1:] -body, url, resolve = None, None, None +body, url, resolve, auth = None, None, None, None +with open(os.environ["STUB_LOG"], "a") as f: + f.write("argv %s\n" % json.dumps(args)) i = 0 while i < len(args): a = args[i] - if a in ("-d", "--data"): - body = args[i + 1]; i += 2; continue + if a in ("-d", "--data", "--data-binary"): + body = sys.stdin.read() if args[i + 1] == "@-" else args[i + 1]; i += 2; continue if a == "--resolve": resolve = args[i + 1]; i += 2; continue - if a in ("-H", "-o", "--max-time"): + if a == "-H": + h = args[i + 1] + lines = open(h[1:]).read().splitlines() if h.startswith("@") else [h] + for line in lines: + if line.lower().startswith("authorization: bearer "): + auth = line.split(" ", 2)[2] + i += 2; continue + if a in ("-o", "--max-time"): i += 2; continue if not a.startswith("-"): url = a i += 1 +with open(os.environ["STUB_LOG"], "a") as f: + f.write("auth %s %s\n" % (url.split("/", 3)[-1] if url else "-", auth)) def msg(content=None, tool_calls=None, finish="stop"): m = {"role": "assistant", "content": content} @@ -78,12 +99,24 @@ def msg(content=None, tool_calls=None, finish="stop"): m["tool_calls"] = tool_calls print(json.dumps({"choices": [{"finish_reason": finish, "message": m}]})) -if url.endswith("/v1/chat/completions"): +if url.endswith("/key/generate"): + req = json.loads(body) + with open(os.environ["STUB_LOG"], "a") as f: + f.write("keygen alias=%s\n" % req.get("key_alias")) + if not os.environ.get("STUB_NO_KEY"): + print(json.dumps({"key": "sk-virtual-stub", "key_alias": req.get("key_alias")})) +elif url.endswith("/key/delete"): + with open(os.environ["STUB_LOG"], "a") as f: + f.write("keydel %s\n" % json.dumps(json.loads(body)["keys"])) +elif url.endswith("/v1/models"): + print(json.dumps({"data": [{"id": i} for i in os.environ.get("STUB_MODELS", "").split(",") if i]})) +elif url.endswith("/v1/chat/completions"): with open(os.environ["STUB_LOG"], "a") as f: f.write(f"resolve {resolve}\n") req = json.loads(body) with open(os.environ["STUB_LOG"], "a") as f: f.write(f"request max_tokens={req.get('max_tokens')} temperature={req.get('temperature')}\n") + f.write(f"model {req.get('model')}\n") prompt = req["messages"][-1]["content"] if req.get("tools"): if os.environ.get("STUB_NO_TOOL"): @@ -138,6 +171,50 @@ check_says "eat-check defaults: 200 tokens, temperature 0" 0 'request max_tokens check_says "eat-check for a shared remote endpoint: sampled and short" 0 'request max_tokens=64 temperature=0.6$' \ EAT_TEMPERATURE=0.6 EAT_MAX_TOKENS=64 check_says "eat-check refuses a non-numeric temperature" fail 'EAT_TEMPERATURE must be a number' EAT_TEMPERATURE=hot +check_says "eat mode: the master key from vault.yml authorizes the chat" 0 'auth v1/chat/completions sk-stub$' +check_says "eat mode: the router token authorizes /execute" 0 'auth execute tok$' +check_says "eat mode: no key on any curl argv" 0 'EAT check passed' +if grep '^argv ' "${WORK}/log" | grep -Eq 'sk-stub|Bearer tok'; then + fail "eat mode: a key reached curl's argv: $(grep '^argv ' "${WORK}/log" | grep -E 'sk-stub|Bearer tok' | head -1)" +else + ok "eat mode: no key in any curl argv" +fi + +# 2b. EAT_MODE=eas: EAS's agent toolkit from `k3d-gpu eas up --agent-toolkit`. +EAS_MODELS="hybrid-router,Qwen3Guard-Gen-0.6B,qwen2-5-coder-7b,bge-embed" +check_says "eas mode passes on a working stack" 0 'EAT check passed \(eas\)' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: eg-gateway's LoadBalancer IP with Host litellm." 0 \ + 'resolve litellm.solutions.ai:443:172.20.0.9' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the domain from EAT_EAS_DOMAIN" 0 'resolve litellm.example.org:443:172.20.0.9' \ + EAT_MODE=eas EAT_EAS_DOMAIN=example.org STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the virtual key is minted with the master key" 0 'auth key/generate sk-master-stub$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: chat goes with the virtual key, not the master key" 0 'auth v1/chat/completions sk-virtual-stub$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the virtual key is deleted after" 0 'keydel \["sk-virtual-stub"\]' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the key alias is this run's" 0 'keygen alias=k3d-gpu-eas-check-[0-9]+-[0-9]+$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +if grep -q '^auth v1/chat/completions sk-master-stub' "${WORK}/log"; then + fail "eas mode: a chat request carried the master key" +else + ok "eas mode: no chat request carries the master key" +fi +if grep '^argv ' "${WORK}/log" | grep -Eq 'sk-master-stub|sk-virtual-stub|Bearer tok'; then + fail "eas mode: a key reached curl's argv" +else + ok "eas mode: no key in any curl argv" +fi +check_says "eas mode: the model is the first chat model the gateway lists" 0 '^model qwen2-5-coder-7b$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: EAT_MODEL wins" 0 '^model my-model$' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" EAT_MODEL=my-model +check_says "eas mode: the sandbox step is unchanged" 0 '"stdout":"385\\n"' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: a failed check still deletes the key" fail 'keydel \["sk-virtual-stub"\]' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" STUB_CODE='print(sum(range(1, 11)))' +check_says "eas mode: no chat model listed stops the check" fail 'lists no chat model' EAT_MODE=eas STUB_MODELS="hybrid-router" +check_says "eas mode: no LoadBalancer IP stops the check" fail 'no LoadBalancer IP' EAT_MODE=eas STUB_NO_LB=1 +check_says "eas mode: no key from /key/generate stops the check" fail 'without a key' EAT_MODE=eas STUB_NO_KEY=1 +check_says "eas mode refuses a domain with shell characters" fail 'must be a domain name' EAT_MODE=eas 'EAT_EAS_DOMAIN=x;id' +check_says "an unknown EAT_MODE is refused" fail 'EAT_MODE must be eat or eas' EAT_MODE=other # 4. eat-remote.sh: a remote endpoint as a LiteLLM model. The key reaches the # cluster only as the Secret on kubectl's stdin, the gateway gets it as a diff --git a/tests/image_pins_test.sh b/tests/image_pins_test.sh index a30ac4a4..ebdd73ca 100755 --- a/tests/image_pins_test.sh +++ b/tests/image_pins_test.sh @@ -155,6 +155,28 @@ EOF ); then ok "images.lock collects integrations/eat values pairs, Dockerfile FROMs and the Dockerfile's *_IMAGE ARGs" else fail "images-lock collect integrations/eat"; sed 's/^/ | /' <<< "${out}"; fi +# ---- images.lock sources: integrations/eas bastion FROM and _image extra vars ---- +mkdir -p "${WORK}/r/integrations/eas" +printf 'FROM ubuntu:24.04@%s\n' "${D1}" > "${WORK}/r/integrations/eas/bastion.Dockerfile" +printf '# a comment naming x_image: nothing\nagent_sandbox_router_image: "us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router:latest-main@%s"\nnot_an_image_key: "busybox:1"\n' "${D2}" \ + > "${WORK}/r/integrations/eas/images.yaml" +printf '{"image": "ignored:1"}\n' > "${WORK}/r/integrations/eas/pins.json" +if out=$(python3 - "${WORK}/r" "${D1}" "${D2}" 2>&1 <<'EOF' +import importlib.machinery, importlib.util, sys +root, d1, d2 = sys.argv[1:] +loader = importlib.machinery.SourceFileLoader("images_lock", root + "/scripts/images-lock") +spec = importlib.util.spec_from_loader("images_lock", loader) +m = importlib.util.module_from_spec(spec) +loader.exec_module(m) +found, _ = m.collect(root) +router = ("us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router", "latest-main") +assert found.get(router) == {"digest": d2, "sources": {"integrations/eas/images.yaml"}}, found.get(router) +assert found[("docker.io/library/ubuntu", "24.04")]["sources"] >= {"integrations/eas/bastion.Dockerfile"}, found +assert not any(k[0].endswith("/busybox") or k[0].endswith("/ignored") for k in found), sorted(found) +EOF +); then ok "images.lock collects integrations/eas: the bastion's FROM and each _image extra var" +else fail "images-lock collect integrations/eas"; sed 's/^/ | /' <<< "${out}"; fi + # ---- repin, with the registry stubbed ---- scratch cat > "${WORK}/r/share/a.yml" < "${eas_pkg}/share/integrations/eas/eas.sh" +check "a packaged launcher finds K3D_GPU_SHARE/integrations/eas" test "$(K3D_GPU_SHARE="${eas_pkg}/share" HOME="${eas_pkg}" bash -c \ + 'source <(sed "/^case \"\\\${1:-help}\" in\$/,\$d" "$1"); eas_dir' _ "${eas_pkg}/bin/k3d-gpu")" = "${eas_pkg}/share/integrations/eas" +rm -rf "${eas_pkg}" +check "overlay.json's patch_dirs are under integrations/, which the packages keep" python3 -c ' +import json, sys +dirs = json.load(open(sys.argv[1]))["patch_dirs"] +sys.exit(not dirs or not all(d.startswith("integrations/") for d in dirs))' "${ROOT}/integrations/eas/overlay.json" check "flake.lock pins nixpkgs to a rev" \ grep -Eq '"rev": "[0-9a-f]{40}"' "${ROOT}/flake.lock" From 10df943153b3bc760011b4195b7bedb4b43e2aeb Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 14:38:34 -0700 Subject: [PATCH 02/19] eas: k3d-gpu eas up, down, status and check k3d-gpu eas runs Intel's es_auto_installer.sh (enterprise-ai-solutions) against a k3d-gpu cluster in its bring-your-own-cluster mode, from a bastion container, for the platform and inference layers and, with --agent-toolkit, EAT 2.0 on top. eas up gathers read-only facts in the schema eas-config.py reads, resolves any ref that is not a pins.json profile or a commit with git ls-remote, runs eas-config.py, and either prints the overlay decisions (--dry-run) or builds the bastion and runs integrations/eas/eas.sh in it. eas.sh clones EAS at the commit, applies the overlay, sets the layer manifest's revs, runs init, merges the edit files into env/k3d-gpu and runs install with the image pins and the overrides as the last extra vars. configure is never run. When the cluster is absent, eas up runs k3d-gpu up first. The bastion (integrations/eas/bastion.Dockerfile) is Ubuntu 24.04 at the EAT bastion's digest with kubectl v1.37.2, helm v3.22.1 and yq v4.54.1 checked by sha256, a hash-locked venv of EAS's requirements.txt at 97f6157 (ansible 12.3.0, kubernetes 35.0.0) and kubernetes.core 6.3.0 by sha256 over the copy ansible bundles. It runs as root because es_auto_installer.sh's ensure_sudo calls sudo -k under set -e unless EUID is 0 (#L866-L875); eas.sh hands the state dir back to the invoking uid:gid on exit. eas down clears AIGatewayRoute finalizers before teardown platform --force (EAS issue #71). eas status prints the layer, pins, overlay decisions, EAS's own status, the gateway and the pods that are not Ready. eas check runs eat-check.sh in its new EAS mode: eg-gateway's address with Host litellm., a LiteLLM virtual key minted from litellm-master-key and deleted after, then the unchanged sandbox step. eat-check.sh now hands keys to curl as header files and request bodies on stdin in both modes. sandbox up refuses while EAS's sandbox-router-deployment exists. The EAT sandbox router and Python runtime images are pinned by digest in integrations/eas/images.yaml, which images.lock now covers. The packages ship integrations/eas, the overlay's patch drafts and eat-check.sh under share/k3d-gpu/integrations/, the layout overlay.py's patch lookup needs. known-issues.json records the upstream issues this path works around. --- Formula/k3d-gpu.rb | 5 + README.md | 2 +- images.lock | 17 + integrations/eas/README.md | 77 ++ integrations/eas/bastion.Dockerfile | 65 ++ integrations/eas/eas.sh | 158 +++++ integrations/eas/images.yaml | 10 + integrations/eas/known-issues.json | 53 ++ integrations/eas/lock.sh | 27 + integrations/eas/requirements.in | 5 + integrations/eas/requirements.lock | 666 ++++++++++++++++++ integrations/eat/README.md | 2 +- integrations/eat/eat-check.sh | 111 ++- packaging/aur/PKGBUILD | 5 + packaging/check.sh | 5 + packaging/nfpm/nfpm.yaml | 34 + packaging/nix/package.nix | 8 + scripts/images-lock | 19 +- scripts/k3d-gpu | 498 +++++++++++++ tests/console/audit/chat-ui-noauth.golden | 2 +- tests/console/audit/chat-ui.golden | 2 +- tests/console/audit/default-json.golden | 4 +- tests/console/audit/default.golden | 2 +- tests/console/audit/image-digest-moved.golden | 2 +- tests/console/audit/image-no-digest.golden | 2 +- tests/console/audit/image-not-in-lock.golden | 2 +- tests/console/audit/ipv6-unauth.golden | 2 +- tests/console/audit/kubeconfig-0644.golden | 2 +- tests/console/audit/loopback-published.golden | 2 +- tests/console/audit/runtime-fetch.golden | 2 +- tests/console/audit/runtime-pull.golden | 2 +- tests/console/audit/unauth-published.golden | 2 +- 32 files changed, 1752 insertions(+), 43 deletions(-) create mode 100644 integrations/eas/README.md create mode 100644 integrations/eas/bastion.Dockerfile create mode 100755 integrations/eas/eas.sh create mode 100644 integrations/eas/images.yaml create mode 100644 integrations/eas/known-issues.json create mode 100755 integrations/eas/lock.sh create mode 100644 integrations/eas/requirements.in create mode 100644 integrations/eas/requirements.lock diff --git a/Formula/k3d-gpu.rb b/Formula/k3d-gpu.rb index d61c781f..42455677 100644 --- a/Formula/k3d-gpu.rb +++ b/Formula/k3d-gpu.rb @@ -21,6 +21,10 @@ def install (pkgshare/"bench/apb").install "bench/apb/apb.py", "bench/apb/capacity.py" (pkgshare/"oracle").install "oracle/bundle.py", "oracle/project.py", "oracle/validate.py", "oracle/share.py" (pkgshare/"oracle/schema").install "oracle/schema/bundle-v1.json" + # k3d-gpu eas, its overlay's patch drafts and eat-check.sh, in the repo's integrations/ layout. + (pkgshare/"integrations/eas").install Dir["integrations/eas/*"] + (pkgshare/"integrations/eat/upstream/patches").install Dir["integrations/eat/upstream/patches/*.patch"] + (pkgshare/"integrations/eat").install "integrations/eat/eat-check.sh" # The launcher reads its manifests from K3D_GPU_SHARE (default /usr/share/k3d-gpu); # k3d-gpu-mcp runs the launcher next to it in libexec and passes its environment on. (bin/"k3d-gpu").write_env_script libexec/"k3d-gpu", K3D_GPU_SHARE: "${K3D_GPU_SHARE:-#{pkgshare}}" @@ -33,6 +37,7 @@ def install assert_match "/x/nvidia-device-plugin.yml", shell_output("K3D_GPU_SHARE=/x #{bin}/k3d-gpu help") assert_path_exists pkgshare/"nvidia-device-plugin.yml" assert_path_exists pkgshare/"console/plan.py" + assert_path_exists pkgshare/"integrations/eas/eas.sh" assert_path_exists bin/"k3d-gpu-mcp" end end diff --git a/README.md b/README.md index b112ab4f..de8ae85c 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ context, so `up` switches that file's current-context to `k3d-`. ### Install Every package installs the same payload: the launcher at `bin/k3d-gpu`, the MCP server at `bin/k3d-gpu-mcp`, -and in `share/k3d-gpu/` the plugin manifests, `console/`, `bench/`, `oracle/` and `images.lock` +and in `share/k3d-gpu/` the plugin manifests, `console/`, `bench/`, `oracle/`, `integrations/` (for `k3d-gpu eas`) and `images.lock` (`facts`, `plan`, `audit`, `tune`, `capacity` and `k3d-gpu-mcp` also need `python3`). The node image `cryptoandcoffee/k3d-gpu` is published for linux/amd64 only, so on arm64 the packages install but `k3d-gpu up` stops unless `K3D_GPU_IMAGE` names a k3s node image built for diff --git a/images.lock b/images.lock index 472f05ed..a64b0fcb 100644 --- a/images.lock +++ b/images.lock @@ -166,6 +166,7 @@ "name": "docker.io/library/ubuntu", "sources": [ "Dockerfile", + "integrations/eas/bastion.Dockerfile", "integrations/eat/bastion.Dockerfile" ], "tag": "24.04" @@ -420,6 +421,14 @@ ], "tag": "v1.0.8" }, + { + "digest": "sha256:bbacb1409a2ec436ee8cecdfb50f626fe66eae35adbf2886df255cbf5a2b10df", + "name": "registry.k8s.io/agent-sandbox/python-runtime-sandbox", + "sources": [ + "integrations/eas/images.yaml" + ], + "tag": "v0.5.0" + }, { "digest": "sha256:41da84809426098cb04cdafbb41855f15e3604f7455ef53816bad666e532e787", "name": "registry.k8s.io/agent-sandbox/python-runtime-sandbox", @@ -436,6 +445,14 @@ ], "tag": "v0.20.1" }, + { + "digest": "sha256:9cebf7d97f98c0ce008aec8524f23637e60b61485db18ad426897eefac156968", + "name": "us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router", + "sources": [ + "integrations/eas/images.yaml" + ], + "tag": "latest-main" + }, { "digest": "sha256:5d064465d751a73a729e7ecdc32c0ba5de0df1d305debb60a3d9a85b4fd3a967", "name": "vault.habana.ai/gaudi-docker/1.24.2/ubuntu24.04/habanalabs/vllm-0.30.0-ptupstream-2.13.0", diff --git a/integrations/eas/README.md b/integrations/eas/README.md new file mode 100644 index 00000000..25c948d5 --- /dev/null +++ b/integrations/eas/README.md @@ -0,0 +1,77 @@ +# Intel Enterprise AI Solutions on k3d-gpu + +`k3d-gpu eas` installs Intel's [Enterprise AI Solutions](https://github.com/intel/enterprise-ai-solutions) +(EAS) on a k3d-gpu cluster with EAS's own installer, `es_auto_installer.sh`, in its +bring-your-own-cluster mode (`existing_kubernetes`). The layers are platform, inference +([enterprise-inference](https://github.com/intel/enterprise-inference), EI) and agent-toolkit +([enterprise-agent-toolkit](https://github.com/intel/enterprise-agent-toolkit) 2.0, EAT). +They run on the Xeon CPU path: `kubernetes_accelerator: cpu` is the only value EAS acts on at +the pinned commit. Gaudi and Arc stay with `k3d-gpu agent up`. + +```bash +k3d-gpu eas up --dry-run --agent-toolkit # facts, choices and overlay decisions; changes nothing +k3d-gpu eas up --agent-toolkit # creates the cluster first if it is absent +k3d-gpu eas check # chat, tool call and a sandbox run through EAS's LiteLLM +k3d-gpu eas status +k3d-gpu eas down # teardown platform; the cluster stays +``` + +`eas up` without `--agent-toolkit` installs platform and inference. `--layer erag` reaches +`eas-config.py`, which refuses it with the reason. + +## What `eas up` does + +1. Reads facts, read-only, into `facts.json` (schema `k3d-gpu.eas-facts/v1`, the one + `eas-config.py` documents): nodes, namespaces, DaemonSets, the agent-sandbox CRD and its + controller image, the server version, rootless docker, the node container's NRI socket + and containerd CNI `bin_dirs` (`docker exec ... test -S` and `crictl info`), MemAvailable + and inotify limits, and `k3d-gpu fit` of each of `eas-config.py`'s model candidates (vLLM + on the CPU). +2. Resolves refs: a profile of `pins.json` (`--eas-ref v1.2.0-rc.1`) and a full SHA go to + `eas-config.py` as given; any other `--eas-ref`, `--ei-ref` or `--eat-ref` becomes a + commit through `git ls-remote` here, once, and is printed. +3. `eas-config.py` writes the overrides, `global_config.edit.yaml`, `models.edit.yaml`, + `repos..yaml` and `choices.txt`, one `fact -> choice [source]` line per decision. + Exit 3 is a refusal: nothing is changed. The commits it resolved (the `# eas|ei|eat` + lines of `choices.txt`) go to `pins.resolved.json`. +4. Builds the bastion (`bastion.Dockerfile`), tagged by the sha256 of the Dockerfile and + `requirements.lock`, or reuses it. +5. Runs `eas.sh up ` in the bastion: clone EAS and check out the commit, apply the + patch overlay (`overlay.py --apply`, with `kubernetes_cpu_policy` from the overrides and + the node count), set each manifest rev from `repos..yaml`, + `init --env k3d-gpu`, merge the two edit files into `env/k3d-gpu/`, then + `install --env k3d-gpu -- -e @k3d-gpu-images.yaml -e @k3d-gpu-overrides.yaml + [your args]`. Extra vars load last in `es_auto_installer.sh` `main()`, so the overrides + win over every config file. `configure` is never run. +6. Prints the gateway: the LoadBalancer IP of eg-gateway's Service (k3s ServiceLB, the node + IP), and with the agent toolkit a `curl --resolve litellm.:443:` + line. `/etc/hosts` is not edited. + +The bastion runs on the host network with the state dir read-write at `/eas-state`, a +kubeconfig holding this cluster's context alone read-only at `/kube/config`, and +`integrations/` read-only at `/k3d-gpu/integrations` (overlay.py finds its patch drafts +relative to the dir above `integrations/`). It runs as root: `es_auto_installer.sh` +`ensure_sudo` runs `sudo -k` under `set -e` unless EUID is 0, and the image has no sudo. +`eas.sh` hands the state dir back to your uid:gid when it exits. `HF_TOKEN` and the proxy +variables reach it by name only, when set. + +State per cluster: `~/.local/state/k3d-gpu/eas-/` (mode 0700): the EAS checkout with +its `env/k3d-gpu/` and `ext/`, `facts.json`, `pins.resolved.json`, `eas-config.py`'s files, +`overlay.txt` and `overlay-decisions.json`, `kubeconfig`, `layer` and `logs/`. + +## Files + +| File | Role | +|------|------| +| `eas.sh` | the in-bastion driver (`up`, `overlay-check`, `down`, `status`) | +| `bastion.Dockerfile` | Ubuntu 24.04 (the EAT bastion's digest), kubectl v1.37.2, helm v3.22.1, yq v4.54.1 by sha256, the installer's venv at `/opt/eas-venv`, kubernetes.core 6.3.0 by sha256: `pins.json`'s bastion profile | +| `requirements.in`, `requirements.lock`, `lock.sh` | EAS `requirements.txt` at the pinned commit, and its hash-locked resolution for the bastion's Python 3.12 | +| `images.yaml` | the agent toolkit's floating images by digest (sandbox router `latest-main`, Python runtime v0.5.0) | +| `known-issues.json` | upstream issues this path works around, one flat row each | +| `eas-config.py`, `pins.json` | the choice generator and the pins it reads | +| `overlay.py`, `overlay.json`, `patches/` | the k3s/k3d patch overlay | + +## Not verified + +No EAS install has run on a k3d-gpu cluster from this code. The agent-toolkit layer has not +run on k3s or k3d; Kubernetes 1.37 is outside what EAS tests (it pins kubectl v1.34.3). diff --git a/integrations/eas/bastion.Dockerfile b/integrations/eas/bastion.Dockerfile new file mode 100644 index 00000000..b22c8e27 --- /dev/null +++ b/integrations/eas/bastion.Dockerfile @@ -0,0 +1,65 @@ +# Bastion for Intel's Enterprise AI Solutions installer (intel/enterprise-ai-solutions, +# es_auto_installer.sh) on a k3d-gpu cluster in its bring-your-own mode. +# +# Everything the installer's `configure` would install with sudo and apt is baked here +# at a pinned version, so `k3d-gpu eas up` never runs `configure`: +# - kubectl, helm and yq from their release downloads, each checked by sha256. The +# sha256 values are the ones the upstream checksum files publish for these releases +# (dl.k8s.io .sha256, get.helm.sh .sha256sum, yq's `checksums` file). +# - the installer's Python requirements (requirements.txt at the EAS commit named in +# requirements.lock) as a hash-locked venv at /opt/eas-venv; the installer reuses a +# venv that has ansible-playbook (es_auto_installer.sh _ensure_venv). +# - kubernetes.core 6.3.0 (EAS collections/requirements.yml) by sha256, into that venv's +# purelib over the 6.2.0 copy the ansible 12.3.0 package bundles, which is where the +# installer's _ensure_collections looks; with 6.3.0 there it makes no network call. +# +# kubectl is v1.37.2, the minor of the node image's K3S_TAG (Dockerfile): EAS's own +# v1.34.3 is outside kubectl's +/-1 minor skew against a v1.37 API server +# (https://kubernetes.io/releases/version-skew-policy/#kubectl). helm is the newest v3 +# (EAS asks for >= v3.20.2); yq the newest v4 (EAS asks for >= v4.53.2). +# +# Runs as root: es_auto_installer.sh ensure_sudo runs `sudo -k` under `set -e` unless +# EUID is 0, and this image has no sudo. eas.sh hands the state dir back to the +# invoking user's uid:gid when it exits. +FROM ubuntu:24.04@sha256:fe81560990fae86dc4743202b2cb4d3df98d0bcf949173cecef15ce1127ef852 + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DEBIAN_FRONTEND=noninteractive + +# The state dir (the EAS checkout and its ext/ clones) belongs to the invoking user, and +# git refuses a repository another uid owns ("detected dubious ownership") unless told. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git python3 python3-venv \ + && rm -rf /var/lib/apt/lists/* \ + && git config --system --add safe.directory '*' + +ARG KUBECTL_VERSION=v1.37.2 +ARG KUBECTL_SHA256=c04487523e7aa7d1493f7808174c899db93d7a9cf57531420cd76b9bd415f8f4 +ARG HELM_VERSION=v3.22.1 +ARG HELM_SHA256=896cf63684a42a723d61c980369d6e588330927deb13676ca0d85006dd6da0ea +ARG YQ_VERSION=v4.54.1 +ARG YQ_SHA256=8e34fc298390875de416e6a4afcb8cabeceb25d9aa8506c1a2f9353cf702ea5f +RUN curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + && echo "${KUBECTL_SHA256} /usr/local/bin/kubectl" | sha256sum -c - \ + && curl -fsSL -o /tmp/helm.tgz "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ + && echo "${HELM_SHA256} /tmp/helm.tgz" | sha256sum -c - \ + && tar -xzf /tmp/helm.tgz -C /usr/local/bin --strip-components=1 linux-amd64/helm \ + && curl -fsSL -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ + && echo "${YQ_SHA256} /usr/local/bin/yq" | sha256sum -c - \ + && chmod 0755 /usr/local/bin/kubectl /usr/local/bin/helm /usr/local/bin/yq \ + && rm /tmp/helm.tgz + +COPY requirements.lock /opt/eas-requirements.lock +RUN python3 -m venv /opt/eas-venv \ + && /opt/eas-venv/bin/pip install --no-cache-dir --disable-pip-version-check --require-hashes --no-deps \ + -r /opt/eas-requirements.lock + +ARG KUBERNETES_CORE_VERSION=6.3.0 +ARG KUBERNETES_CORE_SHA256=86a2a89892b75120a32e58135ab96982e3c02ad9f6b59eb2412671952622e748 +RUN curl -fsSL -o /tmp/kubernetes-core.tar.gz \ + "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/kubernetes-core-${KUBERNETES_CORE_VERSION}.tar.gz" \ + && echo "${KUBERNETES_CORE_SHA256} /tmp/kubernetes-core.tar.gz" | sha256sum -c - \ + && purelib=$(/opt/eas-venv/bin/python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') \ + && /opt/eas-venv/bin/ansible-galaxy collection install --force --offline -p "${purelib}" /tmp/kubernetes-core.tar.gz \ + && rm /tmp/kubernetes-core.tar.gz + +WORKDIR /eas-state diff --git a/integrations/eas/eas.sh b/integrations/eas/eas.sh new file mode 100755 index 00000000..4ea74f5f --- /dev/null +++ b/integrations/eas/eas.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# eas.sh — drives Intel's es_auto_installer.sh (intel/enterprise-ai-solutions) against a +# k3d-gpu cluster in its bring-your-own mode (existing_kubernetes). `k3d-gpu eas` runs it in +# the bastion (bastion.Dockerfile); `overlay-check` also runs on the host for --dry-run. +# +# eas.sh up [-- ] +# 1. clone EAS and check out EAS_SHA (a clean tree: earlier overlay hunks are dropped) +# 2. the patch overlay: overlay.py --apply, decisions in overlay.txt +# 3. the layer manifest's revs as commit SHAs (repos..yaml from eas-config.py) +# 4. es_auto_installer.sh init --env k3d-gpu +# 5. global_config.edit.yaml and models.edit.yaml (eas-config.py) into env/k3d-gpu/ +# 6. es_auto_installer.sh install --env k3d-gpu -- -e @k3d-gpu-images.yaml +# -e @k3d-gpu-overrides.yaml [args]: extra vars load last in es_auto_installer.sh +# main(), so the overrides win +# eas.sh overlay-check steps 1-2 without applying: the overlay decisions only +# eas.sh down es_auto_installer.sh teardown platform --env k3d-gpu --force +# eas.sh status es_auto_installer.sh status --env k3d-gpu +# +# `configure` is never run: it needs a terminal or --force, sudo and apt +# (es_auto_installer.sh@97f6157 #L318-L351), and the bastion bakes everything it installs. +# The installer reuses a venv that has ansible-playbook (#L801-L806), so .venv links to the +# baked /opt/eas-venv, where it also finds kubernetes.core 6.3.0 (#L832-L847). +# +# Environment (set by `k3d-gpu eas`): +# EAS_STATE state dir: the EAS checkout and eas-config.py's files (default /eas-state) +# EAS_SHA the enterprise-ai-solutions commit (choices.txt names it) +# EAS_CPU_POLICY kubernetes_cpu_policy eas-config.py chose; EAS_NODES the node count (overlay.py) +# EAS_VENV the baked venv (default /opt/eas-venv) +# EAS_OWNER uid:gid that owns the state dir when this exits. The bastion runs as root: +# es_auto_installer.sh ensure_sudo runs `sudo -k` under set -e unless EUID +# is 0 (#L866-L875), and the image has no sudo. +# Values from eas-config.py's files reach yq through the environment (strenv), never as +# expression text. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EAS_STATE="${EAS_STATE:-/eas-state}" +EAS_VENV="${EAS_VENV:-/opt/eas-venv}" +EAS_URL="https://github.com/intel/enterprise-ai-solutions" +ENV_NAME="k3d-gpu" +CHECKOUT="${EAS_STATE}/eas" + +err() { echo "ERROR: $*" >&2; exit 1; } +info() { echo "==> $*"; } + +own_state() { + [ -z "${EAS_OWNER:-}" ] || chown -R "${EAS_OWNER}" "${EAS_STATE}" 2>/dev/null || true +} +trap own_state EXIT + +# checkout: EAS at EAS_SHA with no local change to a tracked file and no untracked file +# but the ignored ones (env/, ext/, .kubespray/), so every run starts from upstream's tree. +checkout() { + [[ "${EAS_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || err "EAS_SHA must be a full commit SHA, got '${EAS_SHA:-}'" + if [ ! -d "${CHECKOUT}/.git" ]; then + info "Cloning ${EAS_URL} into ${CHECKOUT}" + git clone -q "${EAS_URL}" "${CHECKOUT}" + fi + git -C "${CHECKOUT}" cat-file -e "${EAS_SHA}^{commit}" 2>/dev/null || git -C "${CHECKOUT}" fetch -q --tags origin + git -C "${CHECKOUT}" checkout -q --force --detach "${EAS_SHA}" + git -C "${CHECKOUT}" clean -q -fd + info "EAS at ${EAS_SHA}" +} + +# overlay [--apply]: overlay.py decides each patch at the checkout's HEAD (and with --apply +# applies the chosen ones); the decisions go to overlay.txt. A STOP exits non-zero first. +overlay() { + [[ "${EAS_CPU_POLICY:-}" =~ ^[a-z-]+$ ]] || err "EAS_CPU_POLICY not set (kubernetes_cpu_policy from eas-config.py)" + [[ "${EAS_NODES:-}" =~ ^[0-9]+$ ]] || err "EAS_NODES not set (the cluster's node count)" + python3 "${HERE}/overlay.py" --eas "${CHECKOUT}" --policy "${EAS_CPU_POLICY}" --nodes "${EAS_NODES}" \ + --json "${EAS_STATE}/overlay-decisions.json" "$@" | tee "${EAS_STATE}/overlay.txt" +} + +# pin_manifest : each row of repos..yaml (layer, url, rev) sets rev on the +# manifest entry with that layer and url, which must exist once. A SHA is a rev +# _checkout_rev accepts (#L684-L689); EAS clones in full, so any published commit resolves. +pin_manifest() { + local m="${CHECKOUT}/configs/repos/repos.$1.yaml" f="${EAS_STATE}/repos.$1.yaml" L U R + [ -f "${f}" ] || err "${f} missing (eas-config.py writes it)" + [ -f "${m}" ] || err "EAS has no configs/repos/repos.$1.yaml at ${EAS_SHA}" + while IFS=$'\t' read -r L U R; do + [[ "${R}" =~ ^[0-9a-f]{40}$ ]] || err "repos.$1.yaml: rev for ${U} is not a commit SHA: '${R}'" + export L U R + yq -e '[.repos[] | select(.layer == strenv(L) and .url == strenv(U))] | length == 1' "${m}" >/dev/null \ + || err "configs/repos/repos.$1.yaml has no single entry for layer ${L}, ${U}" + yq -i '(.repos[] | select(.layer == strenv(L) and .url == strenv(U)) | .rev) = strenv(R)' "${m}" + done < <(yq -r '.repos[] | [.layer, .url, .rev] | @tsv' "${f}") +} + +# env_edits: global_config.edit.yaml merged into env/k3d-gpu/global_config.yaml (where +# _resolve_kubeconfig reads existing_kubernetes, #L1342-L1358), and models.edit.yaml into +# env/k3d-gpu/models.yaml: defaults merge, and each model merges into the catalog entry of +# its name, which must exist once. +env_edits() { + local env="${CHECKOUT}/env/${ENV_NAME}" F N + F="${EAS_STATE}/global_config.edit.yaml" + [ -f "${F}" ] || err "${F} missing (eas-config.py writes it)" + export F + yq -i '. *= load(strenv(F))' "${env}/global_config.yaml" + F="${EAS_STATE}/models.edit.yaml" + [ -f "${F}" ] || err "${F} missing (eas-config.py writes it)" + [ -f "${env}/models.yaml" ] || err "init seeded no env/${ENV_NAME}/models.yaml" + yq -i '.defaults *= load(strenv(F)).defaults' "${env}/models.yaml" + while IFS= read -r N; do + export N + yq -e '[.models[] | select(.name == strenv(N))] | length == 1' "${env}/models.yaml" >/dev/null \ + || err "env/${ENV_NAME}/models.yaml has no single catalog entry named ${N}" + yq -i '(.models[] | select(.name == strenv(N))) *= (load(strenv(F)).models[] | select(.name == strenv(N)))' \ + "${env}/models.yaml" + done < <(yq -r '.models[].name' "${F}") +} + +installer() { (cd "${CHECKOUT}" && ./es_auto_installer.sh "$@"); } + +cmd_up() { + local layer=${1:-} + [ -n "${layer}" ] || err "usage: eas.sh up [-- ]" + shift + [ "${1:-}" != -- ] || shift + checkout + overlay --apply + pin_manifest "${layer}" + ln -sfn "${EAS_VENV}" "${CHECKOUT}/.venv" + if [ -f "${CHECKOUT}/env/${ENV_NAME}/.solutions.yaml" ]; then + # --upgrade moves already-cloned ext/ repos onto the manifest's new revs (never + # rewriting a config); without it a changed pin is refused (#L910-L940). + installer init "${layer}" --env "${ENV_NAME}" --upgrade + else + installer init "${layer}" --env "${ENV_NAME}" + fi + env_edits + installer install "${layer}" --env "${ENV_NAME}" -- \ + -e "@${EAS_STATE}/k3d-gpu-images.yaml" -e "@${EAS_STATE}/k3d-gpu-overrides.yaml" "$@" +} + +cmd_down() { + [ -d "${CHECKOUT}/env/${ENV_NAME}" ] || err "no env/${ENV_NAME} in ${CHECKOUT}: nothing was installed from this state dir" + ln -sfn "${EAS_VENV}" "${CHECKOUT}/.venv" + # teardown's confirm refuses a non-terminal stdin without --force (#L129-L137, #L1486). + installer teardown platform --env "${ENV_NAME}" --force +} + +cmd_status() { + [ -d "${CHECKOUT}/env/${ENV_NAME}" ] || err "no env/${ENV_NAME} in ${CHECKOUT}: run 'k3d-gpu eas up' first" + ln -sfn "${EAS_VENV}" "${CHECKOUT}/.venv" + installer status --env "${ENV_NAME}" +} + +sub=${1:-} +[ $# -eq 0 ] || shift +case "${sub}" in + up) cmd_up "$@" ;; + overlay-check) checkout; overlay ;; + down) cmd_down ;; + status) cmd_status ;; + *) err "usage: eas.sh up [-- args] | overlay-check | down | status" ;; +esac diff --git a/integrations/eas/images.yaml b/integrations/eas/images.yaml new file mode 100644 index 00000000..a97c86ba --- /dev/null +++ b/integrations/eas/images.yaml @@ -0,0 +1,10 @@ +# Extra vars for es_auto_installer.sh install: the agent toolkit's floating images by +# registry digest. `k3d-gpu eas up` passes this file before k3d-gpu-overrides.yaml, so a +# value in the overrides still wins. EAT release-2.0.0 (036b662) roles/agent_sandbox/ +# defaults/main.yaml#L27-L28 names the router by the moving tag latest-main and the +# runtime by tag only; its install.yaml substitutes these values whole into the router +# manifest (${ROUTER_IMAGE}) and the default SandboxTemplate, so a name:tag@sha256 +# reference reaches the pods as written. Digests read from the registries 2026-10-29 +# (scripts/images-lock digest); scripts/images-lock repin moves them. +agent_sandbox_router_image: "us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router:latest-main@sha256:9cebf7d97f98c0ce008aec8524f23637e60b61485db18ad426897eefac156968" +agent_sandbox_python_runtime_image: "registry.k8s.io/agent-sandbox/python-runtime-sandbox:v0.5.0@sha256:bbacb1409a2ec436ee8cecdfb50f626fe66eae35adbf2886df255cbf5a2b10df" diff --git a/integrations/eas/known-issues.json b/integrations/eas/known-issues.json new file mode 100644 index 00000000..ff341b5b --- /dev/null +++ b/integrations/eas/known-issues.json @@ -0,0 +1,53 @@ +{ + "schema": "k3d-gpu.eas-known-issues/v1", + "issues": [ + { + "id": "eas-teardown-aigatewayroute-finalizer", + "upstream": "https://github.com/intel/enterprise-ai-solutions/issues/71", + "symptom": "teardown platform hangs at 'envoy_ai_gateway | Uninstall CRDs Helm release' while a model's AIGatewayRoute still carries aigateway.envoyproxy.io/finalizer: the role strips finalizers only after the controller and the CRDs are gone", + "workaround": "k3d-gpu eas down sets the finalizers of every AIGatewayRoute to [] before it runs teardown platform --force (the issue's own workaround)", + "source": "https://github.com/intel/enterprise-ai-solutions/issues/71 (open); https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/envoy_ai_gateway/tasks/teardown.yaml", + "checked": "2026-10-29" + }, + { + "id": "ei-require-amx-pending", + "upstream": "https://github.com/intel/enterprise-inference", + "symptom": "model pods stay Pending on a node without feature.node.kubernetes.io/cpu-cpuid.AMXBF16=true: models.yaml defaults.require_amx is true and the renderer turns it into a required node affinity on that label", + "workaround": "eas-config.py writes defaults.require_amx to env/k3d-gpu/models.yaml from the nodes' AMXBF16 label (false without it)", + "source": "https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/model_manager/models.yaml#L20; https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/model_manager/core/renderer.py#L90-L104", + "checked": "2026-10-29" + }, + { + "id": "eas-kubectl-skew", + "upstream": "https://github.com/intel/enterprise-ai-solutions", + "symptom": "es_auto_installer.sh configure installs kubectl v1.34.3, three minors behind a k3d-gpu cluster's v1.37 API server, outside kubectl's supported skew of one minor", + "workaround": "the bastion bakes kubectl v1.37.2, the minor of the node image's K3S_TAG, and configure is never run", + "source": "https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L52; https://kubernetes.io/releases/version-skew-policy/#kubectl", + "checked": "2026-10-29" + }, + { + "id": "eat-inotify-minimum", + "upstream": "https://github.com/intel/enterprise-agent-toolkit", + "symptom": "with fs.inotify.max_user_instances at the kernel default of 128, file watchers in the stack's pods fail with 'too many open files'", + "workaround": "eas up warns below 8192 instances or 1048576 watches and prints the sysctl lines; the host is never changed", + "source": "https://github.com/intel/enterprise-agent-toolkit/blob/036b66234801e9bd950afd90830821859780d668/docs/prerequisites.md#L20-L33", + "checked": "2026-10-29" + }, + { + "id": "eat-48-core-minimum", + "upstream": "https://github.com/intel/enterprise-agent-toolkit", + "symptom": "the agent toolkit's documented minimum is 48 cores (96+ recommended); smaller nodes run it with less CPU for the model than the defaults assume", + "workaround": "eas up warns when the node has fewer than 48 allocatable CPUs", + "source": "https://github.com/intel/enterprise-agent-toolkit/blob/036b66234801e9bd950afd90830821859780d668/docs/prerequisites.md#L7", + "checked": "2026-10-29" + }, + { + "id": "ei-skip-models-flag", + "upstream": "https://github.com/intel/enterprise-inference", + "symptom": "the inference docs say './es_auto_installer.sh install inference --skip-models', but es_auto_installer.sh rejects any option it does not know ('Unknown option: --skip-models')", + "workaround": "none needed on the inference layer: EI's seeded config.yaml already sets llm_services_deploy_models: false. On the agent toolkit, EAT's chat model deploys whatever that key says; 'k3d-gpu eas up --agent-toolkit -- -e '{\"llm_services_autodeploy_models\": []}'' skips it (the guard model stays while the guardrail is on)", + "source": "https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/config.yaml#L94-L98; https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/install.yaml#L143-L161; https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L1312", + "checked": "2026-10-29" + } + ] +} diff --git a/integrations/eas/lock.sh b/integrations/eas/lock.sh new file mode 100755 index 00000000..22e52257 --- /dev/null +++ b/integrations/eas/lock.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Regenerate integrations/eas/requirements.lock (hashed) from requirements.in, in the +# bastion's own base image (bastion.Dockerfile FROM: Ubuntu 24.04, Python 3.12), so the +# lock resolves for the interpreter that installs it. +# +# requirements.in is enterprise-ai-solutions' requirements.txt at the commit its first +# line names; copy the new file over it (keeping that line, with the new commit) when the +# EAS pin moves. pip-compile picks the newest release inside those ranges. +# +# Usage: integrations/eas/lock.sh + +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +IMAGE="$(sed -n 's/^FROM \(ubuntu:[^ ]*\).*$/\1/p' "${HERE}/bastion.Dockerfile")" +PIP_TOOLS="pip-tools==7.6.2" +header=$(head -1 "${HERE}/requirements.in") +[[ "${header}" =~ ^#\ enterprise-ai-solutions@[0-9a-f]{40}\ requirements\.txt$ ]] \ + || { echo "requirements.in line 1 must be '# enterprise-ai-solutions@ requirements.txt'" >&2; exit 1; } + +docker run --rm -v "${HERE}:/w" -w /w "${IMAGE}" sh -c \ + "apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends python3-venv >/dev/null && \ + python3 -m venv /tmp/v && /tmp/v/bin/pip install -q '${PIP_TOOLS}' && \ + /tmp/v/bin/pip-compile -q --upgrade --generate-hashes --strip-extras --allow-unsafe --no-emit-index-url \ + --output-file requirements.lock requirements.in && \ + { echo '${header}'; cat requirements.lock; } > /tmp/lock && cat /tmp/lock > requirements.lock && \ + chown $(id -u):$(id -g) requirements.lock" diff --git a/integrations/eas/requirements.in b/integrations/eas/requirements.in new file mode 100644 index 00000000..f27c4fca --- /dev/null +++ b/integrations/eas/requirements.in @@ -0,0 +1,5 @@ +# enterprise-ai-solutions@97f6157d6a72f7173c698c474d4e51d2c79f1fd6 requirements.txt +ansible>=10.0,<13.0 +kubernetes>=29.0,<36.0 +jsonpatch>=1.33,<2.0 +jmespath>=1.0,<2.0 diff --git a/integrations/eas/requirements.lock b/integrations/eas/requirements.lock new file mode 100644 index 00000000..b84c2cea --- /dev/null +++ b/integrations/eas/requirements.lock @@ -0,0 +1,666 @@ +# enterprise-ai-solutions@97f6157d6a72f7173c698c474d4e51d2c79f1fd6 requirements.txt +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --output-file=requirements.lock --strip-extras requirements.in +# +ansible==12.3.0 \ + --hash=sha256:02721f6fb432ddd47f1044ac49b04b5e9ae08890bd427def8df3db1607aeec51 \ + --hash=sha256:cd156f82fa87f7a899212b0efa9f7ca3a24d609212de9f78428a572eb01e5414 + # via -r requirements.in +ansible-core==2.19.14 \ + --hash=sha256:1e7612788eaa6b3c87a3670ce86c64b830eba0ae2b3fcb25a772f276704163b4 \ + --hash=sha256:d518d0e96fa75bedfe9b0a96d5b158aa712d092961824c115140c8677b88ff06 + # via ansible +certifi==2026.10.23 \ + --hash=sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 + # via + # kubernetes + # requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +cryptography==50.0.2 \ + --hash=sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602 \ + --hash=sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2 \ + --hash=sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047 \ + --hash=sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c \ + --hash=sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42 \ + --hash=sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18 \ + --hash=sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51 \ + --hash=sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81 \ + --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 \ + --hash=sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2 \ + --hash=sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de \ + --hash=sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7 \ + --hash=sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd \ + --hash=sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2 \ + --hash=sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be \ + --hash=sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45 \ + --hash=sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0 \ + --hash=sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e \ + --hash=sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c \ + --hash=sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5 \ + --hash=sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452 \ + --hash=sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48 \ + --hash=sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05 \ + --hash=sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1 \ + --hash=sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93 \ + --hash=sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04 \ + --hash=sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e \ + --hash=sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67 \ + --hash=sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7 \ + --hash=sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107 \ + --hash=sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079 \ + --hash=sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134 \ + --hash=sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227 \ + --hash=sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1 \ + --hash=sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539 \ + --hash=sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e \ + --hash=sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d \ + --hash=sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c \ + --hash=sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd \ + --hash=sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020 \ + --hash=sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd \ + --hash=sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94 \ + --hash=sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a \ + --hash=sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408 \ + --hash=sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37 \ + --hash=sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e \ + --hash=sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454 \ + --hash=sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c \ + --hash=sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc \ + --hash=sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37 \ + --hash=sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767 \ + --hash=sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a \ + --hash=sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5 \ + --hash=sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc \ + --hash=sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67 \ + --hash=sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8 \ + --hash=sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480 \ + --hash=sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb \ + --hash=sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b + # via ansible-core +durationpy==0.11 \ + --hash=sha256:181898e1ae282e288f0a2291829656bf1b6b3aadf30a97993b85db4943642905 \ + --hash=sha256:a739fe2b8972c250ff72f8e2c488d18cf25f7b852f49ee76048775d5171df30c + # via kubernetes +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via ansible-core +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via -r requirements.in +jsonpatch==1.38 \ + --hash=sha256:46260c5df952abd4734a098f94512dc00bc38539152ac3ef069ed0475e4f5695 \ + --hash=sha256:debc3e7bdc3d446468d64987f51a0e7a9deea42987ff90a6d1005a489b471c19 + # via -r requirements.in +jsonpointer==3.6 \ + --hash=sha256:82e839a3ba3a9e2b54394fda8b3d276c2459d80408d06ef9a6fbc369d7544ffc \ + --hash=sha256:a45c858f8a9e171f6b3790ed9446c9382782f6c4cb4589176813c5a553f4cfcd + # via jsonpatch +kubernetes==35.0.0 \ + --hash=sha256:39e2b33b46e5834ef6c3985ebfe2047ab39135d41de51ce7641a7ca5b372a13d \ + --hash=sha256:3d00d344944239821458b9efd484d6df9f011da367ecb155dadf9513f05f09ee + # via -r requirements.in +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via jinja2 +oauthlib==4.0.0 \ + --hash=sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd \ + --hash=sha256:efb274799819440f95b4ab3b818869f1ce9ae26c5beacba0201d1a1b76b54f86 + # via requests-oauthlib +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via ansible-core +pycparser==3.12 \ + --hash=sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e \ + --hash=sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6 + # via cffi +python-dateutil==3.0.0 \ + --hash=sha256:3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd \ + --hash=sha256:7ebcb3c29a92f3ceafe79a7a48a5c45677e9e004bbeec5742ac0bbb00560b7a2 + # via kubernetes +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # ansible-core + # kubernetes +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # kubernetes + # requests-oauthlib +requests-oauthlib==2.0.0 \ + --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ + --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 + # via kubernetes +resolvelib==1.2.1 \ + --hash=sha256:7d08a2022f6e16ce405d60b68c390f054efcfd0477d4b9bd019cc941c28fad1c \ + --hash=sha256:fb06b66c8da04172d9e72a21d7d06186d8919e32ae5ab5cdf5b9d920be805ac2 + # via ansible-core +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via + # kubernetes + # python-dateutil +tzdata==2026.6 \ + --hash=sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec \ + --hash=sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d + # via python-dateutil +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # kubernetes + # requests +websocket-client==1.9.2 \ + --hash=sha256:0fcb57545848be86992e128218fd96dd87a6769ffdb1a968dff79632b85604d0 \ + --hash=sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce + # via kubernetes diff --git a/integrations/eat/README.md b/integrations/eat/README.md index 20711d19..da1b83d6 100644 --- a/integrations/eat/README.md +++ b/integrations/eat/README.md @@ -64,7 +64,7 @@ that off the host. | `gateway-charts.py` | writes pinned copies of EAT's GenAI Gateway chart directories; `brownfield.sh` passes them to `deploy-genai-gateway.yml` | | `gateway-values.yaml` | the gateway chart's pins: postgresql and redis chart versions, LiteLLM, PostgreSQL, Redis and init images by digest | | `gateway-trace-values.yaml` | the Langfuse trace stack's pins: Langfuse web and worker, PostgreSQL, Valkey, ClickHouse, ZooKeeper, MinIO by digest | -| `eat-check.sh` | the end-state check below | +| `eat-check.sh` | the end-state check below; with `EAT_MODE=eas` it checks EAS's agent toolkit instead (`k3d-gpu eas check`, see `../eas/README.md`) | `tests/eat_test.sh` runs `eat-check.sh` against stub `kubectl` and `curl` (no cluster) and checks that every script here is committed executable. diff --git a/integrations/eat/eat-check.sh b/integrations/eat/eat-check.sh index d8c8ec94..068f6c37 100755 --- a/integrations/eat/eat-check.sh +++ b/integrations/eat/eat-check.sh @@ -1,26 +1,41 @@ #!/usr/bin/env bash -# eat-check — end-to-end check of an EAT deployment made by eat-up.sh. +# eat-check — end-to-end check of an agent stack with Intel's agent toolkit: EAT 1.x made +# by eat-up.sh (EAT_MODE=eat), or EAS's agent toolkit made by `k3d-gpu eas up +# --agent-toolkit` (EAT_MODE=eas, which `k3d-gpu eas check` sets). # -# 1. chat: a chat completion through EAT's LiteLLM gateway, reached through -# EAT's ingress-nginx on the host IP of its controller pod (Host: cluster_url, TLS) +# 1. chat: a chat completion through the LiteLLM gateway. eat: EAT's ingress-nginx on +# the host IP of its controller pod (Host: cluster_url, TLS). eas: EAS's eg-gateway +# Service on its LoadBalancer IP (k3s ServiceLB: the node IP), Host +# litellm., TLS # 2. tool call: a get_weather tool request must come back as finish_reason=tool_calls -# 3. code run: the model writes Python through the gateway, and EAT's +# 3. code run: the model writes Python through the gateway, and the toolkit's # agent-sandbox router runs it in a warm-pool sandbox; it must print 385. # bash passes the model's text to the router: this is a scripted pipeline, # not an agent loop. # -# EAT_WORK eat-up.sh work dir; its vault.yml holds the LiteLLM master key (default: ./eat-work) +# EAT_MODE eat (default) | eas +# EAT_WORK eat: eat-up.sh work dir; its vault.yml holds the LiteLLM master key (default: ./eat-work) +# EAT_EAS_DOMAIN eas: base_domain_name (default: solutions.ai, EAS's default) # EAT_KUBECONFIG kubeconfig of the cluster (default: $KUBECONFIG, else ~/.kube/config) -# EAT_MODEL model name registered in LiteLLM (default: Qwen/Qwen3-1.7B) +# EAT_MODEL model name registered in LiteLLM (eat default: Qwen/Qwen3-1.7B; eas default: +# the first chat model /v1/models lists, routers, guards, embedders and +# rerankers left out) # EAT_TEMPERATURE sampling temperature (default: 0). A remote endpoint from eat-remote.sh # on a shared fleet: 0.6, never 0 # EAT_MAX_TOKENS max_tokens per request (default: 200; 64 for a shared remote endpoint) +# +# eas: the requests carry a LiteLLM virtual key minted for this run with the master key +# (Secret litellm/litellm-master-key) through POST /key/generate, and deleted at exit +# (POST /key/delete). Keys reach curl as header files and request bodies on its stdin, +# never on an argument list. set -euo pipefail +EAT_MODE="${EAT_MODE:-eat}" EAT_WORK="${EAT_WORK:-${PWD}/eat-work}" +EAT_EAS_DOMAIN="${EAT_EAS_DOMAIN:-solutions.ai}" export KUBECONFIG="${EAT_KUBECONFIG:-${KUBECONFIG:-${HOME}/.kube/config}}" -EAT_MODEL="${EAT_MODEL:-Qwen/Qwen3-1.7B}" +EAT_MODEL="${EAT_MODEL:-}" EAT_TEMPERATURE="${EAT_TEMPERATURE:-0}" EAT_MAX_TOKENS="${EAT_MAX_TOKENS:-200}" CORE="${EAT_WORK}/eat/core" @@ -33,20 +48,65 @@ info() { echo "==> $*"; } [[ "${EAT_TEMPERATURE}" =~ ^[0-9]+([.][0-9]+)?$ ]] || err "EAT_TEMPERATURE must be a number, got '${EAT_TEMPERATURE}'" [[ "${EAT_MAX_TOKENS}" =~ ^[0-9]+$ ]] || err "EAT_MAX_TOKENS must be a whole number, got '${EAT_MAX_TOKENS}'" -host=$(grep -E '^cluster_url=' "${CORE}/inventory/agentic-config.cfg" | cut -d= -f2-) -key=$(grep -E '^litellm_master_key:' "${CORE}/inventory/metadata/vault.yml" | cut -d'"' -f2) -[ -n "${key}" ] || err "no litellm_master_key in ${CORE}/inventory/metadata/vault.yml" -# ingress-nginx binds hostPort 80/443 on the nodes its controller pods run on. -node_ip=$(kubectl -n ingress-nginx get pods \ - -l app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/component=controller \ - --field-selector=status.phase=Running -o jsonpath='{.items[0].status.hostIP}') -[ -n "${node_ip}" ] || err "no running ingress-nginx controller pod" +pf="" vkey="" master="" alias="" +cleanup() { + [ -z "${pf}" ] || kill "${pf}" 2>/dev/null || true + # The run's virtual key goes when the check ends, pass or fail. + if [ -n "${vkey}" ]; then + jq -n --arg k "${vkey}" '{keys: [$k]}' | gw_post /key/delete "${master}" >/dev/null \ + || echo "WARN: could not delete this run's LiteLLM virtual key ${alias}" >&2 + fi +} +trap cleanup EXIT -# chat : POST /v1/chat/completions through the ingress; prints the response body. -chat() { - curl -sfk --max-time 600 --resolve "${host}:443:${node_ip}" "https://${host}/v1/chat/completions" \ - -H "Authorization: Bearer ${key}" -H 'content-type: application/json' -d "$1" +# gw [curl args...]: an HTTPS request to the gateway; the bearer token goes +# to curl as a header file. +gw() { + local path=$1 bearer=$2 + shift 2 + curl -sfk --max-time 600 --resolve "${host}:443:${node_ip}" "https://${host}${path}" \ + -H @<(printf 'Authorization: Bearer %s\n' "${bearer}") "$@" } +# gw_post : POST stdin as the JSON body. +gw_post() { gw "$1" "$2" -H 'content-type: application/json' --data-binary @-; } + +case "${EAT_MODE}" in + eat) + host=$(grep -E '^cluster_url=' "${CORE}/inventory/agentic-config.cfg" | cut -d= -f2-) + key=$(grep -E '^litellm_master_key:' "${CORE}/inventory/metadata/vault.yml" | cut -d'"' -f2) + [ -n "${key}" ] || err "no litellm_master_key in ${CORE}/inventory/metadata/vault.yml" + # ingress-nginx binds hostPort 80/443 on the nodes its controller pods run on. + node_ip=$(kubectl -n ingress-nginx get pods \ + -l app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/component=controller \ + --field-selector=status.phase=Running -o jsonpath='{.items[0].status.hostIP}') + [ -n "${node_ip}" ] || err "no running ingress-nginx controller pod" + EAT_MODEL="${EAT_MODEL:-Qwen/Qwen3-1.7B}" + ;; + eas) + [[ "${EAT_EAS_DOMAIN}" =~ ^[A-Za-z0-9.-]+$ ]] || err "EAT_EAS_DOMAIN must be a domain name, got '${EAT_EAS_DOMAIN}'" + host="litellm.${EAT_EAS_DOMAIN}" + node_ip=$(kubectl -n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway \ + -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') + [ -n "${node_ip}" ] || err "no LoadBalancer IP on the eg-gateway Service in envoy-gateway-system" + master=$(kubectl -n litellm get secret litellm-master-key -o jsonpath='{.data.master_key}' | base64 -d) + [ -n "${master}" ] || err "no master_key in Secret litellm/litellm-master-key" + alias="k3d-gpu-eas-check-$$-${RANDOM}" + vkey=$(jq -n --arg a "${alias}" '{key_alias: $a, duration: "1h"}' | gw_post /key/generate "${master}" | jq -r '.key // empty') \ + || err "LiteLLM did not mint a virtual key (POST https://${host}/key/generate on ${node_ip})" + [ -n "${vkey}" ] || err "LiteLLM answered /key/generate without a key" + key=${vkey} + info "Virtual key ${alias} (deleted at exit)" + if [ -z "${EAT_MODEL}" ]; then + EAT_MODEL=$(gw /v1/models "${key}" | jq -r '[.data[].id | select(test("hybrid-router|vllm-sr|guard|embed|rerank"; "i") | not)][0] // empty') \ + || err "could not list the gateway's models" + [ -n "${EAT_MODEL}" ] || err "the gateway lists no chat model (set EAT_MODEL)" + fi + ;; + *) err "EAT_MODE must be eat or eas, got '${EAT_MODE}'" ;; +esac + +# chat : POST /v1/chat/completions through the gateway; prints the response body. +chat() { gw_post /v1/chat/completions "${key}" <<< "$1"; } # ask : the model's reply text, without Qwen3's empty block. ask() { @@ -55,7 +115,7 @@ ask() { | jq -r '.choices[0].message.content' | sed -e 's###g' -e '/^```/d' -e '/^[[:space:]]*$/d' } -info "Chat through LiteLLM at https://${host} (ingress on ${node_ip})..." +info "Chat through LiteLLM at https://${host} (${EAT_MODE} gateway on ${node_ip}), model ${EAT_MODEL}..." answer=$(ask "In one sentence, what is Kubernetes? /no_think") [ -n "${answer}" ] || err "empty chat reply" echo " ${answer}" @@ -79,15 +139,14 @@ sandbox=$(kubectl -n "${SANDBOX_NS}" get pods -o name | grep -m1 '/python-pool-' token=$(kubectl -n "${SANDBOX_NS}" get secret sandbox-router-auth -o jsonpath='{.data.ROUTER_AUTH_TOKEN}' | base64 -d) kubectl -n "${SANDBOX_NS}" port-forward svc/sandbox-router-svc "${ROUTER_PORT}:8080" >/dev/null 2>&1 & pf=$! -trap 'kill "${pf}" 2>/dev/null || true' EXIT for _ in $(seq 1 40); do curl -s -o /dev/null "127.0.0.1:${ROUTER_PORT}" && break; sleep 0.5; done -result=$(curl -sf "127.0.0.1:${ROUTER_PORT}/execute" -H "Authorization: Bearer ${token}" \ - -H "X-Sandbox-ID: ${sandbox}" -H "X-Sandbox-Namespace: ${SANDBOX_NS}" -H "X-Sandbox-Port: 8888" \ - -H 'content-type: application/json' \ - -d "$(jq -n --arg c "${code}" '{command: ("python3 -c " + ($c | @sh))}')") +result=$(jq -n --arg c "${code}" '{command: ("python3 -c " + ($c | @sh))}' \ + | curl -sf "127.0.0.1:${ROUTER_PORT}/execute" -H @<(printf 'Authorization: Bearer %s\n' "${token}") \ + -H "X-Sandbox-ID: ${sandbox}" -H "X-Sandbox-Namespace: ${SANDBOX_NS}" -H "X-Sandbox-Port: 8888" \ + -H 'content-type: application/json' --data-binary @-) echo " sandbox ${sandbox}: $(jq -c . <<< "${result}")" # @sh single-quotes the code, which both a shlex.split router (agent-sandbox # v0.5.0's python-runtime-sandbox) and a /bin/sh -c one read back unchanged. [ "$(jq -r .exit_code <<< "${result}")" = 0 ] || err "sandbox command failed" [ "$(jq -r .stdout <<< "${result}")" = 385 ] || err "sandbox printed $(jq -c .stdout <<< "${result}"), expected 385" -info "EAT check passed." +info "EAT check passed (${EAT_MODE})." diff --git a/packaging/aur/PKGBUILD b/packaging/aur/PKGBUILD index b20c9716..bec360a6 100644 --- a/packaging/aur/PKGBUILD +++ b/packaging/aur/PKGBUILD @@ -54,6 +54,11 @@ package() { install -Dm644 share/agent-sandbox.yml "$pkgdir/usr/share/k3d-gpu/agent-sandbox.yml" install -Dm644 share/agent-sandbox-k3d.yml "$pkgdir/usr/share/k3d-gpu/agent-sandbox-k3d.yml" install -Dm644 share/model-share.yml "$pkgdir/usr/share/k3d-gpu/model-share.yml" + # k3d-gpu eas, its overlay's patch drafts and eat-check.sh, in the repo's integrations/ layout. + install -d "$pkgdir/usr/share/k3d-gpu/integrations" + cp -r integrations/eas "$pkgdir/usr/share/k3d-gpu/integrations/eas" + install -Dm644 -t "$pkgdir/usr/share/k3d-gpu/integrations/eat/upstream/patches" integrations/eat/upstream/patches/*.patch + install -Dm755 integrations/eat/eat-check.sh "$pkgdir/usr/share/k3d-gpu/integrations/eat/eat-check.sh" install -Dm644 README.md "$pkgdir/usr/share/doc/$pkgname/README.md" install -Dm644 LICENSE.md "$pkgdir/usr/share/licenses/$pkgname/LICENSE.md" } diff --git a/packaging/check.sh b/packaging/check.sh index 9fafe72e..7ef82da6 100755 --- a/packaging/check.sh +++ b/packaging/check.sh @@ -35,6 +35,11 @@ done for f in vs.py llm_load.py competitors.lock; do test "$(sha256sum < "/src/bench/$f")" = "$(sha256sum < "/usr/share/k3d-gpu/bench/$f")" done +for f in /src/integrations/eas/* /src/integrations/eas/patches/* /src/integrations/eat/upstream/patches/*.patch; do + [ -d "$f" ] || test "$(sha256sum < "$f")" = "$(sha256sum < "/usr/share/k3d-gpu/${f#/src/}")" +done +test -x /usr/share/k3d-gpu/integrations/eas/eas.sh +test -x /usr/share/k3d-gpu/integrations/eat/eat-check.sh k3d-gpu help | grep -F "/usr/share/k3d-gpu/nvidia-device-plugin.yml" >/dev/null test -x /usr/bin/k3d-gpu test -x /usr/bin/k3d-gpu-mcp diff --git a/packaging/nfpm/nfpm.yaml b/packaging/nfpm/nfpm.yaml index f4760ca5..97772d7f 100644 --- a/packaging/nfpm/nfpm.yaml +++ b/packaging/nfpm/nfpm.yaml @@ -108,6 +108,20 @@ contents: dst: /usr/share/k3d-gpu/bench/competitors.lock file_info: mode: 0644 + # k3d-gpu eas: integrations/eas, plus the patch drafts its overlay applies and the + # end-state check it runs, in the repo's integrations/ layout: overlay.py finds its + # patches relative to the dir above integrations/. + - src: integrations/eas/ + dst: /usr/share/k3d-gpu/integrations/eas + type: tree + - src: integrations/eat/upstream/patches/*.patch + dst: /usr/share/k3d-gpu/integrations/eat/upstream/patches/ + file_info: + mode: 0644 + - src: integrations/eat/eat-check.sh + dst: /usr/share/k3d-gpu/integrations/eat/eat-check.sh + file_info: + mode: 0755 - src: README.md dst: /usr/share/doc/k3d-gpu/README.md file_info: @@ -163,6 +177,26 @@ contents: packager: rpm file_info: mode: 0755 + - dst: /usr/share/k3d-gpu/integrations + type: dir + packager: rpm + file_info: + mode: 0755 + - dst: /usr/share/k3d-gpu/integrations/eat + type: dir + packager: rpm + file_info: + mode: 0755 + - dst: /usr/share/k3d-gpu/integrations/eat/upstream + type: dir + packager: rpm + file_info: + mode: 0755 + - dst: /usr/share/k3d-gpu/integrations/eat/upstream/patches + type: dir + packager: rpm + file_info: + mode: 0755 - dst: /usr/share/doc/k3d-gpu type: dir packager: rpm diff --git a/packaging/nix/package.nix b/packaging/nix/package.nix index 65ae8d39..bebe4d91 100644 --- a/packaging/nix/package.nix +++ b/packaging/nix/package.nix @@ -43,6 +43,9 @@ stdenvNoCC.mkDerivation { (root + "/images.lock") (root + "/bench/vs.py") (root + "/bench/competitors.lock") + (root + "/integrations/eas") + (root + "/integrations/eat/upstream/patches") + (root + "/integrations/eat/eat-check.sh") (root + "/README.md") (root + "/LICENSE.md") ]; @@ -69,6 +72,11 @@ stdenvNoCC.mkDerivation { install -Dm644 -t "$out/share/k3d-gpu/bench/apb" bench/apb/apb.py bench/apb/capacity.py install -Dm644 -t "$out/share/k3d-gpu/oracle" oracle/bundle.py oracle/project.py oracle/validate.py oracle/share.py install -Dm644 -t "$out/share/k3d-gpu/oracle/schema" oracle/schema/bundle-v1.json + # k3d-gpu eas, its overlay's patch drafts and eat-check.sh, in the repo's integrations/ layout. + mkdir -p "$out/share/k3d-gpu/integrations" + cp -r integrations/eas "$out/share/k3d-gpu/integrations/eas" + install -Dm644 -t "$out/share/k3d-gpu/integrations/eat/upstream/patches" integrations/eat/upstream/patches/*.patch + install -Dm755 integrations/eat/eat-check.sh "$out/share/k3d-gpu/integrations/eat/eat-check.sh" install -Dm644 README.md "$out/share/doc/k3d-gpu/README.md" install -Dm644 LICENSE.md "$out/share/licenses/k3d-gpu/LICENSE.md" runHook postInstall diff --git a/scripts/images-lock b/scripts/images-lock index 39cacb5b..5bf16985 100755 --- a/scripts/images-lock +++ b/scripts/images-lock @@ -32,6 +32,9 @@ Sources of image references (the lock holds all of them and nothing else): - integrations/eat/ the Helm values overlays (*.yaml, *.yml: each repository:/tag: pair) and the bastion's FROM (*.Dockerfile): the images the EAT deploy path pulls + - integrations/eas/ the bastion's FROM (*.Dockerfile) and each `_image:` + extra var (*.yaml): the images `k3d-gpu eas` pins for the + EAS installer - k3s's system images k3s-images.txt of the Dockerfile's K3S_TAG release (coredns, local-path-provisioner, metrics-server, pause, ...): k3s itself starts them in the cluster @@ -112,6 +115,9 @@ ARG_IMAGE_RE = re.compile(r'^ARG[ \t]+[A-Z][A-Z0-9_]*_IMAGE="?([^"\s$]+)"?[ \t]* # One `key: value` line of a Helm values mapping; group 4 is the inline value, unquoted. VALUES_KEY_RE = re.compile(r"""^([ \t]*)(repository|registry|tag):[ \t]*(["']?)([^\s"'#]*)\3[ \t]*(?:#.*)?$""") EAT_DIR = "integrations/eat" +EAS_DIR = "integrations/eas" +# An Ansible extra var naming an image: `agent_sandbox_router_image: "ref"`. +VAR_IMAGE_RE = re.compile(r"""^[a-z][a-z0-9_]*_image:[ \t]*["']?([^"'\s#]+)""", re.M) sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "console")) from audit import parse_ref # noqa: E402 (one image-reference parser for the lock and the audit) @@ -220,6 +226,17 @@ def collect(root): with open(path, encoding="utf-8") as fh: for ref in dockerfile_froms(fh.read())[0]: add(ref, source) + eas = os.path.join(root, EAS_DIR) + for f in sorted(os.listdir(eas)) if os.path.isdir(eas) else []: + path, source = os.path.join(eas, f), EAS_DIR + "/" + f + if f.endswith((".yml", ".yaml")): + with open(path, encoding="utf-8") as fh: + for ref in VAR_IMAGE_RE.findall(fh.read()): + add(ref, source) + elif f.endswith(".Dockerfile"): + with open(path, encoding="utf-8") as fh: + for ref in dockerfile_froms(fh.read())[0]: + add(ref, source) k3s_tag = args.get("K3S_TAG") if not k3s_tag: raise ValueError("Dockerfile has no ARG K3S_TAG default") @@ -559,7 +576,7 @@ def main(argv): if args.command == "pins": print("ok every image reference is pinned by digest or allowed in %s" % ALLOW_FILE) else: - print("ok images.lock matches share/, the Dockerfile, scripts/k3d-gpu, integrations/eat and k3s %s" + print("ok images.lock matches share/, the Dockerfile, scripts/k3d-gpu, integrations/eat, integrations/eas and k3s %s" % collect(args.root)[1]) return 0 diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index 847b5c14..016ba06f 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -58,6 +58,8 @@ # fit size a model to what this host has free now: placement, context x seqs, engine # memory flags, pod memory limit, or why it cannot fit; --explain prints the arithmetic # audit posture checks: unauthenticated listeners off loopback, images not in images.lock +# eas up|down|status|check: Intel Enterprise AI Solutions (platform, inference, agent +# toolkit) through its own installer, in a bastion container # sandbox up|down|status # model-share --image REF --agent-image REF [--replicas N] [--name NAME] # kubernetes-sigs/agent-sandbox with a warm pool of sandboxes pinned to E-cores @@ -293,6 +295,21 @@ Usage: k3d-gpu (no DNS; on a CDN those addresses also serve its other sites), and endpoint in the sandbox's Secret k3d-gpu-llm, which sandbox code can read (use a key you would hand to that code); status checks the pin + eas up [--agent-toolkit] [--model NAME] [--eas-ref R] [--ei-ref R] [--eat-ref R] + [--dry-run] [--explain] [-- ] + Intel Enterprise AI Solutions on this cluster through its own installer + (es_auto_installer.sh, bring-your-own-cluster mode) in a bastion + container: platform + inference (default), or the agent toolkit (EAT 2.0) + on top (--agent-toolkit). Creates the cluster first when it is absent. + Every choice prints as 'fact -> choice [source]' (--explain: full links); + --dry-run prints them and the patch overlay's decisions and changes + nothing. Commits come from integrations/eas/pins.json; a ref given here + is resolved to a commit once and recorded in the state dir + eas status|down|check [--model NAME] + status: EAS's own status, pins, overlay, gateway, pods not Ready. down: + teardown platform (the cluster stays). check (agent toolkit): chat, tool + call and a sandbox run through EAS's LiteLLM gateway with a short-lived + virtual key model-share --image REF --agent-image REF [--replicas N] [--name NAME] Print a Deployment whose N agent replicas mount one model OCI image as a read-only image volume (pipe to kubectl apply -f -). It warns when this @@ -451,6 +468,7 @@ Environment: K3D_GPU_SANDBOX agent-sandbox yaml (default: ${SHARE_DIR}/agent-sandbox.yml) K3D_GPU_SANDBOX_PROFILE E-core sandbox template + warm pool (default: ${SHARE_DIR}/agent-sandbox-k3d.yml) K3D_GPU_SANDBOX_TIMEOUT seconds 'sandbox up' waits for a ready sandbox (default: 180) + K3D_GPU_EAS_DIR integrations/eas for 'eas' (default: the checkout's, then ${SHARE_DIR}/integrations/eas) K3D_GPU_TUNE auto | off: apply this host's tuned profile in 'agent up' (default: auto) K3D_GPU_TUNE_DIR tuned profiles (default: ~/.config/k3d-gpu/tune) K3D_GPU_TUNE_PY tune driver (default: the repo's console/tune.py, then ${SHARE_DIR}/console/tune.py) @@ -6635,6 +6653,11 @@ SANDBOX_CRDS=( sandbox_up() { require kubectl local f ready="" deadline profile fields base model origin + # EAS's agent toolkit (k3d-gpu eas up --agent-toolkit) owns agent-sandbox v0.5.0 here; + # v1.0.8's manifest would replace its CRDs (v1beta1 only) under its controller. + if [ -n "$(kubectl -n agent-sandbox-system get deployment sandbox-router-deployment -o name 2>/dev/null || true)" ]; then + err "agent-sandbox here belongs to EAS's agent toolkit (deployment agent-sandbox-system/sandbox-router-deployment); 'k3d-gpu sandbox up' would replace its CRDs. Remove it first with 'k3d-gpu eas down'" + fi for f in "${SANDBOX_MANIFEST}" "${SANDBOX_PROFILE_MANIFEST}"; do [ -f "${f}" ] || err "${f} not found — set K3D_GPU_SHARE, K3D_GPU_SANDBOX or K3D_GPU_SANDBOX_PROFILE" done @@ -6739,6 +6762,480 @@ cmd_sandbox() { esac } +# ---- eas: Intel Enterprise AI Solutions on this cluster --------------------------- +# intel/enterprise-ai-solutions ships one installer, es_auto_installer.sh, for its +# platform, inference (intel/enterprise-inference) and agent-toolkit +# (intel/enterprise-agent-toolkit 2.0) layers. Its bring-your-own mode +# (existing_kubernetes) installs onto this cluster. The installer runs in a bastion +# container (integrations/eas/bastion.Dockerfile), never on the host: +# integrations/eas/eas.sh the in-bastion driver (clone, overlay, init, edits, install) +# integrations/eas/eas-config.py facts + pins + flags -> overrides, edits, choices.txt +# integrations/eas/overlay.py the patch overlay, decided against the pinned checkout +# integrations/eas/pins.json the EAS, EI and EAT commits, per profile +# State per cluster: ${STATE_DIR}/eas-/ (mode 0700). +EAS_ENV="k3d-gpu" +EAS_GW_NS="envoy-gateway-system" +EAS_GW_SELECTOR="gateway.envoyproxy.io/owning-gateway-name=eg-gateway" +EAS_DEFAULT_DOMAIN="solutions.ai" # base_domain_name in EAS configs/defaults/global_config.yaml +# Environment the bastion gets by name only (docker run -e NAME), never by value. +EAS_PASS_ENV=(HF_TOKEN HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ES_LOG_LEVEL) +EAS_FILES=(eas.sh bastion.Dockerfile requirements.lock images.yaml pins.json eas-config.py overlay.py overlay.json) +# Files eas-config.py writes; each `eas up` starts without them. +EAS_OUTPUTS=(choices.txt k3d-gpu-overrides.yaml global_config.edit.yaml models.edit.yaml + repos.inference.yaml repos.agent-toolkit.yaml pins.resolved.json layer) + +# eas_dir: K3D_GPU_EAS_DIR, then the checkout's integrations/eas, then the packaged copy. +# The packages keep the repo's integrations/ layout, which overlay.py's patch lookup +# (overlay.json patch_dirs, relative to the dir above integrations/) needs. +eas_dir() { + local self d + self=$(readlink -f "${BASH_SOURCE[0]}") + for d in "${K3D_GPU_EAS_DIR:-}" "$(dirname "${self}")/../integrations/eas" "${SHARE_DIR}/integrations/eas"; do + if [ -n "${d}" ] && [ -f "${d}/eas.sh" ]; then (cd "${d}" && pwd); return 0; fi + done + return 1 +} + +eas_state() { echo "${STATE_DIR}/eas-${CLUSTER}"; } + +# eas_bastion_image : the bastion's tag is the sha256 of its Dockerfile and lock, so +# a changed pin builds a new image and an unchanged one is reused. +eas_bastion_image() { + local dir=$1 tag image + tag=$(cat "${dir}/bastion.Dockerfile" "${dir}/requirements.lock" | sha256sum | cut -c1-16) + image="k3d-gpu-eas-bastion:${tag}" + if ! docker image inspect "${image}" >/dev/null 2>&1; then + info "Building the EAS bastion ${image} (kubectl, helm, yq, the installer's venv)..." >&2 + docker build -q -t "${image}" -f "${dir}/bastion.Dockerfile" "${dir}" >/dev/null \ + || err "could not build ${image} from ${dir}/bastion.Dockerfile" + fi + echo "${image}" +} + +# eas_kubeconfig : this cluster's context alone, flattened, for the bastion +# (mounted read-only at /kube/config). A shared kubeconfig's current-context may name +# another cluster; this file names only ${CLUSTER}'s. +eas_kubeconfig() { + local f="$1/kubeconfig" + ( umask 077; kubectl config view --minify --flatten > "${f}.tmp" && mv -f "${f}.tmp" "${f}" ) \ + || err "could not write ${f} for cluster '${CLUSTER}'" +} + +# eas_run : one eas.sh run in the bastion, logged +# to /logs. Host network: the k3d API server's 0.0.0.0: and the node IP are +# reached as from the host. integrations/ is mounted whole, read-only, so overlay.py finds +# the patch drafts in integrations/eat/upstream/patches. Root in the container (eas.sh +# says why); EAS_OWNER hands the state dir back to this user. +eas_run() { + local dir=$1 state=$2 image=$3 v log envs=() + shift 3 + for v in "${EAS_PASS_ENV[@]}"; do [ -z "${!v:-}" ] || envs+=(-e "${v}"); done + mkdir -p "${state}/logs" + log="${state}/logs/eas-$1-$(date +%Y%m%d-%H%M%S).log" + info "eas.sh $* in ${image} (log: ${log})" + docker run --rm --network host --user 0:0 \ + -v "${state}:/eas-state" \ + -v "${state}/kubeconfig:/kube/config:ro" \ + -v "$(dirname "${dir}"):/k3d-gpu/integrations:ro" \ + -e EAS_SHA="${EAS_SHA:-}" -e EAS_CPU_POLICY="${EAS_CPU_POLICY:-}" -e EAS_NODES="${EAS_NODES:-}" \ + -e EAS_OWNER="$(id -u):$(id -g)" \ + ${envs[@]+"${envs[@]}"} \ + "${image}" bash "/k3d-gpu/integrations/$(basename "${dir}")/eas.sh" "$@" 2>&1 | tee -a "${log}" +} + +# eas_ref : a commit SHA for on github.com/, from `git ls-remote` +# (a branch before a tag, as EAS's own _checkout_rev resolves; a tag's peeled commit). +# A full SHA is returned as given. Prints ""; fails on a ref the remote lacks. +eas_ref() { + local repo=$1 ref=$2 out sha + if [[ "${ref}" =~ ^[0-9a-f]{40}$ ]]; then echo "${ref}"; return 0; fi + out=$(git ls-remote -- "https://github.com/${repo}" "${ref}" "${ref}^{}") \ + || err "git ls-remote https://github.com/${repo} ${ref} failed" + sha=$(awk -v r="${ref}" ' + $2 == "refs/heads/" r { h = $1 } $2 == "refs/tags/" r "^{}" { p = $1 } $2 == "refs/tags/" r { t = $1 } + END { print (h != "" ? h : (p != "" ? p : t)) }' <<< "${out}") + [ -n "${sha}" ] || err "https://github.com/${repo} has no branch or tag ${ref}" + echo "${sha}" +} + +# eas_pin_refs : EAS_REF_ARGS for eas-config.py. A profile +# name of pins.json (--eas-ref v1.2.0-rc.1) and a full SHA go as given; any other ref is +# resolved here, once, and printed: a floating ref never reaches the installer. +eas_pin_refs() { + local dir=$1 profiles name repo ref sha + EAS_REF_ARGS=() + profiles=" $(python3 -c 'import json, sys +print(" ".join(sorted({r["profile"] for r in json.load(open(sys.argv[1]))["pins"]})))' "${dir}/pins.json") " \ + || err "cannot read ${dir}/pins.json" + shift + for name in eas ei eat; do + ref=$1; shift + [ -n "${ref}" ] || continue + if [ "${name}" = eas ] && [[ "${profiles}" == *" ${ref} "* ]]; then + EAS_REF_ARGS+=(--eas-ref "${ref}") + echo " eas: profile ${ref} of pins.json" + continue + fi + case "${name}" in + eas) repo="intel/enterprise-ai-solutions" ;; + ei) repo="intel/enterprise-inference" ;; + eat) repo="intel/enterprise-agent-toolkit" ;; + esac + sha=$(eas_ref "${repo}" "${ref}") || exit 1 + EAS_REF_ARGS+=("--${name}-ref" "${sha}") + if [ "${sha}" = "${ref}" ]; then + echo " ${name}: ${repo}@${sha:0:7} (as given)" + else + echo " ${name}: ${repo}@${sha:0:7} (${ref}, git ls-remote https://github.com/${repo} ${ref})" + fi + done +} + +# eas_facts : the facts eas-config.py reads (schema k3d-gpu.eas-facts/v1), all +# read-only: nodes, namespaces, DaemonSets, the agent-sandbox CRD and controller image, the +# server version, rootless docker, the node container's NRI socket and containerd CNI +# bin_dirs, MemAvailable and inotify limits, and `k3d-gpu fit` of each of eas-config.py's +# model candidates (vLLM on the CPU). +eas_facts() { + local state=$1 dir=$2 raw node id rc doc + raw="${state}/facts.d" + rm -rf "${raw}" && mkdir -p "${raw}/fit" + kubectl get nodes -o json > "${raw}/nodes.json" || err "cannot list the nodes of cluster '${CLUSTER}'" + kubectl get daemonsets -A -o json > "${raw}/daemonsets.json" 2>/dev/null || echo '{"items":[]}' > "${raw}/daemonsets.json" + kubectl get namespaces -o json > "${raw}/namespaces.json" 2>/dev/null || echo '{"items":[]}' > "${raw}/namespaces.json" + if kubectl get crd sandboxes.agents.x-k8s.io -o name >/dev/null 2>&1; then + kubectl -n agent-sandbox-system get deployment agent-sandbox-controller \ + -o jsonpath='{.spec.template.spec.containers[0].image}' > "${raw}/sandbox-controller" 2>/dev/null || : + fi + kubectl version -o json > "${raw}/version.json" 2>/dev/null || echo '{}' > "${raw}/version.json" + docker info --format '{{json .SecurityOptions}}' > "${raw}/docker-security.json" 2>/dev/null || echo '[]' > "${raw}/docker-security.json" + # The server node container: whether containerd's NRI socket is there, and the CNI + # bin_dirs containerd hands its plugins (read with crictl, which k3s ships). + node="k3d-${CLUSTER}-server-0" + if docker exec "${node}" test -S /var/run/nri/nri.sock >/dev/null 2>&1; then echo true; else echo false; fi > "${raw}/nri-socket" + docker exec "${node}" crictl info > "${raw}/crictl-info.json" 2>/dev/null || echo '{}' > "${raw}/crictl-info.json" + # Each candidate's fit: exit 3 is "does not fit" with a document, 2 is no answer. + while read -r id; do + [ -n "${id}" ] || continue + rc=0 + doc=$("$(qs_self)" fit --model "${id}" --engine vllm --device cpu --json 2>/dev/null) || rc=$? + if [ "${rc}" -ne 2 ] && [ -n "${doc}" ]; then printf '%s\n' "${doc}" > "${raw}/fit/${id//\//__}.json"; fi + echo "${id}" >> "${raw}/fit/ids" + done < <(python3 - "${dir}/eas-config.py" <<'PY' +import importlib.util, sys +spec = importlib.util.spec_from_file_location("eas_config", sys.argv[1]) +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +for c in m.CANDIDATES: + print(c[1]) +PY +) + python3 - "${raw}" "${state}/facts.json" "${CLUSTER}" "${SYSROOT}" <<'PY' || err "could not assemble ${state}/facts.json" +import json, os, sys +raw, out, cluster, sysroot = sys.argv[1:] +def load(name, default): + try: + with open(os.path.join(raw, name)) as f: + return json.load(f) + except (OSError, ValueError): + return default +def text(name): + try: + with open(os.path.join(raw, name)) as f: + return f.read().strip() + except OSError: + return None +def read_int(path): + try: + with open(sysroot + path) as f: + return int(f.read().split()[0]) + except (OSError, ValueError, IndexError): + return None +def meminfo(key): + try: + with open(sysroot + "/proc/meminfo") as f: + for line in f: + if line.startswith(key + ":"): + return int(line.split()[1]) + except OSError: + pass + return None +def meta(o, *keys): + return {k: o["metadata"].get(k) or {} for k in keys} +cni = (load("crictl-info.json", {}).get("config") or {}).get("cni") or {} +bin_dirs = cni.get("binDirs") or ([cni["binDir"]] if cni.get("binDir") else []) +ids = (text("fit/ids") or "").split() +fit = {i: load("fit/%s.json" % i.replace("/", "__"), None) for i in ids} +controller = text("sandbox-controller") +facts = { + "schema": "k3d-gpu.eas-facts/v1", + "cluster": cluster, + "server_version": load("version.json", {}).get("serverVersion", {}).get("gitVersion"), + "docker_rootless": any("rootless" in s for s in load("docker-security.json", []) or []), + "nri_socket": text("nri-socket") == "true", + "containerd_cni_bin_dirs": bin_dirs, + "host": {"mem_available_kib": meminfo("MemAvailable"), + "inotify_max_user_instances": read_int("/proc/sys/fs/inotify/max_user_instances"), + "inotify_max_user_watches": read_int("/proc/sys/fs/inotify/max_user_watches")}, + "nodes": [dict(name=n["metadata"]["name"], allocatable=n.get("status", {}).get("allocatable", {}), + **meta(n, "labels", "annotations")) for n in load("nodes.json", {"items": []})["items"]], + "namespaces": sorted(o["metadata"]["name"] for o in load("namespaces.json", {"items": []})["items"]), + "daemonsets": [dict(namespace=o["metadata"].get("namespace"), name=o["metadata"]["name"], + **meta(o, "labels", "annotations")) for o in load("daemonsets.json", {"items": []})["items"]], + "agent_sandbox": None if controller is None else {"controller_image": controller or None}, + "fit": fit, +} +with open(out, "w") as f: + json.dump(facts, f, indent=2, sort_keys=True) +PY +} + +# eas_choice_value : a top-level scalar of k3d-gpu-overrides.yaml (eas-config.py +# writes one `key: value` per line), unquoted. +eas_choice_value() { + sed -n "s/^$2:[[:space:]]*\"\{0,1\}\([^\"#]*\)\"\{0,1\}[[:space:]]*\$/\1/p" "$1/k3d-gpu-overrides.yaml" 2>/dev/null | tail -1 +} + +# eas_record_pins : the commits eas-config.py resolved (the `# eas|ei|eat ` +# lines of choices.txt) as /pins.resolved.json, and EAS_SHA. +eas_record_pins() { + EAS_SHA=$(python3 - "$1/choices.txt" "$1/pins.resolved.json" <<'PY' +import json, re, sys +rows, profile = [], None +for line in open(sys.argv[1]): + m = re.match(r"# k3d-gpu eas-config: profile (\S+),", line) + if m: + profile = m.group(1) + m = re.match(r"# (eas|ei|eat) (\S+) ([0-9a-f]{40})$", line.rstrip("\n")) + if m: + rows.append({"name": m.group(1), "repo": m.group(2), "sha": m.group(3)}) +eas = [r["sha"] for r in rows if r["name"] == "eas"] +if not eas: + sys.exit("choices.txt names no EAS commit") +json.dump({"schema": "k3d-gpu.eas-pins-resolved/v1", "profile": profile, "pins": rows}, + open(sys.argv[2], "w"), indent=2) +print(eas[0]) +PY +) || err "eas-config.py's choices.txt names no EAS commit" +} + +# eas_domain : base_domain_name from the overrides, else the env's global_config.yaml, +# else EAS's default. +eas_domain() { + local f d="" + for f in "$1/k3d-gpu-overrides.yaml" "$1/eas/env/${EAS_ENV}/global_config.yaml"; do + [ -f "${f}" ] || continue + d=$(sed -n 's/^base_domain_name:[[:space:]]*["'\'']\{0,1\}\([A-Za-z0-9.-]*\).*$/\1/p' "${f}" | tail -1) + [ -z "${d}" ] || break + done + echo "${d:-${EAS_DEFAULT_DOMAIN}}" +} + +# eas_gateway : where EAS's gateway answers. Envoy Gateway's proxy Service for the +# Gateway eg-gateway carries its owning-gateway label; k3s ServiceLB gives it the node IP. +# The host's /etc/hosts is never edited. LiteLLM is there with the agent toolkit only. +eas_gateway() { + local ip domain + domain=$(eas_domain "$1") + ip=$(kubectl -n "${EAS_GW_NS}" get svc -l "${EAS_GW_SELECTOR}" \ + -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}' 2>/dev/null || true) + if [ -z "${ip}" ]; then + echo " gateway: no LoadBalancer IP on eg-gateway's Service in ${EAS_GW_NS} yet" + return 0 + fi + echo " gateway: eg-gateway at ${ip} (k3s ServiceLB), hosts *.${domain}" + [ "$(cat "$1/layer" 2>/dev/null)" = agent-toolkit ] || return 0 + echo " KEY=\$(${KHINT} -n litellm get secret litellm-master-key -o jsonpath='{.data.master_key}' | base64 -d)" + echo " curl -k --resolve litellm.${domain}:443:${ip} https://litellm.${domain}/v1/models -H \"Authorization: Bearer \${KEY}\"" +} + +eas_require_files() { + local f + for f in "${EAS_FILES[@]}"; do + [ -f "$1/${f}" ] || err "$1/${f} missing: the EAS integration is incomplete (set K3D_GPU_EAS_DIR)" + done +} + +# eas up [--agent-toolkit | --layer L] [--model NAME] [--eas-ref R] [--ei-ref R] +# [--eat-ref R] [--dry-run] [--explain] [-- ] +eas_up() { + local layer=inference model="" eas_ref="" ei_ref="" eat_ref="" dry=0 explain=() extra=() dir state rc v image + while [ $# -gt 0 ]; do + case "$1" in + --agent-toolkit) layer="agent-toolkit" ;; + --layer) [ $# -ge 2 ] || err "--layer needs a value"; layer=$2; shift ;; + --model) [ $# -ge 2 ] || err "--model needs a value"; model=$2; shift ;; + --eas-ref) [ $# -ge 2 ] || err "--eas-ref needs a value"; eas_ref=$2; shift ;; + --ei-ref) [ $# -ge 2 ] || err "--ei-ref needs a value"; ei_ref=$2; shift ;; + --eat-ref) [ $# -ge 2 ] || err "--eat-ref needs a value"; eat_ref=$2; shift ;; + --dry-run) dry=1 ;; + --explain) explain=(--explain) ;; + --) shift; extra=("$@"); break ;; + *) err "unknown eas up option: $1 (see 'k3d-gpu help')" ;; + esac + shift + done + case "${layer}" in + inference|agent-toolkit|erag) ;; # erag: eas-config.py refuses it, with the reason + *) err "--layer must be inference, agent-toolkit or erag, got '${layer}' (platform comes with inference)" ;; + esac + for v in "${eas_ref}" "${ei_ref}" "${eat_ref}" "${model}"; do + [ -z "${v}" ] || [[ "${v}" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] || err "not a ref or model name: '${v}'" + done + require kubectl; require k3d; require git; require python3; require docker + dir=$(eas_dir) || err "integrations/eas not found next to the launcher or in ${SHARE_DIR}/integrations/eas (set K3D_GPU_EAS_DIR)" + eas_require_files "${dir}" + state=$(eas_state) + ( umask 077; mkdir -p "${state}" ) && chmod 0700 "${state}" + + if ! k3d cluster get "${CLUSTER}" >/dev/null 2>&1; then + [ "${dry}" = 0 ] || err "cluster '${CLUSTER}' does not exist; 'k3d-gpu eas up' creates it with 'k3d-gpu up' first, and a dry run reads a running cluster" + info "cluster '${CLUSTER}' does not exist: 'k3d-gpu up' first" + "$(qs_self)" up || err "'k3d-gpu up' failed; fix the error above and run 'k3d-gpu eas up' again" + fi + kubectl get nodes >/dev/null 2>&1 || err "cluster '${CLUSTER}' is not reachable — run 'k3d-gpu doctor'" + + info "EAS ${layer} on cluster '${CLUSTER}' (state ${state})" + if [ -n "${eas_ref}${ei_ref}${eat_ref}" ]; then info "Refs:"; fi + eas_pin_refs "${dir}" "${eas_ref}" "${ei_ref}" "${eat_ref}" + info "Facts (read-only)..." + eas_facts "${state}" "${dir}" + info "Choices (fact -> choice [source]):" + for v in "${EAS_OUTPUTS[@]}"; do rm -f "${state:?}/${v}"; done + rc=0 + python3 "${dir}/eas-config.py" --facts "${state}/facts.json" --out "${state}" --layer "${layer}" \ + --pins "${dir}/pins.json" ${EAS_REF_ARGS[@]+"${EAS_REF_ARGS[@]}"} ${model:+--model "${model}"} \ + ${explain[@]+"${explain[@]}"} || rc=$? + [ "${rc}" -ne 3 ] || err "refused (above); nothing was changed" + [ "${rc}" -eq 0 ] || err "eas-config.py failed (exit ${rc}); nothing was changed" + for v in choices.txt k3d-gpu-overrides.yaml global_config.edit.yaml models.edit.yaml "repos.${layer}.yaml"; do + [ -s "${state}/${v}" ] || err "eas-config.py wrote no ${state}/${v}" + done + eas_record_pins "${state}" + EAS_CPU_POLICY=$(eas_choice_value "${state}" kubernetes_cpu_policy) + EAS_NODES=$(python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))["nodes"]))' "${state}/facts.json") + cp "${dir}/images.yaml" "${state}/k3d-gpu-images.yaml" + eas_kubeconfig "${state}" + + if [ "${dry}" = 1 ]; then + info "Overlay (enterprise-ai-solutions@${EAS_SHA:0:7}, kubernetes_cpu_policy ${EAS_CPU_POLICY}, ${EAS_NODES} node(s)):" + EAS_STATE="${state}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY="${EAS_CPU_POLICY}" EAS_NODES="${EAS_NODES}" \ + bash "${dir}/eas.sh" overlay-check || err "the overlay stops at this EAS commit (above); nothing was changed" + info "Dry run: nothing applied. Under ${state}: facts.json, choices.txt, k3d-gpu-overrides.yaml, global_config.edit.yaml, models.edit.yaml, repos.${layer}.yaml, overlay.txt" + return 0 + fi + image=$(eas_bastion_image "${dir}") + eas_run "${dir}" "${state}" "${image}" up "${layer}" -- ${extra[@]+"${extra[@]}"} \ + || err "es_auto_installer.sh install ${layer} failed (log above, and ${state}/eas/env/${EAS_ENV}/logs)" + echo "${layer}" > "${state}/layer" + info "EAS ${layer} is installed on '${CLUSTER}'" + eas_gateway "${state}" + [ "${layer}" != agent-toolkit ] || echo " check it end to end: k3d-gpu eas check" +} + +eas_installed() { + [ -d "$1/eas/env/${EAS_ENV}" ] || err "nothing installed by 'k3d-gpu eas up' on cluster '${CLUSTER}' (no $1/eas/env/${EAS_ENV})" +} + +eas_down() { + local dir state image ns name + [ $# -eq 0 ] || err "eas down takes no options (got: $*)" + require kubectl; require docker + dir=$(eas_dir) || err "integrations/eas not found (set K3D_GPU_EAS_DIR)" + state=$(eas_state) + eas_installed "${state}" + # EAS issue #71 (open): teardown platform hangs at the Envoy AI Gateway CRD uninstall + # while an AIGatewayRoute keeps its finalizer; the role strips them only after the + # controller is gone. Clear them first, as the issue's workaround does. + while read -r ns name; do + [ -n "${name}" ] || continue + info "clearing the finalizers of AIGatewayRoute ${ns}/${name} (EAS issue #71)" + kubectl -n "${ns}" patch aigatewayroute "${name}" --type=merge -p '{"metadata":{"finalizers":[]}}' >/dev/null \ + || warn "could not clear the finalizers of ${ns}/${name}" + done < <(kubectl get aigatewayroutes.aigateway.envoyproxy.io -A \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' 2>/dev/null || true) + eas_kubeconfig "${state}" + image=$(eas_bastion_image "${dir}") + eas_run "${dir}" "${state}" "${image}" down || err "teardown platform failed (log above)" + rm -f "${state}/layer" + info "EAS removed from '${CLUSTER}'; the cluster stays ('k3d-gpu down' deletes it)" +} + +eas_status() { + local dir state image + [ $# -eq 0 ] || err "eas status takes no options (got: $*)" + require kubectl; require docker; require python3 + dir=$(eas_dir) || err "integrations/eas not found (set K3D_GPU_EAS_DIR)" + state=$(eas_state) + eas_installed "${state}" + info "Layer: $(cat "${state}/layer" 2>/dev/null || echo "unknown (the last eas up did not finish)")" + info "Pins (${state}/pins.resolved.json):" + python3 - "${state}/pins.resolved.json" <<'PY' || echo " (no resolved pins)" +import json, sys +d = json.load(open(sys.argv[1])) +for r in d["pins"]: + print(" %s: %s@%s (profile %s)" % (r["name"], r["repo"], r["sha"][:7], d.get("profile"))) +PY + info "Overlay (${state}/overlay.txt):" + if [ -s "${state}/overlay.txt" ]; then sed 's/^/ /' "${state}/overlay.txt"; else echo " (no decisions recorded)"; fi + info "EAS status:" + eas_kubeconfig "${state}" + image=$(eas_bastion_image "${dir}") + eas_run "${dir}" "${state}" "${image}" status || warn "es_auto_installer.sh status failed (above)" + info "Gateway:" + eas_gateway "${state}" + info "Pods not Ready:" + kubectl get pods -A -o json | python3 -c ' +import json, sys +bad = [] +for p in json.load(sys.stdin)["items"]: + ph = p.get("status", {}).get("phase") + if ph == "Succeeded": + continue + cs = p.get("status", {}).get("containerStatuses", []) + if ph != "Running" or not cs or not all(c.get("ready") for c in cs): + why = [c.get("state", {}).get("waiting", {}).get("reason") for c in cs] + why = ",".join(w for w in why if w) or ph or "?" + bad.append(" %s/%s: %s" % (p["metadata"]["namespace"], p["metadata"]["name"], why)) +print("\n".join(bad) if bad else " none")' +} + +eas_check() { + local dir state chk model="" + while [ $# -gt 0 ]; do + case "$1" in + --model) [ $# -ge 2 ] || err "--model needs a value"; model=$2; shift ;; + *) err "unknown eas check option: $1" ;; + esac + shift + done + require kubectl; require curl; require jq + dir=$(eas_dir) || err "integrations/eas not found (set K3D_GPU_EAS_DIR)" + state=$(eas_state) + eas_installed "${state}" + # LiteLLM and the sandbox come with the agent toolkit (auth_type litellm_vkey); the + # inference layer alone serves models behind a Keycloak token. + [ "$(cat "${state}/layer" 2>/dev/null)" = agent-toolkit ] \ + || err "eas check exercises the agent toolkit (LiteLLM and the sandbox): run 'k3d-gpu eas up --agent-toolkit' first" + chk="${dir}/../eat/eat-check.sh" + [ -f "${chk}" ] || err "${chk} not found" + eas_kubeconfig "${state}" + EAT_MODE=eas EAT_KUBECONFIG="${state}/kubeconfig" EAT_EAS_DOMAIN="$(eas_domain "${state}")" \ + EAT_MODEL="${model:-${EAT_MODEL:-}}" bash "${chk}" +} + +cmd_eas() { + local sub=${1:-} + [ $# -eq 0 ] || shift + case "${sub}" in + up) eas_up "$@" ;; + down) eas_down "$@" ;; + status) eas_status "$@" ;; + check) eas_check "$@" ;; + *) err "usage: k3d-gpu eas up [--agent-toolkit] [--model NAME] [--eas-ref R] [--ei-ref R] [--eat-ref R] [--dry-run] [--explain] [-- ansible args] | status | down | check [--model NAME]" ;; + esac +} + # ---- model-share ---------------------------------------------------------------- # N agent replicas mount one model OCI image as a read-only image volume and mmap # it. Sharing one page-cache copy needs three things, each detected here: @@ -12173,6 +12670,7 @@ case "${1:-help}" in fit) shift; cmd_fit "$@" ;; audit) shift; cmd_audit "$@" ;; sandbox) shift; cmd_sandbox "$@" ;; + eas) shift; cmd_eas "$@" ;; remote) shift; cmd_remote "$@" ;; model-share) shift; cmd_model_share "$@" ;; apply) shift; cmd_apply "$@" ;; diff --git a/tests/console/audit/chat-ui-noauth.golden b/tests/console/audit/chat-ui-noauth.golden index bb79565d..141b6329 100644 --- a/tests/console/audit/chat-ui-noauth.golden +++ b/tests/console/audit/chat-ui-noauth.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 12 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 12 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE); on k3d-gpu-server-0 diff --git a/tests/console/audit/chat-ui.golden b/tests/console/audit/chat-ui.golden index 0e4baf57..a1d403a0 100644 --- a/tests/console/audit/chat-ui.golden +++ b/tests/console/audit/chat-ui.golden @@ -1,4 +1,4 @@ -audit PASS: 0 of 12 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit PASS: 0 of 12 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE); on k3d-gpu-server-0 diff --git a/tests/console/audit/default-json.golden b/tests/console/audit/default-json.golden index 50d812bf..4589b746 100644 --- a/tests/console/audit/default-json.golden +++ b/tests/console/audit/default-json.golden @@ -73,12 +73,12 @@ ], "cluster": "gpu", "lock": { - "images": 55, + "images": 57, "k3s_tag": "v1.37.1-k3s1" }, "schema": "k3d-gpu.audit/v1", "summary": [ - "audit PASS: 0 of 10 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1)" + "audit PASS: 0 of 10 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1)" ], "verdict": "pass" } diff --git a/tests/console/audit/default.golden b/tests/console/audit/default.golden index 9dd398e8..00f3ceb8 100644 --- a/tests/console/audit/default.golden +++ b/tests/console/audit/default.golden @@ -1,4 +1,4 @@ -audit PASS: 0 of 10 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit PASS: 0 of 10 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE); on k3d-gpu-server-0 diff --git a/tests/console/audit/image-digest-moved.golden b/tests/console/audit/image-digest-moved.golden index 60b32c8f..f7dfb48e 100644 --- a/tests/console/audit/image-digest-moved.golden +++ b/tests/console/audit/image-digest-moved.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 10 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 10 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) FAIL image docker.io/intel/intel-gpu-plugin:0.37.1: docker.io/intel/intel-gpu-plugin is sha256:2222222222222222222222222222222222222222222222222222222222222222, images.lock pins sha256:004641b2547d58f1eadce58473fa160ad71377de1eb588cf1f82832387bf3e33 (tag 0.37.1); on k3d-gpu-server-0 fix: redeploy from the pinned digest, or bump the pin with scripts/images-lock update diff --git a/tests/console/audit/image-no-digest.golden b/tests/console/audit/image-no-digest.golden index 33263aa9..5b9be6ea 100644 --- a/tests/console/audit/image-no-digest.golden +++ b/tests/console/audit/image-no-digest.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) FAIL image docker.io/library/app:dev: no registry digest, so it cannot match images.lock; on k3d-gpu-server-0 fix: remove it, or load the image from a registry reference that images.lock pins diff --git a/tests/console/audit/image-not-in-lock.golden b/tests/console/audit/image-not-in-lock.golden index e529e48c..6a96d4f2 100644 --- a/tests/console/audit/image-not-in-lock.golden +++ b/tests/console/audit/image-not-in-lock.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) FAIL image docker.io/bitnami/redis:7.4: not in images.lock: docker.io/bitnami/redis; on k3d-gpu-server-0 fix: remove the workload that runs it, or pin it in the repo and run scripts/images-lock update diff --git a/tests/console/audit/ipv6-unauth.golden b/tests/console/audit/ipv6-unauth.golden index 0f8f1f4c..624d248e 100644 --- a/tests/console/audit/ipv6-unauth.golden +++ b/tests/console/audit/ipv6-unauth.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) FAIL listener [::]:8000 vllm (EAT chart, no auth): unauthenticated listener off loopback: no authentication; off loopback, auth none (fixture: EAT#9 layout) fix: bind it to 127.0.0.1, or put OIDC in front of it (k3d-gpu plan --bind picks oidc off loopback) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) diff --git a/tests/console/audit/kubeconfig-0644.golden b/tests/console/audit/kubeconfig-0644.golden index d02bffb1..bbafe65c 100644 --- a/tests/console/audit/kubeconfig-0644.golden +++ b/tests/console/audit/kubeconfig-0644.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 10 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 10 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE); on k3d-gpu-server-0 diff --git a/tests/console/audit/loopback-published.golden b/tests/console/audit/loopback-published.golden index fee6dd21..814fda77 100644 --- a/tests/console/audit/loopback-published.golden +++ b/tests/console/audit/loopback-published.golden @@ -1,4 +1,4 @@ -audit PASS: 0 of 11 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit PASS: 0 of 11 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok listener 127.0.0.1:8000 published node port 30080/tcp: loopback, auth unknown (docker port 30080/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 diff --git a/tests/console/audit/runtime-fetch.golden b/tests/console/audit/runtime-fetch.golden index 49d6db75..0362db05 100644 --- a/tests/console/audit/runtime-fetch.golden +++ b/tests/console/audit/runtime-fetch.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE); on k3d-gpu-server-0 diff --git a/tests/console/audit/runtime-pull.golden b/tests/console/audit/runtime-pull.golden index e96d3d8a..7ac08abf 100644 --- a/tests/console/audit/runtime-pull.golden +++ b/tests/console/audit/runtime-pull.golden @@ -1,4 +1,4 @@ -audit FAIL: 2 of 12 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 2 of 12 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE); on k3d-gpu-server-0 diff --git a/tests/console/audit/unauth-published.golden b/tests/console/audit/unauth-published.golden index 57c09f13..587f91a7 100644 --- a/tests/console/audit/unauth-published.golden +++ b/tests/console/audit/unauth-published.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 55 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 57 images, k3s v1.37.1-k3s1) FAIL listener 0.0.0.0:8000 published node port 30080/tcp: unauthenticated listener off loopback: no authentication this audit can establish; off loopback, auth unknown (docker port 30080/tcp on k3d-gpu-server-0) fix: bind it to 127.0.0.1, or put OIDC in front of it (k3d-gpu plan --bind picks oidc off loopback) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) From 1f9c2044ad733b260cf60404e72f48596e6629ef Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 14:42:17 -0700 Subject: [PATCH 03/19] eas: eas.sh applies models.edit.yaml's args_append eas-config.py now gives a model whose KV cache it lowers an args_append (fit's --max-model-len) next to its env. eas.sh merges the env into the catalog entry as before and puts args_append after the entry's args, taking each flag it names (and its value) out of the current args first. init never reseeds models.yaml, so a second eas up replaces the flag instead of repeating it, and no args_append key reaches the catalog. Checked with yq v4.54.1 on EI 3e8a244's models.yaml, twice in a row. --- integrations/eas/eas.sh | 35 ++++++++++++++++++++++++++++++----- tests/eas_test.sh | 19 ++++++++++++++++--- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/integrations/eas/eas.sh b/integrations/eas/eas.sh index 4ea74f5f..7b478731 100755 --- a/integrations/eas/eas.sh +++ b/integrations/eas/eas.sh @@ -8,7 +8,8 @@ # 2. the patch overlay: overlay.py --apply, decisions in overlay.txt # 3. the layer manifest's revs as commit SHAs (repos..yaml from eas-config.py) # 4. es_auto_installer.sh init --env k3d-gpu -# 5. global_config.edit.yaml and models.edit.yaml (eas-config.py) into env/k3d-gpu/ +# 5. global_config.edit.yaml and models.edit.yaml (eas-config.py) into env/k3d-gpu/: +# keys merge; a model's env merges and its args_append follows its catalog args # 6. es_auto_installer.sh install --env k3d-gpu -- -e @k3d-gpu-images.yaml # -e @k3d-gpu-overrides.yaml [args]: extra vars load last in es_auto_installer.sh # main(), so the overrides win @@ -88,12 +89,31 @@ pin_manifest() { done < <(yq -r '.repos[] | [.layer, .url, .rev] | @tsv' "${f}") } +# args_merge : the catalog entry's args with +# args_append after them, each --flag of args_append (and its value) taken out of the +# current args first, so a re-run (init never reseeds models.yaml) replaces rather than +# repeats it. Prints JSON. +args_merge() { + python3 -c 'import json, sys +cur, add = json.loads(sys.argv[1]), json.loads(sys.argv[2]) +valued = {t: i + 1 < len(add) and not add[i + 1].startswith("--") for i, t in enumerate(add) if t.startswith("--")} +out, skip = [], False +for t in cur: + if skip: + skip = False + elif t.split("=", 1)[0] in valued: + skip = valued[t.split("=", 1)[0]] and "=" not in t + else: + out.append(t) +print(json.dumps(out + add))' "$1" "$2" +} + # env_edits: global_config.edit.yaml merged into env/k3d-gpu/global_config.yaml (where # _resolve_kubeconfig reads existing_kubernetes, #L1342-L1358), and models.edit.yaml into -# env/k3d-gpu/models.yaml: defaults merge, and each model merges into the catalog entry of -# its name, which must exist once. +# env/k3d-gpu/models.yaml: defaults merge; each model is the catalog entry of its name, +# which must exist once: its env merges, and its args_append goes after its args. env_edits() { - local env="${CHECKOUT}/env/${ENV_NAME}" F N + local env="${CHECKOUT}/env/${ENV_NAME}" F N A cur add F="${EAS_STATE}/global_config.edit.yaml" [ -f "${F}" ] || err "${F} missing (eas-config.py writes it)" export F @@ -106,8 +126,13 @@ env_edits() { export N yq -e '[.models[] | select(.name == strenv(N))] | length == 1' "${env}/models.yaml" >/dev/null \ || err "env/${ENV_NAME}/models.yaml has no single catalog entry named ${N}" - yq -i '(.models[] | select(.name == strenv(N))) *= (load(strenv(F)).models[] | select(.name == strenv(N)))' \ + yq -i '(.models[] | select(.name == strenv(N))).env *= ((load(strenv(F)).models[] | select(.name == strenv(N))).env // {})' \ "${env}/models.yaml" + cur=$(yq -o=json -I=0 '(.models[] | select(.name == strenv(N))).args // []' "${env}/models.yaml") + add=$(yq -o=json -I=0 '(.models[] | select(.name == strenv(N))).args_append // []' "${F}") + A=$(args_merge "${cur}" "${add}") + export A + yq -i '(.models[] | select(.name == strenv(N))).args = (strenv(A) | from_yaml)' "${env}/models.yaml" done < <(yq -r '.models[].name' "${F}") } diff --git a/tests/eas_test.sh b/tests/eas_test.sh index 57a13a26..4e99a69a 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -824,10 +824,17 @@ EOF #!/usr/bin/env python3 import os, re, sys a = sys.argv[1:] -env = " ".join("%s=%s" % (k, os.environ[k]) for k in ("L", "U", "R", "F", "N") if k in os.environ and "strenv(%s)" % k in " ".join(a)) +env = " ".join("%s=%s" % (k, os.environ[k]) for k in ("L", "U", "R", "F", "N", "A") if k in os.environ and "strenv(%s)" % k in " ".join(a)) with open(os.environ["STUB_LOG"], "a") as f: f.write("yq %s%s\n" % (" ".join(a), " [" + env + "]" if env else "")) -if "-r" in a: +if "-o=json" in a: + text = open(a[-1]).read() + if ".args_append" in a[-2]: + m = re.search(r"args_append: (\[.*\])", text) + print(m.group(1) if m else "[]") + else: # the catalog entry's args: one the edit replaces, one it keeps + print('["--block-size","128","--max-model-len","131072","--tensor-parallel-size","1"]') +elif "-r" in a: text = open(a[-1]).read() if "@tsv" in a[a.index("-r") + 1]: for m in re.finditer(r'- layer: "([^"]*)"\n\s+url: "([^"]*)"\n\s+rev: "([^"]*)"', text): @@ -845,7 +852,7 @@ case "$1" in case "$3" in Qwen/Qwen2.5-Coder-14B-Instruct) echo '{"schema": "k3d-gpu.fit/v1", "fits": false, "summary": "fit: does not fit this host", "flags": []}'; exit 3 ;; meta-llama/Llama-3.1-8B-Instruct) exit 2 ;; - *) printf '{"schema": "k3d-gpu.fit/v1", "fits": true, "summary": "fit: %s on CPU via vllm", "flags": ["VLLM_CPU_KVCACHE_SPACE=12"]}\n' "$3" ;; + *) printf '{"schema": "k3d-gpu.fit/v1", "fits": true, "summary": "fit: %s on CPU via vllm", "flags": ["--max-model-len=32768", "--max-num-seqs=4", "VLLM_CPU_KVCACHE_SPACE=12"]}\n' "$3" ;; esac ;; esac EOF @@ -1071,6 +1078,12 @@ print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox cli_before "cli: init, then models.edit.yaml defaults" "${log}" "installer init" \ "yq -i .defaults *= load(strenv(F)).defaults ${ES}/eas/env/k3d-gpu/models.yaml [F=${ES}/models.edit.yaml]" cli_has "cli: the picked model's catalog entry is edited by name" "${log}" "[F=${ES}/models.edit.yaml N=qwen2-5-coder-7b]" + cli_has "cli: the model's env merges into its catalog entry" "${log}" \ + ".env *= ((load(strenv(F)).models[] | select(.name == strenv(N))).env // {}) ${ES}/eas/env/k3d-gpu/models.yaml" + cli_has "cli: args_append follows the catalog args, replacing the flag it names" "${log}" \ + '.args = (strenv(A) | from_yaml) '"${ES}"'/eas/env/k3d-gpu/models.yaml [N=qwen2-5-coder-7b A=["--block-size", "128", "--tensor-parallel-size", "1", "--max-model-len", "32768"]]' + cli_lacks "cli: args_append is not written into the catalog entry as a key" "${log}" \ + ") *= (load(strenv(F)).models[]" cli_before "cli: the edits, then install" "${log}" "load(strenv(F)).defaults" "installer install" cli_has "cli: install with the image pins, then the overrides, then the extra args" "${log}" \ "installer install agent-toolkit --env k3d-gpu -- -e @${ES}/k3d-gpu-images.yaml -e @${ES}/k3d-gpu-overrides.yaml -e x=1" From a054f08852671ef2f12978e24d91bc175c420661 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 14:47:00 -0700 Subject: [PATCH 04/19] packaging: no formula test for integrations/eas until a release carries it packaging/check.sh brew installs the formula's url, the last aur-v release tarball, which has no integrations/eas yet, so an assert_path_exists on it fails until the next release. The install lines stay; brew skips the empty glob with a warning on today's tarball. --- Formula/k3d-gpu.rb | 1 - tests/packaging_test.sh | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Formula/k3d-gpu.rb b/Formula/k3d-gpu.rb index 42455677..f15d2fac 100644 --- a/Formula/k3d-gpu.rb +++ b/Formula/k3d-gpu.rb @@ -37,7 +37,6 @@ def install assert_match "/x/nvidia-device-plugin.yml", shell_output("K3D_GPU_SHARE=/x #{bin}/k3d-gpu help") assert_path_exists pkgshare/"nvidia-device-plugin.yml" assert_path_exists pkgshare/"console/plan.py" - assert_path_exists pkgshare/"integrations/eas/eas.sh" assert_path_exists bin/"k3d-gpu-mcp" end end diff --git a/tests/packaging_test.sh b/tests/packaging_test.sh index fd1d33ea..20e0b4e1 100755 --- a/tests/packaging_test.sh +++ b/tests/packaging_test.sh @@ -165,7 +165,8 @@ check "formula installs integrations/eas" grep -qF '(pkgshare/"integrations/eas" check "formula installs the overlay's patch drafts" \ grep -qF '(pkgshare/"integrations/eat/upstream/patches").install Dir["integrations/eat/upstream/patches/*.patch"]' "${FORMULA}" check "formula installs eat-check.sh" grep -qF '(pkgshare/"integrations/eat").install "integrations/eat/eat-check.sh"' "${FORMULA}" -check "formula test asserts integrations/eas/eas.sh" grep -qF 'assert_path_exists pkgshare/"integrations/eas/eas.sh"' "${FORMULA}" +# No formula test for integrations/eas: packaging/check.sh brew installs the formula's url, +# the last release tarball, which has no integrations/eas until the next release. for f in integrations/eas integrations/eat/upstream/patches integrations/eat/eat-check.sh; do check "nix source includes ${f}" grep -qF "(root + \"/${f}\")" "${NIXPKG}" done From 8ec8875bc45ef56d4b4bd5f48b5f403125f5f1fa Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 17:14:42 -0700 Subject: [PATCH 05/19] eat-check: test that no LiteLLM key reaches jq's argument list (failing) A jq wrapper on the stub PATH logs each argv and runs the real jq. In EAS mode the virtual key reached jq's argv when the check deleted it at exit (--arg k), visible to ps for the life of that jq. --- tests/eat_test.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/eat_test.sh b/tests/eat_test.sh index fe181a26..1af0ac2c 100755 --- a/tests/eat_test.sh +++ b/tests/eat_test.sh @@ -135,7 +135,14 @@ elif url.endswith("/execute"): print(json.dumps({"stdout": p.stdout, "stderr": p.stderr, "exit_code": p.returncode})) # anything else (the router readiness probe) succeeds with no output EOF -chmod +x "${BIN}/kubectl" "${BIN}/curl" +# jq: the real one, with each argv logged (a key must not reach any argument list). +REAL_JQ=$(command -v jq) +cat > "${BIN}/jq" <> "\${STUB_LOG:-/dev/null}" +exec "${REAL_JQ}" "\$@" +EOF +chmod +x "${BIN}/kubectl" "${BIN}/curl" "${BIN}/jq" # check_says [env assignments...] check_says() { @@ -204,6 +211,13 @@ if grep '^argv ' "${WORK}/log" | grep -Eq 'sk-master-stub|sk-virtual-stub|Bearer else ok "eas mode: no key in any curl argv" fi +if ! grep -q '^jqargv ' "${WORK}/log"; then + fail "eas mode: the jq argv log is empty (the jq wrapper did not run)" +elif grep '^jqargv ' "${WORK}/log" | grep -Eq 'sk-master-stub|sk-virtual-stub|Bearer tok'; then + fail "eas mode: a key reached jq's argv: $(grep '^jqargv ' "${WORK}/log" | grep -Eo 'sk-(master|virtual)-stub' | sort -u | tr '\n' ' ')" +else + ok "eas mode: no key in any jq argv" +fi check_says "eas mode: the model is the first chat model the gateway lists" 0 '^model qwen2-5-coder-7b$' \ EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" check_says "eas mode: EAT_MODEL wins" 0 '^model my-model$' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" EAT_MODEL=my-model From 510136575bf3c71c5b4b173487b3ded6b8d9b107 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 17:14:45 -0700 Subject: [PATCH 06/19] eat-check: the virtual key reaches jq through its environment when it is deleted The /key/delete body was built with jq --arg, which put the run's virtual key on an argument list. jq now reads it from env.VKEY. tests/eat_test.sh: 87 passed, 0 failed. --- integrations/eat/eat-check.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integrations/eat/eat-check.sh b/integrations/eat/eat-check.sh index 068f6c37..04473e32 100755 --- a/integrations/eat/eat-check.sh +++ b/integrations/eat/eat-check.sh @@ -27,7 +27,7 @@ # eas: the requests carry a LiteLLM virtual key minted for this run with the master key # (Secret litellm/litellm-master-key) through POST /key/generate, and deleted at exit # (POST /key/delete). Keys reach curl as header files and request bodies on its stdin, -# never on an argument list. +# and jq through its environment, never on an argument list. set -euo pipefail @@ -53,7 +53,8 @@ cleanup() { [ -z "${pf}" ] || kill "${pf}" 2>/dev/null || true # The run's virtual key goes when the check ends, pass or fail. if [ -n "${vkey}" ]; then - jq -n --arg k "${vkey}" '{keys: [$k]}' | gw_post /key/delete "${master}" >/dev/null \ + # The key reaches jq through its environment, not its argument list. + VKEY="${vkey}" jq -n '{keys: [env.VKEY]}' | gw_post /key/delete "${master}" >/dev/null \ || echo "WARN: could not delete this run's LiteLLM virtual key ${alias}" >&2 fi } From c3e34e096760d4f5fa1c27c2fe23faf28641e0b1 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 22:14:14 -0700 Subject: [PATCH 07/19] eas: test the overlay with its scratch index on another filesystem (failing) The wiring smoke on cluster k3dgpu-eas-smoke stopped at the overlay, in --dry-run on the host and in eas up in the bastion: git read-tree --index-output=/tmp/eas-overlay-*/index said "fatal: unable to write new index file". git renames the index into that path from a lock file next to the checkout's own index, so the path must be on the checkout's filesystem. /tmp was a tmpfs with the state dir on disk, and in the bastion /tmp is the container's while the state dir is a bind mount. The case runs with TMPDIR on /dev/shm (or /var/tmp), wherever that is another filesystem than the test's work dir. --- tests/eas_test.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/eas_test.sh b/tests/eas_test.sh index 8f6f0141..adb01d61 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -9,6 +9,8 @@ # - overlay.json resolves every patch, upstream drafts first-class, no copies # - main: 0002 and 0006 apply, 0003 only for nri-balloons, 0004 only on # more than one node, 0005 never; the recorded `verified` cases hold +# - the same decisions with TMPDIR (the scratch index) on another +# filesystem than the checkout # - patches already in the tree: drop-merged (git apply -R --check) # - the fix done another way with the knob present: drop-fixed # - the fix done another way with the knob renamed: STOP @@ -201,6 +203,23 @@ for v in json.load(open(sys.argv[1]))["verified"]: check "main: 0002 resolves to the upstream draft" "${ROOT}/integrations/eat/upstream/patches/0002-eas-istio-global-platform.patch" \ "$(cd "${ROOT}" && realpath "$(jq_ 'd["decisions"][0]["files"][0]')")" check "main: 0002 names the generator knob it needs" istio_platform "$(jq_ 'd["decisions"][0]["knob"]')" + # The scratch index can sit on another filesystem than the checkout: a tmpfs /tmp + # with the state dir on disk, or the bastion's own /tmp with the state dir bind-mounted. + local xdev="" d + for d in /dev/shm /var/tmp; do + [ -d "${d}" ] && [ -w "${d}" ] || continue + [ "$(stat -c %d "${d}")" != "$(stat -c %d "${WORK}")" ] || continue + xdev=$(mktemp -d "${d}/eas-test-xdev.XXXXXX") && break + done + if [ -n "${xdev}" ]; then + got=$(TMPDIR="${xdev}" brief "${WORK}/main" best-effort 1) + check "main with the scratch index on another filesystem (TMPDIR ${d}): the same decisions" \ + "0002=apply:a 0003=skip 0004=skip 0005=skip 0006=apply:a" "${got}" + check "main with the scratch index on another filesystem: exit 0" 0 "$(cat "${WORK}/rc")" + rm -rf "${xdev}" + else + echo "skip no writable /dev/shm or /var/tmp on another filesystem than ${WORK}: the cross-filesystem case did not run" + fi # Text: one `fact -> choice [source]` line per row, sources at the checkout's sha. local sha sha7 From c525300faf8b15dfc98f09fbccb16b78fe4b542c Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 22:14:41 -0700 Subject: [PATCH 08/19] eas: the overlay's scratch index works on any filesystem overlay.py now builds its scratch index with GIT_INDEX_FILE and a plain git read-tree. git writes that file through a lock in the same directory, so the index can live in /tmp whatever filesystem the EAS checkout is on. On the smoke's real checkout (state dir on disk, /tmp a tmpfs) the decisions are 0002 apply, 0006 apply, 0003, 0004 and 0005 skip, as the F3 table predicts. --- integrations/eas/overlay.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integrations/eas/overlay.py b/integrations/eas/overlay.py index 25c5fab7..90431fd4 100755 --- a/integrations/eas/overlay.py +++ b/integrations/eas/overlay.py @@ -155,7 +155,10 @@ def __init__(self, path, rev): self._text = {} self._tmp = tempfile.TemporaryDirectory(prefix="eas-overlay-") self.index = os.path.join(self._tmp.name, "index") - self.git("read-tree", "--index-output=" + self.index, self.sha) + # GIT_INDEX_FILE, not --index-output: git renames --index-output's file in from a + # lock next to the checkout's own index, so it must share the checkout's + # filesystem, and a tmpfs /tmp or the bastion's /tmp does not. + self.git("read-tree", self.sha, env={"GIT_INDEX_FILE": self.index}) def git(self, *args, env=None, check=True): e = dict(os.environ, **(env or {})) From 145d9b1456c8d52e89606afd9b241354cf64b813 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 22:40:56 -0700 Subject: [PATCH 09/19] eas: test that the bastion ships a pinned jq (failing) The second wiring smoke got through the platform and inference layers and 581 tasks of the agent toolkit, then stopped at enterprise-inference's llm_services task "Enumerate autodeploy models from the catalog" (roles/llm_services/tasks/deploy_models.yaml at 3e8a244): it pipes yq into jq on the installer host, and the bastion had no jq ("/bin/bash: line 2: jq: command not found", rc 127). EAS's configure does not install jq either. --- tests/eas_test.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/eas_test.sh b/tests/eas_test.sh index adb01d61..cb54166d 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -53,7 +53,8 @@ # - down clears AIGatewayRoute finalizers before teardown platform --force # - status prints the layer, pins, overlay, EAS status, gateway, pods not Ready # - sandbox up refuses when EAS's sandbox-router-deployment exists -# - the bastion pins what pins.json's bastion profile names +# - the bastion pins what pins.json's bastion profile names, and ships a +# sha256-checked jq (EI's deploy_models.yaml runs jq on the bastion) # Needs bash, git, python3 (with PyYAML, for the config section's shape checks only). set -uo pipefail @@ -1248,13 +1249,21 @@ print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox want=$(python3 - "${EAS_DIR}/pins.json" <<'PY' import json, sys rows = {r["name"]: r for r in json.load(open(sys.argv[1]))["pins"] if r["profile"] == "bastion"} -print("FROM ubuntu:%s@sha256:%s kubectl %s helm %s yq %s kubernetes.core %s" +print("FROM ubuntu:%s@sha256:%s kubectl %s helm %s yq %s jq %s kubernetes.core %s" % (rows["ubuntu"]["ref"], rows["ubuntu"]["sha"], rows["kubectl"]["ref"], rows["helm"]["ref"], rows["yq"]["ref"], - rows["kubernetes.core"]["ref"])) + rows["jq"]["ref"], rows["kubernetes.core"]["ref"])) PY ) - got="$(grep -m1 '^FROM ' "${EAS_DIR}/bastion.Dockerfile") kubectl $(sed -n 's/^ARG KUBECTL_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") helm $(sed -n 's/^ARG HELM_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") yq $(sed -n 's/^ARG YQ_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") kubernetes.core $(sed -n 's/^ARG KUBERNETES_CORE_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile")" + got="$(grep -m1 '^FROM ' "${EAS_DIR}/bastion.Dockerfile") kubectl $(sed -n 's/^ARG KUBECTL_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") helm $(sed -n 's/^ARG HELM_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") yq $(sed -n 's/^ARG YQ_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") jq $(sed -n 's/^ARG JQ_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") kubernetes.core $(sed -n 's/^ARG KUBERNETES_CORE_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile")" cli_eq "cli: bastion.Dockerfile pins what pins.json's bastion profile names" "${got}" "${want}" + # EI's llm_services role pipes yq into jq on the installer host to pick the models to + # deploy (roles/llm_services/tasks/deploy_models.yaml); EAS's configure installs no jq. + if grep -q '^ARG JQ_SHA256=[0-9a-f]\{64\}$' "${EAS_DIR}/bastion.Dockerfile" \ + && grep -q 'echo "${JQ_SHA256} /usr/local/bin/jq" | sha256sum -c -' "${EAS_DIR}/bastion.Dockerfile"; then + ok "cli: the bastion ships jq, checked by sha256 (EI's deploy_models.yaml runs it on the bastion)" + else + fail "cli: the bastion has no sha256-checked jq; EI's deploy_models.yaml fails with 'jq: command not found'" + fi cli_eq "cli: bastion FROM is the EAT bastion's digest" \ "$(grep -m1 '^FROM ' "${EAS_DIR}/bastion.Dockerfile")" "$(grep -m1 '^FROM ' "${ROOT}/integrations/eat/bastion.Dockerfile")" want=$(python3 -c 'import json, sys From 0d0e4721108c6327078310984a7eb0bfe8c88562 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 22:45:31 -0700 Subject: [PATCH 10/19] eas: the bastion ships jq-1.8.2, checked by sha256 enterprise-inference 3e8a244 pipes yq into jq on the installer host to pick the models to deploy (roles/llm_services/tasks/deploy_models.yaml#L108-L114), and EAS's configure installs no jq. jq-1.8.2 (tag commit 34f7186, released 2026-06-20) is the newest; its jq-linux-amd64 sha256 is the one jq's sha256sum.txt and GitHub's asset digest both give. pins.json's bastion profile, the docs table and the README name it. In the new bastion, EI's exact pipeline on the smoke's real models.yaml prints qwen2-5-coder-7b and qwen3guard. --- docs/intel/eas.md | 1 + integrations/eas/README.md | 2 +- integrations/eas/bastion.Dockerfile | 14 ++++++++++---- integrations/eas/pins.json | 3 ++- tests/eas_test.sh | 4 ++-- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/intel/eas.md b/docs/intel/eas.md index 0bf4b4b1..1e20e902 100644 --- a/docs/intel/eas.md +++ b/docs/intel/eas.md @@ -214,6 +214,7 @@ Bastion tools (each binary checked against its upstream checksum file at build t | kubectl | v1.37.2 | the minor of the node's k3s (`K3S_TAG` v1.37.1-k3s1); EAS's own v1.34.3 ([#L52](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L52)) is three minors off, outside the [kubectl skew policy](https://kubernetes.io/releases/version-skew-policy/#kubectl) | | helm | v3.22.1 | latest v3; EAS's floor is v3.20.2 ([#L53](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L53)). Helm v4 is untested with `kubernetes.core` 6.3.0 | | yq | v4.54.1 | EAS's floor is v4.53.2 ([#L51](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L51)) | +| jq | jq-1.8.2 | latest; enterprise-inference pipes yq into jq on the installer host to pick the models to deploy ([deploy_models.yaml#L108-L114](https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/deploy_models.yaml#L108-L114)), and EAS's `configure` installs no jq | | ansible, kubernetes (Python) | 12.3.0, 35.0.0, hashed | newest inside EAS's ranges `ansible>=10.0,<13.0`, `kubernetes>=29.0,<36.0` ([requirements.txt](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/requirements.txt)) | | kubernetes.core | 6.3.0 | EAS's [collections/requirements.yml](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/collections/requirements.yml) | diff --git a/integrations/eas/README.md b/integrations/eas/README.md index 25c948d5..75a712c8 100644 --- a/integrations/eas/README.md +++ b/integrations/eas/README.md @@ -64,7 +64,7 @@ its `env/k3d-gpu/` and `ext/`, `facts.json`, `pins.resolved.json`, `eas-config.p | File | Role | |------|------| | `eas.sh` | the in-bastion driver (`up`, `overlay-check`, `down`, `status`) | -| `bastion.Dockerfile` | Ubuntu 24.04 (the EAT bastion's digest), kubectl v1.37.2, helm v3.22.1, yq v4.54.1 by sha256, the installer's venv at `/opt/eas-venv`, kubernetes.core 6.3.0 by sha256: `pins.json`'s bastion profile | +| `bastion.Dockerfile` | Ubuntu 24.04 (the EAT bastion's digest), kubectl v1.37.2, helm v3.22.1, yq v4.54.1, jq-1.8.2 by sha256, the installer's venv at `/opt/eas-venv`, kubernetes.core 6.3.0 by sha256: `pins.json`'s bastion profile | | `requirements.in`, `requirements.lock`, `lock.sh` | EAS `requirements.txt` at the pinned commit, and its hash-locked resolution for the bastion's Python 3.12 | | `images.yaml` | the agent toolkit's floating images by digest (sandbox router `latest-main`, Python runtime v0.5.0) | | `known-issues.json` | upstream issues this path works around, one flat row each | diff --git a/integrations/eas/bastion.Dockerfile b/integrations/eas/bastion.Dockerfile index b22c8e27..9ed38e5d 100644 --- a/integrations/eas/bastion.Dockerfile +++ b/integrations/eas/bastion.Dockerfile @@ -3,9 +3,11 @@ # # Everything the installer's `configure` would install with sudo and apt is baked here # at a pinned version, so `k3d-gpu eas up` never runs `configure`: -# - kubectl, helm and yq from their release downloads, each checked by sha256. The +# - kubectl, helm, yq and jq from their release downloads, each checked by sha256. The # sha256 values are the ones the upstream checksum files publish for these releases -# (dl.k8s.io .sha256, get.helm.sh .sha256sum, yq's `checksums` file). +# (dl.k8s.io .sha256, get.helm.sh .sha256sum, yq's `checksums` file, +# jq's sha256sum.txt). configure installs no jq, but enterprise-inference's +# roles/llm_services/tasks/deploy_models.yaml pipes yq into jq on the installer host. # - the installer's Python requirements (requirements.txt at the EAS commit named in # requirements.lock) as a hash-locked venv at /opt/eas-venv; the installer reuses a # venv that has ansible-playbook (es_auto_installer.sh _ensure_venv). @@ -16,7 +18,7 @@ # kubectl is v1.37.2, the minor of the node image's K3S_TAG (Dockerfile): EAS's own # v1.34.3 is outside kubectl's +/-1 minor skew against a v1.37 API server # (https://kubernetes.io/releases/version-skew-policy/#kubectl). helm is the newest v3 -# (EAS asks for >= v3.20.2); yq the newest v4 (EAS asks for >= v4.53.2). +# (EAS asks for >= v3.20.2); yq the newest v4 (EAS asks for >= v4.53.2); jq the newest. # # Runs as root: es_auto_installer.sh ensure_sudo runs `sudo -k` under `set -e` unless # EUID is 0, and this image has no sudo. eas.sh hands the state dir back to the @@ -38,6 +40,8 @@ ARG HELM_VERSION=v3.22.1 ARG HELM_SHA256=896cf63684a42a723d61c980369d6e588330927deb13676ca0d85006dd6da0ea ARG YQ_VERSION=v4.54.1 ARG YQ_SHA256=8e34fc298390875de416e6a4afcb8cabeceb25d9aa8506c1a2f9353cf702ea5f +ARG JQ_VERSION=jq-1.8.2 +ARG JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f RUN curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ && echo "${KUBECTL_SHA256} /usr/local/bin/kubectl" | sha256sum -c - \ && curl -fsSL -o /tmp/helm.tgz "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ @@ -45,7 +49,9 @@ RUN curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VE && tar -xzf /tmp/helm.tgz -C /usr/local/bin --strip-components=1 linux-amd64/helm \ && curl -fsSL -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ && echo "${YQ_SHA256} /usr/local/bin/yq" | sha256sum -c - \ - && chmod 0755 /usr/local/bin/kubectl /usr/local/bin/helm /usr/local/bin/yq \ + && curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-amd64" \ + && echo "${JQ_SHA256} /usr/local/bin/jq" | sha256sum -c - \ + && chmod 0755 /usr/local/bin/kubectl /usr/local/bin/helm /usr/local/bin/yq /usr/local/bin/jq \ && rm /tmp/helm.tgz COPY requirements.lock /opt/eas-requirements.lock diff --git a/integrations/eas/pins.json b/integrations/eas/pins.json index 2303dedd..b1d5d529 100644 --- a/integrations/eas/pins.json +++ b/integrations/eas/pins.json @@ -6,7 +6,7 @@ "Each row: one pin. profile groups the rows; name is the component; sha is the commit the ref resolved to on the checked date (git ls-remote, gh api), or for a PyPI package the sha256 of its py3 wheel, or for an image its manifest digest; date is the commit, release or upload date; source is where the pin comes from.", "Profile main freezes what EAS main's configs/repos/repos.agent-toolkit.yaml resolves to today: inference rev main is enterprise-inference 3e8a244, and agent-toolkit rev agentic-toolkit-migration is fb594597, the same tree as release-2.0.0 036b662 (the #19 merge), which is pinned here.", "Profile v1.2.0-rc.1 is the release candidate's component table: https://github.com/intel/enterprise-ai-solutions/releases/tag/v1.2.0-rc.1", - "Profile bastion: the tools the EAS bastion bakes. kubectl matches the k3s minor in the Dockerfile's K3S_TAG (v1.37.1-k3s1); EAS's own v1.34.3 is outside kubectl's one-minor skew. helm is the newest v3 (EAS asks for at least v3.20.2); yq the newest (EAS: at least v4.53.2); kubernetes.core is EAS's exact pin; the Python packages are the newest inside EAS's requirements.txt ranges." + "Profile bastion: the tools the EAS bastion bakes. kubectl matches the k3s minor in the Dockerfile's K3S_TAG (v1.37.1-k3s1); EAS's own v1.34.3 is outside kubectl's one-minor skew. helm is the newest v3 (EAS asks for at least v3.20.2); yq the newest (EAS: at least v4.53.2); jq the newest (enterprise-inference's roles/llm_services/tasks/deploy_models.yaml runs it on the installer host, and EAS's configure installs none); kubernetes.core is EAS's exact pin; the Python packages are the newest inside EAS's requirements.txt ranges." ], "pins": [ {"profile": "main", "name": "eas", "repo": "intel/enterprise-ai-solutions", "ref": "main", "sha": "97f6157d6a72f7173c698c474d4e51d2c79f1fd6", "date": "2026-10-29", "source": "https://github.com/intel/enterprise-ai-solutions/commit/97f6157d6a72f7173c698c474d4e51d2c79f1fd6"}, @@ -19,6 +19,7 @@ {"profile": "bastion", "name": "kubectl", "repo": "kubernetes/kubernetes", "ref": "v1.37.2", "sha": "c50e5395f1ae2d4a2c05f28fdcc38c0916cc1a32", "date": "2026-10-28", "source": "https://github.com/kubernetes/kubernetes/releases/tag/v1.37.2"}, {"profile": "bastion", "name": "helm", "repo": "helm/helm", "ref": "v3.22.1", "sha": "d6c1a0e4027caa6abc7892e7ea9f0cda7447baa3", "date": "2026-10-15", "source": "https://github.com/helm/helm/releases/tag/v3.22.1"}, {"profile": "bastion", "name": "yq", "repo": "mikefarah/yq", "ref": "v4.54.1", "sha": "504fc38780cc46be8444ea1b72fb55919fc0bfb0", "date": "2026-09-29", "source": "https://github.com/mikefarah/yq/releases/tag/v4.54.1"}, + {"profile": "bastion", "name": "jq", "repo": "jqlang/jq", "ref": "jq-1.8.2", "sha": "34f7186b86743a083a589741b6cea95293524108", "date": "2026-06-20", "source": "https://github.com/jqlang/jq/releases/tag/jq-1.8.2"}, {"profile": "bastion", "name": "kubernetes.core", "repo": "ansible-collections/kubernetes.core", "ref": "6.3.0", "sha": "80c33c5573aa63c7fdb315c72d4d7bfac725c337", "date": "2026-02-03", "source": "https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/collections/requirements.yml#L3-L4"}, {"profile": "bastion", "name": "ansible", "repo": "pypi.org/project/ansible", "ref": "12.3.0", "sha": "cd156f82fa87f7a899212b0efa9f7ca3a24d609212de9f78428a572eb01e5414", "date": "2025-12-09", "source": "https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/requirements.txt#L1"}, {"profile": "bastion", "name": "kubernetes", "repo": "pypi.org/project/kubernetes", "ref": "35.0.0", "sha": "39e2b33b46e5834ef6c3985ebfe2047ab39135d41de51ce7641a7ca5b372a13d", "date": "2026-01-16", "source": "https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/requirements.txt#L2"}, diff --git a/tests/eas_test.sh b/tests/eas_test.sh index cb54166d..52591088 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -1286,12 +1286,12 @@ PY else ok "cli: every pin in requirements.lock carries hashes" fi - if grep -E '^ARG [A-Z_]*_(SHA256|VERSION)=' "${EAS_DIR}/bastion.Dockerfile" | grep -Eqv '=(v?[0-9.]+|[0-9a-f]{64})$'; then + if grep -E '^ARG [A-Z_]*_(SHA256|VERSION)=' "${EAS_DIR}/bastion.Dockerfile" | grep -Eqv '=((v|jq-)?[0-9.]+|[0-9a-f]{64})$'; then fail "cli: bastion.Dockerfile has a version or sha256 ARG that is not exact" else ok "cli: bastion.Dockerfile pins every download by version and sha256" fi - cli_eq "cli: every bastion download is checked" "$(grep -c 'sha256sum -c -' "${EAS_DIR}/bastion.Dockerfile")" 4 + cli_eq "cli: every bastion download (kubectl, helm, yq, jq, kubernetes.core) is checked" "$(grep -c 'sha256sum -c -' "${EAS_DIR}/bastion.Dockerfile")" 5 while read -r mode _ _ path; do if [ "${mode}" = 100755 ]; then ok "cli: executable in git: ${path}"; else fail "cli: not executable in git (${mode}): ${path}"; fi done < <(git -C "${ROOT}" ls-files -s -- 'integrations/eas/*.sh') From e4ea3914efed7c8493b2c947d88aaeff52298a1f Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 22:45:40 -0700 Subject: [PATCH 11/19] eas: test that eas down gives the teardown install's extra vars (failing) The second wiring smoke's eas down exited 0 in 217 s, and it deleted k3d-gpu's own node-feature-discovery namespace. EAS resolves teardown components by the same enabled flags as install (filter_plugins/resolve_components.py), and eas.sh down passed none of the overrides, so nfd_enabled fell back to its default of true and EI's nfd teardown removed the namespace. Down must pass the image pins and the overrides as install does, and refuse when the overrides file is gone. --- tests/eas_test.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/eas_test.sh b/tests/eas_test.sh index 52591088..df256431 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -50,7 +50,8 @@ # - refs: a profile and a SHA go as given, any other ref through git ls-remote # - no secret value in an argument list, a log, choices.txt or a state file # - refusals stop before any change -# - down clears AIGatewayRoute finalizers before teardown platform --force +# - down clears AIGatewayRoute finalizers before teardown platform --force, and +# the teardown gets install's extra vars (refused without the overrides) # - status prints the layer, pins, overlay, EAS status, gateway, pods not Ready # - sandbox up refuses when EAS's sandbox-router-deployment exists # - the bastion pins what pins.json's bastion profile names, and ships a @@ -1135,7 +1136,7 @@ print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox local ES="${WORK}/cli/es" mkdir -p "${ES}" command cp -f "${STATE}"/k3d-gpu-overrides.yaml "${STATE}"/global_config.edit.yaml "${STATE}"/models.edit.yaml \ - "${STATE}"/repos.agent-toolkit.yaml "${ES}/" + "${STATE}"/repos.agent-toolkit.yaml "${STATE}"/k3d-gpu-images.yaml "${ES}/" : > "${LOG}" out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit -- -e x=1 2>&1); rc=$? @@ -1183,6 +1184,20 @@ print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox : > "${LOG}" PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" down >/dev/null 2>&1 cli_has "cli: eas.sh down: teardown platform, forced (no terminal)" "$(cat "${LOG}")" "installer teardown platform --env k3d-gpu --force" + # Teardown resolves components by the same enabled flags as install (EAS + # filter_plugins/resolve_components.py), so it gets the same extra vars; without them + # nfd_enabled defaults to true and EI's nfd teardown deletes k3d-gpu's own NFD namespace. + cli_has "cli: eas.sh down: teardown gets install's extra vars (the overrides last)" "$(cat "${LOG}")" \ + "installer teardown platform --env k3d-gpu --force -- -e @${ES}/k3d-gpu-images.yaml -e @${ES}/k3d-gpu-overrides.yaml" + command mv "${ES}/k3d-gpu-overrides.yaml" "${ES}/k3d-gpu-overrides.yaml.away" + : > "${LOG}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" down 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'k3d-gpu-overrides.yaml' <<< "${out}" && ! grep -q 'installer teardown' "${LOG}"; then + ok "cli: eas.sh down refuses without the overrides, before any teardown" + else + fail "cli: eas.sh down without the overrides: rc=${rc} ${out}" + fi + command mv "${ES}/k3d-gpu-overrides.yaml.away" "${ES}/k3d-gpu-overrides.yaml" : > "${LOG}" PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" status >/dev/null 2>&1 cli_has "cli: eas.sh status" "$(cat "${LOG}")" "installer status --env k3d-gpu" From e67ebc52eca507626c5a9cd9ffc15f0f251dd59a Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Thu, 29 Oct 2026 22:47:08 -0700 Subject: [PATCH 12/19] eas: eas down gives the teardown install's extra vars eas.sh down now runs teardown platform with -e @k3d-gpu-images.yaml -e @k3d-gpu-overrides.yaml, as install does, so the components eas up left to k3d-gpu (its NFD) are left alone at teardown too. It refuses, before any teardown, when the overrides file is gone. EAS passes these extra vars to its single BYO pass last (es_auto_installer.sh#L1508-L1509 at 97f6157). docs/intel/eas.md says so. --- docs/intel/eas.md | 4 ++++ integrations/eas/eas.sh | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/intel/eas.md b/docs/intel/eas.md index 1e20e902..56c5a52b 100644 --- a/docs/intel/eas.md +++ b/docs/intel/eas.md @@ -67,6 +67,10 @@ k3d-gpu eas down # EAS teardown; the cluster stays (k3d-gpu down removes i - Arguments after `--` go to `ansible-playbook`, for example `k3d-gpu eas up --agent-toolkit -- -e '{"llm_services_autodeploy_models": []}'` to install without EAT's chat model. +- `eas down` gives the teardown the same image pins and overrides as `eas up` gave the + install. EAS picks teardown components by the same `enabled` flags as install + ([resolve_components.py](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/filter_plugins/resolve_components.py#L245)), + so components `eas up` left to k3d-gpu, such as its NFD, stay. What `eas up` does, in order: diff --git a/integrations/eas/eas.sh b/integrations/eas/eas.sh index 7b478731..72a6f279 100755 --- a/integrations/eas/eas.sh +++ b/integrations/eas/eas.sh @@ -14,7 +14,9 @@ # -e @k3d-gpu-overrides.yaml [args]: extra vars load last in es_auto_installer.sh # main(), so the overrides win # eas.sh overlay-check steps 1-2 without applying: the overlay decisions only -# eas.sh down es_auto_installer.sh teardown platform --env k3d-gpu --force +# eas.sh down es_auto_installer.sh teardown platform --env k3d-gpu --force -- -e +# @k3d-gpu-images.yaml -e @k3d-gpu-overrides.yaml: install's extra +# vars, so teardown skips what install skipped (k3d-gpu's NFD) # eas.sh status es_auto_installer.sh status --env k3d-gpu # # `configure` is never run: it needs a terminal or --force, sudo and apt @@ -161,9 +163,17 @@ cmd_up() { cmd_down() { [ -d "${CHECKOUT}/env/${ENV_NAME}" ] || err "no env/${ENV_NAME} in ${CHECKOUT}: nothing was installed from this state dir" + # Teardown picks its components by the same enabled flags as install + # (filter_plugins/resolve_components.py), so it gets install's extra vars. Without the + # overrides, nfd_enabled is true again and EI's nfd teardown deletes k3d-gpu's own NFD. + [ -f "${EAS_STATE}/k3d-gpu-overrides.yaml" ] \ + || err "${EAS_STATE}/k3d-gpu-overrides.yaml missing: without it teardown also removes what eas up left to k3d-gpu (its NFD); run 'k3d-gpu eas up --dry-run' to write it again" + local vars=() + [ ! -f "${EAS_STATE}/k3d-gpu-images.yaml" ] || vars+=(-e "@${EAS_STATE}/k3d-gpu-images.yaml") + vars+=(-e "@${EAS_STATE}/k3d-gpu-overrides.yaml") ln -sfn "${EAS_VENV}" "${CHECKOUT}/.venv" # teardown's confirm refuses a non-terminal stdin without --force (#L129-L137, #L1486). - installer teardown platform --env "${ENV_NAME}" --force + installer teardown platform --env "${ENV_NAME}" --force -- "${vars[@]}" } cmd_status() { From eed5ebcf76fced6690a11049024b58c88dae4343 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sun, 1 Nov 2026 00:18:33 -0700 Subject: [PATCH 13/19] tests: sandbox up's first reads include EAS's router check main's ownership test expects the controller and CRD reads to come first. On this branch sandbox_up first asks whether EAS's agent-sandbox-system/sandbox-router-deployment exists and refuses if it does, so the fresh-cluster sequence is that read, then the two ownership reads, then the apply. --- tests/detect_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/detect_test.sh b/tests/detect_test.sh index eaae79e1..485c2732 100755 --- a/tests/detect_test.sh +++ b/tests/detect_test.sh @@ -4717,7 +4717,8 @@ first_calls() { [ -n "${PINNED}" ] || { FAILS=$((FAILS + 1)); echo "FAIL the controller image is readable from share/agent-sandbox.yml"; } sandbox_says "sandbox up on a fresh cluster installs agent-sandbox" "${r}" 0 'Installing agent-sandbox' up "${UP_OK[@]}" -first_calls "sandbox up on a fresh cluster adds two reads (controller, CRDs) before today's apply" \ +first_calls "sandbox up on a fresh cluster reads EAS's router, the controller and the CRDs before today's apply" \ + '^-n agent-sandbox-system get deployment sandbox-router-deployment -o name$' \ '^-n agent-sandbox-system get deployment agent-sandbox-controller --ignore-not-found -o jsonpath=' \ '^get crd sandboxes\.agents\.x-k8s\.io .* --ignore-not-found -o jsonpath=' \ 'apply -f .*/agent-sandbox\.yml$' From 61e987a7d9d5e7b552d5c527fd2598c9b4972950 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sun, 1 Nov 2026 01:24:10 -0700 Subject: [PATCH 14/19] eat: test that eat-check reaches the sandbox router on a port kubectl picks (failing) eat-check.sh port-forwards the sandbox router to the fixed local port 18080. On a host where another service already listens there, kubectl cannot bind, the readiness probe gets an answer from that service, and the code and the router token go to it. The stub kubectl now reports the port it bound the way kubectl does, and the stub router answers only there. The new checks want the local port left to the kernel, every router request on the port kubectl reported, /healthz read before /execute, and a stop with nothing sent when the port-forward dies or the endpoint does not answer as the router. --- tests/eat_test.sh | 53 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/tests/eat_test.sh b/tests/eat_test.sh index 00758fba..b154a510 100755 --- a/tests/eat_test.sh +++ b/tests/eat_test.sh @@ -9,6 +9,9 @@ # both for a router that shlex.splits the command (agent-sandbox v0.5.0 # python-runtime-sandbox) and for one that runs it under /bin/sh -c # - the check fails when the sandbox prints anything but 385 +# - the router port-forward takes a local port the kernel picks, read from kubectl's +# own output, and the endpoint must answer /healthz as the router before the code +# and the token go to it # - no key reaches curl's argument list (headers and bodies go as files / stdin) # 2b. eat-check.sh EAT_MODE=eas (k3d-gpu eas check) against stubs: the gateway is # eg-gateway's LoadBalancer IP with Host litellm.; the requests carry a @@ -65,7 +68,14 @@ case "$*" in *"-n litellm get secret litellm-master-key"*"master_key"*) printf 'sk-master-stub' | base64 ;; *"get pods -o name"*) printf 'pod/sandbox-router-1\npod/python-pool-abcde\n' ;; *"get secret sandbox-router-auth"*) printf 'tok' | base64 ;; - *port-forward*) exec sleep 60 ;; + *port-forward*) + # kubectl's own report of the local port it bound (STUB_PF_PORT: the one the + # kernel picked); STUB_PF_DEAD: kubectl exits without binding. + printf 'pf %s\n' "$*" >> "${STUB_LOG}" + if [ -n "${STUB_PF_DEAD:-}" ]; then echo "error: unable to listen on any of the requested ports" >&2; exit 1; fi + printf 'Forwarding from 127.0.0.1:%s -> 8080\nForwarding from [::1]:%s -> 8080\n' \ + "${STUB_PF_PORT:-40123}" "${STUB_PF_PORT:-40123}" + exec sleep 60 ;; *) echo "kubectl stub: unexpected: $*" >&2; exit 1 ;; esac EOF @@ -137,12 +147,21 @@ elif url.endswith("/v1/chat/completions"): msg("\n\n\n\n```python\n" + os.environ["STUB_CODE"] + "\n```") else: msg("\n\n\n\nKubernetes runs containers.") -elif url.endswith("/execute"): - cmd = json.loads(body)["command"] - argv = shlex.split(cmd) if os.environ["STUB_ROUTER"] == "shlex" else ["/bin/sh", "-c", cmd] - p = subprocess.run(argv, capture_output=True, text=True) - print(json.dumps({"stdout": p.stdout, "stderr": p.stderr, "exit_code": p.returncode})) -# anything else (the router readiness probe) succeeds with no output +elif "127.0.0.1:" in url: + # The local end of the router port-forward: only the port kubectl reported is the + # router (STUB_ROUTER_FOREIGN: another local service answers there instead). + hostport, _, path = url.split("://", 1)[-1].partition("/") + with open(os.environ["STUB_LOG"], "a") as f: + f.write("router %s /%s\n" % (hostport, path)) + if hostport != "127.0.0.1:" + os.environ.get("STUB_PF_PORT", "40123") or os.environ.get("STUB_ROUTER_FOREIGN"): + print(json.dumps({"object": "error", "message": "Not Found"})) + elif path == "healthz": + print(json.dumps({"status": "ok"})) + elif path == "execute": + cmd = json.loads(body)["command"] + argv = shlex.split(cmd) if os.environ["STUB_ROUTER"] == "shlex" else ["/bin/sh", "-c", cmd] + p = subprocess.run(argv, capture_output=True, text=True) + print(json.dumps({"stdout": p.stdout, "stderr": p.stderr, "exit_code": p.returncode})) EOF # jq: the real one, with each argv logged (a key must not reach any argument list). REAL_JQ=$(command -v jq) @@ -187,6 +206,26 @@ check_says "eat-check defaults: 200 tokens, temperature 0" 0 'request max_tokens check_says "eat-check for a shared remote endpoint: sampled and short" 0 'request max_tokens=64 temperature=0.6$' \ EAT_TEMPERATURE=0.6 EAT_MAX_TOKENS=64 check_says "eat-check refuses a non-numeric temperature" fail 'EAT_TEMPERATURE must be a number' EAT_TEMPERATURE=hot +# The router port-forward: a local port the kernel picks, read from kubectl's own output, +# and the endpoint confirmed as the sandbox router before /execute (a fixed 18080 reached +# whatever local service held it). +check_says "the router port-forward asks kubectl for a kernel-picked local port" 0 \ + '^pf -n agent-sandbox-system port-forward svc/sandbox-router-svc :8080$' +check_says "/execute goes to the port kubectl reported" 0 '^router 127.0.0.1:43211 /execute$' STUB_PF_PORT=43211 +check_says "the router's /healthz is read on that port before /execute" 0 '^router 127.0.0.1:43211 /healthz$' STUB_PF_PORT=43211 +if grep '^router ' "${WORK}/log" | grep -qv '^router 127.0.0.1:43211 '; then + fail "a router request went to a port kubectl did not report: $(grep '^router ' "${WORK}/log" | grep -v '127.0.0.1:43211' | head -1)" +else + ok "every router request goes to the port kubectl reported" +fi +check_says "a port-forward that reports no port stops the check" fail 'port-forward to svc/sandbox-router-svc exited' STUB_PF_DEAD=1 +check_says "an endpoint that is not the sandbox router stops the check" fail 'does not answer as the sandbox router' \ + STUB_ROUTER_FOREIGN=1 +if grep -q '^router .* /execute$' "${WORK}/log"; then + fail "the code went to an endpoint that is not the sandbox router" +else + ok "nothing is sent to an endpoint that is not the sandbox router" +fi check_says "eat mode: the master key from vault.yml authorizes the chat" 0 'auth v1/chat/completions sk-stub$' check_says "eat mode: the router token authorizes /execute" 0 'auth execute tok$' check_says "eat mode: no key on any curl argv" 0 'EAT check passed' From d966f610f9ec8a0d3daec9edcafbc04b601b17cb Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sun, 1 Nov 2026 01:24:10 -0700 Subject: [PATCH 15/19] eat: eat-check port-forwards the router to a port the kernel picks kubectl port-forward svc/sandbox-router-svc :8080 binds a free local port, and the check reads it from kubectl's own "Forwarding from 127.0.0.1:" line. Before the token and the code go there, GET /healthz must answer {"status":"ok"}, as the agent-sandbox router does. A port-forward that exits or reports no port, and an endpoint that does not answer as the router, stop the check with the reason. The other local ports in eas.sh, eat-check.sh and the eas check path were audited: the router port-forward was the only fixed one. --- integrations/eat/eat-check.sh | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/integrations/eat/eat-check.sh b/integrations/eat/eat-check.sh index 04473e32..16b5b962 100755 --- a/integrations/eat/eat-check.sh +++ b/integrations/eat/eat-check.sh @@ -11,7 +11,8 @@ # 3. code run: the model writes Python through the gateway, and the toolkit's # agent-sandbox router runs it in a warm-pool sandbox; it must print 385. # bash passes the model's text to the router: this is a scripted pipeline, -# not an agent loop. +# not an agent loop. The router is reached through a kubectl port-forward on a +# local port the kernel picks, and must answer GET /healthz as the router does. # # EAT_MODE eat (default) | eas # EAT_WORK eat: eat-up.sh work dir; its vault.yml holds the LiteLLM master key (default: ./eat-work) @@ -40,7 +41,6 @@ EAT_TEMPERATURE="${EAT_TEMPERATURE:-0}" EAT_MAX_TOKENS="${EAT_MAX_TOKENS:-200}" CORE="${EAT_WORK}/eat/core" SANDBOX_NS=agent-sandbox-system -ROUTER_PORT=18080 err() { echo "ERROR: $*" >&2; exit 1; } info() { echo "==> $*"; } @@ -48,9 +48,10 @@ info() { echo "==> $*"; } [[ "${EAT_TEMPERATURE}" =~ ^[0-9]+([.][0-9]+)?$ ]] || err "EAT_TEMPERATURE must be a number, got '${EAT_TEMPERATURE}'" [[ "${EAT_MAX_TOKENS}" =~ ^[0-9]+$ ]] || err "EAT_MAX_TOKENS must be a whole number, got '${EAT_MAX_TOKENS}'" -pf="" vkey="" master="" alias="" +pf="" pflog="" vkey="" master="" alias="" cleanup() { [ -z "${pf}" ] || kill "${pf}" 2>/dev/null || true + [ -z "${pflog}" ] || rm -f "${pflog}" # The run's virtual key goes when the check ends, pass or fail. if [ -n "${vkey}" ]; then # The key reaches jq through its environment, not its argument list. @@ -138,11 +139,33 @@ sed 's/^/ /' <<< "${code}" sandbox=$(kubectl -n "${SANDBOX_NS}" get pods -o name | grep -m1 '/python-pool-' | cut -d/ -f2) \ || err "no python-pool warm sandbox in ${SANDBOX_NS}" token=$(kubectl -n "${SANDBOX_NS}" get secret sandbox-router-auth -o jsonpath='{.data.ROUTER_AUTH_TOKEN}' | base64 -d) -kubectl -n "${SANDBOX_NS}" port-forward svc/sandbox-router-svc "${ROUTER_PORT}:8080" >/dev/null 2>&1 & +# The router's local port is the one the kernel gives kubectl (":8080"), read from kubectl's +# own "Forwarding from 127.0.0.1:" line: a fixed port may be held by another local +# service, which would then get the code and the router token. +pflog=$(mktemp) +kubectl -n "${SANDBOX_NS}" port-forward svc/sandbox-router-svc :8080 > "${pflog}" 2>&1 & pf=$! -for _ in $(seq 1 40); do curl -s -o /dev/null "127.0.0.1:${ROUTER_PORT}" && break; sleep 0.5; done +port="" +for _ in $(seq 1 40); do + port=$(awk '/^Forwarding from 127\.0\.0\.1:[0-9]+ -> 8080$/ { split($3, a, ":"); print a[2]; exit }' "${pflog}") + [ -z "${port}" ] || break + kill -0 "${pf}" 2>/dev/null || err "kubectl port-forward to svc/sandbox-router-svc exited: $(tr '\n' ' ' < "${pflog}")" + sleep 0.5 +done +[ -n "${port}" ] || err "kubectl port-forward to svc/sandbox-router-svc reported no local port in 20 s: $(tr '\n' ' ' < "${pflog}")" +router="http://127.0.0.1:${port}" +# The endpoint must answer as the sandbox router (GET /healthz: {"status":"ok"}) before the +# token and the code go to it. +health="" +for _ in $(seq 1 40); do + health=$(curl -s --max-time 5 "${router}/healthz") || health="" + [ "$(jq -r '.status? // empty' <<< "${health}" 2>/dev/null)" != ok ] || break + sleep 0.5 +done +[ "$(jq -r '.status? // empty' <<< "${health}" 2>/dev/null)" = ok ] \ + || err "${router} (kubectl's port-forward) does not answer as the sandbox router: GET /healthz gave '${health}'" result=$(jq -n --arg c "${code}" '{command: ("python3 -c " + ($c | @sh))}' \ - | curl -sf "127.0.0.1:${ROUTER_PORT}/execute" -H @<(printf 'Authorization: Bearer %s\n' "${token}") \ + | curl -sf "${router}/execute" -H @<(printf 'Authorization: Bearer %s\n' "${token}") \ -H "X-Sandbox-ID: ${sandbox}" -H "X-Sandbox-Namespace: ${SANDBOX_NS}" -H "X-Sandbox-Port: 8888" \ -H 'content-type: application/json' --data-binary @-) echo " sandbox ${sandbox}: $(jq -c . <<< "${result}")" From a0131d5160277fa6897370478a9b2cbcf7d2e0f3 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sun, 1 Nov 2026 01:24:19 -0700 Subject: [PATCH 16/19] eas: test that the LiteLLM master key reaches no argv during the model deploy (failing) The secret scan of the eas up smoke found the master key in sampled ps output. The source is EI's llm_services deploy task: it reads Secret litellm-master-key into _deploy_litellm_secret and hands the decoded key to model-manager as MM_LITELLM_MASTER_KEY in the task's environment:. Ansible builds that environment into the module command, so the key sits in the argv of two /bin/sh -c processes for the whole deploy (a fake-key run of the same task in the bastion's ansible-core 2.19.14 showed it in every sample). Section argv runs EI 3e8a244's two tasks, the MM_LITELLM_MASTER_KEY line verbatim, under ansible-playbook with a fake key while ps samples every argv. The control (no overrides) must show the key; with the generator's k3d-gpu-overrides.yaml no sample may hold it and the deploy must get an empty value. The cli section also wants eas.sh to stop before install when EI's deploy takes the key from anywhere else. --- tests/eas_test.sh | 97 ++++++++++++++++++- tests/fixtures/eas/argv/deploy-models.yml | 31 ++++++ .../eas/argv/library/k8s_secret_stub.py | 10 ++ 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/eas/argv/deploy-models.yml create mode 100644 tests/fixtures/eas/argv/library/k8s_secret_stub.py diff --git a/tests/eas_test.sh b/tests/eas_test.sh index df256431..40034dac 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -36,6 +36,13 @@ # role looks for, --explain prints permalinks, every source it can cite is # a permalink at a full commit SHA, and pins.json has the k3d-gpu.eas-pins/v1 # shape. +# argv the LiteLLM master key on no process argument list during EI's model deploy: +# EI 3e8a244's two tasks that carry it (tests/fixtures/eas/argv, the +# MM_LITELLM_MASTER_KEY line verbatim) run under ansible-playbook with a fake key +# while ps samples every argv. Without the overrides (the control) ansible's +# environment: puts the key on the deploy's /bin/sh -c argv; with the +# generator's k3d-gpu-overrides.yaml no sample holds it and the deploy gets an +# empty MM_LITELLM_MASTER_KEY. Needs ansible-playbook. # cli `k3d-gpu eas up|down|status|check`, eas.sh (the in-bastion driver) and the # bastion, against stub docker, kubectl, k3d, git, yq and `k3d-gpu fit`, with the # real eas-config.py (so the facts it gathers are the facts the generator reads) @@ -47,6 +54,8 @@ # uid:gid, state dir rw, kubeconfig ro, secrets by name only # - eas.sh order: clone at the SHA, overlay, manifest pins, init, the edits, # install with the overrides last; never configure +# - eas.sh stops before install when EI's model deploy takes the master key +# from anything but _deploy_litellm_secret, or has no deploy_models.yaml # - refs: a profile and a SHA go as given, any other ref through git ls-remote # - no secret value in an argument list, a log, choices.txt or a state file # - refusals stop before any change @@ -1175,6 +1184,33 @@ print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit >/dev/null 2>&1 cli_has "cli: a second up re-inits with --upgrade (moved pins)" "$(cat "${LOG}")" "installer init agent-toolkit --env k3d-gpu --upgrade" cli_lacks "cli: a second up does not clone again" "$(cat "${LOG}")" "git clone" + # EI's model deploy: the key from _deploy_litellm_secret (the overrides empty it) passes; + # a key from anywhere else stops the install before it starts. + local DM="${ES}/eas/ext/enterprise.ai-inference/roles/llm_services/tasks/deploy_models.yaml" + mkdir -p "$(dirname "${DM}")" + command cp -f "${ROOT}/tests/fixtures/eas/argv/deploy-models.yml" "${DM}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + cli_eq "cli: eas.sh up passes with EI's deploy taking the key from _deploy_litellm_secret" "${rc}" 0 + sed -i 's/^\( *MM_LITELLM_MASTER_KEY:\).*/\1 "{{ lookup('"'env'"', '"'LITELLM_MASTER_KEY'"') }}"/' "${DM}" + : > "${LOG}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'no longer takes MM_LITELLM_MASTER_KEY from _deploy_litellm_secret' <<< "${out}"; then + ok "cli: eas.sh stops when EI's deploy takes the key from elsewhere" + else + fail "cli: EI deploy taking the key from elsewhere: rc=${rc} ${out}" + fi + cli_lacks "cli: ...before install" "$(cat "${LOG}")" "installer install" + rm -f "${DM}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'EI has no roles/llm_services/tasks/deploy_models.yaml' <<< "${out}"; then + ok "cli: eas.sh stops when EI has no deploy_models.yaml" + else + fail "cli: EI without deploy_models.yaml: rc=${rc} ${out}" + fi + rm -rf "${ES}/eas/ext" out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA=main bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? if [ "${rc}" -ne 0 ] && grep -q 'EAS_SHA must be a full commit SHA' <<< "${out}"; then ok "cli: eas.sh refuses a floating EAS ref"; else fail "cli: eas.sh floating ref: rc=${rc} ${out}"; fi sed -i 's/rev: "3e8a/rev: "main-3e8a/' "${ES}/repos.agent-toolkit.yaml" @@ -1321,7 +1357,66 @@ PY else fail "cli: known-issues.json shape"; fi } -SECTIONS=(overlay config cli) +section_argv() { + local D="${ROOT}/tests/fixtures/eas/argv" A="${WORK}/argv" fake="sk-ARGVPROBE7c1e94d0b3a24f6e8a51" rc pb n hits + local CASE="${ROOT}/tests/fixtures/eas/config/desktop-i9-13900T" args + mkdir -p "${A}/tmp" + if ! command -v ansible-playbook >/dev/null; then + fail "argv: needs ansible-playbook (the bastion runs EAS under it)" + return + fi + # The overrides eas up passes last, from the generator on a golden agent-toolkit case. + read -r -a args < "${CASE}/args" + python3 -S "${ROOT}/integrations/eas/eas-config.py" --facts "${CASE}/facts.json" --out "${A}/gen" "${args[@]}" \ + > /dev/null 2>&1 || { fail "argv: eas-config.py on desktop-i9-13900T"; return; } + # The key as a grep pattern file, so the sampler's own grep never holds it in its argv. + printf '%s\n' "${fake}" > "${A}/pattern" + # play [extra ansible args]: EI's two tasks under ansible-playbook while ps + # samples every process's argv; prints the play's rc and the argv lines holding the key. + play() { + local out=$1 + shift + mkdir -p "${out}" + ARGV_FAKE_KEY="${fake}" ANSIBLE_LIBRARY="${D}/library" ANSIBLE_LOCAL_TEMP="${A}/tmp" \ + ANSIBLE_REMOTE_TEMP="${A}/tmp" ANSIBLE_NOCOLOR=1 ANSIBLE_RETRY_FILES_ENABLED=0 \ + ansible-playbook -i localhost, "${D}/deploy-models.yml" -e "argv_out=${out}" "$@" \ + < /dev/null > "${out}/play.log" 2>&1 & + pb=$! + hits=0 + while kill -0 "${pb}" 2>/dev/null; do + n=$(ps -eo args | grep -c -F -f "${A}/pattern") + hits=$((hits + n)) + sleep 0.2 + done + wait "${pb}" && rc=0 || rc=$? + } + # The control: EI's tasks as they ship put the key on the argv of the /bin/sh -c that + # ansible's environment: wraps around the deploy, for the whole deploy. + play "${A}/control" + if [ "${rc}" -eq 0 ] && [ "${hits}" -gt 0 ]; then + ok "argv: control: without the overrides the key is on a process argv (${hits} sampled lines)" + else + fail "argv: control: rc=${rc}, ${hits} sampled lines hold the key (want rc 0 and > 0): $(tail -5 "${A}/control/play.log")" + fi + play "${A}/fixed" -e "@${A}/gen/k3d-gpu-overrides.yaml" + if [ "${rc}" -eq 0 ] && [ "${hits}" -eq 0 ]; then + ok "argv: with k3d-gpu-overrides.yaml no process argv holds the master key" + else + fail "argv: with the overrides: rc=${rc}, ${hits} sampled lines hold the key: $(tail -5 "${A}/fixed/play.log")" + fi + if [ "$(cat "${A}/control/key" 2>/dev/null)" = set ] && [ "$(cat "${A}/fixed/key" 2>/dev/null)" = empty ]; then + ok "argv: the deploy gets an empty MM_LITELLM_MASTER_KEY with the overrides (model-manager reads the Secret itself)" + else + fail "argv: MM_LITELLM_MASTER_KEY in the deploy: control '$(cat "${A}/control/key" 2>/dev/null)', with the overrides '$(cat "${A}/fixed/key" 2>/dev/null)' (want set, empty)" + fi + if grep -rqF -- "${fake}" "${A}/fixed" "${A}/gen"; then + fail "argv: the key reached a log or the generator's files" + else + ok "argv: the key is in no log and no generated file" + fi +} + +SECTIONS=(overlay config cli argv) [ "$#" -gt 0 ] && SECTIONS=("$@") for s in "${SECTIONS[@]}"; do if declare -F "section_${s}" > /dev/null; then diff --git a/tests/fixtures/eas/argv/deploy-models.yml b/tests/fixtures/eas/argv/deploy-models.yml new file mode 100644 index 00000000..779056ed --- /dev/null +++ b/tests/fixtures/eas/argv/deploy-models.yml @@ -0,0 +1,31 @@ +# EI 3e8a244 roles/llm_services/tasks/deploy_models.yaml, the two tasks that carry the +# LiteLLM master key, for tests/eas_test.sh section argv: +# L81-L90 the Secret read (kubernetes.core.k8s_info replaced by library/k8s_secret_stub.py, +# which returns a Secret holding ARGV_FAKE_KEY) +# L213-L241 the model deploy with its MM_LITELLM_MASTER_KEY line verbatim (L240); the +# `./model-manager deploy` command replaced by a sleep that records whether +# the task hands over a value +# https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/deploy_models.yaml#L81-L90 +# https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/deploy_models.yaml#L213-L241 +- hosts: localhost + connection: local + gather_facts: false + vars: + auth_type: litellm_vkey + tasks: + - name: "Deploy models | Load LiteLLM master key (when auth_type=litellm_vkey)" + k8s_secret_stub: + api_version: v1 + kind: Secret + name: litellm-master-key + namespace: "{{ litellm_namespace | default('litellm') }}" + register: _deploy_litellm_secret + failed_when: false + no_log: true + when: auth_type | default('jwt') == 'litellm_vkey' + + - name: Deploy each catalog model + shell: | + if [ -n "$MM_LITELLM_MASTER_KEY" ]; then echo set; else echo empty; fi > "{{ argv_out }}/key"; sleep 4 + environment: + MM_LITELLM_MASTER_KEY: "{{ (_deploy_litellm_secret.resources[0].data.master_key | b64decode) if (auth_type | default('jwt') == 'litellm_vkey' and _deploy_litellm_secret.resources | default([]) | length > 0) else '' }}" diff --git a/tests/fixtures/eas/argv/library/k8s_secret_stub.py b/tests/fixtures/eas/argv/library/k8s_secret_stub.py new file mode 100644 index 00000000..a23ed5f1 --- /dev/null +++ b/tests/fixtures/eas/argv/library/k8s_secret_stub.py @@ -0,0 +1,10 @@ +#!/usr/bin/python +# kubernetes.core.k8s_info's result for Secret litellm/litellm-master-key, its master_key +# the fake key in ARGV_FAKE_KEY (base64, as the API returns Secret data). +import base64 +import json +import os + +key = os.environ["ARGV_FAKE_KEY"].encode() +print(json.dumps({"changed": False, "api_found": True, + "resources": [{"data": {"master_key": base64.b64encode(key).decode()}}]})) From 207f84de04942a79d8e955db367e50b58691ab56 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sun, 1 Nov 2026 01:24:19 -0700 Subject: [PATCH 17/19] eas: keep the LiteLLM master key off the model deploy's command line eas-config.py sets _deploy_litellm_secret to "" in k3d-gpu-overrides.yaml. Extra vars win over the task's register, so EI's MM_LITELLM_MASTER_KEY expression sees no Secret and yields "", and model-manager falls back to reading the Secret with kubectl (_litellm_key in model_manager/core/routing.py, the same at EI 3e8a244 and 170eb3a). The key then appears on no command line. choices.txt names the override and its source. eas.sh checks EI's deploy_models.yaml after init: every MM_LITELLM_MASTER_KEY line must take the key from _deploy_litellm_secret and the task must register it, or the install stops, since the override would no longer cover it. The goldens gain the one override and its choice line. --- integrations/eas/eas-config.py | 11 +++++++++++ integrations/eas/eas.sh | 19 ++++++++++++++++++- .../config/default-cni/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../desktop-i9-13900T/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../eas/config/eas-rerun/expected/choices.txt | 1 + .../eas-rerun/expected/k3d-gpu-overrides.yaml | 1 + .../config/fit-missing/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../eas/config/gaudi/expected/choices.txt | 1 + .../gaudi/expected/k3d-gpu-overrides.yaml | 1 + .../inference-layer/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../config/model-nofit/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../model-override/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../eas/config/no-nri/expected/choices.txt | 1 + .../no-nri/expected/k3d-gpu-overrides.yaml | 1 + .../config/rc1-profile/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../rootless-docker/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../stock-k3s-model/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + .../eas/config/tiny-host/expected/choices.txt | 1 + .../tiny-host/expected/k3d-gpu-overrides.yaml | 1 + .../xeon-amx-external/expected/choices.txt | 1 + .../expected/k3d-gpu-overrides.yaml | 1 + 30 files changed, 57 insertions(+), 1 deletion(-) diff --git a/integrations/eas/eas-config.py b/integrations/eas/eas-config.py index 1e76cd82..f3c8ea60 100755 --- a/integrations/eas/eas-config.py +++ b/integrations/eas/eas-config.py @@ -107,6 +107,7 @@ "cluster_cidr": _K3S + "pkg/cli/cmds/server.go#L137-L141", "service_cidr": _K3S + "pkg/cli/cmds/server.go#L142-L146", "kata": _EAS + "roles/kata_containers/tasks/install.yaml#L9-L22", + "argv_master_key": _EI + "roles/llm_services/tasks/deploy_models.yaml#L240", "accel_cpu": _EAS + "configs/defaults/global_config.yaml#L106", "accel_xpu": _EAT + "README.md#L36", "sandbox": _EAT + "config.yaml#L56", @@ -432,6 +433,16 @@ def plan(facts, layer, pins, model_flag): p.choose("bring-your-own cluster", "kata_containers_enabled: false", "kata") p.set("kata_containers_enabled", False) + # EI's model deploy task hands model-manager the LiteLLM master key in its ansible + # environment:, which ansible writes into the /bin/sh -c command line it runs the + # deploy under, readable in `ps` for the whole deploy. An empty Secret read leaves + # MM_LITELLM_MASTER_KEY empty, and model-manager then reads the Secret itself. + p.choose("EI's model deploy passes the LiteLLM master key through ansible environment: " + "(a /bin/sh -c argument while each model deploys)", + "_deploy_litellm_secret: \"\" (model-manager reads Secret litellm/litellm-master-key itself)", + "argv_master_key") + p.set("_deploy_litellm_secret", "") + devices = [] for n in nodes: for res, val in sorted((n.get("allocatable") or {}).items()): diff --git a/integrations/eas/eas.sh b/integrations/eas/eas.sh index 72a6f279..38cc3434 100755 --- a/integrations/eas/eas.sh +++ b/integrations/eas/eas.sh @@ -7,7 +7,8 @@ # 1. clone EAS and check out EAS_SHA (a clean tree: earlier overlay hunks are dropped) # 2. the patch overlay: overlay.py --apply, decisions in overlay.txt # 3. the layer manifest's revs as commit SHAs (repos..yaml from eas-config.py) -# 4. es_auto_installer.sh init --env k3d-gpu +# 4. es_auto_installer.sh init --env k3d-gpu; EI's model deploy must take the +# LiteLLM master key from the Secret read the overrides empty (key_argv_guard) # 5. global_config.edit.yaml and models.edit.yaml (eas-config.py) into env/k3d-gpu/: # keys merge; a model's env merges and its args_append follows its catalog args # 6. es_auto_installer.sh install --env k3d-gpu -- -e @k3d-gpu-images.yaml @@ -140,6 +141,21 @@ env_edits() { installer() { (cd "${CHECKOUT}" && ./es_auto_installer.sh "$@"); } +# key_argv_guard: EI's model deploy hands model-manager the LiteLLM master key as +# MM_LITELLM_MASTER_KEY in its ansible environment:, which ansible writes into the deploy's +# /bin/sh -c command line. k3d-gpu-overrides.yaml empties the Secret read that value comes +# from (_deploy_litellm_secret, eas-config.py argv_master_key); an EI that takes the key +# from anything else would put it back on that command line, so it stops the install. +key_argv_guard() { + local f="${CHECKOUT}/ext/enterprise.ai-inference/roles/llm_services/tasks/deploy_models.yaml" + [ -d "${CHECKOUT}/ext/enterprise.ai-inference" ] || return 0 + [ -f "${f}" ] || err "EI has no roles/llm_services/tasks/deploy_models.yaml: check how it now hands model-manager the LiteLLM master key (eas-config.py argv_master_key)" + awk '/^[[:space:]]*MM_LITELLM_MASTER_KEY:/ { any = 1; if ($0 !~ /_deploy_litellm_secret/) bad = 1 } + /^[[:space:]]*register:[[:space:]]*_deploy_litellm_secret[[:space:]]*$/ { reg = 1 } + END { exit (bad || (any && !reg)) ? 1 : 0 }' "${f}" \ + || err "EI's deploy_models.yaml no longer takes MM_LITELLM_MASTER_KEY from _deploy_litellm_secret alone: k3d-gpu-overrides.yaml would not keep the master key off the deploy's command line (eas-config.py argv_master_key)" +} + cmd_up() { local layer=${1:-} [ -n "${layer}" ] || err "usage: eas.sh up [-- ]" @@ -156,6 +172,7 @@ cmd_up() { else installer init "${layer}" --env "${ENV_NAME}" fi + key_argv_guard env_edits installer install "${layer}" --env "${ENV_NAME}" -- \ -e "@${EAS_STATE}/k3d-gpu-images.yaml" -e "@${EAS_STATE}/k3d-gpu-overrides.yaml" "$@" diff --git a/tests/fixtures/eas/config/default-cni/expected/choices.txt b/tests/fixtures/eas/config/default-cni/expected/choices.txt index 389d0777..36f68ea2 100644 --- a/tests/fixtures/eas/config/default-cni/expected/choices.txt +++ b/tests/fixtures/eas/config/default-cni/expected/choices.txt @@ -12,6 +12,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] pin ei main (profile main) -> repos.inference.yaml inference rev: 3e8a24486b6ee812adf2bf452f8ba5751c16c8bd [intel/enterprise-ai-solutions@97f6157 configs/repos/repos.agent-toolkit.yaml:12] WARN: node has 32 CPUs, under EAT's 48-core minimum -> continuing [intel/enterprise-agent-toolkit@036b662 docs/prerequisites.md:7] diff --git a/tests/fixtures/eas/config/default-cni/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/default-cni/expected/k3d-gpu-overrides.yaml index c7b3c3cc..4f780aaf 100644 --- a/tests/fixtures/eas/config/default-cni/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/default-cni/expected/k3d-gpu-overrides.yaml @@ -7,4 +7,5 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" diff --git a/tests/fixtures/eas/config/desktop-i9-13900T/expected/choices.txt b/tests/fixtures/eas/config/desktop-i9-13900T/expected/choices.txt index 7897681b..762a1c5e 100644 --- a/tests/fixtures/eas/config/desktop-i9-13900T/expected/choices.txt +++ b/tests/fixtures/eas/config/desktop-i9-13900T/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 125.3 GiB; qwen2-5-coder-14b needs 131 GiB; qwen2-5-coder-7b needs 99 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-7b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/desktop-i9-13900T/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/desktop-i9-13900T/expected/k3d-gpu-overrides.yaml index 58481ff1..5195c268 100644 --- a/tests/fixtures/eas/config/desktop-i9-13900T/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/desktop-i9-13900T/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-7b"] diff --git a/tests/fixtures/eas/config/eas-rerun/expected/choices.txt b/tests/fixtures/eas/config/eas-rerun/expected/choices.txt index 7983f5ff..73c5a779 100644 --- a/tests/fixtures/eas/config/eas-rerun/expected/choices.txt +++ b/tests/fixtures/eas/config/eas-rerun/expected/choices.txt @@ -13,6 +13,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= k3s.io/node-args --cluster-cidr 10.52.0.0/16 -> kubernetes_cluster_cidr: 10.52.0.0/16 [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] k3s.io/node-args --service-cidr 10.53.0.0/16 -> kubernetes_service_cidr: 10.53.0.0/16 [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] agent-sandbox v0.5.0 installed (EAT's own) -> agent_sandbox_enabled: true [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 512 GiB; qwen2-5-coder-14b needs 131 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/eas-rerun/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/eas-rerun/expected/k3d-gpu-overrides.yaml index b57e3136..705903b3 100644 --- a/tests/fixtures/eas/config/eas-rerun/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/eas-rerun/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: true kubernetes_cluster_cidr: "10.52.0.0/16" kubernetes_service_cidr: "10.53.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/fixtures/eas/config/fit-missing/expected/choices.txt b/tests/fixtures/eas/config/fit-missing/expected/choices.txt index f43f455a..e74fe653 100644 --- a/tests/fixtures/eas/config/fit-missing/expected/choices.txt +++ b/tests/fixtures/eas/config/fit-missing/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 125.3 GiB; qwen2-5-coder-14b needs 131 GiB; qwen2-5-coder-7b: k3d-gpu fit gave no result; llama-3-1-8b needs 99 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [llama-3-1-8b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/fit-missing/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/fit-missing/expected/k3d-gpu-overrides.yaml index 07011ae2..23eee106 100644 --- a/tests/fixtures/eas/config/fit-missing/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/fit-missing/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["llama-3-1-8b"] diff --git a/tests/fixtures/eas/config/gaudi/expected/choices.txt b/tests/fixtures/eas/config/gaudi/expected/choices.txt index f543fd4a..4cea16e2 100644 --- a/tests/fixtures/eas/config/gaudi/expected/choices.txt +++ b/tests/fixtures/eas/config/gaudi/expected/choices.txt @@ -12,6 +12,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] habana.ai/gaudi allocatable 8 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 1007 GiB; qwen2-5-coder-14b needs 131 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/gaudi/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/gaudi/expected/k3d-gpu-overrides.yaml index 6b01df56..93d41d97 100644 --- a/tests/fixtures/eas/config/gaudi/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/gaudi/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/fixtures/eas/config/inference-layer/expected/choices.txt b/tests/fixtures/eas/config/inference-layer/expected/choices.txt index 7c532221..833d9d1d 100644 --- a/tests/fixtures/eas/config/inference-layer/expected/choices.txt +++ b/tests/fixtures/eas/config/inference-layer/expected/choices.txt @@ -12,6 +12,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] pin ei main (profile main) -> repos.inference.yaml inference rev: 3e8a24486b6ee812adf2bf452f8ba5751c16c8bd [intel/enterprise-ai-solutions@97f6157 configs/repos/repos.agent-toolkit.yaml:12] WARN: node has 32 CPUs, under EAT's 48-core minimum -> continuing [intel/enterprise-agent-toolkit@036b662 docs/prerequisites.md:7] diff --git a/tests/fixtures/eas/config/inference-layer/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/inference-layer/expected/k3d-gpu-overrides.yaml index 463cc163..44b9acec 100644 --- a/tests/fixtures/eas/config/inference-layer/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/inference-layer/expected/k3d-gpu-overrides.yaml @@ -7,4 +7,5 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" diff --git a/tests/fixtures/eas/config/model-nofit/expected/choices.txt b/tests/fixtures/eas/config/model-nofit/expected/choices.txt index 99d516d1..18de13ca 100644 --- a/tests/fixtures/eas/config/model-nofit/expected/choices.txt +++ b/tests/fixtures/eas/config/model-nofit/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] --model qwen2-5-coder-7b -> llm_services_autodeploy_models: [qwen2-5-coder-7b] [intel/enterprise-agent-toolkit@036b662 config.yaml:32-35] diff --git a/tests/fixtures/eas/config/model-nofit/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/model-nofit/expected/k3d-gpu-overrides.yaml index 58481ff1..5195c268 100644 --- a/tests/fixtures/eas/config/model-nofit/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/model-nofit/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-7b"] diff --git a/tests/fixtures/eas/config/model-override/expected/choices.txt b/tests/fixtures/eas/config/model-override/expected/choices.txt index da2f2ad9..90b470f9 100644 --- a/tests/fixtures/eas/config/model-override/expected/choices.txt +++ b/tests/fixtures/eas/config/model-override/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] --model qwen2-5-coder-14b -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 config.yaml:32-35] diff --git a/tests/fixtures/eas/config/model-override/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/model-override/expected/k3d-gpu-overrides.yaml index 6b01df56..93d41d97 100644 --- a/tests/fixtures/eas/config/model-override/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/model-override/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/fixtures/eas/config/no-nri/expected/choices.txt b/tests/fixtures/eas/config/no-nri/expected/choices.txt index 22ef51cf..6e426488 100644 --- a/tests/fixtures/eas/config/no-nri/expected/choices.txt +++ b/tests/fixtures/eas/config/no-nri/expected/choices.txt @@ -12,6 +12,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] pin ei main (profile main) -> repos.inference.yaml inference rev: 3e8a24486b6ee812adf2bf452f8ba5751c16c8bd [intel/enterprise-ai-solutions@97f6157 configs/repos/repos.agent-toolkit.yaml:12] WARN: node has 32 CPUs, under EAT's 48-core minimum -> continuing [intel/enterprise-agent-toolkit@036b662 docs/prerequisites.md:7] diff --git a/tests/fixtures/eas/config/no-nri/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/no-nri/expected/k3d-gpu-overrides.yaml index 463cc163..44b9acec 100644 --- a/tests/fixtures/eas/config/no-nri/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/no-nri/expected/k3d-gpu-overrides.yaml @@ -7,4 +7,5 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" diff --git a/tests/fixtures/eas/config/rc1-profile/expected/choices.txt b/tests/fixtures/eas/config/rc1-profile/expected/choices.txt index 2ed5e56b..10045d3b 100644 --- a/tests/fixtures/eas/config/rc1-profile/expected/choices.txt +++ b/tests/fixtures/eas/config/rc1-profile/expected/choices.txt @@ -13,6 +13,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 512 GiB; qwen2-5-coder-14b needs 131 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/rc1-profile/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/rc1-profile/expected/k3d-gpu-overrides.yaml index 5cbd2a2e..41a58e48 100644 --- a/tests/fixtures/eas/config/rc1-profile/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/rc1-profile/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/fixtures/eas/config/rootless-docker/expected/choices.txt b/tests/fixtures/eas/config/rootless-docker/expected/choices.txt index fc88a536..51b8c13a 100644 --- a/tests/fixtures/eas/config/rootless-docker/expected/choices.txt +++ b/tests/fixtures/eas/config/rootless-docker/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 125.3 GiB; qwen2-5-coder-14b needs 131 GiB; qwen2-5-coder-7b: fit: Qwen/Qwen2.5-Coder-7B-Instruct does not fit this host: 6.1 GiB free for the KV cache, needs 7.3 GiB; llama-3-1-8b needs 99 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [llama-3-1-8b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/rootless-docker/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/rootless-docker/expected/k3d-gpu-overrides.yaml index 07011ae2..23eee106 100644 --- a/tests/fixtures/eas/config/rootless-docker/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/rootless-docker/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["llama-3-1-8b"] diff --git a/tests/fixtures/eas/config/stock-k3s-model/expected/choices.txt b/tests/fixtures/eas/config/stock-k3s-model/expected/choices.txt index 40eab205..33cfa642 100644 --- a/tests/fixtures/eas/config/stock-k3s-model/expected/choices.txt +++ b/tests/fixtures/eas/config/stock-k3s-model/expected/choices.txt @@ -12,6 +12,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] --model qwen3-coder-30b -> llm_services_autodeploy_models: [qwen3-coder-30b] [intel/enterprise-inference@3e8a244 roles/llm_services/defaults/main.yaml:84] pin ei main (profile main) -> repos.inference.yaml inference rev: 3e8a24486b6ee812adf2bf452f8ba5751c16c8bd [intel/enterprise-ai-solutions@97f6157 configs/repos/repos.agent-toolkit.yaml:12] diff --git a/tests/fixtures/eas/config/stock-k3s-model/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/stock-k3s-model/expected/k3d-gpu-overrides.yaml index de8bad5c..c261a081 100644 --- a/tests/fixtures/eas/config/stock-k3s-model/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/stock-k3s-model/expected/k3d-gpu-overrides.yaml @@ -7,5 +7,6 @@ nfd_enabled: true kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" llm_services_autodeploy_models: ["qwen3-coder-30b"] diff --git a/tests/fixtures/eas/config/tiny-host/expected/choices.txt b/tests/fixtures/eas/config/tiny-host/expected/choices.txt index 8280327e..2b88825b 100644 --- a/tests/fixtures/eas/config/tiny-host/expected/choices.txt +++ b/tests/fixtures/eas/config/tiny-host/expected/choices.txt @@ -14,6 +14,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 30 GiB; qwen2-5-coder-14b needs 131 GiB; qwen2-5-coder-7b needs 99 GiB; llama-3-1-8b needs 99 GiB -> llm_services_autodeploy_models: [] (deploy one later with ./model-manager deploy ) [intel/enterprise-agent-toolkit@036b662 config.yaml:32-35] diff --git a/tests/fixtures/eas/config/tiny-host/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/tiny-host/expected/k3d-gpu-overrides.yaml index 5742b8f0..9ae6c264 100644 --- a/tests/fixtures/eas/config/tiny-host/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/tiny-host/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: [] diff --git a/tests/fixtures/eas/config/xeon-amx-external/expected/choices.txt b/tests/fixtures/eas/config/xeon-amx-external/expected/choices.txt index e1c50153..8227e246 100644 --- a/tests/fixtures/eas/config/xeon-amx-external/expected/choices.txt +++ b/tests/fixtures/eas/config/xeon-amx-external/expected/choices.txt @@ -13,6 +13,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 512 GiB; qwen2-5-coder-14b needs 131 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/xeon-amx-external/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/xeon-amx-external/expected/k3d-gpu-overrides.yaml index 5cbd2a2e..41a58e48 100644 --- a/tests/fixtures/eas/config/xeon-amx-external/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/xeon-amx-external/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] From 0593ef5f4d03d489e3d69ea1c1b922698a7fe4e7 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sun, 1 Nov 2026 01:53:08 -0700 Subject: [PATCH 18/19] eas: test that eas up stops when EI hands the master key over under another name (failing) key_argv_guard only looked at lines starting with MM_LITELLM_MASTER_KEY:. An EI that reads the Secret into another registered variable and passes it as, say, MM_LITELLM_KEY would put the key back on the deploy's /bin/sh -c command line, and the guard would let the install go ahead. The new case renames both and expects the install to stop before it starts. The existing case's expected message now names the master key in general. --- tests/eas_test.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/eas_test.sh b/tests/eas_test.sh index 40034dac..f8571596 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -1196,12 +1196,26 @@ print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox : > "${LOG}" out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? - if [ "${rc}" -ne 0 ] && grep -q 'no longer takes MM_LITELLM_MASTER_KEY from _deploy_litellm_secret' <<< "${out}"; then + if [ "${rc}" -ne 0 ] && grep -q 'no longer takes the master key from _deploy_litellm_secret' <<< "${out}"; then ok "cli: eas.sh stops when EI's deploy takes the key from elsewhere" else fail "cli: EI deploy taking the key from elsewhere: rc=${rc} ${out}" fi cli_lacks "cli: ...before install" "$(cat "${LOG}")" "installer install" + # The same under another variable name: the master key from a Secret read the + # overrides do not empty, handed over as MM_LITELLM_KEY. + command cp -f "${ROOT}/tests/fixtures/eas/argv/deploy-models.yml" "${DM}" + sed -i -e 's/register: _deploy_litellm_secret$/register: _mm_litellm_secret/' \ + -e 's/^\( *\)MM_LITELLM_MASTER_KEY:.*/\1MM_LITELLM_KEY: "{{ _mm_litellm_secret.resources[0].data.master_key | b64decode }}"/' "${DM}" + : > "${LOG}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'no longer takes the master key from _deploy_litellm_secret' <<< "${out}"; then + ok "cli: eas.sh stops when EI's deploy hands the key over under another variable name" + else + fail "cli: EI deploy handing the key over as MM_LITELLM_KEY: rc=${rc} ${out}" + fi + cli_lacks "cli: ...before install" "$(cat "${LOG}")" "installer install" rm -f "${DM}" out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? From 3bcb1676d2da6268da3b25a23f9bbd92da0036e7 Mon Sep 17 00:00:00 2001 From: Andrew Mello Date: Sun, 1 Nov 2026 01:53:13 -0700 Subject: [PATCH 19/19] eas: key_argv_guard checks every line that names the master key The guard now covers every non-comment line of EI's deploy_models.yaml that names master_key in any case: MM_LITELLM_MASTER_KEY, the Secret's .data.master_key and litellm_master_key. Each must take the key from _deploy_litellm_secret, the variable k3d-gpu-overrides.yaml empties. A shell's $MM_LITELLM_MASTER_KEY is left out because it reads the value the environment: line sets. EI at 3e8a244 and 170eb3a passes under gawk and under the bastion's mawk; the renamed variable from the new test case stops the install. --- integrations/eas/eas.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integrations/eas/eas.sh b/integrations/eas/eas.sh index 38cc3434..17a16291 100755 --- a/integrations/eas/eas.sh +++ b/integrations/eas/eas.sh @@ -150,10 +150,14 @@ key_argv_guard() { local f="${CHECKOUT}/ext/enterprise.ai-inference/roles/llm_services/tasks/deploy_models.yaml" [ -d "${CHECKOUT}/ext/enterprise.ai-inference" ] || return 0 [ -f "${f}" ] || err "EI has no roles/llm_services/tasks/deploy_models.yaml: check how it now hands model-manager the LiteLLM master key (eas-config.py argv_master_key)" - awk '/^[[:space:]]*MM_LITELLM_MASTER_KEY:/ { any = 1; if ($0 !~ /_deploy_litellm_secret/) bad = 1 } + # Every line naming the master key (MM_LITELLM_MASTER_KEY, the Secret's .data.master_key, + # litellm_master_key), under any variable name, must take it from _deploy_litellm_secret; + # a shell's $MM_LITELLM_MASTER_KEY reads the value that line sets. + awk '{ l = $0; gsub(/\$\{?MM_LITELLM_MASTER_KEY\}?/, "", l) } + tolower(l) ~ /master_key/ && l !~ /^[[:space:]]*#/ { any = 1; if (l !~ /_deploy_litellm_secret/) bad = 1 } /^[[:space:]]*register:[[:space:]]*_deploy_litellm_secret[[:space:]]*$/ { reg = 1 } END { exit (bad || (any && !reg)) ? 1 : 0 }' "${f}" \ - || err "EI's deploy_models.yaml no longer takes MM_LITELLM_MASTER_KEY from _deploy_litellm_secret alone: k3d-gpu-overrides.yaml would not keep the master key off the deploy's command line (eas-config.py argv_master_key)" + || err "EI's deploy_models.yaml no longer takes the master key from _deploy_litellm_secret alone: k3d-gpu-overrides.yaml would not keep the master key off the deploy's command line (eas-config.py argv_master_key)" } cmd_up() {