diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index d9ebe3c9..68e724cc 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -50,6 +50,9 @@ on: - 'oracle/validate.py' - 'oracle/share.py' - 'oracle/schema/bundle-v1.json' + - 'integrations/eas/**' + - 'integrations/eat/eat-check.sh' + - 'integrations/eat/upstream/patches/**' workflow_dispatch: inputs: mode: diff --git a/Formula/k3d-gpu.rb b/Formula/k3d-gpu.rb index 6a67c10d..ce0db496 100644 --- a/Formula/k3d-gpu.rb +++ b/Formula/k3d-gpu.rb @@ -21,6 +21,10 @@ def install (pkgshare/"bench/apb").install "bench/apb/apb.py", "bench/apb/capacity.py" (pkgshare/"oracle").install "oracle/bundle.py", "oracle/project.py", "oracle/validate.py", "oracle/share.py" (pkgshare/"oracle/schema").install "oracle/schema/bundle-v1.json" + # k3d-gpu eas, its overlay's patch drafts and eat-check.sh, in the repo's integrations/ layout. + (pkgshare/"integrations/eas").install Dir["integrations/eas/*"] + (pkgshare/"integrations/eat/upstream/patches").install Dir["integrations/eat/upstream/patches/*.patch"] + (pkgshare/"integrations/eat").install "integrations/eat/eat-check.sh" # The launcher reads its manifests from K3D_GPU_SHARE (default /usr/share/k3d-gpu); # k3d-gpu-mcp runs the launcher next to it in libexec and passes its environment on. (bin/"k3d-gpu").write_env_script libexec/"k3d-gpu", K3D_GPU_SHARE: "${K3D_GPU_SHARE:-#{pkgshare}}" diff --git a/README.md b/README.md index 9231b017..7b7cbb96 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ context, so `up` switches that file's current-context to `k3d-`. ### Install Every package installs the same payload: the launcher at `bin/k3d-gpu`, the MCP server at `bin/k3d-gpu-mcp`, -and in `share/k3d-gpu/` the plugin manifests, `console/`, `bench/`, `oracle/` and `images.lock` +and in `share/k3d-gpu/` the plugin manifests, `console/`, `bench/`, `oracle/`, `integrations/` (for `k3d-gpu eas`) and `images.lock` (`facts`, `plan`, `audit`, `tune`, `capacity` and `k3d-gpu-mcp` also need `python3`). The node image `cryptoandcoffee/k3d-gpu` is published for linux/amd64 only, so on arm64 the packages install but `k3d-gpu up` stops unless `K3D_GPU_IMAGE` names a k3s node image built for diff --git a/docs/intel/eas.md b/docs/intel/eas.md index 0bf4b4b1..56c5a52b 100644 --- a/docs/intel/eas.md +++ b/docs/intel/eas.md @@ -67,6 +67,10 @@ k3d-gpu eas down # EAS teardown; the cluster stays (k3d-gpu down removes i - Arguments after `--` go to `ansible-playbook`, for example `k3d-gpu eas up --agent-toolkit -- -e '{"llm_services_autodeploy_models": []}'` to install without EAT's chat model. +- `eas down` gives the teardown the same image pins and overrides as `eas up` gave the + install. EAS picks teardown components by the same `enabled` flags as install + ([resolve_components.py](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/filter_plugins/resolve_components.py#L245)), + so components `eas up` left to k3d-gpu, such as its NFD, stay. What `eas up` does, in order: @@ -214,6 +218,7 @@ Bastion tools (each binary checked against its upstream checksum file at build t | kubectl | v1.37.2 | the minor of the node's k3s (`K3S_TAG` v1.37.1-k3s1); EAS's own v1.34.3 ([#L52](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L52)) is three minors off, outside the [kubectl skew policy](https://kubernetes.io/releases/version-skew-policy/#kubectl) | | helm | v3.22.1 | latest v3; EAS's floor is v3.20.2 ([#L53](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L53)). Helm v4 is untested with `kubernetes.core` 6.3.0 | | yq | v4.54.1 | EAS's floor is v4.53.2 ([#L51](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L51)) | +| jq | jq-1.8.2 | latest; enterprise-inference pipes yq into jq on the installer host to pick the models to deploy ([deploy_models.yaml#L108-L114](https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/deploy_models.yaml#L108-L114)), and EAS's `configure` installs no jq | | ansible, kubernetes (Python) | 12.3.0, 35.0.0, hashed | newest inside EAS's ranges `ansible>=10.0,<13.0`, `kubernetes>=29.0,<36.0` ([requirements.txt](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/requirements.txt)) | | kubernetes.core | 6.3.0 | EAS's [collections/requirements.yml](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/collections/requirements.yml) | diff --git a/hack/release-paths b/hack/release-paths index a3bc94e3..e167d362 100644 --- a/hack/release-paths +++ b/hack/release-paths @@ -27,3 +27,6 @@ oracle/project.py oracle/validate.py oracle/share.py oracle/schema/bundle-v1.json +integrations/eas/** +integrations/eat/eat-check.sh +integrations/eat/upstream/patches/** diff --git a/images.lock b/images.lock index 5052f203..af22ab99 100644 --- a/images.lock +++ b/images.lock @@ -166,6 +166,7 @@ "name": "docker.io/library/ubuntu", "sources": [ "Dockerfile", + "integrations/eas/bastion.Dockerfile", "integrations/eat/bastion.Dockerfile" ], "tag": "24.04" @@ -413,6 +414,14 @@ ], "tag": "v1.0.8" }, + { + "digest": "sha256:bbacb1409a2ec436ee8cecdfb50f626fe66eae35adbf2886df255cbf5a2b10df", + "name": "registry.k8s.io/agent-sandbox/python-runtime-sandbox", + "sources": [ + "integrations/eas/images.yaml" + ], + "tag": "v0.5.0" + }, { "digest": "sha256:41da84809426098cb04cdafbb41855f15e3604f7455ef53816bad666e532e787", "name": "registry.k8s.io/agent-sandbox/python-runtime-sandbox", @@ -429,6 +438,14 @@ ], "tag": "v0.20.1" }, + { + "digest": "sha256:9cebf7d97f98c0ce008aec8524f23637e60b61485db18ad426897eefac156968", + "name": "us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router", + "sources": [ + "integrations/eas/images.yaml" + ], + "tag": "latest-main" + }, { "digest": "sha256:5d064465d751a73a729e7ecdc32c0ba5de0df1d305debb60a3d9a85b4fd3a967", "name": "vault.habana.ai/gaudi-docker/1.24.2/ubuntu24.04/habanalabs/vllm-0.30.0-ptupstream-2.13.0", diff --git a/integrations/eas/README.md b/integrations/eas/README.md new file mode 100644 index 00000000..75a712c8 --- /dev/null +++ b/integrations/eas/README.md @@ -0,0 +1,77 @@ +# Intel Enterprise AI Solutions on k3d-gpu + +`k3d-gpu eas` installs Intel's [Enterprise AI Solutions](https://github.com/intel/enterprise-ai-solutions) +(EAS) on a k3d-gpu cluster with EAS's own installer, `es_auto_installer.sh`, in its +bring-your-own-cluster mode (`existing_kubernetes`). The layers are platform, inference +([enterprise-inference](https://github.com/intel/enterprise-inference), EI) and agent-toolkit +([enterprise-agent-toolkit](https://github.com/intel/enterprise-agent-toolkit) 2.0, EAT). +They run on the Xeon CPU path: `kubernetes_accelerator: cpu` is the only value EAS acts on at +the pinned commit. Gaudi and Arc stay with `k3d-gpu agent up`. + +```bash +k3d-gpu eas up --dry-run --agent-toolkit # facts, choices and overlay decisions; changes nothing +k3d-gpu eas up --agent-toolkit # creates the cluster first if it is absent +k3d-gpu eas check # chat, tool call and a sandbox run through EAS's LiteLLM +k3d-gpu eas status +k3d-gpu eas down # teardown platform; the cluster stays +``` + +`eas up` without `--agent-toolkit` installs platform and inference. `--layer erag` reaches +`eas-config.py`, which refuses it with the reason. + +## What `eas up` does + +1. Reads facts, read-only, into `facts.json` (schema `k3d-gpu.eas-facts/v1`, the one + `eas-config.py` documents): nodes, namespaces, DaemonSets, the agent-sandbox CRD and its + controller image, the server version, rootless docker, the node container's NRI socket + and containerd CNI `bin_dirs` (`docker exec ... test -S` and `crictl info`), MemAvailable + and inotify limits, and `k3d-gpu fit` of each of `eas-config.py`'s model candidates (vLLM + on the CPU). +2. Resolves refs: a profile of `pins.json` (`--eas-ref v1.2.0-rc.1`) and a full SHA go to + `eas-config.py` as given; any other `--eas-ref`, `--ei-ref` or `--eat-ref` becomes a + commit through `git ls-remote` here, once, and is printed. +3. `eas-config.py` writes the overrides, `global_config.edit.yaml`, `models.edit.yaml`, + `repos..yaml` and `choices.txt`, one `fact -> choice [source]` line per decision. + Exit 3 is a refusal: nothing is changed. The commits it resolved (the `# eas|ei|eat` + lines of `choices.txt`) go to `pins.resolved.json`. +4. Builds the bastion (`bastion.Dockerfile`), tagged by the sha256 of the Dockerfile and + `requirements.lock`, or reuses it. +5. Runs `eas.sh up ` in the bastion: clone EAS and check out the commit, apply the + patch overlay (`overlay.py --apply`, with `kubernetes_cpu_policy` from the overrides and + the node count), set each manifest rev from `repos..yaml`, + `init --env k3d-gpu`, merge the two edit files into `env/k3d-gpu/`, then + `install --env k3d-gpu -- -e @k3d-gpu-images.yaml -e @k3d-gpu-overrides.yaml + [your args]`. Extra vars load last in `es_auto_installer.sh` `main()`, so the overrides + win over every config file. `configure` is never run. +6. Prints the gateway: the LoadBalancer IP of eg-gateway's Service (k3s ServiceLB, the node + IP), and with the agent toolkit a `curl --resolve litellm.:443:` + line. `/etc/hosts` is not edited. + +The bastion runs on the host network with the state dir read-write at `/eas-state`, a +kubeconfig holding this cluster's context alone read-only at `/kube/config`, and +`integrations/` read-only at `/k3d-gpu/integrations` (overlay.py finds its patch drafts +relative to the dir above `integrations/`). It runs as root: `es_auto_installer.sh` +`ensure_sudo` runs `sudo -k` under `set -e` unless EUID is 0, and the image has no sudo. +`eas.sh` hands the state dir back to your uid:gid when it exits. `HF_TOKEN` and the proxy +variables reach it by name only, when set. + +State per cluster: `~/.local/state/k3d-gpu/eas-/` (mode 0700): the EAS checkout with +its `env/k3d-gpu/` and `ext/`, `facts.json`, `pins.resolved.json`, `eas-config.py`'s files, +`overlay.txt` and `overlay-decisions.json`, `kubeconfig`, `layer` and `logs/`. + +## Files + +| File | Role | +|------|------| +| `eas.sh` | the in-bastion driver (`up`, `overlay-check`, `down`, `status`) | +| `bastion.Dockerfile` | Ubuntu 24.04 (the EAT bastion's digest), kubectl v1.37.2, helm v3.22.1, yq v4.54.1, jq-1.8.2 by sha256, the installer's venv at `/opt/eas-venv`, kubernetes.core 6.3.0 by sha256: `pins.json`'s bastion profile | +| `requirements.in`, `requirements.lock`, `lock.sh` | EAS `requirements.txt` at the pinned commit, and its hash-locked resolution for the bastion's Python 3.12 | +| `images.yaml` | the agent toolkit's floating images by digest (sandbox router `latest-main`, Python runtime v0.5.0) | +| `known-issues.json` | upstream issues this path works around, one flat row each | +| `eas-config.py`, `pins.json` | the choice generator and the pins it reads | +| `overlay.py`, `overlay.json`, `patches/` | the k3s/k3d patch overlay | + +## Not verified + +No EAS install has run on a k3d-gpu cluster from this code. The agent-toolkit layer has not +run on k3s or k3d; Kubernetes 1.37 is outside what EAS tests (it pins kubectl v1.34.3). diff --git a/integrations/eas/bastion.Dockerfile b/integrations/eas/bastion.Dockerfile new file mode 100644 index 00000000..9ed38e5d --- /dev/null +++ b/integrations/eas/bastion.Dockerfile @@ -0,0 +1,71 @@ +# Bastion for Intel's Enterprise AI Solutions installer (intel/enterprise-ai-solutions, +# es_auto_installer.sh) on a k3d-gpu cluster in its bring-your-own mode. +# +# Everything the installer's `configure` would install with sudo and apt is baked here +# at a pinned version, so `k3d-gpu eas up` never runs `configure`: +# - kubectl, helm, yq and jq from their release downloads, each checked by sha256. The +# sha256 values are the ones the upstream checksum files publish for these releases +# (dl.k8s.io .sha256, get.helm.sh .sha256sum, yq's `checksums` file, +# jq's sha256sum.txt). configure installs no jq, but enterprise-inference's +# roles/llm_services/tasks/deploy_models.yaml pipes yq into jq on the installer host. +# - the installer's Python requirements (requirements.txt at the EAS commit named in +# requirements.lock) as a hash-locked venv at /opt/eas-venv; the installer reuses a +# venv that has ansible-playbook (es_auto_installer.sh _ensure_venv). +# - kubernetes.core 6.3.0 (EAS collections/requirements.yml) by sha256, into that venv's +# purelib over the 6.2.0 copy the ansible 12.3.0 package bundles, which is where the +# installer's _ensure_collections looks; with 6.3.0 there it makes no network call. +# +# kubectl is v1.37.2, the minor of the node image's K3S_TAG (Dockerfile): EAS's own +# v1.34.3 is outside kubectl's +/-1 minor skew against a v1.37 API server +# (https://kubernetes.io/releases/version-skew-policy/#kubectl). helm is the newest v3 +# (EAS asks for >= v3.20.2); yq the newest v4 (EAS asks for >= v4.53.2); jq the newest. +# +# Runs as root: es_auto_installer.sh ensure_sudo runs `sudo -k` under `set -e` unless +# EUID is 0, and this image has no sudo. eas.sh hands the state dir back to the +# invoking user's uid:gid when it exits. +FROM ubuntu:24.04@sha256:fe81560990fae86dc4743202b2cb4d3df98d0bcf949173cecef15ce1127ef852 + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DEBIAN_FRONTEND=noninteractive + +# The state dir (the EAS checkout and its ext/ clones) belongs to the invoking user, and +# git refuses a repository another uid owns ("detected dubious ownership") unless told. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git python3 python3-venv \ + && rm -rf /var/lib/apt/lists/* \ + && git config --system --add safe.directory '*' + +ARG KUBECTL_VERSION=v1.37.2 +ARG KUBECTL_SHA256=c04487523e7aa7d1493f7808174c899db93d7a9cf57531420cd76b9bd415f8f4 +ARG HELM_VERSION=v3.22.1 +ARG HELM_SHA256=896cf63684a42a723d61c980369d6e588330927deb13676ca0d85006dd6da0ea +ARG YQ_VERSION=v4.54.1 +ARG YQ_SHA256=8e34fc298390875de416e6a4afcb8cabeceb25d9aa8506c1a2f9353cf702ea5f +ARG JQ_VERSION=jq-1.8.2 +ARG JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f +RUN curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ + && echo "${KUBECTL_SHA256} /usr/local/bin/kubectl" | sha256sum -c - \ + && curl -fsSL -o /tmp/helm.tgz "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ + && echo "${HELM_SHA256} /tmp/helm.tgz" | sha256sum -c - \ + && tar -xzf /tmp/helm.tgz -C /usr/local/bin --strip-components=1 linux-amd64/helm \ + && curl -fsSL -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ + && echo "${YQ_SHA256} /usr/local/bin/yq" | sha256sum -c - \ + && curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-amd64" \ + && echo "${JQ_SHA256} /usr/local/bin/jq" | sha256sum -c - \ + && chmod 0755 /usr/local/bin/kubectl /usr/local/bin/helm /usr/local/bin/yq /usr/local/bin/jq \ + && rm /tmp/helm.tgz + +COPY requirements.lock /opt/eas-requirements.lock +RUN python3 -m venv /opt/eas-venv \ + && /opt/eas-venv/bin/pip install --no-cache-dir --disable-pip-version-check --require-hashes --no-deps \ + -r /opt/eas-requirements.lock + +ARG KUBERNETES_CORE_VERSION=6.3.0 +ARG KUBERNETES_CORE_SHA256=86a2a89892b75120a32e58135ab96982e3c02ad9f6b59eb2412671952622e748 +RUN curl -fsSL -o /tmp/kubernetes-core.tar.gz \ + "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/kubernetes-core-${KUBERNETES_CORE_VERSION}.tar.gz" \ + && echo "${KUBERNETES_CORE_SHA256} /tmp/kubernetes-core.tar.gz" | sha256sum -c - \ + && purelib=$(/opt/eas-venv/bin/python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') \ + && /opt/eas-venv/bin/ansible-galaxy collection install --force --offline -p "${purelib}" /tmp/kubernetes-core.tar.gz \ + && rm /tmp/kubernetes-core.tar.gz + +WORKDIR /eas-state diff --git a/integrations/eas/eas-config.py b/integrations/eas/eas-config.py index 1e76cd82..f3c8ea60 100755 --- a/integrations/eas/eas-config.py +++ b/integrations/eas/eas-config.py @@ -107,6 +107,7 @@ "cluster_cidr": _K3S + "pkg/cli/cmds/server.go#L137-L141", "service_cidr": _K3S + "pkg/cli/cmds/server.go#L142-L146", "kata": _EAS + "roles/kata_containers/tasks/install.yaml#L9-L22", + "argv_master_key": _EI + "roles/llm_services/tasks/deploy_models.yaml#L240", "accel_cpu": _EAS + "configs/defaults/global_config.yaml#L106", "accel_xpu": _EAT + "README.md#L36", "sandbox": _EAT + "config.yaml#L56", @@ -432,6 +433,16 @@ def plan(facts, layer, pins, model_flag): p.choose("bring-your-own cluster", "kata_containers_enabled: false", "kata") p.set("kata_containers_enabled", False) + # EI's model deploy task hands model-manager the LiteLLM master key in its ansible + # environment:, which ansible writes into the /bin/sh -c command line it runs the + # deploy under, readable in `ps` for the whole deploy. An empty Secret read leaves + # MM_LITELLM_MASTER_KEY empty, and model-manager then reads the Secret itself. + p.choose("EI's model deploy passes the LiteLLM master key through ansible environment: " + "(a /bin/sh -c argument while each model deploys)", + "_deploy_litellm_secret: \"\" (model-manager reads Secret litellm/litellm-master-key itself)", + "argv_master_key") + p.set("_deploy_litellm_secret", "") + devices = [] for n in nodes: for res, val in sorted((n.get("allocatable") or {}).items()): diff --git a/integrations/eas/eas.sh b/integrations/eas/eas.sh new file mode 100755 index 00000000..17a16291 --- /dev/null +++ b/integrations/eas/eas.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# eas.sh — drives Intel's es_auto_installer.sh (intel/enterprise-ai-solutions) against a +# k3d-gpu cluster in its bring-your-own mode (existing_kubernetes). `k3d-gpu eas` runs it in +# the bastion (bastion.Dockerfile); `overlay-check` also runs on the host for --dry-run. +# +# eas.sh up [-- ] +# 1. clone EAS and check out EAS_SHA (a clean tree: earlier overlay hunks are dropped) +# 2. the patch overlay: overlay.py --apply, decisions in overlay.txt +# 3. the layer manifest's revs as commit SHAs (repos..yaml from eas-config.py) +# 4. es_auto_installer.sh init --env k3d-gpu; EI's model deploy must take the +# LiteLLM master key from the Secret read the overrides empty (key_argv_guard) +# 5. global_config.edit.yaml and models.edit.yaml (eas-config.py) into env/k3d-gpu/: +# keys merge; a model's env merges and its args_append follows its catalog args +# 6. es_auto_installer.sh install --env k3d-gpu -- -e @k3d-gpu-images.yaml +# -e @k3d-gpu-overrides.yaml [args]: extra vars load last in es_auto_installer.sh +# main(), so the overrides win +# eas.sh overlay-check steps 1-2 without applying: the overlay decisions only +# eas.sh down es_auto_installer.sh teardown platform --env k3d-gpu --force -- -e +# @k3d-gpu-images.yaml -e @k3d-gpu-overrides.yaml: install's extra +# vars, so teardown skips what install skipped (k3d-gpu's NFD) +# eas.sh status es_auto_installer.sh status --env k3d-gpu +# +# `configure` is never run: it needs a terminal or --force, sudo and apt +# (es_auto_installer.sh@97f6157 #L318-L351), and the bastion bakes everything it installs. +# The installer reuses a venv that has ansible-playbook (#L801-L806), so .venv links to the +# baked /opt/eas-venv, where it also finds kubernetes.core 6.3.0 (#L832-L847). +# +# Environment (set by `k3d-gpu eas`): +# EAS_STATE state dir: the EAS checkout and eas-config.py's files (default /eas-state) +# EAS_SHA the enterprise-ai-solutions commit (choices.txt names it) +# EAS_CPU_POLICY kubernetes_cpu_policy eas-config.py chose; EAS_NODES the node count (overlay.py) +# EAS_VENV the baked venv (default /opt/eas-venv) +# EAS_OWNER uid:gid that owns the state dir when this exits. The bastion runs as root: +# es_auto_installer.sh ensure_sudo runs `sudo -k` under set -e unless EUID +# is 0 (#L866-L875), and the image has no sudo. +# Values from eas-config.py's files reach yq through the environment (strenv), never as +# expression text. + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EAS_STATE="${EAS_STATE:-/eas-state}" +EAS_VENV="${EAS_VENV:-/opt/eas-venv}" +EAS_URL="https://github.com/intel/enterprise-ai-solutions" +ENV_NAME="k3d-gpu" +CHECKOUT="${EAS_STATE}/eas" + +err() { echo "ERROR: $*" >&2; exit 1; } +info() { echo "==> $*"; } + +own_state() { + [ -z "${EAS_OWNER:-}" ] || chown -R "${EAS_OWNER}" "${EAS_STATE}" 2>/dev/null || true +} +trap own_state EXIT + +# checkout: EAS at EAS_SHA with no local change to a tracked file and no untracked file +# but the ignored ones (env/, ext/, .kubespray/), so every run starts from upstream's tree. +checkout() { + [[ "${EAS_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || err "EAS_SHA must be a full commit SHA, got '${EAS_SHA:-}'" + if [ ! -d "${CHECKOUT}/.git" ]; then + info "Cloning ${EAS_URL} into ${CHECKOUT}" + git clone -q "${EAS_URL}" "${CHECKOUT}" + fi + git -C "${CHECKOUT}" cat-file -e "${EAS_SHA}^{commit}" 2>/dev/null || git -C "${CHECKOUT}" fetch -q --tags origin + git -C "${CHECKOUT}" checkout -q --force --detach "${EAS_SHA}" + git -C "${CHECKOUT}" clean -q -fd + info "EAS at ${EAS_SHA}" +} + +# overlay [--apply]: overlay.py decides each patch at the checkout's HEAD (and with --apply +# applies the chosen ones); the decisions go to overlay.txt. A STOP exits non-zero first. +overlay() { + [[ "${EAS_CPU_POLICY:-}" =~ ^[a-z-]+$ ]] || err "EAS_CPU_POLICY not set (kubernetes_cpu_policy from eas-config.py)" + [[ "${EAS_NODES:-}" =~ ^[0-9]+$ ]] || err "EAS_NODES not set (the cluster's node count)" + python3 "${HERE}/overlay.py" --eas "${CHECKOUT}" --policy "${EAS_CPU_POLICY}" --nodes "${EAS_NODES}" \ + --json "${EAS_STATE}/overlay-decisions.json" "$@" | tee "${EAS_STATE}/overlay.txt" +} + +# pin_manifest : each row of repos..yaml (layer, url, rev) sets rev on the +# manifest entry with that layer and url, which must exist once. A SHA is a rev +# _checkout_rev accepts (#L684-L689); EAS clones in full, so any published commit resolves. +pin_manifest() { + local m="${CHECKOUT}/configs/repos/repos.$1.yaml" f="${EAS_STATE}/repos.$1.yaml" L U R + [ -f "${f}" ] || err "${f} missing (eas-config.py writes it)" + [ -f "${m}" ] || err "EAS has no configs/repos/repos.$1.yaml at ${EAS_SHA}" + while IFS=$'\t' read -r L U R; do + [[ "${R}" =~ ^[0-9a-f]{40}$ ]] || err "repos.$1.yaml: rev for ${U} is not a commit SHA: '${R}'" + export L U R + yq -e '[.repos[] | select(.layer == strenv(L) and .url == strenv(U))] | length == 1' "${m}" >/dev/null \ + || err "configs/repos/repos.$1.yaml has no single entry for layer ${L}, ${U}" + yq -i '(.repos[] | select(.layer == strenv(L) and .url == strenv(U)) | .rev) = strenv(R)' "${m}" + done < <(yq -r '.repos[] | [.layer, .url, .rev] | @tsv' "${f}") +} + +# args_merge : the catalog entry's args with +# args_append after them, each --flag of args_append (and its value) taken out of the +# current args first, so a re-run (init never reseeds models.yaml) replaces rather than +# repeats it. Prints JSON. +args_merge() { + python3 -c 'import json, sys +cur, add = json.loads(sys.argv[1]), json.loads(sys.argv[2]) +valued = {t: i + 1 < len(add) and not add[i + 1].startswith("--") for i, t in enumerate(add) if t.startswith("--")} +out, skip = [], False +for t in cur: + if skip: + skip = False + elif t.split("=", 1)[0] in valued: + skip = valued[t.split("=", 1)[0]] and "=" not in t + else: + out.append(t) +print(json.dumps(out + add))' "$1" "$2" +} + +# env_edits: global_config.edit.yaml merged into env/k3d-gpu/global_config.yaml (where +# _resolve_kubeconfig reads existing_kubernetes, #L1342-L1358), and models.edit.yaml into +# env/k3d-gpu/models.yaml: defaults merge; each model is the catalog entry of its name, +# which must exist once: its env merges, and its args_append goes after its args. +env_edits() { + local env="${CHECKOUT}/env/${ENV_NAME}" F N A cur add + F="${EAS_STATE}/global_config.edit.yaml" + [ -f "${F}" ] || err "${F} missing (eas-config.py writes it)" + export F + yq -i '. *= load(strenv(F))' "${env}/global_config.yaml" + F="${EAS_STATE}/models.edit.yaml" + [ -f "${F}" ] || err "${F} missing (eas-config.py writes it)" + [ -f "${env}/models.yaml" ] || err "init seeded no env/${ENV_NAME}/models.yaml" + yq -i '.defaults *= load(strenv(F)).defaults' "${env}/models.yaml" + while IFS= read -r N; do + export N + yq -e '[.models[] | select(.name == strenv(N))] | length == 1' "${env}/models.yaml" >/dev/null \ + || err "env/${ENV_NAME}/models.yaml has no single catalog entry named ${N}" + yq -i '(.models[] | select(.name == strenv(N))).env *= ((load(strenv(F)).models[] | select(.name == strenv(N))).env // {})' \ + "${env}/models.yaml" + cur=$(yq -o=json -I=0 '(.models[] | select(.name == strenv(N))).args // []' "${env}/models.yaml") + add=$(yq -o=json -I=0 '(.models[] | select(.name == strenv(N))).args_append // []' "${F}") + A=$(args_merge "${cur}" "${add}") + export A + yq -i '(.models[] | select(.name == strenv(N))).args = (strenv(A) | from_yaml)' "${env}/models.yaml" + done < <(yq -r '.models[].name' "${F}") +} + +installer() { (cd "${CHECKOUT}" && ./es_auto_installer.sh "$@"); } + +# key_argv_guard: EI's model deploy hands model-manager the LiteLLM master key as +# MM_LITELLM_MASTER_KEY in its ansible environment:, which ansible writes into the deploy's +# /bin/sh -c command line. k3d-gpu-overrides.yaml empties the Secret read that value comes +# from (_deploy_litellm_secret, eas-config.py argv_master_key); an EI that takes the key +# from anything else would put it back on that command line, so it stops the install. +key_argv_guard() { + local f="${CHECKOUT}/ext/enterprise.ai-inference/roles/llm_services/tasks/deploy_models.yaml" + [ -d "${CHECKOUT}/ext/enterprise.ai-inference" ] || return 0 + [ -f "${f}" ] || err "EI has no roles/llm_services/tasks/deploy_models.yaml: check how it now hands model-manager the LiteLLM master key (eas-config.py argv_master_key)" + # Every line naming the master key (MM_LITELLM_MASTER_KEY, the Secret's .data.master_key, + # litellm_master_key), under any variable name, must take it from _deploy_litellm_secret; + # a shell's $MM_LITELLM_MASTER_KEY reads the value that line sets. + awk '{ l = $0; gsub(/\$\{?MM_LITELLM_MASTER_KEY\}?/, "", l) } + tolower(l) ~ /master_key/ && l !~ /^[[:space:]]*#/ { any = 1; if (l !~ /_deploy_litellm_secret/) bad = 1 } + /^[[:space:]]*register:[[:space:]]*_deploy_litellm_secret[[:space:]]*$/ { reg = 1 } + END { exit (bad || (any && !reg)) ? 1 : 0 }' "${f}" \ + || err "EI's deploy_models.yaml no longer takes the master key from _deploy_litellm_secret alone: k3d-gpu-overrides.yaml would not keep the master key off the deploy's command line (eas-config.py argv_master_key)" +} + +cmd_up() { + local layer=${1:-} + [ -n "${layer}" ] || err "usage: eas.sh up [-- ]" + shift + [ "${1:-}" != -- ] || shift + checkout + overlay --apply + pin_manifest "${layer}" + ln -sfn "${EAS_VENV}" "${CHECKOUT}/.venv" + if [ -f "${CHECKOUT}/env/${ENV_NAME}/.solutions.yaml" ]; then + # --upgrade moves already-cloned ext/ repos onto the manifest's new revs (never + # rewriting a config); without it a changed pin is refused (#L910-L940). + installer init "${layer}" --env "${ENV_NAME}" --upgrade + else + installer init "${layer}" --env "${ENV_NAME}" + fi + key_argv_guard + env_edits + installer install "${layer}" --env "${ENV_NAME}" -- \ + -e "@${EAS_STATE}/k3d-gpu-images.yaml" -e "@${EAS_STATE}/k3d-gpu-overrides.yaml" "$@" +} + +cmd_down() { + [ -d "${CHECKOUT}/env/${ENV_NAME}" ] || err "no env/${ENV_NAME} in ${CHECKOUT}: nothing was installed from this state dir" + # Teardown picks its components by the same enabled flags as install + # (filter_plugins/resolve_components.py), so it gets install's extra vars. Without the + # overrides, nfd_enabled is true again and EI's nfd teardown deletes k3d-gpu's own NFD. + [ -f "${EAS_STATE}/k3d-gpu-overrides.yaml" ] \ + || err "${EAS_STATE}/k3d-gpu-overrides.yaml missing: without it teardown also removes what eas up left to k3d-gpu (its NFD); run 'k3d-gpu eas up --dry-run' to write it again" + local vars=() + [ ! -f "${EAS_STATE}/k3d-gpu-images.yaml" ] || vars+=(-e "@${EAS_STATE}/k3d-gpu-images.yaml") + vars+=(-e "@${EAS_STATE}/k3d-gpu-overrides.yaml") + ln -sfn "${EAS_VENV}" "${CHECKOUT}/.venv" + # teardown's confirm refuses a non-terminal stdin without --force (#L129-L137, #L1486). + installer teardown platform --env "${ENV_NAME}" --force -- "${vars[@]}" +} + +cmd_status() { + [ -d "${CHECKOUT}/env/${ENV_NAME}" ] || err "no env/${ENV_NAME} in ${CHECKOUT}: run 'k3d-gpu eas up' first" + ln -sfn "${EAS_VENV}" "${CHECKOUT}/.venv" + installer status --env "${ENV_NAME}" +} + +sub=${1:-} +[ $# -eq 0 ] || shift +case "${sub}" in + up) cmd_up "$@" ;; + overlay-check) checkout; overlay ;; + down) cmd_down ;; + status) cmd_status ;; + *) err "usage: eas.sh up [-- args] | overlay-check | down | status" ;; +esac diff --git a/integrations/eas/images.yaml b/integrations/eas/images.yaml new file mode 100644 index 00000000..a97c86ba --- /dev/null +++ b/integrations/eas/images.yaml @@ -0,0 +1,10 @@ +# Extra vars for es_auto_installer.sh install: the agent toolkit's floating images by +# registry digest. `k3d-gpu eas up` passes this file before k3d-gpu-overrides.yaml, so a +# value in the overrides still wins. EAT release-2.0.0 (036b662) roles/agent_sandbox/ +# defaults/main.yaml#L27-L28 names the router by the moving tag latest-main and the +# runtime by tag only; its install.yaml substitutes these values whole into the router +# manifest (${ROUTER_IMAGE}) and the default SandboxTemplate, so a name:tag@sha256 +# reference reaches the pods as written. Digests read from the registries 2026-10-29 +# (scripts/images-lock digest); scripts/images-lock repin moves them. +agent_sandbox_router_image: "us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router:latest-main@sha256:9cebf7d97f98c0ce008aec8524f23637e60b61485db18ad426897eefac156968" +agent_sandbox_python_runtime_image: "registry.k8s.io/agent-sandbox/python-runtime-sandbox:v0.5.0@sha256:bbacb1409a2ec436ee8cecdfb50f626fe66eae35adbf2886df255cbf5a2b10df" diff --git a/integrations/eas/known-issues.json b/integrations/eas/known-issues.json new file mode 100644 index 00000000..ff341b5b --- /dev/null +++ b/integrations/eas/known-issues.json @@ -0,0 +1,53 @@ +{ + "schema": "k3d-gpu.eas-known-issues/v1", + "issues": [ + { + "id": "eas-teardown-aigatewayroute-finalizer", + "upstream": "https://github.com/intel/enterprise-ai-solutions/issues/71", + "symptom": "teardown platform hangs at 'envoy_ai_gateway | Uninstall CRDs Helm release' while a model's AIGatewayRoute still carries aigateway.envoyproxy.io/finalizer: the role strips finalizers only after the controller and the CRDs are gone", + "workaround": "k3d-gpu eas down sets the finalizers of every AIGatewayRoute to [] before it runs teardown platform --force (the issue's own workaround)", + "source": "https://github.com/intel/enterprise-ai-solutions/issues/71 (open); https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/envoy_ai_gateway/tasks/teardown.yaml", + "checked": "2026-10-29" + }, + { + "id": "ei-require-amx-pending", + "upstream": "https://github.com/intel/enterprise-inference", + "symptom": "model pods stay Pending on a node without feature.node.kubernetes.io/cpu-cpuid.AMXBF16=true: models.yaml defaults.require_amx is true and the renderer turns it into a required node affinity on that label", + "workaround": "eas-config.py writes defaults.require_amx to env/k3d-gpu/models.yaml from the nodes' AMXBF16 label (false without it)", + "source": "https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/model_manager/models.yaml#L20; https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/model_manager/core/renderer.py#L90-L104", + "checked": "2026-10-29" + }, + { + "id": "eas-kubectl-skew", + "upstream": "https://github.com/intel/enterprise-ai-solutions", + "symptom": "es_auto_installer.sh configure installs kubectl v1.34.3, three minors behind a k3d-gpu cluster's v1.37 API server, outside kubectl's supported skew of one minor", + "workaround": "the bastion bakes kubectl v1.37.2, the minor of the node image's K3S_TAG, and configure is never run", + "source": "https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L52; https://kubernetes.io/releases/version-skew-policy/#kubectl", + "checked": "2026-10-29" + }, + { + "id": "eat-inotify-minimum", + "upstream": "https://github.com/intel/enterprise-agent-toolkit", + "symptom": "with fs.inotify.max_user_instances at the kernel default of 128, file watchers in the stack's pods fail with 'too many open files'", + "workaround": "eas up warns below 8192 instances or 1048576 watches and prints the sysctl lines; the host is never changed", + "source": "https://github.com/intel/enterprise-agent-toolkit/blob/036b66234801e9bd950afd90830821859780d668/docs/prerequisites.md#L20-L33", + "checked": "2026-10-29" + }, + { + "id": "eat-48-core-minimum", + "upstream": "https://github.com/intel/enterprise-agent-toolkit", + "symptom": "the agent toolkit's documented minimum is 48 cores (96+ recommended); smaller nodes run it with less CPU for the model than the defaults assume", + "workaround": "eas up warns when the node has fewer than 48 allocatable CPUs", + "source": "https://github.com/intel/enterprise-agent-toolkit/blob/036b66234801e9bd950afd90830821859780d668/docs/prerequisites.md#L7", + "checked": "2026-10-29" + }, + { + "id": "ei-skip-models-flag", + "upstream": "https://github.com/intel/enterprise-inference", + "symptom": "the inference docs say './es_auto_installer.sh install inference --skip-models', but es_auto_installer.sh rejects any option it does not know ('Unknown option: --skip-models')", + "workaround": "none needed on the inference layer: EI's seeded config.yaml already sets llm_services_deploy_models: false. On the agent toolkit, EAT's chat model deploys whatever that key says; 'k3d-gpu eas up --agent-toolkit -- -e '{\"llm_services_autodeploy_models\": []}'' skips it (the guard model stays while the guardrail is on)", + "source": "https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/config.yaml#L94-L98; https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/install.yaml#L143-L161; https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L1312", + "checked": "2026-10-29" + } + ] +} diff --git a/integrations/eas/lock.sh b/integrations/eas/lock.sh new file mode 100755 index 00000000..22e52257 --- /dev/null +++ b/integrations/eas/lock.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Regenerate integrations/eas/requirements.lock (hashed) from requirements.in, in the +# bastion's own base image (bastion.Dockerfile FROM: Ubuntu 24.04, Python 3.12), so the +# lock resolves for the interpreter that installs it. +# +# requirements.in is enterprise-ai-solutions' requirements.txt at the commit its first +# line names; copy the new file over it (keeping that line, with the new commit) when the +# EAS pin moves. pip-compile picks the newest release inside those ranges. +# +# Usage: integrations/eas/lock.sh + +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +IMAGE="$(sed -n 's/^FROM \(ubuntu:[^ ]*\).*$/\1/p' "${HERE}/bastion.Dockerfile")" +PIP_TOOLS="pip-tools==7.6.2" +header=$(head -1 "${HERE}/requirements.in") +[[ "${header}" =~ ^#\ enterprise-ai-solutions@[0-9a-f]{40}\ requirements\.txt$ ]] \ + || { echo "requirements.in line 1 must be '# enterprise-ai-solutions@ requirements.txt'" >&2; exit 1; } + +docker run --rm -v "${HERE}:/w" -w /w "${IMAGE}" sh -c \ + "apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends python3-venv >/dev/null && \ + python3 -m venv /tmp/v && /tmp/v/bin/pip install -q '${PIP_TOOLS}' && \ + /tmp/v/bin/pip-compile -q --upgrade --generate-hashes --strip-extras --allow-unsafe --no-emit-index-url \ + --output-file requirements.lock requirements.in && \ + { echo '${header}'; cat requirements.lock; } > /tmp/lock && cat /tmp/lock > requirements.lock && \ + chown $(id -u):$(id -g) requirements.lock" diff --git a/integrations/eas/overlay.py b/integrations/eas/overlay.py index 25c5fab7..90431fd4 100755 --- a/integrations/eas/overlay.py +++ b/integrations/eas/overlay.py @@ -155,7 +155,10 @@ def __init__(self, path, rev): self._text = {} self._tmp = tempfile.TemporaryDirectory(prefix="eas-overlay-") self.index = os.path.join(self._tmp.name, "index") - self.git("read-tree", "--index-output=" + self.index, self.sha) + # GIT_INDEX_FILE, not --index-output: git renames --index-output's file in from a + # lock next to the checkout's own index, so it must share the checkout's + # filesystem, and a tmpfs /tmp or the bastion's /tmp does not. + self.git("read-tree", self.sha, env={"GIT_INDEX_FILE": self.index}) def git(self, *args, env=None, check=True): e = dict(os.environ, **(env or {})) diff --git a/integrations/eas/pins.json b/integrations/eas/pins.json index 2303dedd..b1d5d529 100644 --- a/integrations/eas/pins.json +++ b/integrations/eas/pins.json @@ -6,7 +6,7 @@ "Each row: one pin. profile groups the rows; name is the component; sha is the commit the ref resolved to on the checked date (git ls-remote, gh api), or for a PyPI package the sha256 of its py3 wheel, or for an image its manifest digest; date is the commit, release or upload date; source is where the pin comes from.", "Profile main freezes what EAS main's configs/repos/repos.agent-toolkit.yaml resolves to today: inference rev main is enterprise-inference 3e8a244, and agent-toolkit rev agentic-toolkit-migration is fb594597, the same tree as release-2.0.0 036b662 (the #19 merge), which is pinned here.", "Profile v1.2.0-rc.1 is the release candidate's component table: https://github.com/intel/enterprise-ai-solutions/releases/tag/v1.2.0-rc.1", - "Profile bastion: the tools the EAS bastion bakes. kubectl matches the k3s minor in the Dockerfile's K3S_TAG (v1.37.1-k3s1); EAS's own v1.34.3 is outside kubectl's one-minor skew. helm is the newest v3 (EAS asks for at least v3.20.2); yq the newest (EAS: at least v4.53.2); kubernetes.core is EAS's exact pin; the Python packages are the newest inside EAS's requirements.txt ranges." + "Profile bastion: the tools the EAS bastion bakes. kubectl matches the k3s minor in the Dockerfile's K3S_TAG (v1.37.1-k3s1); EAS's own v1.34.3 is outside kubectl's one-minor skew. helm is the newest v3 (EAS asks for at least v3.20.2); yq the newest (EAS: at least v4.53.2); jq the newest (enterprise-inference's roles/llm_services/tasks/deploy_models.yaml runs it on the installer host, and EAS's configure installs none); kubernetes.core is EAS's exact pin; the Python packages are the newest inside EAS's requirements.txt ranges." ], "pins": [ {"profile": "main", "name": "eas", "repo": "intel/enterprise-ai-solutions", "ref": "main", "sha": "97f6157d6a72f7173c698c474d4e51d2c79f1fd6", "date": "2026-10-29", "source": "https://github.com/intel/enterprise-ai-solutions/commit/97f6157d6a72f7173c698c474d4e51d2c79f1fd6"}, @@ -19,6 +19,7 @@ {"profile": "bastion", "name": "kubectl", "repo": "kubernetes/kubernetes", "ref": "v1.37.2", "sha": "c50e5395f1ae2d4a2c05f28fdcc38c0916cc1a32", "date": "2026-10-28", "source": "https://github.com/kubernetes/kubernetes/releases/tag/v1.37.2"}, {"profile": "bastion", "name": "helm", "repo": "helm/helm", "ref": "v3.22.1", "sha": "d6c1a0e4027caa6abc7892e7ea9f0cda7447baa3", "date": "2026-10-15", "source": "https://github.com/helm/helm/releases/tag/v3.22.1"}, {"profile": "bastion", "name": "yq", "repo": "mikefarah/yq", "ref": "v4.54.1", "sha": "504fc38780cc46be8444ea1b72fb55919fc0bfb0", "date": "2026-09-29", "source": "https://github.com/mikefarah/yq/releases/tag/v4.54.1"}, + {"profile": "bastion", "name": "jq", "repo": "jqlang/jq", "ref": "jq-1.8.2", "sha": "34f7186b86743a083a589741b6cea95293524108", "date": "2026-06-20", "source": "https://github.com/jqlang/jq/releases/tag/jq-1.8.2"}, {"profile": "bastion", "name": "kubernetes.core", "repo": "ansible-collections/kubernetes.core", "ref": "6.3.0", "sha": "80c33c5573aa63c7fdb315c72d4d7bfac725c337", "date": "2026-02-03", "source": "https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/collections/requirements.yml#L3-L4"}, {"profile": "bastion", "name": "ansible", "repo": "pypi.org/project/ansible", "ref": "12.3.0", "sha": "cd156f82fa87f7a899212b0efa9f7ca3a24d609212de9f78428a572eb01e5414", "date": "2025-12-09", "source": "https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/requirements.txt#L1"}, {"profile": "bastion", "name": "kubernetes", "repo": "pypi.org/project/kubernetes", "ref": "35.0.0", "sha": "39e2b33b46e5834ef6c3985ebfe2047ab39135d41de51ce7641a7ca5b372a13d", "date": "2026-01-16", "source": "https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/requirements.txt#L2"}, diff --git a/integrations/eas/requirements.in b/integrations/eas/requirements.in new file mode 100644 index 00000000..f27c4fca --- /dev/null +++ b/integrations/eas/requirements.in @@ -0,0 +1,5 @@ +# enterprise-ai-solutions@97f6157d6a72f7173c698c474d4e51d2c79f1fd6 requirements.txt +ansible>=10.0,<13.0 +kubernetes>=29.0,<36.0 +jsonpatch>=1.33,<2.0 +jmespath>=1.0,<2.0 diff --git a/integrations/eas/requirements.lock b/integrations/eas/requirements.lock new file mode 100644 index 00000000..b84c2cea --- /dev/null +++ b/integrations/eas/requirements.lock @@ -0,0 +1,666 @@ +# enterprise-ai-solutions@97f6157d6a72f7173c698c474d4e51d2c79f1fd6 requirements.txt +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --output-file=requirements.lock --strip-extras requirements.in +# +ansible==12.3.0 \ + --hash=sha256:02721f6fb432ddd47f1044ac49b04b5e9ae08890bd427def8df3db1607aeec51 \ + --hash=sha256:cd156f82fa87f7a899212b0efa9f7ca3a24d609212de9f78428a572eb01e5414 + # via -r requirements.in +ansible-core==2.19.14 \ + --hash=sha256:1e7612788eaa6b3c87a3670ce86c64b830eba0ae2b3fcb25a772f276704163b4 \ + --hash=sha256:d518d0e96fa75bedfe9b0a96d5b158aa712d092961824c115140c8677b88ff06 + # via ansible +certifi==2026.10.23 \ + --hash=sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 + # via + # kubernetes + # requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +cryptography==50.0.2 \ + --hash=sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602 \ + --hash=sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2 \ + --hash=sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047 \ + --hash=sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c \ + --hash=sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42 \ + --hash=sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18 \ + --hash=sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51 \ + --hash=sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81 \ + --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 \ + --hash=sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2 \ + --hash=sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de \ + --hash=sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7 \ + --hash=sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd \ + --hash=sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2 \ + --hash=sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be \ + --hash=sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45 \ + --hash=sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0 \ + --hash=sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e \ + --hash=sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c \ + --hash=sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5 \ + --hash=sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452 \ + --hash=sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48 \ + --hash=sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05 \ + --hash=sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1 \ + --hash=sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93 \ + --hash=sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04 \ + --hash=sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e \ + --hash=sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67 \ + --hash=sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7 \ + --hash=sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107 \ + --hash=sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079 \ + --hash=sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134 \ + --hash=sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227 \ + --hash=sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1 \ + --hash=sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539 \ + --hash=sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e \ + --hash=sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d \ + --hash=sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c \ + --hash=sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd \ + --hash=sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020 \ + --hash=sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd \ + --hash=sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94 \ + --hash=sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a \ + --hash=sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408 \ + --hash=sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37 \ + --hash=sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e \ + --hash=sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454 \ + --hash=sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c \ + --hash=sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc \ + --hash=sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37 \ + --hash=sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767 \ + --hash=sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a \ + --hash=sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5 \ + --hash=sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc \ + --hash=sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67 \ + --hash=sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8 \ + --hash=sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480 \ + --hash=sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb \ + --hash=sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b + # via ansible-core +durationpy==0.11 \ + --hash=sha256:181898e1ae282e288f0a2291829656bf1b6b3aadf30a97993b85db4943642905 \ + --hash=sha256:a739fe2b8972c250ff72f8e2c488d18cf25f7b852f49ee76048775d5171df30c + # via kubernetes +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via requests +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via ansible-core +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via -r requirements.in +jsonpatch==1.38 \ + --hash=sha256:46260c5df952abd4734a098f94512dc00bc38539152ac3ef069ed0475e4f5695 \ + --hash=sha256:debc3e7bdc3d446468d64987f51a0e7a9deea42987ff90a6d1005a489b471c19 + # via -r requirements.in +jsonpointer==3.6 \ + --hash=sha256:82e839a3ba3a9e2b54394fda8b3d276c2459d80408d06ef9a6fbc369d7544ffc \ + --hash=sha256:a45c858f8a9e171f6b3790ed9446c9382782f6c4cb4589176813c5a553f4cfcd + # via jsonpatch +kubernetes==35.0.0 \ + --hash=sha256:39e2b33b46e5834ef6c3985ebfe2047ab39135d41de51ce7641a7ca5b372a13d \ + --hash=sha256:3d00d344944239821458b9efd484d6df9f011da367ecb155dadf9513f05f09ee + # via -r requirements.in +markupsafe==3.0.4 \ + --hash=sha256:007e1ffd9bf65bb6ee96df7b258fc632a4868dd5566037986c64781f35a36e98 \ + --hash=sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002 \ + --hash=sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b \ + --hash=sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653 \ + --hash=sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c \ + --hash=sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e \ + --hash=sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc \ + --hash=sha256:0764a13d34cae40db7bbf3a09b7e9b491bf4603e20b263a7a9d6b8e324975d0a \ + --hash=sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92 \ + --hash=sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f \ + --hash=sha256:0cee7cb0f9a1b6892ea482237d9403b3d1b4603aee057d0ff01f0fac2d019a97 \ + --hash=sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4 \ + --hash=sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7 \ + --hash=sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691 \ + --hash=sha256:14bd2d845d62ab678eaf81da89d7b621b51756c72346745c1a594c09d49207a2 \ + --hash=sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc \ + --hash=sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde \ + --hash=sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99 \ + --hash=sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9 \ + --hash=sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df \ + --hash=sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5 \ + --hash=sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17 \ + --hash=sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8 \ + --hash=sha256:2a6ef68ae94aed8721934072b27a3b654ea2100b97e4ab864cf1489c90926fbc \ + --hash=sha256:2b2b1e18af909b448bb3cf9e3433366f7a8726271fc214e8b10e0f62a78c724b \ + --hash=sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea \ + --hash=sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248 \ + --hash=sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741 \ + --hash=sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5 \ + --hash=sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6 \ + --hash=sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7 \ + --hash=sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1 \ + --hash=sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67 \ + --hash=sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f \ + --hash=sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9 \ + --hash=sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c \ + --hash=sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc \ + --hash=sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba \ + --hash=sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17 \ + --hash=sha256:3d23795802fc8bd72534836d64489bbf0f67c088959091bdb22e10735a5107bf \ + --hash=sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6 \ + --hash=sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2 \ + --hash=sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163 \ + --hash=sha256:4a540e2d3192792fc84eced57bef37851ccb2b41f73291bb17408eea77bcd278 \ + --hash=sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d \ + --hash=sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b \ + --hash=sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634 \ + --hash=sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38 \ + --hash=sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed \ + --hash=sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c \ + --hash=sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148 \ + --hash=sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a \ + --hash=sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7 \ + --hash=sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f \ + --hash=sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811 \ + --hash=sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e \ + --hash=sha256:57f9947a7e57a081c1e3e0a2dd0d2dcf290a4531450e6f611e30084c222a7295 \ + --hash=sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2 \ + --hash=sha256:5c22873ad1f0532ba40fa1727f3c0fc1bbbaab6d373d4cbe3f0dc74b2e2521c7 \ + --hash=sha256:5e8b3d0b18fd623afa12ecb2ce8d8becef69f9b5440c6330c7972200e0bb84b0 \ + --hash=sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6 \ + --hash=sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed \ + --hash=sha256:6669c1bf34080161ce49c589cc512ef24d4c704ac9d2b2d3667f519c60418378 \ + --hash=sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0 \ + --hash=sha256:6768d67d1bce64270e0fdc2e69309d68b9b18ae56ddf6c711d168e9d051c2cac \ + --hash=sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b \ + --hash=sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96 \ + --hash=sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59 \ + --hash=sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808 \ + --hash=sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2 \ + --hash=sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb \ + --hash=sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65 \ + --hash=sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72 \ + --hash=sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8 \ + --hash=sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e \ + --hash=sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91 \ + --hash=sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a \ + --hash=sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2 \ + --hash=sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e \ + --hash=sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707 \ + --hash=sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21 \ + --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef \ + --hash=sha256:8698d70a8081ee8c090dbb394768b5789a1da8b131b5499f89d071dd3cfaf6be \ + --hash=sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453 \ + --hash=sha256:88d59b473bfb03259722600839af9bbd7fa13a2eb514beefeedb95997882f69a \ + --hash=sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6 \ + --hash=sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977 \ + --hash=sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978 \ + --hash=sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581 \ + --hash=sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692 \ + --hash=sha256:9240187afb63d2f9ddc3e032c670356fe941f6e20662ea168a5dc3f1f317e1b3 \ + --hash=sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369 \ + --hash=sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a \ + --hash=sha256:9388003072b95f2f1e3fd908604194d653ba21330d811961a78b7da1a77e9e36 \ + --hash=sha256:9438a2648b2195980cb2dd8e53ed7b8df91319e2d0b70ae61a9e1d1bc8d3bec9 \ + --hash=sha256:94e4c421742086aeee4c32a506eec8859d7634aad943f7e6aacf70f813478768 \ + --hash=sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916 \ + --hash=sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b \ + --hash=sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f \ + --hash=sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346 \ + --hash=sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c \ + --hash=sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464 \ + --hash=sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9 \ + --hash=sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee \ + --hash=sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300 \ + --hash=sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6 \ + --hash=sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d \ + --hash=sha256:ac0c7c9f1609b0c4c114feb1d7a3409564c7fb77e360bed9e97e5d25dfeaf868 \ + --hash=sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46 \ + --hash=sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97 \ + --hash=sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733 \ + --hash=sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe \ + --hash=sha256:b61687d0828e72bf5cda24a2690188f37170bd31c9359ac97e4e66569f120a16 \ + --hash=sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429 \ + --hash=sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39 \ + --hash=sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894 \ + --hash=sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c \ + --hash=sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c \ + --hash=sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169 \ + --hash=sha256:bf053da3c97a4bc5ecfbb218cdd2983febd91c617be8367d139882aa11e490aa \ + --hash=sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77 \ + --hash=sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe \ + --hash=sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad \ + --hash=sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85 \ + --hash=sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e \ + --hash=sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34 \ + --hash=sha256:cf63c214fe879a65e69a386f915e36104fc84254ab141240f8854602d8e0be2a \ + --hash=sha256:d1aca03ede943eb80ab3d63bb082c84b7aab85ea83bd0fd0c200260945fb49d9 \ + --hash=sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c \ + --hash=sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749 \ + --hash=sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214 \ + --hash=sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932 \ + --hash=sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494 \ + --hash=sha256:dd8ea6ebee7aedbf7c749fa80521d9ccf1ba473e0d1e14805caafbaad281c889 \ + --hash=sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1 \ + --hash=sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0 \ + --hash=sha256:dff05cb7016dff1e9fd68f4122c127b65dfc59de5306cfb7ad92f956f230bee2 \ + --hash=sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786 \ + --hash=sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78 \ + --hash=sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e \ + --hash=sha256:e841068dc0be4cb6dfb5c890eb88cbdcff2f4a332393c7ec94e8e618bd32c1a8 \ + --hash=sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289 \ + --hash=sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c \ + --hash=sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe \ + --hash=sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237 \ + --hash=sha256:f291bcf42ae98eb5107edb162c3c998b4a89648fd8e99ed4cbd12705292788cd \ + --hash=sha256:f61efe1d2fe0de16158a5fe1d1cf3c14bdb6aecd54d8938fd26512c525c1f624 \ + --hash=sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19 \ + --hash=sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977 \ + --hash=sha256:fd9f8797427910198f95bced71ddfed61130d7e349213bfb8466c9c99e2c46a8 \ + --hash=sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3 + # via jinja2 +oauthlib==4.0.0 \ + --hash=sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd \ + --hash=sha256:efb274799819440f95b4ab3b818869f1ce9ae26c5beacba0201d1a1b76b54f86 + # via requests-oauthlib +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via ansible-core +pycparser==3.12 \ + --hash=sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e \ + --hash=sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6 + # via cffi +python-dateutil==3.0.0 \ + --hash=sha256:3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd \ + --hash=sha256:7ebcb3c29a92f3ceafe79a7a48a5c45677e9e004bbeec5742ac0bbb00560b7a2 + # via kubernetes +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via + # ansible-core + # kubernetes +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # kubernetes + # requests-oauthlib +requests-oauthlib==2.0.0 \ + --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ + --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 + # via kubernetes +resolvelib==1.2.1 \ + --hash=sha256:7d08a2022f6e16ce405d60b68c390f054efcfd0477d4b9bd019cc941c28fad1c \ + --hash=sha256:fb06b66c8da04172d9e72a21d7d06186d8919e32ae5ab5cdf5b9d920be805ac2 + # via ansible-core +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via + # kubernetes + # python-dateutil +tzdata==2026.6 \ + --hash=sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec \ + --hash=sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d + # via python-dateutil +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # kubernetes + # requests +websocket-client==1.9.2 \ + --hash=sha256:0fcb57545848be86992e128218fd96dd87a6769ffdb1a968dff79632b85604d0 \ + --hash=sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce + # via kubernetes diff --git a/integrations/eat/README.md b/integrations/eat/README.md index 3888cb52..942f4bf1 100644 --- a/integrations/eat/README.md +++ b/integrations/eat/README.md @@ -74,7 +74,7 @@ that off the host. | `guard-values.py` | with `deploy_guardrails=on`, puts the guard's `vllm/vllm-openai-cpu` image at `vllm-values.yaml`'s digest in a copy of EAT's charts; `brownfield.sh` passes that copy to `deploy-guardrails.yml` | | `gateway-values.yaml` | the gateway chart's pins: postgresql and redis chart versions, LiteLLM, PostgreSQL, Redis and init images by digest | | `gateway-trace-values.yaml` | the Langfuse trace stack's pins: Langfuse web and worker, PostgreSQL, Valkey, ClickHouse, ZooKeeper, MinIO by digest | -| `eat-check.sh` | the end-state check below | +| `eat-check.sh` | the end-state check below; with `EAT_MODE=eas` it checks EAS's agent toolkit instead (`k3d-gpu eas check`, see `../eas/README.md`) | `tests/eat_test.sh` runs `eat-check.sh` against stub `kubectl` and `curl` (no cluster) and checks that every script here is committed executable. diff --git a/integrations/eat/eat-check.sh b/integrations/eat/eat-check.sh index d8c8ec94..16b5b962 100755 --- a/integrations/eat/eat-check.sh +++ b/integrations/eat/eat-check.sh @@ -1,31 +1,46 @@ #!/usr/bin/env bash -# eat-check — end-to-end check of an EAT deployment made by eat-up.sh. +# eat-check — end-to-end check of an agent stack with Intel's agent toolkit: EAT 1.x made +# by eat-up.sh (EAT_MODE=eat), or EAS's agent toolkit made by `k3d-gpu eas up +# --agent-toolkit` (EAT_MODE=eas, which `k3d-gpu eas check` sets). # -# 1. chat: a chat completion through EAT's LiteLLM gateway, reached through -# EAT's ingress-nginx on the host IP of its controller pod (Host: cluster_url, TLS) +# 1. chat: a chat completion through the LiteLLM gateway. eat: EAT's ingress-nginx on +# the host IP of its controller pod (Host: cluster_url, TLS). eas: EAS's eg-gateway +# Service on its LoadBalancer IP (k3s ServiceLB: the node IP), Host +# litellm., TLS # 2. tool call: a get_weather tool request must come back as finish_reason=tool_calls -# 3. code run: the model writes Python through the gateway, and EAT's +# 3. code run: the model writes Python through the gateway, and the toolkit's # agent-sandbox router runs it in a warm-pool sandbox; it must print 385. # bash passes the model's text to the router: this is a scripted pipeline, -# not an agent loop. +# not an agent loop. The router is reached through a kubectl port-forward on a +# local port the kernel picks, and must answer GET /healthz as the router does. # -# EAT_WORK eat-up.sh work dir; its vault.yml holds the LiteLLM master key (default: ./eat-work) +# EAT_MODE eat (default) | eas +# EAT_WORK eat: eat-up.sh work dir; its vault.yml holds the LiteLLM master key (default: ./eat-work) +# EAT_EAS_DOMAIN eas: base_domain_name (default: solutions.ai, EAS's default) # EAT_KUBECONFIG kubeconfig of the cluster (default: $KUBECONFIG, else ~/.kube/config) -# EAT_MODEL model name registered in LiteLLM (default: Qwen/Qwen3-1.7B) +# EAT_MODEL model name registered in LiteLLM (eat default: Qwen/Qwen3-1.7B; eas default: +# the first chat model /v1/models lists, routers, guards, embedders and +# rerankers left out) # EAT_TEMPERATURE sampling temperature (default: 0). A remote endpoint from eat-remote.sh # on a shared fleet: 0.6, never 0 # EAT_MAX_TOKENS max_tokens per request (default: 200; 64 for a shared remote endpoint) +# +# eas: the requests carry a LiteLLM virtual key minted for this run with the master key +# (Secret litellm/litellm-master-key) through POST /key/generate, and deleted at exit +# (POST /key/delete). Keys reach curl as header files and request bodies on its stdin, +# and jq through its environment, never on an argument list. set -euo pipefail +EAT_MODE="${EAT_MODE:-eat}" EAT_WORK="${EAT_WORK:-${PWD}/eat-work}" +EAT_EAS_DOMAIN="${EAT_EAS_DOMAIN:-solutions.ai}" export KUBECONFIG="${EAT_KUBECONFIG:-${KUBECONFIG:-${HOME}/.kube/config}}" -EAT_MODEL="${EAT_MODEL:-Qwen/Qwen3-1.7B}" +EAT_MODEL="${EAT_MODEL:-}" EAT_TEMPERATURE="${EAT_TEMPERATURE:-0}" EAT_MAX_TOKENS="${EAT_MAX_TOKENS:-200}" CORE="${EAT_WORK}/eat/core" SANDBOX_NS=agent-sandbox-system -ROUTER_PORT=18080 err() { echo "ERROR: $*" >&2; exit 1; } info() { echo "==> $*"; } @@ -33,20 +48,67 @@ info() { echo "==> $*"; } [[ "${EAT_TEMPERATURE}" =~ ^[0-9]+([.][0-9]+)?$ ]] || err "EAT_TEMPERATURE must be a number, got '${EAT_TEMPERATURE}'" [[ "${EAT_MAX_TOKENS}" =~ ^[0-9]+$ ]] || err "EAT_MAX_TOKENS must be a whole number, got '${EAT_MAX_TOKENS}'" -host=$(grep -E '^cluster_url=' "${CORE}/inventory/agentic-config.cfg" | cut -d= -f2-) -key=$(grep -E '^litellm_master_key:' "${CORE}/inventory/metadata/vault.yml" | cut -d'"' -f2) -[ -n "${key}" ] || err "no litellm_master_key in ${CORE}/inventory/metadata/vault.yml" -# ingress-nginx binds hostPort 80/443 on the nodes its controller pods run on. -node_ip=$(kubectl -n ingress-nginx get pods \ - -l app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/component=controller \ - --field-selector=status.phase=Running -o jsonpath='{.items[0].status.hostIP}') -[ -n "${node_ip}" ] || err "no running ingress-nginx controller pod" +pf="" pflog="" vkey="" master="" alias="" +cleanup() { + [ -z "${pf}" ] || kill "${pf}" 2>/dev/null || true + [ -z "${pflog}" ] || rm -f "${pflog}" + # The run's virtual key goes when the check ends, pass or fail. + if [ -n "${vkey}" ]; then + # The key reaches jq through its environment, not its argument list. + VKEY="${vkey}" jq -n '{keys: [env.VKEY]}' | gw_post /key/delete "${master}" >/dev/null \ + || echo "WARN: could not delete this run's LiteLLM virtual key ${alias}" >&2 + fi +} +trap cleanup EXIT -# chat : POST /v1/chat/completions through the ingress; prints the response body. -chat() { - curl -sfk --max-time 600 --resolve "${host}:443:${node_ip}" "https://${host}/v1/chat/completions" \ - -H "Authorization: Bearer ${key}" -H 'content-type: application/json' -d "$1" +# gw [curl args...]: an HTTPS request to the gateway; the bearer token goes +# to curl as a header file. +gw() { + local path=$1 bearer=$2 + shift 2 + curl -sfk --max-time 600 --resolve "${host}:443:${node_ip}" "https://${host}${path}" \ + -H @<(printf 'Authorization: Bearer %s\n' "${bearer}") "$@" } +# gw_post : POST stdin as the JSON body. +gw_post() { gw "$1" "$2" -H 'content-type: application/json' --data-binary @-; } + +case "${EAT_MODE}" in + eat) + host=$(grep -E '^cluster_url=' "${CORE}/inventory/agentic-config.cfg" | cut -d= -f2-) + key=$(grep -E '^litellm_master_key:' "${CORE}/inventory/metadata/vault.yml" | cut -d'"' -f2) + [ -n "${key}" ] || err "no litellm_master_key in ${CORE}/inventory/metadata/vault.yml" + # ingress-nginx binds hostPort 80/443 on the nodes its controller pods run on. + node_ip=$(kubectl -n ingress-nginx get pods \ + -l app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/component=controller \ + --field-selector=status.phase=Running -o jsonpath='{.items[0].status.hostIP}') + [ -n "${node_ip}" ] || err "no running ingress-nginx controller pod" + EAT_MODEL="${EAT_MODEL:-Qwen/Qwen3-1.7B}" + ;; + eas) + [[ "${EAT_EAS_DOMAIN}" =~ ^[A-Za-z0-9.-]+$ ]] || err "EAT_EAS_DOMAIN must be a domain name, got '${EAT_EAS_DOMAIN}'" + host="litellm.${EAT_EAS_DOMAIN}" + node_ip=$(kubectl -n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway \ + -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') + [ -n "${node_ip}" ] || err "no LoadBalancer IP on the eg-gateway Service in envoy-gateway-system" + master=$(kubectl -n litellm get secret litellm-master-key -o jsonpath='{.data.master_key}' | base64 -d) + [ -n "${master}" ] || err "no master_key in Secret litellm/litellm-master-key" + alias="k3d-gpu-eas-check-$$-${RANDOM}" + vkey=$(jq -n --arg a "${alias}" '{key_alias: $a, duration: "1h"}' | gw_post /key/generate "${master}" | jq -r '.key // empty') \ + || err "LiteLLM did not mint a virtual key (POST https://${host}/key/generate on ${node_ip})" + [ -n "${vkey}" ] || err "LiteLLM answered /key/generate without a key" + key=${vkey} + info "Virtual key ${alias} (deleted at exit)" + if [ -z "${EAT_MODEL}" ]; then + EAT_MODEL=$(gw /v1/models "${key}" | jq -r '[.data[].id | select(test("hybrid-router|vllm-sr|guard|embed|rerank"; "i") | not)][0] // empty') \ + || err "could not list the gateway's models" + [ -n "${EAT_MODEL}" ] || err "the gateway lists no chat model (set EAT_MODEL)" + fi + ;; + *) err "EAT_MODE must be eat or eas, got '${EAT_MODE}'" ;; +esac + +# chat : POST /v1/chat/completions through the gateway; prints the response body. +chat() { gw_post /v1/chat/completions "${key}" <<< "$1"; } # ask : the model's reply text, without Qwen3's empty block. ask() { @@ -55,7 +117,7 @@ ask() { | jq -r '.choices[0].message.content' | sed -e 's###g' -e '/^```/d' -e '/^[[:space:]]*$/d' } -info "Chat through LiteLLM at https://${host} (ingress on ${node_ip})..." +info "Chat through LiteLLM at https://${host} (${EAT_MODE} gateway on ${node_ip}), model ${EAT_MODEL}..." answer=$(ask "In one sentence, what is Kubernetes? /no_think") [ -n "${answer}" ] || err "empty chat reply" echo " ${answer}" @@ -77,17 +139,38 @@ sed 's/^/ /' <<< "${code}" sandbox=$(kubectl -n "${SANDBOX_NS}" get pods -o name | grep -m1 '/python-pool-' | cut -d/ -f2) \ || err "no python-pool warm sandbox in ${SANDBOX_NS}" token=$(kubectl -n "${SANDBOX_NS}" get secret sandbox-router-auth -o jsonpath='{.data.ROUTER_AUTH_TOKEN}' | base64 -d) -kubectl -n "${SANDBOX_NS}" port-forward svc/sandbox-router-svc "${ROUTER_PORT}:8080" >/dev/null 2>&1 & +# The router's local port is the one the kernel gives kubectl (":8080"), read from kubectl's +# own "Forwarding from 127.0.0.1:" line: a fixed port may be held by another local +# service, which would then get the code and the router token. +pflog=$(mktemp) +kubectl -n "${SANDBOX_NS}" port-forward svc/sandbox-router-svc :8080 > "${pflog}" 2>&1 & pf=$! -trap 'kill "${pf}" 2>/dev/null || true' EXIT -for _ in $(seq 1 40); do curl -s -o /dev/null "127.0.0.1:${ROUTER_PORT}" && break; sleep 0.5; done -result=$(curl -sf "127.0.0.1:${ROUTER_PORT}/execute" -H "Authorization: Bearer ${token}" \ - -H "X-Sandbox-ID: ${sandbox}" -H "X-Sandbox-Namespace: ${SANDBOX_NS}" -H "X-Sandbox-Port: 8888" \ - -H 'content-type: application/json' \ - -d "$(jq -n --arg c "${code}" '{command: ("python3 -c " + ($c | @sh))}')") +port="" +for _ in $(seq 1 40); do + port=$(awk '/^Forwarding from 127\.0\.0\.1:[0-9]+ -> 8080$/ { split($3, a, ":"); print a[2]; exit }' "${pflog}") + [ -z "${port}" ] || break + kill -0 "${pf}" 2>/dev/null || err "kubectl port-forward to svc/sandbox-router-svc exited: $(tr '\n' ' ' < "${pflog}")" + sleep 0.5 +done +[ -n "${port}" ] || err "kubectl port-forward to svc/sandbox-router-svc reported no local port in 20 s: $(tr '\n' ' ' < "${pflog}")" +router="http://127.0.0.1:${port}" +# The endpoint must answer as the sandbox router (GET /healthz: {"status":"ok"}) before the +# token and the code go to it. +health="" +for _ in $(seq 1 40); do + health=$(curl -s --max-time 5 "${router}/healthz") || health="" + [ "$(jq -r '.status? // empty' <<< "${health}" 2>/dev/null)" != ok ] || break + sleep 0.5 +done +[ "$(jq -r '.status? // empty' <<< "${health}" 2>/dev/null)" = ok ] \ + || err "${router} (kubectl's port-forward) does not answer as the sandbox router: GET /healthz gave '${health}'" +result=$(jq -n --arg c "${code}" '{command: ("python3 -c " + ($c | @sh))}' \ + | curl -sf "${router}/execute" -H @<(printf 'Authorization: Bearer %s\n' "${token}") \ + -H "X-Sandbox-ID: ${sandbox}" -H "X-Sandbox-Namespace: ${SANDBOX_NS}" -H "X-Sandbox-Port: 8888" \ + -H 'content-type: application/json' --data-binary @-) echo " sandbox ${sandbox}: $(jq -c . <<< "${result}")" # @sh single-quotes the code, which both a shlex.split router (agent-sandbox # v0.5.0's python-runtime-sandbox) and a /bin/sh -c one read back unchanged. [ "$(jq -r .exit_code <<< "${result}")" = 0 ] || err "sandbox command failed" [ "$(jq -r .stdout <<< "${result}")" = 385 ] || err "sandbox printed $(jq -c .stdout <<< "${result}"), expected 385" -info "EAT check passed." +info "EAT check passed (${EAT_MODE})." diff --git a/packaging/aur/PKGBUILD b/packaging/aur/PKGBUILD index 13dd5129..1d98f75f 100644 --- a/packaging/aur/PKGBUILD +++ b/packaging/aur/PKGBUILD @@ -54,6 +54,11 @@ package() { install -Dm644 share/agent-sandbox.yml "$pkgdir/usr/share/k3d-gpu/agent-sandbox.yml" install -Dm644 share/agent-sandbox-k3d.yml "$pkgdir/usr/share/k3d-gpu/agent-sandbox-k3d.yml" install -Dm644 share/model-share.yml "$pkgdir/usr/share/k3d-gpu/model-share.yml" + # k3d-gpu eas, its overlay's patch drafts and eat-check.sh, in the repo's integrations/ layout. + install -d "$pkgdir/usr/share/k3d-gpu/integrations" + cp -r integrations/eas "$pkgdir/usr/share/k3d-gpu/integrations/eas" + install -Dm644 -t "$pkgdir/usr/share/k3d-gpu/integrations/eat/upstream/patches" integrations/eat/upstream/patches/*.patch + install -Dm755 integrations/eat/eat-check.sh "$pkgdir/usr/share/k3d-gpu/integrations/eat/eat-check.sh" install -Dm644 README.md "$pkgdir/usr/share/doc/$pkgname/README.md" install -Dm644 LICENSE.md "$pkgdir/usr/share/licenses/$pkgname/LICENSE.md" } diff --git a/packaging/check.sh b/packaging/check.sh index 9659c2c9..89d1b88c 100755 --- a/packaging/check.sh +++ b/packaging/check.sh @@ -35,6 +35,11 @@ done for f in vs.py llm_load.py competitors.lock; do test "$(sha256sum < "/src/bench/$f")" = "$(sha256sum < "/usr/share/k3d-gpu/bench/$f")" done +for f in /src/integrations/eas/* /src/integrations/eas/patches/* /src/integrations/eat/upstream/patches/*.patch; do + [ -d "$f" ] || test "$(sha256sum < "$f")" = "$(sha256sum < "/usr/share/k3d-gpu/${f#/src/}")" +done +test -x /usr/share/k3d-gpu/integrations/eas/eas.sh +test -x /usr/share/k3d-gpu/integrations/eat/eat-check.sh k3d-gpu help | grep -F "/usr/share/k3d-gpu/nvidia-device-plugin.yml" >/dev/null test -x /usr/bin/k3d-gpu test -x /usr/bin/k3d-gpu-mcp diff --git a/packaging/nfpm/nfpm.yaml b/packaging/nfpm/nfpm.yaml index f4760ca5..97772d7f 100644 --- a/packaging/nfpm/nfpm.yaml +++ b/packaging/nfpm/nfpm.yaml @@ -108,6 +108,20 @@ contents: dst: /usr/share/k3d-gpu/bench/competitors.lock file_info: mode: 0644 + # k3d-gpu eas: integrations/eas, plus the patch drafts its overlay applies and the + # end-state check it runs, in the repo's integrations/ layout: overlay.py finds its + # patches relative to the dir above integrations/. + - src: integrations/eas/ + dst: /usr/share/k3d-gpu/integrations/eas + type: tree + - src: integrations/eat/upstream/patches/*.patch + dst: /usr/share/k3d-gpu/integrations/eat/upstream/patches/ + file_info: + mode: 0644 + - src: integrations/eat/eat-check.sh + dst: /usr/share/k3d-gpu/integrations/eat/eat-check.sh + file_info: + mode: 0755 - src: README.md dst: /usr/share/doc/k3d-gpu/README.md file_info: @@ -163,6 +177,26 @@ contents: packager: rpm file_info: mode: 0755 + - dst: /usr/share/k3d-gpu/integrations + type: dir + packager: rpm + file_info: + mode: 0755 + - dst: /usr/share/k3d-gpu/integrations/eat + type: dir + packager: rpm + file_info: + mode: 0755 + - dst: /usr/share/k3d-gpu/integrations/eat/upstream + type: dir + packager: rpm + file_info: + mode: 0755 + - dst: /usr/share/k3d-gpu/integrations/eat/upstream/patches + type: dir + packager: rpm + file_info: + mode: 0755 - dst: /usr/share/doc/k3d-gpu type: dir packager: rpm diff --git a/packaging/nix/package.nix b/packaging/nix/package.nix index 65ae8d39..bebe4d91 100644 --- a/packaging/nix/package.nix +++ b/packaging/nix/package.nix @@ -43,6 +43,9 @@ stdenvNoCC.mkDerivation { (root + "/images.lock") (root + "/bench/vs.py") (root + "/bench/competitors.lock") + (root + "/integrations/eas") + (root + "/integrations/eat/upstream/patches") + (root + "/integrations/eat/eat-check.sh") (root + "/README.md") (root + "/LICENSE.md") ]; @@ -69,6 +72,11 @@ stdenvNoCC.mkDerivation { install -Dm644 -t "$out/share/k3d-gpu/bench/apb" bench/apb/apb.py bench/apb/capacity.py install -Dm644 -t "$out/share/k3d-gpu/oracle" oracle/bundle.py oracle/project.py oracle/validate.py oracle/share.py install -Dm644 -t "$out/share/k3d-gpu/oracle/schema" oracle/schema/bundle-v1.json + # k3d-gpu eas, its overlay's patch drafts and eat-check.sh, in the repo's integrations/ layout. + mkdir -p "$out/share/k3d-gpu/integrations" + cp -r integrations/eas "$out/share/k3d-gpu/integrations/eas" + install -Dm644 -t "$out/share/k3d-gpu/integrations/eat/upstream/patches" integrations/eat/upstream/patches/*.patch + install -Dm755 integrations/eat/eat-check.sh "$out/share/k3d-gpu/integrations/eat/eat-check.sh" install -Dm644 README.md "$out/share/doc/k3d-gpu/README.md" install -Dm644 LICENSE.md "$out/share/licenses/k3d-gpu/LICENSE.md" runHook postInstall diff --git a/scripts/images-lock b/scripts/images-lock index ff6c4c02..ebfb36e0 100755 --- a/scripts/images-lock +++ b/scripts/images-lock @@ -47,6 +47,9 @@ Sources of image references (the lock holds all of them and nothing else): - integrations/eat/ the Helm values overlays (*.yaml, *.yml: each repository:/tag: pair) and the bastion's FROM (*.Dockerfile): the images the EAT deploy path pulls + - integrations/eas/ the bastion's FROM (*.Dockerfile) and each `_image:` + extra var (*.yaml): the images `k3d-gpu eas` pins for the + EAS installer - k3s's system images k3s-images.txt of the Dockerfile's K3S_TAG release (coredns, local-path-provisioner, metrics-server, pause, ...): k3s itself starts them in the cluster @@ -134,6 +137,9 @@ ARG_IMAGE_RE = re.compile(r'^ARG[ \t]+[A-Z][A-Z0-9_]*_IMAGE="?([^"\s$]+)"?[ \t]* # One `key: value` line of a Helm values mapping; group 4 is the inline value, unquoted. VALUES_KEY_RE = re.compile(r"""^([ \t]*)(repository|registry|tag):[ \t]*(["']?)([^\s"'#]*)\3[ \t]*(?:#.*)?$""") EAT_DIR = "integrations/eat" +EAS_DIR = "integrations/eas" +# An Ansible extra var naming an image: `agent_sandbox_router_image: "ref"`. +VAR_IMAGE_RE = re.compile(r"""^[a-z][a-z0-9_]*_image:[ \t]*["']?([^"'\s#]+)""", re.M) sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "console")) from audit import parse_ref # noqa: E402 (one image-reference parser for the lock and the audit) @@ -275,6 +281,17 @@ def collect(root): with open(path, encoding="utf-8") as fh: for ref in dockerfile_froms(fh.read())[0]: add(ref, source) + eas = os.path.join(root, EAS_DIR) + for f in sorted(os.listdir(eas)) if os.path.isdir(eas) else []: + path, source = os.path.join(eas, f), EAS_DIR + "/" + f + if f.endswith((".yml", ".yaml")): + with open(path, encoding="utf-8") as fh: + for ref in VAR_IMAGE_RE.findall(fh.read()): + add(ref, source) + elif f.endswith(".Dockerfile"): + with open(path, encoding="utf-8") as fh: + for ref in dockerfile_froms(fh.read())[0]: + add(ref, source) k3s_tag = args.get("K3S_TAG") if not k3s_tag: raise ValueError("Dockerfile has no ARG K3S_TAG default") @@ -684,7 +701,7 @@ def main(argv): if args.command == "pins": print("ok every image reference is pinned by digest or allowed in %s" % ALLOW_FILE) else: - print("ok images.lock matches share/, the Dockerfile, scripts/k3d-gpu, integrations/eat and k3s %s" + print("ok images.lock matches share/, the Dockerfile, scripts/k3d-gpu, integrations/eat, integrations/eas and k3s %s" % collect(args.root)[1]) return 0 diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index d562c971..3c6f926d 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -58,6 +58,8 @@ # fit size a model to what this host has free now: placement, context x seqs, engine # memory flags, pod memory limit, or why it cannot fit; --explain prints the arithmetic # audit posture checks: unauthenticated listeners off loopback, images not in images.lock +# eas up|down|status|check: Intel Enterprise AI Solutions (platform, inference, agent +# toolkit) through its own installer, in a bastion container # sandbox up [--adopt]|down|status # kubernetes-sigs/agent-sandbox with a warm pool of sandboxes pinned to E-cores; # --adopt keeps another installer's controller and adds only the E-core profile @@ -300,6 +302,21 @@ Usage: k3d-gpu (no DNS; on a CDN those addresses also serve its other sites), and endpoint in the sandbox's Secret k3d-gpu-llm, which sandbox code can read (use a key you would hand to that code); status checks the pin + eas up [--agent-toolkit] [--model NAME] [--eas-ref R] [--ei-ref R] [--eat-ref R] + [--dry-run] [--explain] [-- ] + Intel Enterprise AI Solutions on this cluster through its own installer + (es_auto_installer.sh, bring-your-own-cluster mode) in a bastion + container: platform + inference (default), or the agent toolkit (EAT 2.0) + on top (--agent-toolkit). Creates the cluster first when it is absent. + Every choice prints as 'fact -> choice [source]' (--explain: full links); + --dry-run prints them and the patch overlay's decisions and changes + nothing. Commits come from integrations/eas/pins.json; a ref given here + is resolved to a commit once and recorded in the state dir + eas status|down|check [--model NAME] + status: EAS's own status, pins, overlay, gateway, pods not Ready. down: + teardown platform (the cluster stays). check (agent toolkit): chat, tool + call and a sandbox run through EAS's LiteLLM gateway with a short-lived + virtual key model-share --image REF --agent-image REF [--replicas N] [--name NAME] Print a Deployment whose N agent replicas mount one model OCI image as a read-only image volume (pipe to kubectl apply -f -). It warns when this @@ -459,6 +476,7 @@ Environment: K3D_GPU_SANDBOX agent-sandbox yaml (default: ${SHARE_DIR}/agent-sandbox.yml) K3D_GPU_SANDBOX_PROFILE E-core sandbox template + warm pool (default: ${SHARE_DIR}/agent-sandbox-k3d.yml) K3D_GPU_SANDBOX_TIMEOUT seconds 'sandbox up' waits for a ready sandbox (default: 180) + K3D_GPU_EAS_DIR integrations/eas for 'eas' (default: the checkout's, then ${SHARE_DIR}/integrations/eas) K3D_GPU_TUNE auto | off: apply this host's tuned profile in 'agent up' (default: auto) K3D_GPU_TUNE_DIR tuned profiles (default: ~/.config/k3d-gpu/tune) K3D_GPU_TUNE_PY tune driver (default: the repo's console/tune.py, then ${SHARE_DIR}/console/tune.py) @@ -6725,6 +6743,11 @@ sandbox_foreign_crds() { sandbox_up() { require kubectl local f ready="" deadline profile fields base model origin ctl whose crds theirs found crd served + # EAS's agent toolkit (k3d-gpu eas up --agent-toolkit) owns agent-sandbox v0.5.0 here; + # v1.0.8's manifest would replace its CRDs (v1beta1 only) under its controller. + if [ -n "$(kubectl -n agent-sandbox-system get deployment sandbox-router-deployment -o name 2>/dev/null || true)" ]; then + err "agent-sandbox here belongs to EAS's agent toolkit (deployment agent-sandbox-system/sandbox-router-deployment); 'k3d-gpu sandbox up' would replace its CRDs. Remove it first with 'k3d-gpu eas down'" + fi for f in "${SANDBOX_MANIFEST}" "${SANDBOX_PROFILE_MANIFEST}"; do [ -f "${f}" ] || err "${f} not found — set K3D_GPU_SHARE, K3D_GPU_SANDBOX or K3D_GPU_SANDBOX_PROFILE" done @@ -6903,6 +6926,480 @@ cmd_sandbox() { esac } +# ---- eas: Intel Enterprise AI Solutions on this cluster --------------------------- +# intel/enterprise-ai-solutions ships one installer, es_auto_installer.sh, for its +# platform, inference (intel/enterprise-inference) and agent-toolkit +# (intel/enterprise-agent-toolkit 2.0) layers. Its bring-your-own mode +# (existing_kubernetes) installs onto this cluster. The installer runs in a bastion +# container (integrations/eas/bastion.Dockerfile), never on the host: +# integrations/eas/eas.sh the in-bastion driver (clone, overlay, init, edits, install) +# integrations/eas/eas-config.py facts + pins + flags -> overrides, edits, choices.txt +# integrations/eas/overlay.py the patch overlay, decided against the pinned checkout +# integrations/eas/pins.json the EAS, EI and EAT commits, per profile +# State per cluster: ${STATE_DIR}/eas-/ (mode 0700). +EAS_ENV="k3d-gpu" +EAS_GW_NS="envoy-gateway-system" +EAS_GW_SELECTOR="gateway.envoyproxy.io/owning-gateway-name=eg-gateway" +EAS_DEFAULT_DOMAIN="solutions.ai" # base_domain_name in EAS configs/defaults/global_config.yaml +# Environment the bastion gets by name only (docker run -e NAME), never by value. +EAS_PASS_ENV=(HF_TOKEN HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy ES_LOG_LEVEL) +EAS_FILES=(eas.sh bastion.Dockerfile requirements.lock images.yaml pins.json eas-config.py overlay.py overlay.json) +# Files eas-config.py writes; each `eas up` starts without them. +EAS_OUTPUTS=(choices.txt k3d-gpu-overrides.yaml global_config.edit.yaml models.edit.yaml + repos.inference.yaml repos.agent-toolkit.yaml pins.resolved.json layer) + +# eas_dir: K3D_GPU_EAS_DIR, then the checkout's integrations/eas, then the packaged copy. +# The packages keep the repo's integrations/ layout, which overlay.py's patch lookup +# (overlay.json patch_dirs, relative to the dir above integrations/) needs. +eas_dir() { + local self d + self=$(readlink -f "${BASH_SOURCE[0]}") + for d in "${K3D_GPU_EAS_DIR:-}" "$(dirname "${self}")/../integrations/eas" "${SHARE_DIR}/integrations/eas"; do + if [ -n "${d}" ] && [ -f "${d}/eas.sh" ]; then (cd "${d}" && pwd); return 0; fi + done + return 1 +} + +eas_state() { echo "${STATE_DIR}/eas-${CLUSTER}"; } + +# eas_bastion_image : the bastion's tag is the sha256 of its Dockerfile and lock, so +# a changed pin builds a new image and an unchanged one is reused. +eas_bastion_image() { + local dir=$1 tag image + tag=$(cat "${dir}/bastion.Dockerfile" "${dir}/requirements.lock" | sha256sum | cut -c1-16) + image="k3d-gpu-eas-bastion:${tag}" + if ! docker image inspect "${image}" >/dev/null 2>&1; then + info "Building the EAS bastion ${image} (kubectl, helm, yq, the installer's venv)..." >&2 + docker build -q -t "${image}" -f "${dir}/bastion.Dockerfile" "${dir}" >/dev/null \ + || err "could not build ${image} from ${dir}/bastion.Dockerfile" + fi + echo "${image}" +} + +# eas_kubeconfig : this cluster's context alone, flattened, for the bastion +# (mounted read-only at /kube/config). A shared kubeconfig's current-context may name +# another cluster; this file names only ${CLUSTER}'s. +eas_kubeconfig() { + local f="$1/kubeconfig" + ( umask 077; kubectl config view --minify --flatten > "${f}.tmp" && mv -f "${f}.tmp" "${f}" ) \ + || err "could not write ${f} for cluster '${CLUSTER}'" +} + +# eas_run : one eas.sh run in the bastion, logged +# to /logs. Host network: the k3d API server's 0.0.0.0: and the node IP are +# reached as from the host. integrations/ is mounted whole, read-only, so overlay.py finds +# the patch drafts in integrations/eat/upstream/patches. Root in the container (eas.sh +# says why); EAS_OWNER hands the state dir back to this user. +eas_run() { + local dir=$1 state=$2 image=$3 v log envs=() + shift 3 + for v in "${EAS_PASS_ENV[@]}"; do [ -z "${!v:-}" ] || envs+=(-e "${v}"); done + mkdir -p "${state}/logs" + log="${state}/logs/eas-$1-$(date +%Y%m%d-%H%M%S).log" + info "eas.sh $* in ${image} (log: ${log})" + docker run --rm --network host --user 0:0 \ + -v "${state}:/eas-state" \ + -v "${state}/kubeconfig:/kube/config:ro" \ + -v "$(dirname "${dir}"):/k3d-gpu/integrations:ro" \ + -e EAS_SHA="${EAS_SHA:-}" -e EAS_CPU_POLICY="${EAS_CPU_POLICY:-}" -e EAS_NODES="${EAS_NODES:-}" \ + -e EAS_OWNER="$(id -u):$(id -g)" \ + ${envs[@]+"${envs[@]}"} \ + "${image}" bash "/k3d-gpu/integrations/$(basename "${dir}")/eas.sh" "$@" 2>&1 | tee -a "${log}" +} + +# eas_ref : a commit SHA for on github.com/, from `git ls-remote` +# (a branch before a tag, as EAS's own _checkout_rev resolves; a tag's peeled commit). +# A full SHA is returned as given. Prints ""; fails on a ref the remote lacks. +eas_ref() { + local repo=$1 ref=$2 out sha + if [[ "${ref}" =~ ^[0-9a-f]{40}$ ]]; then echo "${ref}"; return 0; fi + out=$(git ls-remote -- "https://github.com/${repo}" "${ref}" "${ref}^{}") \ + || err "git ls-remote https://github.com/${repo} ${ref} failed" + sha=$(awk -v r="${ref}" ' + $2 == "refs/heads/" r { h = $1 } $2 == "refs/tags/" r "^{}" { p = $1 } $2 == "refs/tags/" r { t = $1 } + END { print (h != "" ? h : (p != "" ? p : t)) }' <<< "${out}") + [ -n "${sha}" ] || err "https://github.com/${repo} has no branch or tag ${ref}" + echo "${sha}" +} + +# eas_pin_refs : EAS_REF_ARGS for eas-config.py. A profile +# name of pins.json (--eas-ref v1.2.0-rc.1) and a full SHA go as given; any other ref is +# resolved here, once, and printed: a floating ref never reaches the installer. +eas_pin_refs() { + local dir=$1 profiles name repo ref sha + EAS_REF_ARGS=() + profiles=" $(python3 -c 'import json, sys +print(" ".join(sorted({r["profile"] for r in json.load(open(sys.argv[1]))["pins"]})))' "${dir}/pins.json") " \ + || err "cannot read ${dir}/pins.json" + shift + for name in eas ei eat; do + ref=$1; shift + [ -n "${ref}" ] || continue + if [ "${name}" = eas ] && [[ "${profiles}" == *" ${ref} "* ]]; then + EAS_REF_ARGS+=(--eas-ref "${ref}") + echo " eas: profile ${ref} of pins.json" + continue + fi + case "${name}" in + eas) repo="intel/enterprise-ai-solutions" ;; + ei) repo="intel/enterprise-inference" ;; + eat) repo="intel/enterprise-agent-toolkit" ;; + esac + sha=$(eas_ref "${repo}" "${ref}") || exit 1 + EAS_REF_ARGS+=("--${name}-ref" "${sha}") + if [ "${sha}" = "${ref}" ]; then + echo " ${name}: ${repo}@${sha:0:7} (as given)" + else + echo " ${name}: ${repo}@${sha:0:7} (${ref}, git ls-remote https://github.com/${repo} ${ref})" + fi + done +} + +# eas_facts : the facts eas-config.py reads (schema k3d-gpu.eas-facts/v1), all +# read-only: nodes, namespaces, DaemonSets, the agent-sandbox CRD and controller image, the +# server version, rootless docker, the node container's NRI socket and containerd CNI +# bin_dirs, MemAvailable and inotify limits, and `k3d-gpu fit` of each of eas-config.py's +# model candidates (vLLM on the CPU). +eas_facts() { + local state=$1 dir=$2 raw node id rc doc + raw="${state}/facts.d" + rm -rf "${raw}" && mkdir -p "${raw}/fit" + kubectl get nodes -o json > "${raw}/nodes.json" || err "cannot list the nodes of cluster '${CLUSTER}'" + kubectl get daemonsets -A -o json > "${raw}/daemonsets.json" 2>/dev/null || echo '{"items":[]}' > "${raw}/daemonsets.json" + kubectl get namespaces -o json > "${raw}/namespaces.json" 2>/dev/null || echo '{"items":[]}' > "${raw}/namespaces.json" + if kubectl get crd sandboxes.agents.x-k8s.io -o name >/dev/null 2>&1; then + kubectl -n agent-sandbox-system get deployment agent-sandbox-controller \ + -o jsonpath='{.spec.template.spec.containers[0].image}' > "${raw}/sandbox-controller" 2>/dev/null || : + fi + kubectl version -o json > "${raw}/version.json" 2>/dev/null || echo '{}' > "${raw}/version.json" + docker info --format '{{json .SecurityOptions}}' > "${raw}/docker-security.json" 2>/dev/null || echo '[]' > "${raw}/docker-security.json" + # The server node container: whether containerd's NRI socket is there, and the CNI + # bin_dirs containerd hands its plugins (read with crictl, which k3s ships). + node="k3d-${CLUSTER}-server-0" + if docker exec "${node}" test -S /var/run/nri/nri.sock >/dev/null 2>&1; then echo true; else echo false; fi > "${raw}/nri-socket" + docker exec "${node}" crictl info > "${raw}/crictl-info.json" 2>/dev/null || echo '{}' > "${raw}/crictl-info.json" + # Each candidate's fit: exit 3 is "does not fit" with a document, 2 is no answer. + while read -r id; do + [ -n "${id}" ] || continue + rc=0 + doc=$("$(qs_self)" fit --model "${id}" --engine vllm --device cpu --json 2>/dev/null) || rc=$? + if [ "${rc}" -ne 2 ] && [ -n "${doc}" ]; then printf '%s\n' "${doc}" > "${raw}/fit/${id//\//__}.json"; fi + echo "${id}" >> "${raw}/fit/ids" + done < <(python3 - "${dir}/eas-config.py" <<'PY' +import importlib.util, sys +spec = importlib.util.spec_from_file_location("eas_config", sys.argv[1]) +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +for c in m.CANDIDATES: + print(c[1]) +PY +) + python3 - "${raw}" "${state}/facts.json" "${CLUSTER}" "${SYSROOT}" <<'PY' || err "could not assemble ${state}/facts.json" +import json, os, sys +raw, out, cluster, sysroot = sys.argv[1:] +def load(name, default): + try: + with open(os.path.join(raw, name)) as f: + return json.load(f) + except (OSError, ValueError): + return default +def text(name): + try: + with open(os.path.join(raw, name)) as f: + return f.read().strip() + except OSError: + return None +def read_int(path): + try: + with open(sysroot + path) as f: + return int(f.read().split()[0]) + except (OSError, ValueError, IndexError): + return None +def meminfo(key): + try: + with open(sysroot + "/proc/meminfo") as f: + for line in f: + if line.startswith(key + ":"): + return int(line.split()[1]) + except OSError: + pass + return None +def meta(o, *keys): + return {k: o["metadata"].get(k) or {} for k in keys} +cni = (load("crictl-info.json", {}).get("config") or {}).get("cni") or {} +bin_dirs = cni.get("binDirs") or ([cni["binDir"]] if cni.get("binDir") else []) +ids = (text("fit/ids") or "").split() +fit = {i: load("fit/%s.json" % i.replace("/", "__"), None) for i in ids} +controller = text("sandbox-controller") +facts = { + "schema": "k3d-gpu.eas-facts/v1", + "cluster": cluster, + "server_version": load("version.json", {}).get("serverVersion", {}).get("gitVersion"), + "docker_rootless": any("rootless" in s for s in load("docker-security.json", []) or []), + "nri_socket": text("nri-socket") == "true", + "containerd_cni_bin_dirs": bin_dirs, + "host": {"mem_available_kib": meminfo("MemAvailable"), + "inotify_max_user_instances": read_int("/proc/sys/fs/inotify/max_user_instances"), + "inotify_max_user_watches": read_int("/proc/sys/fs/inotify/max_user_watches")}, + "nodes": [dict(name=n["metadata"]["name"], allocatable=n.get("status", {}).get("allocatable", {}), + **meta(n, "labels", "annotations")) for n in load("nodes.json", {"items": []})["items"]], + "namespaces": sorted(o["metadata"]["name"] for o in load("namespaces.json", {"items": []})["items"]), + "daemonsets": [dict(namespace=o["metadata"].get("namespace"), name=o["metadata"]["name"], + **meta(o, "labels", "annotations")) for o in load("daemonsets.json", {"items": []})["items"]], + "agent_sandbox": None if controller is None else {"controller_image": controller or None}, + "fit": fit, +} +with open(out, "w") as f: + json.dump(facts, f, indent=2, sort_keys=True) +PY +} + +# eas_choice_value : a top-level scalar of k3d-gpu-overrides.yaml (eas-config.py +# writes one `key: value` per line), unquoted. +eas_choice_value() { + sed -n "s/^$2:[[:space:]]*\"\{0,1\}\([^\"#]*\)\"\{0,1\}[[:space:]]*\$/\1/p" "$1/k3d-gpu-overrides.yaml" 2>/dev/null | tail -1 +} + +# eas_record_pins : the commits eas-config.py resolved (the `# eas|ei|eat ` +# lines of choices.txt) as /pins.resolved.json, and EAS_SHA. +eas_record_pins() { + EAS_SHA=$(python3 - "$1/choices.txt" "$1/pins.resolved.json" <<'PY' +import json, re, sys +rows, profile = [], None +for line in open(sys.argv[1]): + m = re.match(r"# k3d-gpu eas-config: profile (\S+),", line) + if m: + profile = m.group(1) + m = re.match(r"# (eas|ei|eat) (\S+) ([0-9a-f]{40})$", line.rstrip("\n")) + if m: + rows.append({"name": m.group(1), "repo": m.group(2), "sha": m.group(3)}) +eas = [r["sha"] for r in rows if r["name"] == "eas"] +if not eas: + sys.exit("choices.txt names no EAS commit") +json.dump({"schema": "k3d-gpu.eas-pins-resolved/v1", "profile": profile, "pins": rows}, + open(sys.argv[2], "w"), indent=2) +print(eas[0]) +PY +) || err "eas-config.py's choices.txt names no EAS commit" +} + +# eas_domain : base_domain_name from the overrides, else the env's global_config.yaml, +# else EAS's default. +eas_domain() { + local f d="" + for f in "$1/k3d-gpu-overrides.yaml" "$1/eas/env/${EAS_ENV}/global_config.yaml"; do + [ -f "${f}" ] || continue + d=$(sed -n 's/^base_domain_name:[[:space:]]*["'\'']\{0,1\}\([A-Za-z0-9.-]*\).*$/\1/p' "${f}" | tail -1) + [ -z "${d}" ] || break + done + echo "${d:-${EAS_DEFAULT_DOMAIN}}" +} + +# eas_gateway : where EAS's gateway answers. Envoy Gateway's proxy Service for the +# Gateway eg-gateway carries its owning-gateway label; k3s ServiceLB gives it the node IP. +# The host's /etc/hosts is never edited. LiteLLM is there with the agent toolkit only. +eas_gateway() { + local ip domain + domain=$(eas_domain "$1") + ip=$(kubectl -n "${EAS_GW_NS}" get svc -l "${EAS_GW_SELECTOR}" \ + -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}' 2>/dev/null || true) + if [ -z "${ip}" ]; then + echo " gateway: no LoadBalancer IP on eg-gateway's Service in ${EAS_GW_NS} yet" + return 0 + fi + echo " gateway: eg-gateway at ${ip} (k3s ServiceLB), hosts *.${domain}" + [ "$(cat "$1/layer" 2>/dev/null)" = agent-toolkit ] || return 0 + echo " KEY=\$(${KHINT} -n litellm get secret litellm-master-key -o jsonpath='{.data.master_key}' | base64 -d)" + echo " curl -k --resolve litellm.${domain}:443:${ip} https://litellm.${domain}/v1/models -H \"Authorization: Bearer \${KEY}\"" +} + +eas_require_files() { + local f + for f in "${EAS_FILES[@]}"; do + [ -f "$1/${f}" ] || err "$1/${f} missing: the EAS integration is incomplete (set K3D_GPU_EAS_DIR)" + done +} + +# eas up [--agent-toolkit | --layer L] [--model NAME] [--eas-ref R] [--ei-ref R] +# [--eat-ref R] [--dry-run] [--explain] [-- ] +eas_up() { + local layer=inference model="" eas_ref="" ei_ref="" eat_ref="" dry=0 explain=() extra=() dir state rc v image + while [ $# -gt 0 ]; do + case "$1" in + --agent-toolkit) layer="agent-toolkit" ;; + --layer) [ $# -ge 2 ] || err "--layer needs a value"; layer=$2; shift ;; + --model) [ $# -ge 2 ] || err "--model needs a value"; model=$2; shift ;; + --eas-ref) [ $# -ge 2 ] || err "--eas-ref needs a value"; eas_ref=$2; shift ;; + --ei-ref) [ $# -ge 2 ] || err "--ei-ref needs a value"; ei_ref=$2; shift ;; + --eat-ref) [ $# -ge 2 ] || err "--eat-ref needs a value"; eat_ref=$2; shift ;; + --dry-run) dry=1 ;; + --explain) explain=(--explain) ;; + --) shift; extra=("$@"); break ;; + *) err "unknown eas up option: $1 (see 'k3d-gpu help')" ;; + esac + shift + done + case "${layer}" in + inference|agent-toolkit|erag) ;; # erag: eas-config.py refuses it, with the reason + *) err "--layer must be inference, agent-toolkit or erag, got '${layer}' (platform comes with inference)" ;; + esac + for v in "${eas_ref}" "${ei_ref}" "${eat_ref}" "${model}"; do + [ -z "${v}" ] || [[ "${v}" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] || err "not a ref or model name: '${v}'" + done + require kubectl; require k3d; require git; require python3; require docker + dir=$(eas_dir) || err "integrations/eas not found next to the launcher or in ${SHARE_DIR}/integrations/eas (set K3D_GPU_EAS_DIR)" + eas_require_files "${dir}" + state=$(eas_state) + ( umask 077; mkdir -p "${state}" ) && chmod 0700 "${state}" + + if ! k3d cluster get "${CLUSTER}" >/dev/null 2>&1; then + [ "${dry}" = 0 ] || err "cluster '${CLUSTER}' does not exist; 'k3d-gpu eas up' creates it with 'k3d-gpu up' first, and a dry run reads a running cluster" + info "cluster '${CLUSTER}' does not exist: 'k3d-gpu up' first" + "$(qs_self)" up || err "'k3d-gpu up' failed; fix the error above and run 'k3d-gpu eas up' again" + fi + kubectl get nodes >/dev/null 2>&1 || err "cluster '${CLUSTER}' is not reachable — run 'k3d-gpu doctor'" + + info "EAS ${layer} on cluster '${CLUSTER}' (state ${state})" + if [ -n "${eas_ref}${ei_ref}${eat_ref}" ]; then info "Refs:"; fi + eas_pin_refs "${dir}" "${eas_ref}" "${ei_ref}" "${eat_ref}" + info "Facts (read-only)..." + eas_facts "${state}" "${dir}" + info "Choices (fact -> choice [source]):" + for v in "${EAS_OUTPUTS[@]}"; do rm -f "${state:?}/${v}"; done + rc=0 + python3 "${dir}/eas-config.py" --facts "${state}/facts.json" --out "${state}" --layer "${layer}" \ + --pins "${dir}/pins.json" ${EAS_REF_ARGS[@]+"${EAS_REF_ARGS[@]}"} ${model:+--model "${model}"} \ + ${explain[@]+"${explain[@]}"} || rc=$? + [ "${rc}" -ne 3 ] || err "refused (above); nothing was changed" + [ "${rc}" -eq 0 ] || err "eas-config.py failed (exit ${rc}); nothing was changed" + for v in choices.txt k3d-gpu-overrides.yaml global_config.edit.yaml models.edit.yaml "repos.${layer}.yaml"; do + [ -s "${state}/${v}" ] || err "eas-config.py wrote no ${state}/${v}" + done + eas_record_pins "${state}" + EAS_CPU_POLICY=$(eas_choice_value "${state}" kubernetes_cpu_policy) + EAS_NODES=$(python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))["nodes"]))' "${state}/facts.json") + cp "${dir}/images.yaml" "${state}/k3d-gpu-images.yaml" + eas_kubeconfig "${state}" + + if [ "${dry}" = 1 ]; then + info "Overlay (enterprise-ai-solutions@${EAS_SHA:0:7}, kubernetes_cpu_policy ${EAS_CPU_POLICY}, ${EAS_NODES} node(s)):" + EAS_STATE="${state}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY="${EAS_CPU_POLICY}" EAS_NODES="${EAS_NODES}" \ + bash "${dir}/eas.sh" overlay-check || err "the overlay stops at this EAS commit (above); nothing was changed" + info "Dry run: nothing applied. Under ${state}: facts.json, choices.txt, k3d-gpu-overrides.yaml, global_config.edit.yaml, models.edit.yaml, repos.${layer}.yaml, overlay.txt" + return 0 + fi + image=$(eas_bastion_image "${dir}") + eas_run "${dir}" "${state}" "${image}" up "${layer}" -- ${extra[@]+"${extra[@]}"} \ + || err "es_auto_installer.sh install ${layer} failed (log above, and ${state}/eas/env/${EAS_ENV}/logs)" + echo "${layer}" > "${state}/layer" + info "EAS ${layer} is installed on '${CLUSTER}'" + eas_gateway "${state}" + [ "${layer}" != agent-toolkit ] || echo " check it end to end: k3d-gpu eas check" +} + +eas_installed() { + [ -d "$1/eas/env/${EAS_ENV}" ] || err "nothing installed by 'k3d-gpu eas up' on cluster '${CLUSTER}' (no $1/eas/env/${EAS_ENV})" +} + +eas_down() { + local dir state image ns name + [ $# -eq 0 ] || err "eas down takes no options (got: $*)" + require kubectl; require docker + dir=$(eas_dir) || err "integrations/eas not found (set K3D_GPU_EAS_DIR)" + state=$(eas_state) + eas_installed "${state}" + # EAS issue #71 (open): teardown platform hangs at the Envoy AI Gateway CRD uninstall + # while an AIGatewayRoute keeps its finalizer; the role strips them only after the + # controller is gone. Clear them first, as the issue's workaround does. + while read -r ns name; do + [ -n "${name}" ] || continue + info "clearing the finalizers of AIGatewayRoute ${ns}/${name} (EAS issue #71)" + kubectl -n "${ns}" patch aigatewayroute "${name}" --type=merge -p '{"metadata":{"finalizers":[]}}' >/dev/null \ + || warn "could not clear the finalizers of ${ns}/${name}" + done < <(kubectl get aigatewayroutes.aigateway.envoyproxy.io -A \ + -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' 2>/dev/null || true) + eas_kubeconfig "${state}" + image=$(eas_bastion_image "${dir}") + eas_run "${dir}" "${state}" "${image}" down || err "teardown platform failed (log above)" + rm -f "${state}/layer" + info "EAS removed from '${CLUSTER}'; the cluster stays ('k3d-gpu down' deletes it)" +} + +eas_status() { + local dir state image + [ $# -eq 0 ] || err "eas status takes no options (got: $*)" + require kubectl; require docker; require python3 + dir=$(eas_dir) || err "integrations/eas not found (set K3D_GPU_EAS_DIR)" + state=$(eas_state) + eas_installed "${state}" + info "Layer: $(cat "${state}/layer" 2>/dev/null || echo "unknown (the last eas up did not finish)")" + info "Pins (${state}/pins.resolved.json):" + python3 - "${state}/pins.resolved.json" <<'PY' || echo " (no resolved pins)" +import json, sys +d = json.load(open(sys.argv[1])) +for r in d["pins"]: + print(" %s: %s@%s (profile %s)" % (r["name"], r["repo"], r["sha"][:7], d.get("profile"))) +PY + info "Overlay (${state}/overlay.txt):" + if [ -s "${state}/overlay.txt" ]; then sed 's/^/ /' "${state}/overlay.txt"; else echo " (no decisions recorded)"; fi + info "EAS status:" + eas_kubeconfig "${state}" + image=$(eas_bastion_image "${dir}") + eas_run "${dir}" "${state}" "${image}" status || warn "es_auto_installer.sh status failed (above)" + info "Gateway:" + eas_gateway "${state}" + info "Pods not Ready:" + kubectl get pods -A -o json | python3 -c ' +import json, sys +bad = [] +for p in json.load(sys.stdin)["items"]: + ph = p.get("status", {}).get("phase") + if ph == "Succeeded": + continue + cs = p.get("status", {}).get("containerStatuses", []) + if ph != "Running" or not cs or not all(c.get("ready") for c in cs): + why = [c.get("state", {}).get("waiting", {}).get("reason") for c in cs] + why = ",".join(w for w in why if w) or ph or "?" + bad.append(" %s/%s: %s" % (p["metadata"]["namespace"], p["metadata"]["name"], why)) +print("\n".join(bad) if bad else " none")' +} + +eas_check() { + local dir state chk model="" + while [ $# -gt 0 ]; do + case "$1" in + --model) [ $# -ge 2 ] || err "--model needs a value"; model=$2; shift ;; + *) err "unknown eas check option: $1" ;; + esac + shift + done + require kubectl; require curl; require jq + dir=$(eas_dir) || err "integrations/eas not found (set K3D_GPU_EAS_DIR)" + state=$(eas_state) + eas_installed "${state}" + # LiteLLM and the sandbox come with the agent toolkit (auth_type litellm_vkey); the + # inference layer alone serves models behind a Keycloak token. + [ "$(cat "${state}/layer" 2>/dev/null)" = agent-toolkit ] \ + || err "eas check exercises the agent toolkit (LiteLLM and the sandbox): run 'k3d-gpu eas up --agent-toolkit' first" + chk="${dir}/../eat/eat-check.sh" + [ -f "${chk}" ] || err "${chk} not found" + eas_kubeconfig "${state}" + EAT_MODE=eas EAT_KUBECONFIG="${state}/kubeconfig" EAT_EAS_DOMAIN="$(eas_domain "${state}")" \ + EAT_MODEL="${model:-${EAT_MODEL:-}}" bash "${chk}" +} + +cmd_eas() { + local sub=${1:-} + [ $# -eq 0 ] || shift + case "${sub}" in + up) eas_up "$@" ;; + down) eas_down "$@" ;; + status) eas_status "$@" ;; + check) eas_check "$@" ;; + *) err "usage: k3d-gpu eas up [--agent-toolkit] [--model NAME] [--eas-ref R] [--ei-ref R] [--eat-ref R] [--dry-run] [--explain] [-- ansible args] | status | down | check [--model NAME]" ;; + esac +} + # ---- model-share ---------------------------------------------------------------- # N agent replicas mount one model OCI image as a read-only image volume and mmap # it. Sharing one page-cache copy needs three things, each detected here: @@ -12356,6 +12853,7 @@ case "${1:-help}" in fit) shift; cmd_fit "$@" ;; audit) shift; cmd_audit "$@" ;; sandbox) shift; cmd_sandbox "$@" ;; + eas) shift; cmd_eas "$@" ;; remote) shift; cmd_remote "$@" ;; model-share) shift; cmd_model_share "$@" ;; apply) shift; cmd_apply "$@" ;; diff --git a/tests/console/audit/chat-ui-noauth.golden b/tests/console/audit/chat-ui-noauth.golden index c7400842..50d1896a 100644 --- a/tests/console/audit/chat-ui-noauth.golden +++ b/tests/console/audit/chat-ui-noauth.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 12 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 12 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE, scripts/k3d-gpu NPU_UMD_IMAGE, share/npu-embeddings.yml); on k3d-gpu-server-0 diff --git a/tests/console/audit/chat-ui.golden b/tests/console/audit/chat-ui.golden index b49dbceb..ff6cbd1a 100644 --- a/tests/console/audit/chat-ui.golden +++ b/tests/console/audit/chat-ui.golden @@ -1,4 +1,4 @@ -audit PASS: 0 of 12 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit PASS: 0 of 12 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE, scripts/k3d-gpu NPU_UMD_IMAGE, share/npu-embeddings.yml); on k3d-gpu-server-0 diff --git a/tests/console/audit/default-json.golden b/tests/console/audit/default-json.golden index d134bfe5..4c328606 100644 --- a/tests/console/audit/default-json.golden +++ b/tests/console/audit/default-json.golden @@ -73,12 +73,12 @@ ], "cluster": "gpu", "lock": { - "images": 54, + "images": 56, "k3s_tag": "v1.37.1-k3s1" }, "schema": "k3d-gpu.audit/v1", "summary": [ - "audit PASS: 0 of 10 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1)" + "audit PASS: 0 of 10 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1)" ], "verdict": "pass" } diff --git a/tests/console/audit/default.golden b/tests/console/audit/default.golden index 61669446..0e404200 100644 --- a/tests/console/audit/default.golden +++ b/tests/console/audit/default.golden @@ -1,4 +1,4 @@ -audit PASS: 0 of 10 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit PASS: 0 of 10 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE, scripts/k3d-gpu NPU_UMD_IMAGE, share/npu-embeddings.yml); on k3d-gpu-server-0 diff --git a/tests/console/audit/image-digest-moved.golden b/tests/console/audit/image-digest-moved.golden index ae7bb42e..76d01b24 100644 --- a/tests/console/audit/image-digest-moved.golden +++ b/tests/console/audit/image-digest-moved.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 10 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 10 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) FAIL image docker.io/intel/intel-gpu-plugin:0.37.1: docker.io/intel/intel-gpu-plugin is sha256:2222222222222222222222222222222222222222222222222222222222222222, images.lock pins sha256:004641b2547d58f1eadce58473fa160ad71377de1eb588cf1f82832387bf3e33 (tag 0.37.1); on k3d-gpu-server-0 fix: redeploy from the pinned digest, or bump the pin with scripts/images-lock update diff --git a/tests/console/audit/image-no-digest.golden b/tests/console/audit/image-no-digest.golden index b2430703..a2143ddd 100644 --- a/tests/console/audit/image-no-digest.golden +++ b/tests/console/audit/image-no-digest.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) FAIL image docker.io/library/app:dev: no registry digest, so it cannot match images.lock; on k3d-gpu-server-0 fix: remove it, or load the image from a registry reference that images.lock pins diff --git a/tests/console/audit/image-not-in-lock.golden b/tests/console/audit/image-not-in-lock.golden index c0d9d3a6..8bf92b54 100644 --- a/tests/console/audit/image-not-in-lock.golden +++ b/tests/console/audit/image-not-in-lock.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) FAIL image docker.io/bitnami/redis:7.4: not in images.lock: docker.io/bitnami/redis; on k3d-gpu-server-0 fix: remove the workload that runs it, or pin it in the repo and run scripts/images-lock update diff --git a/tests/console/audit/ipv6-unauth.golden b/tests/console/audit/ipv6-unauth.golden index 3710cf04..c8f4787f 100644 --- a/tests/console/audit/ipv6-unauth.golden +++ b/tests/console/audit/ipv6-unauth.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) FAIL listener [::]:8000 vllm (EAT chart, no auth): unauthenticated listener off loopback: no authentication; off loopback, auth none (fixture: EAT#9 layout) fix: bind it to 127.0.0.1, or put OIDC in front of it (k3d-gpu plan --bind picks oidc off loopback) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) diff --git a/tests/console/audit/kubeconfig-0644.golden b/tests/console/audit/kubeconfig-0644.golden index 2921e599..860a8a2a 100644 --- a/tests/console/audit/kubeconfig-0644.golden +++ b/tests/console/audit/kubeconfig-0644.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 10 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 10 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE, scripts/k3d-gpu NPU_UMD_IMAGE, share/npu-embeddings.yml); on k3d-gpu-server-0 diff --git a/tests/console/audit/loopback-published.golden b/tests/console/audit/loopback-published.golden index c2c87a52..1c4e50b5 100644 --- a/tests/console/audit/loopback-published.golden +++ b/tests/console/audit/loopback-published.golden @@ -1,4 +1,4 @@ -audit PASS: 0 of 11 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit PASS: 0 of 11 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok listener 127.0.0.1:8000 published node port 30080/tcp: loopback, auth unknown (docker port 30080/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 diff --git a/tests/console/audit/runtime-fetch.golden b/tests/console/audit/runtime-fetch.golden index c6734f5a..abd5bfd6 100644 --- a/tests/console/audit/runtime-fetch.golden +++ b/tests/console/audit/runtime-fetch.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE, scripts/k3d-gpu NPU_UMD_IMAGE, share/npu-embeddings.yml); on k3d-gpu-server-0 diff --git a/tests/console/audit/runtime-pull.golden b/tests/console/audit/runtime-pull.golden index 3de318b3..cf09dc4f 100644 --- a/tests/console/audit/runtime-pull.golden +++ b/tests/console/audit/runtime-pull.golden @@ -1,4 +1,4 @@ -audit FAIL: 2 of 12 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 2 of 12 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) ok image docker.io/intel/intel-gpu-plugin:0.37.1: sha256:004641b2547d in images.lock (share/intel-gpu-plugin.yml, share/intel-gpu-wsl-plugin.yml); on k3d-gpu-server-0 ok image docker.io/openvino/model_server@sha256:a928903fe429f1e3e27107cc45932d30d95febfd2d1159dc50869c3eac76fc58: sha256:a928903fe429 in images.lock (scripts/k3d-gpu AGENT_OVMS_IMAGE, scripts/k3d-gpu NPU_UMD_IMAGE, share/npu-embeddings.yml); on k3d-gpu-server-0 diff --git a/tests/console/audit/unauth-published.golden b/tests/console/audit/unauth-published.golden index 9a68e42b..a6c65a26 100644 --- a/tests/console/audit/unauth-published.golden +++ b/tests/console/audit/unauth-published.golden @@ -1,4 +1,4 @@ -audit FAIL: 1 of 11 checks failed (images.lock: 54 images, k3s v1.37.1-k3s1) +audit FAIL: 1 of 11 checks failed (images.lock: 56 images, k3s v1.37.1-k3s1) FAIL listener 0.0.0.0:8000 published node port 30080/tcp: unauthenticated listener off loopback: no authentication this audit can establish; off loopback, auth unknown (docker port 30080/tcp on k3d-gpu-server-0) fix: bind it to 127.0.0.1, or put OIDC in front of it (k3d-gpu plan --bind picks oidc off loopback) ok listener 0.0.0.0:39651 kube-apiserver: off loopback, auth client-cert (docker port 6443/tcp on k3d-gpu-server-0) diff --git a/tests/detect_test.sh b/tests/detect_test.sh index eaae79e1..485c2732 100755 --- a/tests/detect_test.sh +++ b/tests/detect_test.sh @@ -4717,7 +4717,8 @@ first_calls() { [ -n "${PINNED}" ] || { FAILS=$((FAILS + 1)); echo "FAIL the controller image is readable from share/agent-sandbox.yml"; } sandbox_says "sandbox up on a fresh cluster installs agent-sandbox" "${r}" 0 'Installing agent-sandbox' up "${UP_OK[@]}" -first_calls "sandbox up on a fresh cluster adds two reads (controller, CRDs) before today's apply" \ +first_calls "sandbox up on a fresh cluster reads EAS's router, the controller and the CRDs before today's apply" \ + '^-n agent-sandbox-system get deployment sandbox-router-deployment -o name$' \ '^-n agent-sandbox-system get deployment agent-sandbox-controller --ignore-not-found -o jsonpath=' \ '^get crd sandboxes\.agents\.x-k8s\.io .* --ignore-not-found -o jsonpath=' \ 'apply -f .*/agent-sandbox\.yml$' diff --git a/tests/eas_test.sh b/tests/eas_test.sh index 610835a8..f8571596 100755 --- a/tests/eas_test.sh +++ b/tests/eas_test.sh @@ -9,6 +9,8 @@ # - overlay.json resolves every patch, upstream drafts first-class, no copies # - main: 0002 and 0006 apply, 0003 only for nri-balloons, 0004 only on # more than one node, 0005 never; the recorded `verified` cases hold +# - the same decisions with TMPDIR (the scratch index) on another +# filesystem than the checkout # - patches already in the tree: drop-merged (git apply -R --check) # - the fix done another way with the knob present: drop-fixed # - the fix done another way with the knob renamed: STOP @@ -34,6 +36,35 @@ # role looks for, --explain prints permalinks, every source it can cite is # a permalink at a full commit SHA, and pins.json has the k3d-gpu.eas-pins/v1 # shape. +# argv the LiteLLM master key on no process argument list during EI's model deploy: +# EI 3e8a244's two tasks that carry it (tests/fixtures/eas/argv, the +# MM_LITELLM_MASTER_KEY line verbatim) run under ansible-playbook with a fake key +# while ps samples every argv. Without the overrides (the control) ansible's +# environment: puts the key on the deploy's /bin/sh -c argv; with the +# generator's k3d-gpu-overrides.yaml no sample holds it and the deploy gets an +# empty MM_LITELLM_MASTER_KEY. Needs ansible-playbook. +# cli `k3d-gpu eas up|down|status|check`, eas.sh (the in-bastion driver) and the +# bastion, against stub docker, kubectl, k3d, git, yq and `k3d-gpu fit`, with the +# real eas-config.py (so the facts it gathers are the facts the generator reads) +# and a recording overlay.py: +# - up --dry-run prints the choices and the overlay decisions and makes no +# docker or kubectl change +# - up creates the cluster with `k3d-gpu up` when it is absent +# - the bastion run: host network, root with the state handed back to this +# uid:gid, state dir rw, kubeconfig ro, secrets by name only +# - eas.sh order: clone at the SHA, overlay, manifest pins, init, the edits, +# install with the overrides last; never configure +# - eas.sh stops before install when EI's model deploy takes the master key +# from anything but _deploy_litellm_secret, or has no deploy_models.yaml +# - refs: a profile and a SHA go as given, any other ref through git ls-remote +# - no secret value in an argument list, a log, choices.txt or a state file +# - refusals stop before any change +# - down clears AIGatewayRoute finalizers before teardown platform --force, and +# the teardown gets install's extra vars (refused without the overrides) +# - status prints the layer, pins, overlay, EAS status, gateway, pods not Ready +# - sandbox up refuses when EAS's sandbox-router-deployment exists +# - the bastion pins what pins.json's bastion profile names, and ships a +# sha256-checked jq (EI's deploy_models.yaml runs jq on the bastion) # Needs bash, git, python3 (with PyYAML, for the config section's shape checks only). set -uo pipefail @@ -183,6 +214,23 @@ for v in json.load(open(sys.argv[1]))["verified"]: check "main: 0002 resolves to the upstream draft" "${ROOT}/integrations/eat/upstream/patches/0002-eas-istio-global-platform.patch" \ "$(cd "${ROOT}" && realpath "$(jq_ 'd["decisions"][0]["files"][0]')")" check "main: 0002 names the generator knob it needs" istio_platform "$(jq_ 'd["decisions"][0]["knob"]')" + # The scratch index can sit on another filesystem than the checkout: a tmpfs /tmp + # with the state dir on disk, or the bastion's own /tmp with the state dir bind-mounted. + local xdev="" d + for d in /dev/shm /var/tmp; do + [ -d "${d}" ] && [ -w "${d}" ] || continue + [ "$(stat -c %d "${d}")" != "$(stat -c %d "${WORK}")" ] || continue + xdev=$(mktemp -d "${d}/eas-test-xdev.XXXXXX") && break + done + if [ -n "${xdev}" ]; then + got=$(TMPDIR="${xdev}" brief "${WORK}/main" best-effort 1) + check "main with the scratch index on another filesystem (TMPDIR ${d}): the same decisions" \ + "0002=apply:a 0003=skip 0004=skip 0005=skip 0006=apply:a" "${got}" + check "main with the scratch index on another filesystem: exit 0" 0 "$(cat "${WORK}/rc")" + rm -rf "${xdev}" + else + echo "skip no writable /dev/shm or /var/tmp on another filesystem than ${WORK}: the cross-filesystem case did not run" + fi # Text: one `fact -> choice [source]` line per row, sources at the checkout's sha. local sha sha7 @@ -748,7 +796,641 @@ PY fi } -SECTIONS=(overlay config) +section_cli() { + local B="${WORK}/cli/bin" K="${WORK}/cli/k" S="${WORK}/cli/s" LOG="${WORK}/cli/log" + local EAS_DIR="${ROOT}/integrations/eas" LAUNCHER="${ROOT}/scripts/k3d-gpu" + local SECRET="hf_SECRET_7f3a9c" EAS_SHA="97f6157d6a72f7173c698c474d4e51d2c79f1fd6" + local EI_SHA="3e8a24486b6ee812adf2bf452f8ba5751c16c8bd" EAT_SHA="036b66234801e9bd950afd90830821859780d668" + local STATE="${S}/k3d-gpu-state/eas-easc" + mkdir -p "${B}" "${K}/eas" "${K}/eat" "${S}/fx" "${S}/home" "${S}/host/proc/sys/fs/inotify" + + # Helpers named apart from the other sections' (has there takes the needle first). + cli_eq() { if [ "$2" = "$3" ]; then ok "$1"; else fail "$1: got '$2', want '$3'"; fi; } + cli_has() { if grep -Fq -- "$3" <<< "$2"; then ok "$1"; else fail "$1: missing '$3' in: $(head -c 700 <<< "$2")"; fi; } + cli_lacks() { if grep -Fq -- "$3" <<< "$2"; then fail "$1: unexpected '$3'"; else ok "$1"; fi; } + # cli_before : occurs, and before . + cli_before() { + local a b + a=$(grep -nF -- "$3" <<< "$2" | head -1 | cut -d: -f1) + b=$(grep -nF -- "$4" <<< "$2" | head -1 | cut -d: -f1) + if [ -n "${a}" ] && [ -n "${b}" ] && [ "${a}" -lt "${b}" ]; then ok "$1"; else fail "$1: '$3' (line ${a:-none}) not before '$4' (line ${b:-none})"; fi + } + + # The integration dir: the real files, eas-config.py included (the facts this section + # gathers must be what it reads), with overlay.py replaced by a recorder (the overlay + # section tests it on real EAS trees). + cp "${EAS_DIR}"/eas.sh "${EAS_DIR}"/eas-config.py "${EAS_DIR}"/pins.json "${EAS_DIR}"/overlay.json \ + "${EAS_DIR}"/bastion.Dockerfile "${EAS_DIR}"/requirements.lock "${EAS_DIR}"/images.yaml "${K}/eas/" + cat > "${K}/eas/overlay.py" <<'EOF' +import os, sys +a = sys.argv[1:] +with open(os.environ["STUB_LOG"], "a") as f: + f.write("overlay %s\n" % " ".join(a)) +if os.environ.get("STUB_OVERLAY_STOP"): + print("STOP: the overlay does not fit eas@97f6157; regenerate the rows above that say STOP. Nothing was applied.", file=sys.stderr) + sys.exit(3) +print("# overlay at eas@97f6157 (HEAD), kubernetes_cpu_policy=%s, nodes=%s" % (a[a.index("--policy") + 1], a[a.index("--nodes") + 1])) +print("0002 istio global.platform -> apply [stub]\n0006 envoy control-plane affinity -> apply (variant a) [stub]") +EOF + cat > "${K}/eat/eat-check.sh" <<'EOF' +echo "eat-check mode=${EAT_MODE} kubeconfig=${EAT_KUBECONFIG} domain=${EAT_EAS_DOMAIN} model=${EAT_MODEL}" >> "${STUB_LOG}" +echo "EAT check passed (${EAT_MODE})." +EOF + + # ---- stubs: every call is one line in ${STUB_LOG} ------------------------------------- + cat > "${B}/docker" <<'EOF' +#!/usr/bin/env bash +echo "docker $*" >> "${STUB_LOG}" +case "$*" in + "info "*) echo '["name=seccomp,profile=builtin","name=cgroupns"]' ;; + "exec k3d-easc-server-0 test -S /var/run/nri/nri.sock") exit 0 ;; + "exec k3d-easc-server-0 crictl info") echo '{"config": {"cni": {"binDirs": ["/usr/bin"], "confDir": "/var/lib/rancher/k3s/agent/etc/cni/net.d"}}}' ;; + "image inspect "*) [ -e "${STUB_STATE}/image-present" ] ;; + "build "*) touch "${STUB_STATE}/image-present" ;; + "run "*) echo "bastion: ran" ;; + *) echo "docker stub: unexpected: $*" >&2; exit 1 ;; +esac +EOF + cat > "${B}/k3d" <<'EOF' +#!/usr/bin/env bash +echo "k3d $*" >> "${STUB_LOG}" +case "$*" in + "cluster get "*) [ -e "${STUB_STATE}/cluster" ] ;; + "kubeconfig get "*) cat "${STUB_STATE}/kubeconfig-src" ;; +esac +EOF + cat > "${B}/kubectl" <<'EOF' +#!/usr/bin/env bash +echo "kubectl $*" >> "${STUB_LOG}" +F="${STUB_STATE}/fx" +case "$*" in + *"config view --minify --flatten"*) echo "apiVersion: v1 # minified for ${STUB_CLUSTER}" ;; + *"get nodes -o json"*) cat "${F}/nodes.json" ;; + *"get nodes"*) echo "node/k3d-easc-server-0" ;; + *"get daemonsets -A -o json"*) cat "${F}/daemonsets.json" ;; + *"get namespaces -o json"*) cat "${F}/namespaces.json" ;; + *"get crd sandboxes.agents.x-k8s.io -o name"*) echo 'Error from server (NotFound)' >&2; exit 1 ;; + *"version -o json"*) echo '{"serverVersion": {"gitVersion": "v1.37.1+k3s1"}}' ;; + *"get aigatewayroutes.aigateway.envoyproxy.io -A"*) printf 'llm-inference aigw-qwen2-5-coder-7b\nllm-inference aigw-qwen3guard\n' ;; + *"-n agent-sandbox-system get deployment sandbox-router-deployment -o name"*) + [ -n "${STUB_EAS_SANDBOX:-}" ] || { echo 'Error from server (NotFound)' >&2; exit 1; } + echo deployment.apps/sandbox-router-deployment ;; + *"-n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway"*) echo 172.18.0.2 ;; + *"get pods -A -o json"*) cat "${F}/pods.json" ;; + *" patch "*|*" apply "*|*" delete "*) ;; + *) echo "kubectl stub: unexpected: $*" >&2; exit 1 ;; +esac +EOF + cat > "${B}/git" <<'EOF' +#!/usr/bin/env bash +echo "git $*" >> "${STUB_LOG}" +if [ "$1" = ls-remote ]; then + case "$4" in + release-1.1) printf '170eb3abb1956de0787727f21e356aef2b13ff24\trefs/heads/release-1.1\n' ;; + v9.9.9) printf '1111111111111111111111111111111111111111\trefs/tags/v9.9.9\n2222222222222222222222222222222222222222\trefs/tags/v9.9.9^{}\n' ;; + esac + exit 0 +fi +if [ "$1" = clone ]; then + # A checkout whose installer records its argv; init seeds env/ as the real one does. + d=$4 + mkdir -p "${d}/.git" "${d}/configs/repos" + : > "${d}/configs/repos/repos.agent-toolkit.yaml" + : > "${d}/configs/repos/repos.inference.yaml" + cat > "${d}/es_auto_installer.sh" <<'EOS' +#!/usr/bin/env bash +echo "installer $*" >> "${STUB_LOG}" +if [ "$1" = init ]; then + mkdir -p env/k3d-gpu && : > env/k3d-gpu/global_config.yaml && : > env/k3d-gpu/models.yaml && : > env/k3d-gpu/.solutions.yaml +fi +EOS + chmod +x "${d}/es_auto_installer.sh" +fi +exit 0 +EOF + # yq: records its argv and the strenv() values; answers the row listings eas.sh reads + # from eas-config.py's files (one `key: "value"` per line, as it writes them). + cat > "${B}/yq" <<'EOF' +#!/usr/bin/env python3 +import os, re, sys +a = sys.argv[1:] +env = " ".join("%s=%s" % (k, os.environ[k]) for k in ("L", "U", "R", "F", "N", "A") if k in os.environ and "strenv(%s)" % k in " ".join(a)) +with open(os.environ["STUB_LOG"], "a") as f: + f.write("yq %s%s\n" % (" ".join(a), " [" + env + "]" if env else "")) +if "-o=json" in a: + text = open(a[-1]).read() + if ".args_append" in a[-2]: + m = re.search(r"args_append: (\[.*\])", text) + print(m.group(1) if m else "[]") + else: # the catalog entry's args: one the edit replaces, one it keeps + print('["--block-size","128","--max-model-len","131072","--tensor-parallel-size","1"]') +elif "-r" in a: + text = open(a[-1]).read() + if "@tsv" in a[a.index("-r") + 1]: + for m in re.finditer(r'- layer: "([^"]*)"\n\s+url: "([^"]*)"\n\s+rev: "([^"]*)"', text): + print("\t".join(m.groups())) + elif ".models[].name" in a: + for m in re.finditer(r'^ - name: "([^"]*)"', text, re.M): + print(m.group(1)) +EOF + cat > "${B}/self" <<'EOF' +#!/usr/bin/env bash +echo "self $*" >> "${STUB_LOG}" +case "$1" in + up) touch "${STUB_STATE}/cluster" ;; + fit) + case "$3" in + Qwen/Qwen2.5-Coder-14B-Instruct) echo '{"schema": "k3d-gpu.fit/v1", "fits": false, "summary": "fit: does not fit this host", "flags": []}'; exit 3 ;; + meta-llama/Llama-3.1-8B-Instruct) exit 2 ;; + *) printf '{"schema": "k3d-gpu.fit/v1", "fits": true, "summary": "fit: %s on CPU via vllm", "flags": ["--max-model-len=32768", "--max-num-seqs=4", "VLLM_CPU_KVCACHE_SPACE=12"]}\n' "$3" ;; + esac ;; +esac +EOF + chmod +x "${B}"/* + + # ---- fixtures: a k3d-gpu cluster on a 32-CPU, 125 GiB hybrid desktop -------------------- + cat > "${S}/fx/nodes.json" <<'EOF' +{"items": [{"metadata": {"name": "k3d-easc-server-0", + "labels": {"feature.node.kubernetes.io/cpu-cpuid.AVXVNNI": "true", "k3d-gpu.io/hybrid-cpu": "true", + "node-role.kubernetes.io/control-plane": "true"}, + "annotations": {"k3s.io/node-args": "[\"server\", \"--disable\", \"traefik\", \"--tls-san\", \"0.0.0.0\"]"}}, + "status": {"allocatable": {"cpu": "32", "memory": "131419028Ki", "gpu.intel.com/xe": "1", "pods": "110"}}}]} +EOF + cat > "${S}/fx/daemonsets.json" <<'EOF' +{"items": [ + {"metadata": {"namespace": "kube-system", "name": "nri-resource-policy-balloons", + "labels": {"app.kubernetes.io/managed-by": "Helm"}, + "annotations": {"meta.helm.sh/release-name": "nri-resource-policy-balloons", + "objectset.rio.cattle.io/owner-gvk": "k3s.cattle.io/v1, Kind=Addon", "objectset.rio.cattle.io/owner-name": "nri-balloons"}}}, + {"metadata": {"namespace": "node-feature-discovery", "name": "nfd-worker", "labels": {"app": "nfd"}, + "annotations": {"objectset.rio.cattle.io/owner-gvk": "k3s.cattle.io/v1, Kind=Addon", "objectset.rio.cattle.io/owner-name": "node-feature-discovery"}}}]} +EOF + echo '{"items": [{"metadata": {"name": "default"}}, {"metadata": {"name": "kube-system"}}, {"metadata": {"name": "node-feature-discovery"}}]}' > "${S}/fx/namespaces.json" + cp "${S}/fx/namespaces.json" "${S}/fx/namespaces.ok" + cat > "${S}/fx/pods.json" <<'EOF' +{"items": [ + {"metadata": {"namespace": "litellm", "name": "litellm-0"}, "status": {"phase": "Running", "containerStatuses": [{"ready": true}]}}, + {"metadata": {"namespace": "llm-inference", "name": "vllm-qwen-0"}, "status": {"phase": "Pending", "containerStatuses": [{"ready": false, "state": {"waiting": {"reason": "ContainerCreating"}}}]}}, + {"metadata": {"namespace": "kube-system", "name": "helm-install-x"}, "status": {"phase": "Succeeded"}}]} +EOF + echo "apiVersion: v1 # easc" > "${S}/kubeconfig-src" + mkdir -p "${S}/k3d-gpu-state" + cp "${S}/kubeconfig-src" "${S}/k3d-gpu-state/kubeconfig-easc" + touch "${S}/cluster" + printf 'MemTotal: 131500000 kB\nMemAvailable: 120000000 kB\n' > "${S}/host/proc/meminfo" + echo 128 > "${S}/host/proc/sys/fs/inotify/max_user_instances" + echo 1048576 > "${S}/host/proc/sys/fs/inotify/max_user_watches" + + # cli [args...]: a launcher function in a subshell, the stubs first on PATH, the + # launcher's own `up` and `fit` replaced by the self stub. + cli() { + (export PATH="${B}:${PATH}" STUB_LOG="${LOG}" STUB_STATE="${S}" STUB_CLUSTER=easc HOME="${S}/home" \ + K3D_GPU_STATE_DIR="${S}/k3d-gpu-state" K3D_GPU_CLUSTER=easc K3D_GPU_EAS_DIR="${K}/eas" \ + K3D_GPU_SHARE="${ROOT}/share" + unset KUBECONFIG K3D_GPU_KUBECONFIG K3D_GPU_CONTEXT + # shellcheck disable=SC1090 + source <(sed '/^case "\${1:-help}" in$/,$d' "${LAUNCHER}") + # shellcheck disable=SC2034 + SYSROOT="${S}/host" + qs_self() { echo "${B}/self"; } + "$@") 2>&1 + } + local out rc log + + # ---- up --dry-run ------------------------------------------------------------------------- + : > "${LOG}" + out=$(cli cmd_eas up --agent-toolkit --dry-run); rc=$? + log=$(cat "${LOG}") + cli_eq "cli: dry run exits 0" "${rc}" 0 + cli_has "cli: dry run prints eas-config's choices" "${out}" "ServiceLB on (no --disable servicelb in k3s.io/node-args) -> metallb_enabled: false" + cli_has "cli: the facts carry the baked balloons policy to eas-config" "${out}" "-> kubernetes_cpu_policy: best-effort (one NRI policy per node)" + cli_has "cli: the facts carry the node's CNI bin_dirs to eas-config" "${out}" 'containerd CNI bin_dirs ["/usr/bin"] -> istio_platform: k3d' + cli_has "cli: the facts carry the fit documents to eas-config" "${out}" "qwen2-5-coder-7b needs 99 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-7b]" + cli_has "cli: the inotify warning reaches the user" "${out}" "WARN: fs.inotify.max_user_instances 128 < 8192" + cli_has "cli: dry run prints the overlay decisions" "${out}" "0006 envoy control-plane affinity -> apply (variant a)" + cli_has "cli: the overlay gets eas-config's cpu policy and the node count" "${log}" \ + "overlay --eas ${STATE}/eas --policy best-effort --nodes 1 --json ${STATE}/overlay-decisions.json" + cli_lacks "cli: dry run: the overlay is not applied" "${log}" "--apply" + cli_has "cli: dry run says nothing was applied" "${out}" "Dry run: nothing applied" + if grep -E '^docker ' <<< "${log}" | grep -Ev '^docker (info --format|exec k3d-easc-server-0 (test -S /var/run/nri/nri.sock|crictl info)$)' | grep -q .; then + fail "cli: dry run: a docker call that is not a read: $(grep -E '^docker ' <<< "${log}" | grep -Ev '^docker (info|exec)' | head -1)" + else + ok "cli: dry run: every docker call is a read (info, exec test, exec crictl info)" + fi + if grep -E '^kubectl ' <<< "${log}" | grep -Eqv ' (get|version|config view) '; then + fail "cli: dry run: a kubectl call that is not a read: $(grep -E '^kubectl ' <<< "${log}" | grep -Ev ' (get|version|config view) ' | head -1)" + else + ok "cli: dry run: every kubectl call is a read" + fi + cli_lacks "cli: dry run: no installer call" "${log}" "installer " + cli_eq "cli: state dir is 0700" "$(stat -c %a "${STATE}")" 700 + cli_eq "cli: the bastion's kubeconfig is 0600" "$(stat -c %a "${STATE}/kubeconfig")" 600 + cli_has "cli: the bastion's kubeconfig is this cluster's context alone" "${log}" "kubectl --context k3d-easc config view --minify --flatten" + cli_has "cli: eas-config writes its five files" "$(ls "${STATE}")" "repos.agent-toolkit.yaml" + local facts + facts=$(python3 -c 'import json, sys +f = json.load(open(sys.argv[1])) +print(sorted(f), f["nri_socket"], f["containerd_cni_bin_dirs"], f["agent_sandbox"], f["docker_rootless"], sorted(f["host"]), + sorted(f["nodes"][0]), sorted(f["daemonsets"][0]), f["namespaces"], f["fit"]["meta-llama/Llama-3.1-8B-Instruct"], + f["fit"]["Qwen/Qwen2.5-Coder-14B-Instruct"]["fits"])' "${STATE}/facts.json" 2>&1) + cli_eq "cli: facts.json has the k3d-gpu.eas-facts/v1 shape eas-config.py documents" "${facts}" \ + "['agent_sandbox', 'cluster', 'containerd_cni_bin_dirs', 'daemonsets', 'docker_rootless', 'fit', 'host', 'namespaces', 'nodes', 'nri_socket', 'schema', 'server_version'] True ['/usr/bin'] None False ['inotify_max_user_instances', 'inotify_max_user_watches', 'mem_available_kib'] ['allocatable', 'annotations', 'labels', 'name'] ['annotations', 'labels', 'name', 'namespace'] ['default', 'kube-system', 'node-feature-discovery'] None False" + cli_has "cli: fit runs per eas-config candidate, vLLM on the CPU" "${log}" "self fit --model Qwen/Qwen2.5-Coder-7B-Instruct --engine vllm --device cpu --json" + cli_has "cli: pins.resolved.json records eas-config's commits" "$(cat "${STATE}/pins.resolved.json")" "\"sha\": \"${EAS_SHA}\"" + + # ---- refs ----------------------------------------------------------------------------------- + : > "${LOG}" + out=$(cli cmd_eas up --dry-run --eas-ref v1.2.0-rc.1); rc=$? + cli_eq "cli: --eas-ref v1.2.0-rc.1 (a profile) runs" "${rc}" 0 + cli_has "cli: a profile goes to eas-config as given" "${out}" "# k3d-gpu eas-config: profile v1.2.0-rc.1, layer inference" + cli_has "cli: the rc profile's EAS commit drives the overlay" "${out}" "enterprise-ai-solutions@5c27fb3" + cli_lacks "cli: a profile needs no ls-remote" "$(cat "${LOG}")" "git ls-remote" + : > "${LOG}" + out=$(cli cmd_eas up --dry-run --agent-toolkit --ei-ref release-1.1 --eat-ref v9.9.9); rc=$? + cli_eq "cli: refs resolve" "${rc}" 0 + cli_has "cli: a branch resolves through git ls-remote" "$(cat "${LOG}")" "git ls-remote -- https://github.com/intel/enterprise-inference release-1.1 release-1.1^{}" + cli_has "cli: the resolved commit is printed with its ref" "${out}" "ei: intel/enterprise-inference@170eb3a (release-1.1, git ls-remote" + cli_has "cli: eas-config gets the commit, not the ref" "${out}" "# ei intel/enterprise-inference 170eb3abb1956de0787727f21e356aef2b13ff24" + cli_has "cli: an annotated tag resolves to its commit" "$(cat "${STATE}/repos.agent-toolkit.yaml")" 'rev: "2222222222222222222222222222222222222222"' + : > "${LOG}" + out=$(cli cmd_eas up --dry-run --eas-ref 0123456789abcdef0123456789abcdef01234567); rc=$? + cli_has "cli: a full SHA is taken as given" "${out}" "eas: intel/enterprise-ai-solutions@0123456 (as given)" + cli_lacks "cli: a full SHA needs no ls-remote" "$(cat "${LOG}")" "git ls-remote" + out=$(cli cmd_eas up --dry-run --ei-ref no-such-branch); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'has no branch or tag no-such-branch' <<< "${out}"; then ok "cli: an unknown ref stops"; else fail "cli: an unknown ref: rc=${rc} ${out}"; fi + + # ---- refusals --------------------------------------------------------------------------------- + # cli_refused : non-zero, the message, and no change. + cli_refused() { + local name=$1 msg=$2 o r + shift 2 + : > "${LOG}" + o=$(cli cmd_eas "$@"); r=$? + if [ "${r}" -ne 0 ] && grep -qF -- "${msg}" <<< "${o}" \ + && ! grep -Eq '^docker (run|build)|^kubectl .* (patch|apply|delete|create) |^installer ' "${LOG}"; then + ok "cli: refused before any change: ${name}" + else + fail "cli: refused before any change: ${name} (rc=${r}): $(head -c 600 <<< "${o}") | $(grep -E '^docker (run|build)|^installer' "${LOG}" | head -2)" + fi + } + cli_refused "--layer erag (eas-config's reason)" "chatqna_intel_gpu is EXPERIMENTAL" up --layer erag + cli_refused "--layer erag reads as a refusal" "refused (above); nothing was changed" up --layer erag + cli_refused "an unknown layer" "--layer must be inference, agent-toolkit or erag" up --layer platform + cli_refused "an unknown option" "unknown eas up option: --nope" up --nope + cli_refused "a ref that is an option" "not a ref or model name: '-x'" up --eas-ref -x + cli_refused "a ref with shell characters" "not a ref or model name" up --ei-ref 'main;id' + cli_refused "no subcommand" "usage: k3d-gpu eas up" "" + cli_refused "down with an option" "eas down takes no options" down --force + echo '{"items": [{"metadata": {"name": "genai-gateway"}}, {"metadata": {"name": "ingress-nginx"}}]}' > "${S}/fx/namespaces.json" + cli_refused "EAT 1.x present (eas-config refuses)" "both stacks would own LiteLLM" up --agent-toolkit + cp "${S}/fx/namespaces.ok" "${S}/fx/namespaces.json" + STUB_OVERLAY_STOP=1 cli_refused "an overlay STOP" "the overlay stops at this EAS commit" up --dry-run + rm "${S}/cluster" + cli_refused "a dry run on an absent cluster" "a dry run reads a running cluster" up --dry-run + touch "${S}/cluster" + mv "${K}/eas/overlay.py" "${K}/eas/overlay.py.off" + cli_refused "an incomplete integration dir" "overlay.py missing" up + mv "${K}/eas/overlay.py.off" "${K}/eas/overlay.py" + + # ---- up: the cluster, the bastion and its run ------------------------------------------------ + rm -f "${S}/cluster" "${S}/image-present" + : > "${LOG}" + out=$(HF_TOKEN="${SECRET}" HTTPS_PROXY=http://proxy.test:3128 cli cmd_eas up --agent-toolkit -- -e '{"llm_services_autodeploy_models": []}'); rc=$? + log=$(cat "${LOG}") + cli_eq "cli: up exits 0" "${rc}" 0 + cli_has "cli: up creates the absent cluster with 'k3d-gpu up'" "${log}" "self up" + cli_before "cli: the cluster comes before the facts" "${log}" "self up" "get nodes -o json" + local tag runline + tag=$(cat "${EAS_DIR}/bastion.Dockerfile" "${EAS_DIR}/requirements.lock" | sha256sum | cut -c1-16) + cli_has "cli: the bastion is built when absent, tagged by Dockerfile + lock" "${log}" \ + "docker build -q -t k3d-gpu-eas-bastion:${tag} -f ${K}/eas/bastion.Dockerfile ${K}/eas" + runline=$(grep '^docker run' <<< "${log}") + cli_has "cli: bastion: host network" "${runline}" "--network host" + cli_has "cli: bastion: root (ensure_sudo), the state handed back to this uid:gid" "${runline}" "--user 0:0" + cli_has "cli: bastion: EAS_OWNER is this uid:gid" "${runline}" "-e EAS_OWNER=$(id -u):$(id -g)" + cli_has "cli: bastion: state dir rw" "${runline}" "-v ${STATE}:/eas-state " + cli_has "cli: bastion: kubeconfig ro" "${runline}" "-v ${STATE}/kubeconfig:/kube/config:ro" + cli_has "cli: bastion: integrations/ ro, as overlay.py's patch lookup needs" "${runline}" "-v ${K}:/k3d-gpu/integrations:ro" + cli_has "cli: bastion: the EAS commit eas-config resolved" "${runline}" "-e EAS_SHA=${EAS_SHA}" + cli_has "cli: bastion: eas-config's cpu policy and the node count for the overlay" "${runline}" "-e EAS_CPU_POLICY=best-effort -e EAS_NODES=1" + cli_has "cli: bastion: HF_TOKEN by name only" "${runline}" "-e HF_TOKEN " + cli_has "cli: bastion: a proxy by name only" "${runline}" "-e HTTPS_PROXY " + cli_has "cli: bastion: eas.sh up with the layer and the extra args" "${runline}" \ + "k3d-gpu-eas-bastion:${tag} bash /k3d-gpu/integrations/eas/eas.sh up agent-toolkit -- -e {\"llm_services_autodeploy_models\": []}" + cli_has "cli: up prints the gateway with its --resolve line" "${out}" "curl -k --resolve litellm.solutions.ai:443:172.18.0.2 https://litellm.solutions.ai/v1/models" + local leak + leak=$(grep -rlF -- "${SECRET}" "${S}" "${LOG}" 2>/dev/null; grep -F -- "${SECRET}" <<< "${out}") + cli_eq "cli: no secret value in argv, logs, choices, state files or output" "${leak}" "" + cli_has "cli: the image pins go to the state dir" "$(cat "${STATE}/k3d-gpu-images.yaml" 2>/dev/null)" "agent_sandbox_router_image:" + cli_eq "cli: the installed layer is recorded" "$(cat "${STATE}/layer" 2>/dev/null)" agent-toolkit + : > "${LOG}" + out=$(cli cmd_eas up --agent-toolkit); rc=$? + cli_lacks "cli: a present cluster is not recreated" "$(cat "${LOG}")" "self up" + cli_lacks "cli: a present bastion is reused, not rebuilt" "$(cat "${LOG}")" "docker build" + : > "${LOG}" + out=$(cli cmd_eas up); rc=$? + cli_eq "cli: up of the inference layer exits 0" "${rc}" 0 + cli_lacks "cli: the inference layer has no LiteLLM line" "${out}" "litellm." + cli_has "cli: the inference layer prints the gateway" "${out}" "gateway: eg-gateway at 172.18.0.2" + mkdir -p "${STATE}/eas/env/k3d-gpu" + out=$(cli cmd_eas check); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'run .k3d-gpu eas up --agent-toolkit. first' <<< "${out}"; then + ok "cli: check refuses without the agent toolkit (no LiteLLM)" + else + fail "cli: check without the agent toolkit: rc=${rc} ${out}" + fi + cli cmd_eas up --agent-toolkit >/dev/null + + # ---- eas.sh in the bastion: its steps, in order ------------------------------------------------ + local ES="${WORK}/cli/es" + mkdir -p "${ES}" + command cp -f "${STATE}"/k3d-gpu-overrides.yaml "${STATE}"/global_config.edit.yaml "${STATE}"/models.edit.yaml \ + "${STATE}"/repos.agent-toolkit.yaml "${STATE}"/k3d-gpu-images.yaml "${ES}/" + : > "${LOG}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit -- -e x=1 2>&1); rc=$? + log=$(cat "${LOG}") + cli_eq "cli: eas.sh up exits 0" "${rc}" 0 + cli_has "cli: eas.sh clones EAS" "${log}" "git clone -q https://github.com/intel/enterprise-ai-solutions ${ES}/eas" + cli_has "cli: eas.sh checks out the SHA, clean" "${log}" "git -C ${ES}/eas checkout -q --force --detach ${EAS_SHA}" + cli_before "cli: clone, then checkout" "${log}" "git clone" "checkout -q --force --detach" + cli_before "cli: checkout, then clean" "${log}" "checkout -q --force --detach" "git -C ${ES}/eas clean -q -fd" + cli_before "cli: checkout, then the overlay, applied" "${log}" "checkout -q --force --detach" \ + "overlay --eas ${ES}/eas --policy best-effort --nodes 1 --json ${ES}/overlay-decisions.json --apply" + cli_has "cli: overlay.txt holds the decisions" "$(cat "${ES}/overlay.txt")" "0002 istio global.platform -> apply" + cli_before "cli: the overlay, then the manifest pins" "${log}" "overlay --eas" \ + "yq -i (.repos[] | select(.layer == strenv(L) and .url == strenv(U)) | .rev) = strenv(R) ${ES}/eas/configs/repos/repos.agent-toolkit.yaml [L=inference U=https://github.com/intel/enterprise-inference R=${EI_SHA}]" + cli_has "cli: the EAT repo is pinned too" "${log}" "[L=agent-toolkit U=https://github.com/intel/enterprise-agent-toolkit R=${EAT_SHA}]" + cli_before "cli: the manifest pins, then init" "${log}" "strenv(R) ${ES}/eas/configs/repos" "installer init" + cli_has "cli: init --env k3d-gpu" "${log}" "installer init agent-toolkit --env k3d-gpu" + cli_before "cli: init, then global_config.edit.yaml" "${log}" "installer init" \ + "yq -i . *= load(strenv(F)) ${ES}/eas/env/k3d-gpu/global_config.yaml [F=${ES}/global_config.edit.yaml]" + cli_before "cli: init, then models.edit.yaml defaults" "${log}" "installer init" \ + "yq -i .defaults *= load(strenv(F)).defaults ${ES}/eas/env/k3d-gpu/models.yaml [F=${ES}/models.edit.yaml]" + cli_has "cli: the picked model's catalog entry is edited by name" "${log}" "[F=${ES}/models.edit.yaml N=qwen2-5-coder-7b]" + cli_has "cli: the model's env merges into its catalog entry" "${log}" \ + ".env *= ((load(strenv(F)).models[] | select(.name == strenv(N))).env // {}) ${ES}/eas/env/k3d-gpu/models.yaml" + cli_has "cli: args_append follows the catalog args, replacing the flag it names" "${log}" \ + '.args = (strenv(A) | from_yaml) '"${ES}"'/eas/env/k3d-gpu/models.yaml [N=qwen2-5-coder-7b A=["--block-size", "128", "--tensor-parallel-size", "1", "--max-model-len", "32768"]]' + cli_lacks "cli: args_append is not written into the catalog entry as a key" "${log}" \ + ") *= (load(strenv(F)).models[]" + cli_before "cli: the edits, then install" "${log}" "load(strenv(F)).defaults" "installer install" + cli_has "cli: install with the image pins, then the overrides, then the extra args" "${log}" \ + "installer install agent-toolkit --env k3d-gpu -- -e @${ES}/k3d-gpu-images.yaml -e @${ES}/k3d-gpu-overrides.yaml -e x=1" + cli_lacks "cli: configure is never called" "${log}" "installer configure" + cli_eq "cli: .venv links to the baked venv" "$(readlink "${ES}/eas/.venv")" "${WORK}/cli/venv" + : > "${LOG}" + PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit >/dev/null 2>&1 + cli_has "cli: a second up re-inits with --upgrade (moved pins)" "$(cat "${LOG}")" "installer init agent-toolkit --env k3d-gpu --upgrade" + cli_lacks "cli: a second up does not clone again" "$(cat "${LOG}")" "git clone" + # EI's model deploy: the key from _deploy_litellm_secret (the overrides empty it) passes; + # a key from anywhere else stops the install before it starts. + local DM="${ES}/eas/ext/enterprise.ai-inference/roles/llm_services/tasks/deploy_models.yaml" + mkdir -p "$(dirname "${DM}")" + command cp -f "${ROOT}/tests/fixtures/eas/argv/deploy-models.yml" "${DM}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + cli_eq "cli: eas.sh up passes with EI's deploy taking the key from _deploy_litellm_secret" "${rc}" 0 + sed -i 's/^\( *MM_LITELLM_MASTER_KEY:\).*/\1 "{{ lookup('"'env'"', '"'LITELLM_MASTER_KEY'"') }}"/' "${DM}" + : > "${LOG}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'no longer takes the master key from _deploy_litellm_secret' <<< "${out}"; then + ok "cli: eas.sh stops when EI's deploy takes the key from elsewhere" + else + fail "cli: EI deploy taking the key from elsewhere: rc=${rc} ${out}" + fi + cli_lacks "cli: ...before install" "$(cat "${LOG}")" "installer install" + # The same under another variable name: the master key from a Secret read the + # overrides do not empty, handed over as MM_LITELLM_KEY. + command cp -f "${ROOT}/tests/fixtures/eas/argv/deploy-models.yml" "${DM}" + sed -i -e 's/register: _deploy_litellm_secret$/register: _mm_litellm_secret/' \ + -e 's/^\( *\)MM_LITELLM_MASTER_KEY:.*/\1MM_LITELLM_KEY: "{{ _mm_litellm_secret.resources[0].data.master_key | b64decode }}"/' "${DM}" + : > "${LOG}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'no longer takes the master key from _deploy_litellm_secret' <<< "${out}"; then + ok "cli: eas.sh stops when EI's deploy hands the key over under another variable name" + else + fail "cli: EI deploy handing the key over as MM_LITELLM_KEY: rc=${rc} ${out}" + fi + cli_lacks "cli: ...before install" "$(cat "${LOG}")" "installer install" + rm -f "${DM}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'EI has no roles/llm_services/tasks/deploy_models.yaml' <<< "${out}"; then + ok "cli: eas.sh stops when EI has no deploy_models.yaml" + else + fail "cli: EI without deploy_models.yaml: rc=${rc} ${out}" + fi + rm -rf "${ES}/eas/ext" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA=main bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'EAS_SHA must be a full commit SHA' <<< "${out}"; then ok "cli: eas.sh refuses a floating EAS ref"; else fail "cli: eas.sh floating ref: rc=${rc} ${out}"; fi + sed -i 's/rev: "3e8a/rev: "main-3e8a/' "${ES}/repos.agent-toolkit.yaml" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_SHA="${EAS_SHA}" EAS_CPU_POLICY=best-effort EAS_NODES=1 \ + EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" up agent-toolkit 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'is not a commit SHA' <<< "${out}"; then ok "cli: eas.sh refuses a manifest rev that is not a SHA"; else fail "cli: eas.sh manifest rev: rc=${rc} ${out}"; fi + : > "${LOG}" + PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" down >/dev/null 2>&1 + cli_has "cli: eas.sh down: teardown platform, forced (no terminal)" "$(cat "${LOG}")" "installer teardown platform --env k3d-gpu --force" + # Teardown resolves components by the same enabled flags as install (EAS + # filter_plugins/resolve_components.py), so it gets the same extra vars; without them + # nfd_enabled defaults to true and EI's nfd teardown deletes k3d-gpu's own NFD namespace. + cli_has "cli: eas.sh down: teardown gets install's extra vars (the overrides last)" "$(cat "${LOG}")" \ + "installer teardown platform --env k3d-gpu --force -- -e @${ES}/k3d-gpu-images.yaml -e @${ES}/k3d-gpu-overrides.yaml" + command mv "${ES}/k3d-gpu-overrides.yaml" "${ES}/k3d-gpu-overrides.yaml.away" + : > "${LOG}" + out=$(PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" down 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'k3d-gpu-overrides.yaml' <<< "${out}" && ! grep -q 'installer teardown' "${LOG}"; then + ok "cli: eas.sh down refuses without the overrides, before any teardown" + else + fail "cli: eas.sh down without the overrides: rc=${rc} ${out}" + fi + command mv "${ES}/k3d-gpu-overrides.yaml.away" "${ES}/k3d-gpu-overrides.yaml" + : > "${LOG}" + PATH="${B}:${PATH}" STUB_LOG="${LOG}" EAS_STATE="${ES}" EAS_VENV="${WORK}/cli/venv" bash "${K}/eas/eas.sh" status >/dev/null 2>&1 + cli_has "cli: eas.sh status" "$(cat "${LOG}")" "installer status --env k3d-gpu" + + # ---- down ---------------------------------------------------------------------------------------- + mkdir -p "${STATE}/eas/env/k3d-gpu" + : > "${LOG}" + out=$(cli cmd_eas down); rc=$? + log=$(cat "${LOG}") + cli_eq "cli: down exits 0" "${rc}" 0 + cli_has "cli: down clears each AIGatewayRoute's finalizers (EAS #71)" "${log}" \ + 'kubectl --context k3d-easc -n llm-inference patch aigatewayroute aigw-qwen2-5-coder-7b --type=merge -p {"metadata":{"finalizers":[]}}' + cli_has "cli: down clears the guard model's route too" "${log}" "patch aigatewayroute aigw-qwen3guard" + cli_before "cli: finalizers cleared before teardown" "${log}" "patch aigatewayroute aigw-qwen3guard" "eas.sh down" + cli_has "cli: down keeps the cluster" "${out}" "the cluster stays" + cli_lacks "cli: down does not delete the cluster" "${log}" "cluster delete" + + # ---- status -------------------------------------------------------------------------------------- + echo agent-toolkit > "${STATE}/layer" + command cp -f "${ES}/overlay.txt" "${STATE}/overlay.txt" # what eas.sh up writes in the bastion + : > "${LOG}" + out=$(cli cmd_eas status); rc=$? + cli_eq "cli: status exits 0" "${rc}" 0 + cli_has "cli: status: the layer" "${out}" "Layer: agent-toolkit" + cli_has "cli: status: pins" "${out}" "eas: intel/enterprise-ai-solutions@97f6157 (profile main)" + cli_has "cli: status: overlay decisions" "${out}" "0006 envoy control-plane affinity -> apply (variant a)" + cli_has "cli: status: EAS's own status" "$(cat "${LOG}")" "eas.sh status" + cli_has "cli: status: the gateway with its --resolve line" "${out}" "--resolve litellm.solutions.ai:443:172.18.0.2" + cli_has "cli: status: a pod not Ready" "${out}" "llm-inference/vllm-qwen-0: ContainerCreating" + cli_lacks "cli: status: a Ready pod is not listed" "${out}" "litellm/litellm-0" + cli_lacks "cli: status: a finished pod is not listed" "${out}" "helm-install-x" + + # ---- check --------------------------------------------------------------------------------------- + : > "${LOG}" + out=$(cli cmd_eas check --model qwen2-5-coder-7b); rc=$? + cli_eq "cli: check exits 0" "${rc}" 0 + cli_has "cli: check runs eat-check.sh in eas mode on this cluster's kubeconfig and domain" "$(cat "${LOG}")" \ + "eat-check mode=eas kubeconfig=${STATE}/kubeconfig domain=solutions.ai model=qwen2-5-coder-7b" + + # ---- nothing installed --------------------------------------------------------------------------- + rm -rf "${STATE}/eas" + cli_refused "down with nothing installed" "nothing installed by 'k3d-gpu eas up'" down + cli_refused "status with nothing installed" "nothing installed by 'k3d-gpu eas up'" status + + # ---- sandbox up next to EAS's agent sandbox -------------------------------------------------------- + : > "${LOG}" + out=$(STUB_EAS_SANDBOX=1 cli sandbox_up); rc=$? + if [ "${rc}" -ne 0 ] && grep -q "belongs to EAS's agent toolkit" <<< "${out}" && ! grep -q ' apply ' "${LOG}"; then + ok "cli: sandbox up refuses when sandbox-router-deployment exists, before any apply" + else + fail "cli: sandbox up next to EAS: rc=${rc} ${out}" + fi + + # ---- the launcher's dispatcher and help ------------------------------------------------------------ + out=$(K3D_GPU_STATE_DIR="${S}/k3d-gpu-state" HOME="${S}/home" "${LAUNCHER}" eas 2>&1); rc=$? + if [ "${rc}" -ne 0 ] && grep -q 'usage: k3d-gpu eas up' <<< "${out}"; then ok "cli: dispatcher: 'eas' reaches cmd_eas"; else fail "cli: dispatcher: rc=${rc} ${out}"; fi + cli_has "cli: help documents eas up" "$("${LAUNCHER}" help)" "eas up [--agent-toolkit] [--model NAME]" + + # ---- the bastion: its pins are pins.json's bastion profile ----------------------------------------- + local k3s kc want got + k3s=$(sed -n 's/^ARG K3S_TAG="\{0,1\}v\([0-9]*\.[0-9]*\)\..*/\1/p' "${ROOT}/Dockerfile") + kc=$(sed -n 's/^ARG KUBECTL_VERSION=v\([0-9]*\.[0-9]*\)\..*/\1/p' "${EAS_DIR}/bastion.Dockerfile") + if [ -n "${k3s}" ] && [ "${kc}" = "${k3s}" ]; then ok "cli: bastion kubectl minor ${kc} is the K3S_TAG minor"; else fail "cli: bastion kubectl minor '${kc}' vs K3S_TAG minor '${k3s}'"; fi + want=$(python3 - "${EAS_DIR}/pins.json" <<'PY' +import json, sys +rows = {r["name"]: r for r in json.load(open(sys.argv[1]))["pins"] if r["profile"] == "bastion"} +print("FROM ubuntu:%s@sha256:%s kubectl %s helm %s yq %s jq %s kubernetes.core %s" + % (rows["ubuntu"]["ref"], rows["ubuntu"]["sha"], rows["kubectl"]["ref"], rows["helm"]["ref"], rows["yq"]["ref"], + rows["jq"]["ref"], rows["kubernetes.core"]["ref"])) +PY +) + got="$(grep -m1 '^FROM ' "${EAS_DIR}/bastion.Dockerfile") kubectl $(sed -n 's/^ARG KUBECTL_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") helm $(sed -n 's/^ARG HELM_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") yq $(sed -n 's/^ARG YQ_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") jq $(sed -n 's/^ARG JQ_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile") kubernetes.core $(sed -n 's/^ARG KUBERNETES_CORE_VERSION=//p' "${EAS_DIR}/bastion.Dockerfile")" + cli_eq "cli: bastion.Dockerfile pins what pins.json's bastion profile names" "${got}" "${want}" + # EI's llm_services role pipes yq into jq on the installer host to pick the models to + # deploy (roles/llm_services/tasks/deploy_models.yaml); EAS's configure installs no jq. + if grep -q '^ARG JQ_SHA256=[0-9a-f]\{64\}$' "${EAS_DIR}/bastion.Dockerfile" \ + && grep -q 'echo "${JQ_SHA256} /usr/local/bin/jq" | sha256sum -c -' "${EAS_DIR}/bastion.Dockerfile"; then + ok "cli: the bastion ships jq, checked by sha256 (EI's deploy_models.yaml runs it on the bastion)" + else + fail "cli: the bastion has no sha256-checked jq; EI's deploy_models.yaml fails with 'jq: command not found'" + fi + cli_eq "cli: bastion FROM is the EAT bastion's digest" \ + "$(grep -m1 '^FROM ' "${EAS_DIR}/bastion.Dockerfile")" "$(grep -m1 '^FROM ' "${ROOT}/integrations/eat/bastion.Dockerfile")" + want=$(python3 -c 'import json, sys +print(next(r["sha"] for r in json.load(open(sys.argv[1]))["pins"] if r["profile"] == "main" and r["name"] == "eas"))' "${EAS_DIR}/pins.json") + cli_eq "cli: requirements.lock is for the pinned EAS commit" "$(head -1 "${EAS_DIR}/requirements.lock")" "# enterprise-ai-solutions@${want} requirements.txt" + cli_eq "cli: requirements.in names the same commit" "$(head -1 "${EAS_DIR}/requirements.in")" "$(head -1 "${EAS_DIR}/requirements.lock")" + if python3 - "${EAS_DIR}/pins.json" "${EAS_DIR}/requirements.lock" <<'PY' +import json, re, sys +lock = open(sys.argv[2]).read() +for r in json.load(open(sys.argv[1]))["pins"]: + if r["profile"] == "bastion" and r["repo"].startswith("pypi.org/project/"): + name = r["repo"].rsplit("/", 1)[1] + block = re.search(r"(?ms)^%s==%s \\\n(.*?)(?=^\S)" % (re.escape(name), re.escape(r["ref"])), lock + "\nEND") + assert block and "--hash=sha256:" + r["sha"] in block.group(1), (name, r["ref"]) +PY + then ok "cli: requirements.lock pins each pins.json Python package at its version, with its wheel's sha256" + else fail "cli: requirements.lock and pins.json's bastion Python rows disagree"; fi + if grep -E '^[A-Za-z0-9_.-]+==' "${EAS_DIR}/requirements.lock" | grep -qv -- ' \\$'; then + fail "cli: requirements.lock has a pin without hashes" + else + ok "cli: every pin in requirements.lock carries hashes" + fi + if grep -E '^ARG [A-Z_]*_(SHA256|VERSION)=' "${EAS_DIR}/bastion.Dockerfile" | grep -Eqv '=((v|jq-)?[0-9.]+|[0-9a-f]{64})$'; then + fail "cli: bastion.Dockerfile has a version or sha256 ARG that is not exact" + else + ok "cli: bastion.Dockerfile pins every download by version and sha256" + fi + cli_eq "cli: every bastion download (kubectl, helm, yq, jq, kubernetes.core) is checked" "$(grep -c 'sha256sum -c -' "${EAS_DIR}/bastion.Dockerfile")" 5 + while read -r mode _ _ path; do + if [ "${mode}" = 100755 ]; then ok "cli: executable in git: ${path}"; else fail "cli: not executable in git (${mode}): ${path}"; fi + done < <(git -C "${ROOT}" ls-files -s -- 'integrations/eas/*.sh') + if python3 - "${EAS_DIR}/known-issues.json" <<'PY' +import json, sys +rows = json.load(open(sys.argv[1]))["issues"] +keys = {"id", "upstream", "symptom", "workaround", "source", "checked"} +assert rows and all(set(r) == keys and all(isinstance(v, str) and v for v in r.values()) for r in rows), rows +assert len({r["id"] for r in rows}) == len(rows) +PY + then ok "cli: known-issues.json rows are flat {id, upstream, symptom, workaround, source, checked}" + else fail "cli: known-issues.json shape"; fi +} + +section_argv() { + local D="${ROOT}/tests/fixtures/eas/argv" A="${WORK}/argv" fake="sk-ARGVPROBE7c1e94d0b3a24f6e8a51" rc pb n hits + local CASE="${ROOT}/tests/fixtures/eas/config/desktop-i9-13900T" args + mkdir -p "${A}/tmp" + if ! command -v ansible-playbook >/dev/null; then + fail "argv: needs ansible-playbook (the bastion runs EAS under it)" + return + fi + # The overrides eas up passes last, from the generator on a golden agent-toolkit case. + read -r -a args < "${CASE}/args" + python3 -S "${ROOT}/integrations/eas/eas-config.py" --facts "${CASE}/facts.json" --out "${A}/gen" "${args[@]}" \ + > /dev/null 2>&1 || { fail "argv: eas-config.py on desktop-i9-13900T"; return; } + # The key as a grep pattern file, so the sampler's own grep never holds it in its argv. + printf '%s\n' "${fake}" > "${A}/pattern" + # play [extra ansible args]: EI's two tasks under ansible-playbook while ps + # samples every process's argv; prints the play's rc and the argv lines holding the key. + play() { + local out=$1 + shift + mkdir -p "${out}" + ARGV_FAKE_KEY="${fake}" ANSIBLE_LIBRARY="${D}/library" ANSIBLE_LOCAL_TEMP="${A}/tmp" \ + ANSIBLE_REMOTE_TEMP="${A}/tmp" ANSIBLE_NOCOLOR=1 ANSIBLE_RETRY_FILES_ENABLED=0 \ + ansible-playbook -i localhost, "${D}/deploy-models.yml" -e "argv_out=${out}" "$@" \ + < /dev/null > "${out}/play.log" 2>&1 & + pb=$! + hits=0 + while kill -0 "${pb}" 2>/dev/null; do + n=$(ps -eo args | grep -c -F -f "${A}/pattern") + hits=$((hits + n)) + sleep 0.2 + done + wait "${pb}" && rc=0 || rc=$? + } + # The control: EI's tasks as they ship put the key on the argv of the /bin/sh -c that + # ansible's environment: wraps around the deploy, for the whole deploy. + play "${A}/control" + if [ "${rc}" -eq 0 ] && [ "${hits}" -gt 0 ]; then + ok "argv: control: without the overrides the key is on a process argv (${hits} sampled lines)" + else + fail "argv: control: rc=${rc}, ${hits} sampled lines hold the key (want rc 0 and > 0): $(tail -5 "${A}/control/play.log")" + fi + play "${A}/fixed" -e "@${A}/gen/k3d-gpu-overrides.yaml" + if [ "${rc}" -eq 0 ] && [ "${hits}" -eq 0 ]; then + ok "argv: with k3d-gpu-overrides.yaml no process argv holds the master key" + else + fail "argv: with the overrides: rc=${rc}, ${hits} sampled lines hold the key: $(tail -5 "${A}/fixed/play.log")" + fi + if [ "$(cat "${A}/control/key" 2>/dev/null)" = set ] && [ "$(cat "${A}/fixed/key" 2>/dev/null)" = empty ]; then + ok "argv: the deploy gets an empty MM_LITELLM_MASTER_KEY with the overrides (model-manager reads the Secret itself)" + else + fail "argv: MM_LITELLM_MASTER_KEY in the deploy: control '$(cat "${A}/control/key" 2>/dev/null)', with the overrides '$(cat "${A}/fixed/key" 2>/dev/null)' (want set, empty)" + fi + if grep -rqF -- "${fake}" "${A}/fixed" "${A}/gen"; then + fail "argv: the key reached a log or the generator's files" + else + ok "argv: the key is in no log and no generated file" + fi +} + +SECTIONS=(overlay config cli argv) [ "$#" -gt 0 ] && SECTIONS=("$@") for s in "${SECTIONS[@]}"; do if declare -F "section_${s}" > /dev/null; then diff --git a/tests/eat_test.sh b/tests/eat_test.sh index 9864ea34..b154a510 100755 --- a/tests/eat_test.sh +++ b/tests/eat_test.sh @@ -9,6 +9,15 @@ # both for a router that shlex.splits the command (agent-sandbox v0.5.0 # python-runtime-sandbox) and for one that runs it under /bin/sh -c # - the check fails when the sandbox prints anything but 385 +# - the router port-forward takes a local port the kernel picks, read from kubectl's +# own output, and the endpoint must answer /healthz as the router before the code +# and the token go to it +# - no key reaches curl's argument list (headers and bodies go as files / stdin) +# 2b. eat-check.sh EAT_MODE=eas (k3d-gpu eas check) against stubs: the gateway is +# eg-gateway's LoadBalancer IP with Host litellm.; the requests carry a +# LiteLLM virtual key minted from Secret litellm/litellm-master-key and deleted at +# exit (pass or fail); the model defaults to the first chat model /v1/models lists; +# the sandbox step is the same as in eat mode. # 3. eat-up.sh refuses deploy_openshell=on, unverified on k3d-gpu, and a config with no # deploy_guardrails value, on which release-1.1.0 deploys its guardrail. It also # refuses a deploy_nri_balloon_policy other than off (a second balloons policy), @@ -54,32 +63,54 @@ cat > "${BIN}/kubectl" <<'EOF' case "$*" in *"get nodes"*) echo 172.20.0.2 ;; # first labelled node: not where ingress runs *"-n ingress-nginx get pods"*"app.kubernetes.io/component=controller"*"status.phase=Running"*) echo 172.20.0.3 ;; + *"-n envoy-gateway-system get svc -l gateway.envoyproxy.io/owning-gateway-name=eg-gateway"*"loadBalancer.ingress[0].ip"*) + [ -n "${STUB_NO_LB:-}" ] || echo 172.20.0.9 ;; + *"-n litellm get secret litellm-master-key"*"master_key"*) printf 'sk-master-stub' | base64 ;; *"get pods -o name"*) printf 'pod/sandbox-router-1\npod/python-pool-abcde\n' ;; *"get secret sandbox-router-auth"*) printf 'tok' | base64 ;; - *port-forward*) exec sleep 60 ;; + *port-forward*) + # kubectl's own report of the local port it bound (STUB_PF_PORT: the one the + # kernel picked); STUB_PF_DEAD: kubectl exits without binding. + printf 'pf %s\n' "$*" >> "${STUB_LOG}" + if [ -n "${STUB_PF_DEAD:-}" ]; then echo "error: unable to listen on any of the requested ports" >&2; exit 1; fi + printf 'Forwarding from 127.0.0.1:%s -> 8080\nForwarding from [::1]:%s -> 8080\n' \ + "${STUB_PF_PORT:-40123}" "${STUB_PF_PORT:-40123}" + exec sleep 60 ;; *) echo "kubectl stub: unexpected: $*" >&2; exit 1 ;; esac EOF cat > "${BIN}/curl" <<'EOF' #!/usr/bin/env python3 -"""curl stub: answers the LiteLLM gateway and the sandbox router.""" +"""curl stub: answers the LiteLLM gateway (chat, /key/generate, /key/delete, /v1/models) +and the sandbox router. Logs every argv and the bearer token each request carries.""" import json, os, shlex, subprocess, sys args = sys.argv[1:] -body, url, resolve = None, None, None +body, url, resolve, auth = None, None, None, None +with open(os.environ["STUB_LOG"], "a") as f: + f.write("argv %s\n" % json.dumps(args)) i = 0 while i < len(args): a = args[i] - if a in ("-d", "--data"): - body = args[i + 1]; i += 2; continue + if a in ("-d", "--data", "--data-binary"): + body = sys.stdin.read() if args[i + 1] == "@-" else args[i + 1]; i += 2; continue if a == "--resolve": resolve = args[i + 1]; i += 2; continue - if a in ("-H", "-o", "--max-time"): + if a == "-H": + h = args[i + 1] + lines = open(h[1:]).read().splitlines() if h.startswith("@") else [h] + for line in lines: + if line.lower().startswith("authorization: bearer "): + auth = line.split(" ", 2)[2] + i += 2; continue + if a in ("-o", "--max-time"): i += 2; continue if not a.startswith("-"): url = a i += 1 +with open(os.environ["STUB_LOG"], "a") as f: + f.write("auth %s %s\n" % (url.split("/", 3)[-1] if url else "-", auth)) def msg(content=None, tool_calls=None, finish="stop"): m = {"role": "assistant", "content": content} @@ -87,12 +118,24 @@ def msg(content=None, tool_calls=None, finish="stop"): m["tool_calls"] = tool_calls print(json.dumps({"choices": [{"finish_reason": finish, "message": m}]})) -if url.endswith("/v1/chat/completions"): +if url.endswith("/key/generate"): + req = json.loads(body) + with open(os.environ["STUB_LOG"], "a") as f: + f.write("keygen alias=%s\n" % req.get("key_alias")) + if not os.environ.get("STUB_NO_KEY"): + print(json.dumps({"key": "sk-virtual-stub", "key_alias": req.get("key_alias")})) +elif url.endswith("/key/delete"): + with open(os.environ["STUB_LOG"], "a") as f: + f.write("keydel %s\n" % json.dumps(json.loads(body)["keys"])) +elif url.endswith("/v1/models"): + print(json.dumps({"data": [{"id": i} for i in os.environ.get("STUB_MODELS", "").split(",") if i]})) +elif url.endswith("/v1/chat/completions"): with open(os.environ["STUB_LOG"], "a") as f: f.write(f"resolve {resolve}\n") req = json.loads(body) with open(os.environ["STUB_LOG"], "a") as f: f.write(f"request max_tokens={req.get('max_tokens')} temperature={req.get('temperature')}\n") + f.write(f"model {req.get('model')}\n") prompt = req["messages"][-1]["content"] if req.get("tools"): if os.environ.get("STUB_NO_TOOL"): @@ -104,14 +147,30 @@ if url.endswith("/v1/chat/completions"): msg("\n\n\n\n```python\n" + os.environ["STUB_CODE"] + "\n```") else: msg("\n\n\n\nKubernetes runs containers.") -elif url.endswith("/execute"): - cmd = json.loads(body)["command"] - argv = shlex.split(cmd) if os.environ["STUB_ROUTER"] == "shlex" else ["/bin/sh", "-c", cmd] - p = subprocess.run(argv, capture_output=True, text=True) - print(json.dumps({"stdout": p.stdout, "stderr": p.stderr, "exit_code": p.returncode})) -# anything else (the router readiness probe) succeeds with no output +elif "127.0.0.1:" in url: + # The local end of the router port-forward: only the port kubectl reported is the + # router (STUB_ROUTER_FOREIGN: another local service answers there instead). + hostport, _, path = url.split("://", 1)[-1].partition("/") + with open(os.environ["STUB_LOG"], "a") as f: + f.write("router %s /%s\n" % (hostport, path)) + if hostport != "127.0.0.1:" + os.environ.get("STUB_PF_PORT", "40123") or os.environ.get("STUB_ROUTER_FOREIGN"): + print(json.dumps({"object": "error", "message": "Not Found"})) + elif path == "healthz": + print(json.dumps({"status": "ok"})) + elif path == "execute": + cmd = json.loads(body)["command"] + argv = shlex.split(cmd) if os.environ["STUB_ROUTER"] == "shlex" else ["/bin/sh", "-c", cmd] + p = subprocess.run(argv, capture_output=True, text=True) + print(json.dumps({"stdout": p.stdout, "stderr": p.stderr, "exit_code": p.returncode})) EOF -chmod +x "${BIN}/kubectl" "${BIN}/curl" +# jq: the real one, with each argv logged (a key must not reach any argument list). +REAL_JQ=$(command -v jq) +cat > "${BIN}/jq" <> "\${STUB_LOG:-/dev/null}" +exec "${REAL_JQ}" "\$@" +EOF +chmod +x "${BIN}/kubectl" "${BIN}/curl" "${BIN}/jq" # check_says [env assignments...] check_says() { @@ -147,6 +206,77 @@ check_says "eat-check defaults: 200 tokens, temperature 0" 0 'request max_tokens check_says "eat-check for a shared remote endpoint: sampled and short" 0 'request max_tokens=64 temperature=0.6$' \ EAT_TEMPERATURE=0.6 EAT_MAX_TOKENS=64 check_says "eat-check refuses a non-numeric temperature" fail 'EAT_TEMPERATURE must be a number' EAT_TEMPERATURE=hot +# The router port-forward: a local port the kernel picks, read from kubectl's own output, +# and the endpoint confirmed as the sandbox router before /execute (a fixed 18080 reached +# whatever local service held it). +check_says "the router port-forward asks kubectl for a kernel-picked local port" 0 \ + '^pf -n agent-sandbox-system port-forward svc/sandbox-router-svc :8080$' +check_says "/execute goes to the port kubectl reported" 0 '^router 127.0.0.1:43211 /execute$' STUB_PF_PORT=43211 +check_says "the router's /healthz is read on that port before /execute" 0 '^router 127.0.0.1:43211 /healthz$' STUB_PF_PORT=43211 +if grep '^router ' "${WORK}/log" | grep -qv '^router 127.0.0.1:43211 '; then + fail "a router request went to a port kubectl did not report: $(grep '^router ' "${WORK}/log" | grep -v '127.0.0.1:43211' | head -1)" +else + ok "every router request goes to the port kubectl reported" +fi +check_says "a port-forward that reports no port stops the check" fail 'port-forward to svc/sandbox-router-svc exited' STUB_PF_DEAD=1 +check_says "an endpoint that is not the sandbox router stops the check" fail 'does not answer as the sandbox router' \ + STUB_ROUTER_FOREIGN=1 +if grep -q '^router .* /execute$' "${WORK}/log"; then + fail "the code went to an endpoint that is not the sandbox router" +else + ok "nothing is sent to an endpoint that is not the sandbox router" +fi +check_says "eat mode: the master key from vault.yml authorizes the chat" 0 'auth v1/chat/completions sk-stub$' +check_says "eat mode: the router token authorizes /execute" 0 'auth execute tok$' +check_says "eat mode: no key on any curl argv" 0 'EAT check passed' +if grep '^argv ' "${WORK}/log" | grep -Eq 'sk-stub|Bearer tok'; then + fail "eat mode: a key reached curl's argv: $(grep '^argv ' "${WORK}/log" | grep -E 'sk-stub|Bearer tok' | head -1)" +else + ok "eat mode: no key in any curl argv" +fi + +# 2b. EAT_MODE=eas: EAS's agent toolkit from `k3d-gpu eas up --agent-toolkit`. +EAS_MODELS="hybrid-router,Qwen3Guard-Gen-0.6B,qwen2-5-coder-7b,bge-embed" +check_says "eas mode passes on a working stack" 0 'EAT check passed \(eas\)' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: eg-gateway's LoadBalancer IP with Host litellm." 0 \ + 'resolve litellm.solutions.ai:443:172.20.0.9' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the domain from EAT_EAS_DOMAIN" 0 'resolve litellm.example.org:443:172.20.0.9' \ + EAT_MODE=eas EAT_EAS_DOMAIN=example.org STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the virtual key is minted with the master key" 0 'auth key/generate sk-master-stub$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: chat goes with the virtual key, not the master key" 0 'auth v1/chat/completions sk-virtual-stub$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the virtual key is deleted after" 0 'keydel \["sk-virtual-stub"\]' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: the key alias is this run's" 0 'keygen alias=k3d-gpu-eas-check-[0-9]+-[0-9]+$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +if grep -q '^auth v1/chat/completions sk-master-stub' "${WORK}/log"; then + fail "eas mode: a chat request carried the master key" +else + ok "eas mode: no chat request carries the master key" +fi +if grep '^argv ' "${WORK}/log" | grep -Eq 'sk-master-stub|sk-virtual-stub|Bearer tok'; then + fail "eas mode: a key reached curl's argv" +else + ok "eas mode: no key in any curl argv" +fi +if ! grep -q '^jqargv ' "${WORK}/log"; then + fail "eas mode: the jq argv log is empty (the jq wrapper did not run)" +elif grep '^jqargv ' "${WORK}/log" | grep -Eq 'sk-master-stub|sk-virtual-stub|Bearer tok'; then + fail "eas mode: a key reached jq's argv: $(grep '^jqargv ' "${WORK}/log" | grep -Eo 'sk-(master|virtual)-stub' | sort -u | tr '\n' ' ')" +else + ok "eas mode: no key in any jq argv" +fi +check_says "eas mode: the model is the first chat model the gateway lists" 0 '^model qwen2-5-coder-7b$' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: EAT_MODEL wins" 0 '^model my-model$' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" EAT_MODEL=my-model +check_says "eas mode: the sandbox step is unchanged" 0 '"stdout":"385\\n"' EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" +check_says "eas mode: a failed check still deletes the key" fail 'keydel \["sk-virtual-stub"\]' \ + EAT_MODE=eas STUB_MODELS="${EAS_MODELS}" STUB_CODE='print(sum(range(1, 11)))' +check_says "eas mode: no chat model listed stops the check" fail 'lists no chat model' EAT_MODE=eas STUB_MODELS="hybrid-router" +check_says "eas mode: no LoadBalancer IP stops the check" fail 'no LoadBalancer IP' EAT_MODE=eas STUB_NO_LB=1 +check_says "eas mode: no key from /key/generate stops the check" fail 'without a key' EAT_MODE=eas STUB_NO_KEY=1 +check_says "eas mode refuses a domain with shell characters" fail 'must be a domain name' EAT_MODE=eas 'EAT_EAS_DOMAIN=x;id' +check_says "an unknown EAT_MODE is refused" fail 'EAT_MODE must be eat or eas' EAT_MODE=other # 4. eat-remote.sh: a remote endpoint as a LiteLLM model. The key reaches the # cluster only as the Secret on kubectl's stdin, the gateway gets it as a diff --git a/tests/fixtures/eas/argv/deploy-models.yml b/tests/fixtures/eas/argv/deploy-models.yml new file mode 100644 index 00000000..779056ed --- /dev/null +++ b/tests/fixtures/eas/argv/deploy-models.yml @@ -0,0 +1,31 @@ +# EI 3e8a244 roles/llm_services/tasks/deploy_models.yaml, the two tasks that carry the +# LiteLLM master key, for tests/eas_test.sh section argv: +# L81-L90 the Secret read (kubernetes.core.k8s_info replaced by library/k8s_secret_stub.py, +# which returns a Secret holding ARGV_FAKE_KEY) +# L213-L241 the model deploy with its MM_LITELLM_MASTER_KEY line verbatim (L240); the +# `./model-manager deploy` command replaced by a sleep that records whether +# the task hands over a value +# https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/deploy_models.yaml#L81-L90 +# https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/deploy_models.yaml#L213-L241 +- hosts: localhost + connection: local + gather_facts: false + vars: + auth_type: litellm_vkey + tasks: + - name: "Deploy models | Load LiteLLM master key (when auth_type=litellm_vkey)" + k8s_secret_stub: + api_version: v1 + kind: Secret + name: litellm-master-key + namespace: "{{ litellm_namespace | default('litellm') }}" + register: _deploy_litellm_secret + failed_when: false + no_log: true + when: auth_type | default('jwt') == 'litellm_vkey' + + - name: Deploy each catalog model + shell: | + if [ -n "$MM_LITELLM_MASTER_KEY" ]; then echo set; else echo empty; fi > "{{ argv_out }}/key"; sleep 4 + environment: + MM_LITELLM_MASTER_KEY: "{{ (_deploy_litellm_secret.resources[0].data.master_key | b64decode) if (auth_type | default('jwt') == 'litellm_vkey' and _deploy_litellm_secret.resources | default([]) | length > 0) else '' }}" diff --git a/tests/fixtures/eas/argv/library/k8s_secret_stub.py b/tests/fixtures/eas/argv/library/k8s_secret_stub.py new file mode 100644 index 00000000..a23ed5f1 --- /dev/null +++ b/tests/fixtures/eas/argv/library/k8s_secret_stub.py @@ -0,0 +1,10 @@ +#!/usr/bin/python +# kubernetes.core.k8s_info's result for Secret litellm/litellm-master-key, its master_key +# the fake key in ARGV_FAKE_KEY (base64, as the API returns Secret data). +import base64 +import json +import os + +key = os.environ["ARGV_FAKE_KEY"].encode() +print(json.dumps({"changed": False, "api_found": True, + "resources": [{"data": {"master_key": base64.b64encode(key).decode()}}]})) diff --git a/tests/fixtures/eas/config/default-cni/expected/choices.txt b/tests/fixtures/eas/config/default-cni/expected/choices.txt index 389d0777..36f68ea2 100644 --- a/tests/fixtures/eas/config/default-cni/expected/choices.txt +++ b/tests/fixtures/eas/config/default-cni/expected/choices.txt @@ -12,6 +12,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] pin ei main (profile main) -> repos.inference.yaml inference rev: 3e8a24486b6ee812adf2bf452f8ba5751c16c8bd [intel/enterprise-ai-solutions@97f6157 configs/repos/repos.agent-toolkit.yaml:12] WARN: node has 32 CPUs, under EAT's 48-core minimum -> continuing [intel/enterprise-agent-toolkit@036b662 docs/prerequisites.md:7] diff --git a/tests/fixtures/eas/config/default-cni/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/default-cni/expected/k3d-gpu-overrides.yaml index c7b3c3cc..4f780aaf 100644 --- a/tests/fixtures/eas/config/default-cni/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/default-cni/expected/k3d-gpu-overrides.yaml @@ -7,4 +7,5 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" diff --git a/tests/fixtures/eas/config/desktop-i9-13900T/expected/choices.txt b/tests/fixtures/eas/config/desktop-i9-13900T/expected/choices.txt index 7897681b..762a1c5e 100644 --- a/tests/fixtures/eas/config/desktop-i9-13900T/expected/choices.txt +++ b/tests/fixtures/eas/config/desktop-i9-13900T/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 125.3 GiB; qwen2-5-coder-14b needs 131 GiB; qwen2-5-coder-7b needs 99 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-7b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/desktop-i9-13900T/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/desktop-i9-13900T/expected/k3d-gpu-overrides.yaml index 58481ff1..5195c268 100644 --- a/tests/fixtures/eas/config/desktop-i9-13900T/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/desktop-i9-13900T/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-7b"] diff --git a/tests/fixtures/eas/config/eas-rerun/expected/choices.txt b/tests/fixtures/eas/config/eas-rerun/expected/choices.txt index 7983f5ff..73c5a779 100644 --- a/tests/fixtures/eas/config/eas-rerun/expected/choices.txt +++ b/tests/fixtures/eas/config/eas-rerun/expected/choices.txt @@ -13,6 +13,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= k3s.io/node-args --cluster-cidr 10.52.0.0/16 -> kubernetes_cluster_cidr: 10.52.0.0/16 [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] k3s.io/node-args --service-cidr 10.53.0.0/16 -> kubernetes_service_cidr: 10.53.0.0/16 [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] agent-sandbox v0.5.0 installed (EAT's own) -> agent_sandbox_enabled: true [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 512 GiB; qwen2-5-coder-14b needs 131 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/eas-rerun/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/eas-rerun/expected/k3d-gpu-overrides.yaml index b57e3136..705903b3 100644 --- a/tests/fixtures/eas/config/eas-rerun/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/eas-rerun/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: true kubernetes_cluster_cidr: "10.52.0.0/16" kubernetes_service_cidr: "10.53.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/fixtures/eas/config/fit-missing/expected/choices.txt b/tests/fixtures/eas/config/fit-missing/expected/choices.txt index f43f455a..e74fe653 100644 --- a/tests/fixtures/eas/config/fit-missing/expected/choices.txt +++ b/tests/fixtures/eas/config/fit-missing/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 125.3 GiB; qwen2-5-coder-14b needs 131 GiB; qwen2-5-coder-7b: k3d-gpu fit gave no result; llama-3-1-8b needs 99 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [llama-3-1-8b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/fit-missing/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/fit-missing/expected/k3d-gpu-overrides.yaml index 07011ae2..23eee106 100644 --- a/tests/fixtures/eas/config/fit-missing/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/fit-missing/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["llama-3-1-8b"] diff --git a/tests/fixtures/eas/config/gaudi/expected/choices.txt b/tests/fixtures/eas/config/gaudi/expected/choices.txt index f543fd4a..4cea16e2 100644 --- a/tests/fixtures/eas/config/gaudi/expected/choices.txt +++ b/tests/fixtures/eas/config/gaudi/expected/choices.txt @@ -12,6 +12,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] habana.ai/gaudi allocatable 8 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 1007 GiB; qwen2-5-coder-14b needs 131 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/gaudi/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/gaudi/expected/k3d-gpu-overrides.yaml index 6b01df56..93d41d97 100644 --- a/tests/fixtures/eas/config/gaudi/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/gaudi/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/fixtures/eas/config/inference-layer/expected/choices.txt b/tests/fixtures/eas/config/inference-layer/expected/choices.txt index 7c532221..833d9d1d 100644 --- a/tests/fixtures/eas/config/inference-layer/expected/choices.txt +++ b/tests/fixtures/eas/config/inference-layer/expected/choices.txt @@ -12,6 +12,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] pin ei main (profile main) -> repos.inference.yaml inference rev: 3e8a24486b6ee812adf2bf452f8ba5751c16c8bd [intel/enterprise-ai-solutions@97f6157 configs/repos/repos.agent-toolkit.yaml:12] WARN: node has 32 CPUs, under EAT's 48-core minimum -> continuing [intel/enterprise-agent-toolkit@036b662 docs/prerequisites.md:7] diff --git a/tests/fixtures/eas/config/inference-layer/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/inference-layer/expected/k3d-gpu-overrides.yaml index 463cc163..44b9acec 100644 --- a/tests/fixtures/eas/config/inference-layer/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/inference-layer/expected/k3d-gpu-overrides.yaml @@ -7,4 +7,5 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" diff --git a/tests/fixtures/eas/config/model-nofit/expected/choices.txt b/tests/fixtures/eas/config/model-nofit/expected/choices.txt index 99d516d1..18de13ca 100644 --- a/tests/fixtures/eas/config/model-nofit/expected/choices.txt +++ b/tests/fixtures/eas/config/model-nofit/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] --model qwen2-5-coder-7b -> llm_services_autodeploy_models: [qwen2-5-coder-7b] [intel/enterprise-agent-toolkit@036b662 config.yaml:32-35] diff --git a/tests/fixtures/eas/config/model-nofit/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/model-nofit/expected/k3d-gpu-overrides.yaml index 58481ff1..5195c268 100644 --- a/tests/fixtures/eas/config/model-nofit/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/model-nofit/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-7b"] diff --git a/tests/fixtures/eas/config/model-override/expected/choices.txt b/tests/fixtures/eas/config/model-override/expected/choices.txt index da2f2ad9..90b470f9 100644 --- a/tests/fixtures/eas/config/model-override/expected/choices.txt +++ b/tests/fixtures/eas/config/model-override/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] --model qwen2-5-coder-14b -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 config.yaml:32-35] diff --git a/tests/fixtures/eas/config/model-override/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/model-override/expected/k3d-gpu-overrides.yaml index 6b01df56..93d41d97 100644 --- a/tests/fixtures/eas/config/model-override/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/model-override/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/fixtures/eas/config/no-nri/expected/choices.txt b/tests/fixtures/eas/config/no-nri/expected/choices.txt index 22ef51cf..6e426488 100644 --- a/tests/fixtures/eas/config/no-nri/expected/choices.txt +++ b/tests/fixtures/eas/config/no-nri/expected/choices.txt @@ -12,6 +12,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] pin ei main (profile main) -> repos.inference.yaml inference rev: 3e8a24486b6ee812adf2bf452f8ba5751c16c8bd [intel/enterprise-ai-solutions@97f6157 configs/repos/repos.agent-toolkit.yaml:12] WARN: node has 32 CPUs, under EAT's 48-core minimum -> continuing [intel/enterprise-agent-toolkit@036b662 docs/prerequisites.md:7] diff --git a/tests/fixtures/eas/config/no-nri/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/no-nri/expected/k3d-gpu-overrides.yaml index 463cc163..44b9acec 100644 --- a/tests/fixtures/eas/config/no-nri/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/no-nri/expected/k3d-gpu-overrides.yaml @@ -7,4 +7,5 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" diff --git a/tests/fixtures/eas/config/rc1-profile/expected/choices.txt b/tests/fixtures/eas/config/rc1-profile/expected/choices.txt index 2ed5e56b..10045d3b 100644 --- a/tests/fixtures/eas/config/rc1-profile/expected/choices.txt +++ b/tests/fixtures/eas/config/rc1-profile/expected/choices.txt @@ -13,6 +13,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 512 GiB; qwen2-5-coder-14b needs 131 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/rc1-profile/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/rc1-profile/expected/k3d-gpu-overrides.yaml index 5cbd2a2e..41a58e48 100644 --- a/tests/fixtures/eas/config/rc1-profile/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/rc1-profile/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/fixtures/eas/config/rootless-docker/expected/choices.txt b/tests/fixtures/eas/config/rootless-docker/expected/choices.txt index fc88a536..51b8c13a 100644 --- a/tests/fixtures/eas/config/rootless-docker/expected/choices.txt +++ b/tests/fixtures/eas/config/rootless-docker/expected/choices.txt @@ -13,6 +13,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] gpu.intel.com/xe allocatable 1 -> kubernetes_accelerator: cpu (serve the device with k3d-gpu agent up) [intel/enterprise-agent-toolkit@036b662 README.md:36] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 125.3 GiB; qwen2-5-coder-14b needs 131 GiB; qwen2-5-coder-7b: fit: Qwen/Qwen2.5-Coder-7B-Instruct does not fit this host: 6.1 GiB free for the KV cache, needs 7.3 GiB; llama-3-1-8b needs 99 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [llama-3-1-8b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/rootless-docker/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/rootless-docker/expected/k3d-gpu-overrides.yaml index 07011ae2..23eee106 100644 --- a/tests/fixtures/eas/config/rootless-docker/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/rootless-docker/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["llama-3-1-8b"] diff --git a/tests/fixtures/eas/config/stock-k3s-model/expected/choices.txt b/tests/fixtures/eas/config/stock-k3s-model/expected/choices.txt index 40eab205..33cfa642 100644 --- a/tests/fixtures/eas/config/stock-k3s-model/expected/choices.txt +++ b/tests/fixtures/eas/config/stock-k3s-model/expected/choices.txt @@ -12,6 +12,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] --model qwen3-coder-30b -> llm_services_autodeploy_models: [qwen3-coder-30b] [intel/enterprise-inference@3e8a244 roles/llm_services/defaults/main.yaml:84] pin ei main (profile main) -> repos.inference.yaml inference rev: 3e8a24486b6ee812adf2bf452f8ba5751c16c8bd [intel/enterprise-ai-solutions@97f6157 configs/repos/repos.agent-toolkit.yaml:12] diff --git a/tests/fixtures/eas/config/stock-k3s-model/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/stock-k3s-model/expected/k3d-gpu-overrides.yaml index de8bad5c..c261a081 100644 --- a/tests/fixtures/eas/config/stock-k3s-model/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/stock-k3s-model/expected/k3d-gpu-overrides.yaml @@ -7,5 +7,6 @@ nfd_enabled: true kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" llm_services_autodeploy_models: ["qwen3-coder-30b"] diff --git a/tests/fixtures/eas/config/tiny-host/expected/choices.txt b/tests/fixtures/eas/config/tiny-host/expected/choices.txt index 8280327e..2b88825b 100644 --- a/tests/fixtures/eas/config/tiny-host/expected/choices.txt +++ b/tests/fixtures/eas/config/tiny-host/expected/choices.txt @@ -14,6 +14,7 @@ no node with AMXBF16 -> VLLM_CPU_SGL_KERNEL: as shipped (vLLM v0.28.0 checks for no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 30 GiB; qwen2-5-coder-14b needs 131 GiB; qwen2-5-coder-7b needs 99 GiB; llama-3-1-8b needs 99 GiB -> llm_services_autodeploy_models: [] (deploy one later with ./model-manager deploy ) [intel/enterprise-agent-toolkit@036b662 config.yaml:32-35] diff --git a/tests/fixtures/eas/config/tiny-host/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/tiny-host/expected/k3d-gpu-overrides.yaml index 5742b8f0..9ae6c264 100644 --- a/tests/fixtures/eas/config/tiny-host/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/tiny-host/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: [] diff --git a/tests/fixtures/eas/config/xeon-amx-external/expected/choices.txt b/tests/fixtures/eas/config/xeon-amx-external/expected/choices.txt index e1c50153..8227e246 100644 --- a/tests/fixtures/eas/config/xeon-amx-external/expected/choices.txt +++ b/tests/fixtures/eas/config/xeon-amx-external/expected/choices.txt @@ -13,6 +13,7 @@ node k3d-k3d-gpu-server-0 labelled feature.node.kubernetes.io/cpu-cpuid.AMXBF16= no --cluster-cidr in k3s.io/node-args -> kubernetes_cluster_cidr: 10.42.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:137-141] no --service-cidr in k3s.io/node-args -> kubernetes_service_cidr: 10.43.0.0/16 (the k3s default) [k3s-io/k3s@356c025 pkg/cli/cmds/server.go:142-146] bring-your-own cluster -> kata_containers_enabled: false [intel/enterprise-ai-solutions@97f6157 roles/kata_containers/tasks/install.yaml:9-22] +EI's model deploy passes the LiteLLM master key through ansible environment: (a /bin/sh -c argument while each model deploys) -> _deploy_litellm_secret: "" (model-manager reads Secret litellm/litellm-master-key itself) [intel/enterprise-inference@3e8a244 roles/llm_services/tasks/deploy_models.yaml:240] no accelerator resources -> kubernetes_accelerator: cpu [intel/enterprise-ai-solutions@97f6157 configs/defaults/global_config.yaml:106] no agent-sandbox CRD -> agent_sandbox_enabled: true (EAT installs agent-sandbox v0.5.0) [intel/enterprise-agent-toolkit@036b662 config.yaml:56] node memory 512 GiB; qwen2-5-coder-14b needs 131 GiB and k3d-gpu fit says it fits -> llm_services_autodeploy_models: [qwen2-5-coder-14b] [intel/enterprise-agent-toolkit@036b662 docs/single-node-deployment.md:84] diff --git a/tests/fixtures/eas/config/xeon-amx-external/expected/k3d-gpu-overrides.yaml b/tests/fixtures/eas/config/xeon-amx-external/expected/k3d-gpu-overrides.yaml index 5cbd2a2e..41a58e48 100644 --- a/tests/fixtures/eas/config/xeon-amx-external/expected/k3d-gpu-overrides.yaml +++ b/tests/fixtures/eas/config/xeon-amx-external/expected/k3d-gpu-overrides.yaml @@ -7,6 +7,7 @@ nfd_enabled: false kubernetes_cluster_cidr: "10.42.0.0/16" kubernetes_service_cidr: "10.43.0.0/16" kata_containers_enabled: false +_deploy_litellm_secret: "" kubernetes_accelerator: "cpu" agent_sandbox_enabled: true llm_services_autodeploy_models: ["qwen2-5-coder-14b"] diff --git a/tests/image_pins_test.sh b/tests/image_pins_test.sh index a3d3e6b5..5dac89fa 100755 --- a/tests/image_pins_test.sh +++ b/tests/image_pins_test.sh @@ -165,6 +165,28 @@ EOF ); then ok "images.lock collects integrations/eat values pairs, Dockerfile FROMs and the Dockerfile's *_IMAGE ARGs" else fail "images-lock collect integrations/eat"; sed 's/^/ | /' <<< "${out}"; fi +# ---- images.lock sources: integrations/eas bastion FROM and _image extra vars ---- +mkdir -p "${WORK}/r/integrations/eas" +printf 'FROM ubuntu:24.04@%s\n' "${D1}" > "${WORK}/r/integrations/eas/bastion.Dockerfile" +printf '# a comment naming x_image: nothing\nagent_sandbox_router_image: "us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router:latest-main@%s"\nnot_an_image_key: "busybox:1"\n' "${D2}" \ + > "${WORK}/r/integrations/eas/images.yaml" +printf '{"image": "ignored:1"}\n' > "${WORK}/r/integrations/eas/pins.json" +if out=$(python3 - "${WORK}/r" "${D1}" "${D2}" 2>&1 <<'EOF' +import importlib.machinery, importlib.util, sys +root, d1, d2 = sys.argv[1:] +loader = importlib.machinery.SourceFileLoader("images_lock", root + "/scripts/images-lock") +spec = importlib.util.spec_from_loader("images_lock", loader) +m = importlib.util.module_from_spec(spec) +loader.exec_module(m) +found, _ = m.collect(root) +router = ("us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router", "latest-main") +assert found.get(router) == {"digest": d2, "sources": {"integrations/eas/images.yaml"}}, found.get(router) +assert found[("docker.io/library/ubuntu", "24.04")]["sources"] >= {"integrations/eas/bastion.Dockerfile"}, found +assert not any(k[0].endswith("/busybox") or k[0].endswith("/ignored") for k in found), sorted(found) +EOF +); then ok "images.lock collects integrations/eas: the bastion's FROM and each _image extra var" +else fail "images-lock collect integrations/eas"; sed 's/^/ | /' <<< "${out}"; fi + # ---- repin, with the registry stubbed ---- scratch cat > "${WORK}/r/share/a.yml" < "${eas_pkg}/share/integrations/eas/eas.sh" +check "a packaged launcher finds K3D_GPU_SHARE/integrations/eas" test "$(K3D_GPU_SHARE="${eas_pkg}/share" HOME="${eas_pkg}" bash -c \ + 'source <(sed "/^case \"\\\${1:-help}\" in\$/,\$d" "$1"); eas_dir' _ "${eas_pkg}/bin/k3d-gpu")" = "${eas_pkg}/share/integrations/eas" +rm -rf "${eas_pkg}" +check "overlay.json's patch_dirs are under integrations/, which the packages keep" python3 -c ' +import json, sys +dirs = json.load(open(sys.argv[1]))["patch_dirs"] +sys.exit(not dirs or not all(d.startswith("integrations/") for d in dirs))' "${ROOT}/integrations/eas/overlay.json" check "flake.lock pins nixpkgs to a rev" \ grep -Eq '"rev": "[0-9a-f]{40}"' "${ROOT}/flake.lock"