eas: k3d-gpu eas up, down, status and check (EAS with EAT 2.0 on a k3d-gpu cluster)

.github/workflows/auto-release.yml

@@ -50,6 +50,9 @@ on:
       - 'oracle/validate.py'
       - 'oracle/share.py'
       - 'oracle/schema/bundle-v1.json'
+      - 'integrations/eas/**'
+      - 'integrations/eat/eat-check.sh'
+      - 'integrations/eat/upstream/patches/**'
   workflow_dispatch:
     inputs:
       mode:

Formula/k3d-gpu.rb

@@ -21,6 +21,10 @@ def install
     (pkgshare/"bench/apb").install "bench/apb/apb.py", "bench/apb/capacity.py"
     (pkgshare/"oracle").install "oracle/bundle.py", "oracle/project.py", "oracle/validate.py", "oracle/share.py"
     (pkgshare/"oracle/schema").install "oracle/schema/bundle-v1.json"
+    # k3d-gpu eas, its overlay's patch drafts and eat-check.sh, in the repo's integrations/ layout.
+    (pkgshare/"integrations/eas").install Dir["integrations/eas/*"]
+    (pkgshare/"integrations/eat/upstream/patches").install Dir["integrations/eat/upstream/patches/*.patch"]
+    (pkgshare/"integrations/eat").install "integrations/eat/eat-check.sh"
     # The launcher reads its manifests from K3D_GPU_SHARE (default /usr/share/k3d-gpu);
     # k3d-gpu-mcp runs the launcher next to it in libexec and passes its environment on.
     (bin/"k3d-gpu").write_env_script libexec/"k3d-gpu", K3D_GPU_SHARE: "${K3D_GPU_SHARE:-#{pkgshare}}"

README.md

@@ -88,7 +88,7 @@ context, so `up` switches that file's current-context to `k3d-<cluster>`.
 ### Install
 
 Every package installs the same payload: the launcher at `bin/k3d-gpu`, the MCP server at `bin/k3d-gpu-mcp`,
-and in `share/k3d-gpu/` the plugin manifests, `console/`, `bench/`, `oracle/` and `images.lock`
+and in `share/k3d-gpu/` the plugin manifests, `console/`, `bench/`, `oracle/`, `integrations/` (for `k3d-gpu eas`) and `images.lock`
 (`facts`, `plan`, `audit`, `tune`, `capacity` and `k3d-gpu-mcp` also need `python3`).
 The node image `cryptoandcoffee/k3d-gpu` is published for linux/amd64 only, so on arm64 the
 packages install but `k3d-gpu up` stops unless `K3D_GPU_IMAGE` names a k3s node image built for

docs/intel/eas.md

@@ -67,6 +67,10 @@ k3d-gpu eas down       # EAS teardown; the cluster stays (k3d-gpu down removes i
 - Arguments after `--` go to `ansible-playbook`, for example
   `k3d-gpu eas up --agent-toolkit -- -e '{"llm_services_autodeploy_models": []}'` to
   install without EAT's chat model.
+- `eas down` gives the teardown the same image pins and overrides as `eas up` gave the
+  install. EAS picks teardown components by the same `enabled` flags as install
+  ([resolve_components.py](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/filter_plugins/resolve_components.py#L245)),
+  so components `eas up` left to k3d-gpu, such as its NFD, stay.
 
 What `eas up` does, in order:
 
@@ -214,6 +218,7 @@ Bastion tools (each binary checked against its upstream checksum file at build t
 | kubectl | v1.37.2 | the minor of the node's k3s (`K3S_TAG` v1.37.1-k3s1); EAS's own v1.34.3 ([#L52](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L52)) is three minors off, outside the [kubectl skew policy](https://kubernetes.io/releases/version-skew-policy/#kubectl) |
 | helm | v3.22.1 | latest v3; EAS's floor is v3.20.2 ([#L53](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L53)). Helm v4 is untested with `kubernetes.core` 6.3.0 |
 | yq | v4.54.1 | EAS's floor is v4.53.2 ([#L51](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/es_auto_installer.sh#L51)) |
+| jq | jq-1.8.2 | latest; enterprise-inference pipes yq into jq on the installer host to pick the models to deploy ([deploy_models.yaml#L108-L114](https://github.com/intel/enterprise-inference/blob/3e8a24486b6ee812adf2bf452f8ba5751c16c8bd/roles/llm_services/tasks/deploy_models.yaml#L108-L114)), and EAS's `configure` installs no jq |
 | ansible, kubernetes (Python) | 12.3.0, 35.0.0, hashed | newest inside EAS's ranges `ansible>=10.0,<13.0`, `kubernetes>=29.0,<36.0` ([requirements.txt](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/requirements.txt)) |
 | kubernetes.core | 6.3.0 | EAS's [collections/requirements.yml](https://github.com/intel/enterprise-ai-solutions/blob/97f6157d6a72f7173c698c474d4e51d2c79f1fd6/collections/requirements.yml) |
 

hack/release-paths

@@ -27,3 +27,6 @@ oracle/project.py
 oracle/validate.py
 oracle/share.py
 oracle/schema/bundle-v1.json
+integrations/eas/**
+integrations/eat/eat-check.sh
+integrations/eat/upstream/patches/**

images.lock

@@ -166,6 +166,7 @@
       "name": "docker.io/library/ubuntu",
       "sources": [
         "Dockerfile",
+        "integrations/eas/bastion.Dockerfile",
         "integrations/eat/bastion.Dockerfile"
       ],
       "tag": "24.04"
@@ -413,6 +414,14 @@
       ],
       "tag": "v1.0.8"
     },
+    {
+      "digest": "sha256:bbacb1409a2ec436ee8cecdfb50f626fe66eae35adbf2886df255cbf5a2b10df",
+      "name": "registry.k8s.io/agent-sandbox/python-runtime-sandbox",
+      "sources": [
+        "integrations/eas/images.yaml"
+      ],
+      "tag": "v0.5.0"
+    },
     {
       "digest": "sha256:41da84809426098cb04cdafbb41855f15e3604f7455ef53816bad666e532e787",
       "name": "registry.k8s.io/agent-sandbox/python-runtime-sandbox",
@@ -429,6 +438,14 @@
       ],
       "tag": "v0.20.1"
     },
+    {
+      "digest": "sha256:9cebf7d97f98c0ce008aec8524f23637e60b61485db18ad426897eefac156968",
+      "name": "us-central1-docker.pkg.dev/k8s-staging-images/agent-sandbox/sandbox-router",
+      "sources": [
+        "integrations/eas/images.yaml"
+      ],
+      "tag": "latest-main"
+    },
     {
       "digest": "sha256:5d064465d751a73a729e7ecdc32c0ba5de0df1d305debb60a3d9a85b4fd3a967",
       "name": "vault.habana.ai/gaudi-docker/1.24.2/ubuntu24.04/habanalabs/vllm-0.30.0-ptupstream-2.13.0",

integrations/eas/README.md

@@ -0,0 +1,77 @@
+# Intel Enterprise AI Solutions on k3d-gpu
+
+`k3d-gpu eas` installs Intel's [Enterprise AI Solutions](https://github.com/intel/enterprise-ai-solutions)
+(EAS) on a k3d-gpu cluster with EAS's own installer, `es_auto_installer.sh`, in its
+bring-your-own-cluster mode (`existing_kubernetes`). The layers are platform, inference
+([enterprise-inference](https://github.com/intel/enterprise-inference), EI) and agent-toolkit
+([enterprise-agent-toolkit](https://github.com/intel/enterprise-agent-toolkit) 2.0, EAT).
+They run on the Xeon CPU path: `kubernetes_accelerator: cpu` is the only value EAS acts on at
+the pinned commit. Gaudi and Arc stay with `k3d-gpu agent up`.
+
+```bash
+k3d-gpu eas up --dry-run --agent-toolkit   # facts, choices and overlay decisions; changes nothing
+k3d-gpu eas up --agent-toolkit             # creates the cluster first if it is absent
+k3d-gpu eas check                          # chat, tool call and a sandbox run through EAS's LiteLLM
+k3d-gpu eas status
+k3d-gpu eas down                           # teardown platform; the cluster stays
+```
+
+`eas up` without `--agent-toolkit` installs platform and inference. `--layer erag` reaches
+`eas-config.py`, which refuses it with the reason.
+
+## What `eas up` does
+
+1. Reads facts, read-only, into `facts.json` (schema `k3d-gpu.eas-facts/v1`, the one
+   `eas-config.py` documents): nodes, namespaces, DaemonSets, the agent-sandbox CRD and its
+   controller image, the server version, rootless docker, the node container's NRI socket
+   and containerd CNI `bin_dirs` (`docker exec ... test -S` and `crictl info`), MemAvailable
+   and inotify limits, and `k3d-gpu fit` of each of `eas-config.py`'s model candidates (vLLM
+   on the CPU).
+2. Resolves refs: a profile of `pins.json` (`--eas-ref v1.2.0-rc.1`) and a full SHA go to
+   `eas-config.py` as given; any other `--eas-ref`, `--ei-ref` or `--eat-ref` becomes a
+   commit through `git ls-remote` here, once, and is printed.
+3. `eas-config.py` writes the overrides, `global_config.edit.yaml`, `models.edit.yaml`,
+   `repos.<layer>.yaml` and `choices.txt`, one `fact -> choice [source]` line per decision.
+   Exit 3 is a refusal: nothing is changed. The commits it resolved (the `# eas|ei|eat`
+   lines of `choices.txt`) go to `pins.resolved.json`.
+4. Builds the bastion (`bastion.Dockerfile`), tagged by the sha256 of the Dockerfile and
+   `requirements.lock`, or reuses it.
+5. Runs `eas.sh up <layer>` in the bastion: clone EAS and check out the commit, apply the
+   patch overlay (`overlay.py --apply`, with `kubernetes_cpu_policy` from the overrides and
+   the node count), set each manifest rev from `repos.<layer>.yaml`,
+   `init <layer> --env k3d-gpu`, merge the two edit files into `env/k3d-gpu/`, then
+   `install <layer> --env k3d-gpu -- -e @k3d-gpu-images.yaml -e @k3d-gpu-overrides.yaml
+   [your args]`. Extra vars load last in `es_auto_installer.sh` `main()`, so the overrides
+   win over every config file. `configure` is never run.
+6. Prints the gateway: the LoadBalancer IP of eg-gateway's Service (k3s ServiceLB, the node
+   IP), and with the agent toolkit a `curl --resolve litellm.<base_domain_name>:443:<ip>`
+   line. `/etc/hosts` is not edited.
+
+The bastion runs on the host network with the state dir read-write at `/eas-state`, a
+kubeconfig holding this cluster's context alone read-only at `/kube/config`, and
+`integrations/` read-only at `/k3d-gpu/integrations` (overlay.py finds its patch drafts
+relative to the dir above `integrations/`). It runs as root: `es_auto_installer.sh`
+`ensure_sudo` runs `sudo -k` under `set -e` unless EUID is 0, and the image has no sudo.
+`eas.sh` hands the state dir back to your uid:gid when it exits. `HF_TOKEN` and the proxy
+variables reach it by name only, when set.
+
+State per cluster: `~/.local/state/k3d-gpu/eas-<cluster>/` (mode 0700): the EAS checkout with
+its `env/k3d-gpu/` and `ext/`, `facts.json`, `pins.resolved.json`, `eas-config.py`'s files,
+`overlay.txt` and `overlay-decisions.json`, `kubeconfig`, `layer` and `logs/`.
+
+## Files
+
+| File | Role |
+|------|------|
+| `eas.sh` | the in-bastion driver (`up`, `overlay-check`, `down`, `status`) |
+| `bastion.Dockerfile` | Ubuntu 24.04 (the EAT bastion's digest), kubectl v1.37.2, helm v3.22.1, yq v4.54.1, jq-1.8.2 by sha256, the installer's venv at `/opt/eas-venv`, kubernetes.core 6.3.0 by sha256: `pins.json`'s bastion profile |
+| `requirements.in`, `requirements.lock`, `lock.sh` | EAS `requirements.txt` at the pinned commit, and its hash-locked resolution for the bastion's Python 3.12 |
+| `images.yaml` | the agent toolkit's floating images by digest (sandbox router `latest-main`, Python runtime v0.5.0) |
+| `known-issues.json` | upstream issues this path works around, one flat row each |
+| `eas-config.py`, `pins.json` | the choice generator and the pins it reads |
+| `overlay.py`, `overlay.json`, `patches/` | the k3s/k3d patch overlay |
+
+## Not verified
+
+No EAS install has run on a k3d-gpu cluster from this code. The agent-toolkit layer has not
+run on k3s or k3d; Kubernetes 1.37 is outside what EAS tests (it pins kubectl v1.34.3).

integrations/eas/bastion.Dockerfile

@@ -0,0 +1,71 @@
+# Bastion for Intel's Enterprise AI Solutions installer (intel/enterprise-ai-solutions,
+# es_auto_installer.sh) on a k3d-gpu cluster in its bring-your-own mode.
+#
+# Everything the installer's `configure` would install with sudo and apt is baked here
+# at a pinned version, so `k3d-gpu eas up` never runs `configure`:
+#   - kubectl, helm, yq and jq from their release downloads, each checked by sha256. The
+#     sha256 values are the ones the upstream checksum files publish for these releases
+#     (dl.k8s.io <file>.sha256, get.helm.sh <file>.sha256sum, yq's `checksums` file,
+#     jq's sha256sum.txt). configure installs no jq, but enterprise-inference's
+#     roles/llm_services/tasks/deploy_models.yaml pipes yq into jq on the installer host.
+#   - the installer's Python requirements (requirements.txt at the EAS commit named in
+#     requirements.lock) as a hash-locked venv at /opt/eas-venv; the installer reuses a
+#     venv that has ansible-playbook (es_auto_installer.sh _ensure_venv).
+#   - kubernetes.core 6.3.0 (EAS collections/requirements.yml) by sha256, into that venv's
+#     purelib over the 6.2.0 copy the ansible 12.3.0 package bundles, which is where the
+#     installer's _ensure_collections looks; with 6.3.0 there it makes no network call.
+#
+# kubectl is v1.37.2, the minor of the node image's K3S_TAG (Dockerfile): EAS's own
+# v1.34.3 is outside kubectl's +/-1 minor skew against a v1.37 API server
+# (https://kubernetes.io/releases/version-skew-policy/#kubectl). helm is the newest v3
+# (EAS asks for >= v3.20.2); yq the newest v4 (EAS asks for >= v4.53.2); jq the newest.
+#
+# Runs as root: es_auto_installer.sh ensure_sudo runs `sudo -k` under `set -e` unless
+# EUID is 0, and this image has no sudo. eas.sh hands the state dir back to the
+# invoking user's uid:gid when it exits.
+FROM ubuntu:24.04@sha256:fe81560990fae86dc4743202b2cb4d3df98d0bcf949173cecef15ce1127ef852
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DEBIAN_FRONTEND=noninteractive
+
+# The state dir (the EAS checkout and its ext/ clones) belongs to the invoking user, and
+# git refuses a repository another uid owns ("detected dubious ownership") unless told.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates curl git python3 python3-venv \
+    && rm -rf /var/lib/apt/lists/* \
+    && git config --system --add safe.directory '*'
+
+ARG KUBECTL_VERSION=v1.37.2
+ARG KUBECTL_SHA256=c04487523e7aa7d1493f7808174c899db93d7a9cf57531420cd76b9bd415f8f4
+ARG HELM_VERSION=v3.22.1
+ARG HELM_SHA256=896cf63684a42a723d61c980369d6e588330927deb13676ca0d85006dd6da0ea
+ARG YQ_VERSION=v4.54.1
+ARG YQ_SHA256=8e34fc298390875de416e6a4afcb8cabeceb25d9aa8506c1a2f9353cf702ea5f
+ARG JQ_VERSION=jq-1.8.2
+ARG JQ_SHA256=b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f
+RUN curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
+    && echo "${KUBECTL_SHA256}  /usr/local/bin/kubectl" | sha256sum -c - \
+    && curl -fsSL -o /tmp/helm.tgz "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \
+    && echo "${HELM_SHA256}  /tmp/helm.tgz" | sha256sum -c - \
+    && tar -xzf /tmp/helm.tgz -C /usr/local/bin --strip-components=1 linux-amd64/helm \
+    && curl -fsSL -o /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
+    && echo "${YQ_SHA256}  /usr/local/bin/yq" | sha256sum -c - \
+    && curl -fsSL -o /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/${JQ_VERSION}/jq-linux-amd64" \
+    && echo "${JQ_SHA256}  /usr/local/bin/jq" | sha256sum -c - \
+    && chmod 0755 /usr/local/bin/kubectl /usr/local/bin/helm /usr/local/bin/yq /usr/local/bin/jq \
+    && rm /tmp/helm.tgz
+
+COPY requirements.lock /opt/eas-requirements.lock
+RUN python3 -m venv /opt/eas-venv \
+    && /opt/eas-venv/bin/pip install --no-cache-dir --disable-pip-version-check --require-hashes --no-deps \
+       -r /opt/eas-requirements.lock
+
+ARG KUBERNETES_CORE_VERSION=6.3.0
+ARG KUBERNETES_CORE_SHA256=86a2a89892b75120a32e58135ab96982e3c02ad9f6b59eb2412671952622e748
+RUN curl -fsSL -o /tmp/kubernetes-core.tar.gz \
+       "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/kubernetes-core-${KUBERNETES_CORE_VERSION}.tar.gz" \
+    && echo "${KUBERNETES_CORE_SHA256}  /tmp/kubernetes-core.tar.gz" | sha256sum -c - \
+    && purelib=$(/opt/eas-venv/bin/python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') \
+    && /opt/eas-venv/bin/ansible-galaxy collection install --force --offline -p "${purelib}" /tmp/kubernetes-core.tar.gz \
+    && rm /tmp/kubernetes-core.tar.gz
+
+WORKDIR /eas-state

integrations/eas/eas-config.py

@@ -107,6 +107,7 @@
     "cluster_cidr": _K3S + "pkg/cli/cmds/server.go#L137-L141",
     "service_cidr": _K3S + "pkg/cli/cmds/server.go#L142-L146",
     "kata": _EAS + "roles/kata_containers/tasks/install.yaml#L9-L22",
+    "argv_master_key": _EI + "roles/llm_services/tasks/deploy_models.yaml#L240",
     "accel_cpu": _EAS + "configs/defaults/global_config.yaml#L106",
     "accel_xpu": _EAT + "README.md#L36",
     "sandbox": _EAT + "config.yaml#L56",
@@ -432,6 +433,16 @@ def plan(facts, layer, pins, model_flag):
     p.choose("bring-your-own cluster", "kata_containers_enabled: false", "kata")
     p.set("kata_containers_enabled", False)
 
+    # EI's model deploy task hands model-manager the LiteLLM master key in its ansible
+    # environment:, which ansible writes into the /bin/sh -c command line it runs the
+    # deploy under, readable in `ps` for the whole deploy. An empty Secret read leaves
+    # MM_LITELLM_MASTER_KEY empty, and model-manager then reads the Secret itself.
+    p.choose("EI's model deploy passes the LiteLLM master key through ansible environment: "
+             "(a /bin/sh -c argument while each model deploys)",
+             "_deploy_litellm_secret: \"\" (model-manager reads Secret litellm/litellm-master-key itself)",
+             "argv_master_key")
+    p.set("_deploy_litellm_secret", "")
+
     devices = []
     for n in nodes:
         for res, val in sorted((n.get("allocatable") or {}).items()):