diff --git a/README.md b/README.md index 8e7aad3f..2ac2e4aa 100644 --- a/README.md +++ b/README.md @@ -1521,7 +1521,28 @@ That is placement, not isolation: package power, LLC and DRAM bandwidth stay sha and E-core load still slowed CPU-served prefill on the P-cores [ledger:H3]. `up` waits for a warm sandbox, then runs `status`, which reads each sandbox's `Cpus_allowed_list` and fails if it is not inside the host's E-cores. -`k3d-gpu sandbox down` removes it all. +`k3d-gpu sandbox down` removes the profile, then the namespace `agent-sandbox-system`, +the agent-sandbox CRDs and the controller, unless something else uses them. + +Other installers put agent-sandbox in the same namespace: Intel's Enterprise Agent +Toolkit installs v0.5.0 into `agent-sandbox-system` with its own warm pool and router. +k3d-gpu counts the controller as its own when the Deployment carries +`app.kubernetes.io/managed-by=k3d-gpu` (`up` labels the Deployment, the namespace and +the CRDs after it installs them) or runs the controller image k3d-gpu pins, with no +other installer's `managed-by` label or Helm release on it. + +- `up` stops before applying anything when another installer's controller is there, + or a CRD carries another installer's `managed-by` label or Helm release, and names + the image or the CRDs. +- `up --adopt` keeps that controller and applies only the E-core profile. The + SandboxTemplate and SandboxWarmPool CRDs must serve `extensions.agents.x-k8s.io/v1beta1`; + v0.5.0's do. +- `down` keeps the namespace, the CRDs and the controller when the controller is + another installer's, when a CRD carries another installer's label or Helm release, + when a Sandbox, SandboxClaim, SandboxTemplate or SandboxWarmPool exists outside + namespace `agent-sandbox`, or when a pod other than the controller's runs in + `agent-sandbox-system`. It prints what it kept and why, and exits 0. Deleting a CRD + deletes every object of that kind in every namespace. ```bash k3d-gpu sandbox up diff --git a/scripts/k3d-gpu b/scripts/k3d-gpu index 4134a048..f12d1039 100755 --- a/scripts/k3d-gpu +++ b/scripts/k3d-gpu @@ -58,9 +58,10 @@ # fit size a model to what this host has free now: placement, context x seqs, engine # memory flags, pod memory limit, or why it cannot fit; --explain prints the arithmetic # audit posture checks: unauthenticated listeners off loopback, images not in images.lock -# sandbox up|down|status +# sandbox up [--adopt]|down|status +# kubernetes-sigs/agent-sandbox with a warm pool of sandboxes pinned to E-cores; +# --adopt keeps another installer's controller and adds only the E-core profile # model-share --image REF --agent-image REF [--replicas N] [--name NAME] -# kubernetes-sigs/agent-sandbox with a warm pool of sandboxes pinned to E-cores # tune run|show|space|env|fingerprint|reset: search the OVMS agent's serving config on # this host; keep a winner only if it beats the default; agent up applies it # capacity agents per box: closed-loop agent ramp against the cluster's chat endpoint @@ -286,9 +287,15 @@ Usage: k3d-gpu Posture checks of the running cluster: fails on an unauthenticated listener off loopback, an image not pinned in images.lock, a runtime pull, a key file readable by others, or Open WebUI without sign-in - sandbox up|down|status + sandbox up [--adopt]|down|status Install kubernetes-sigs/agent-sandbox with a warm pool of sandboxes on E-cores (k3d-gpu.io/cores=efficient), check where they run, or remove it. + up stops when another installer's controller (Intel's Enterprise Agent + Toolkit installs v0.5.0 in agent-sandbox-system) or CRDs are installed; + up --adopt keeps that controller and adds only the E-core profile (its + CRDs must serve v1beta1). down removes the profile, and removes the namespace + agent-sandbox-system, the CRDs and the controller only when k3d-gpu + installed them and no other installer's sandboxes or pods use them. up --remote [URL]: egress only to the endpoint's addresses and port (no DNS; on a CDN those addresses also serve its other sites), and endpoint in the sandbox's Secret k3d-gpu-llm, which sandbox code can @@ -6632,13 +6639,111 @@ SANDBOX_CRDS=( crd/sandboxclaims.extensions.agents.x-k8s.io crd/sandboxwarmpools.extensions.agents.x-k8s.io ) +SANDBOX_ADOPT="" # sandbox up --adopt: use the agent-sandbox controller already installed + +# Ownership. Other installers put agent-sandbox in the same namespace (Intel's EAT +# installs v0.5.0 there, with its own warm pool and router). The controller is k3d-gpu's +# when its Deployment carries app.kubernetes.io/managed-by=k3d-gpu (`sandbox up` sets it +# on the Deployment, the Namespace and the CRDs after an install), or when it runs the +# controller image pinned in ${SANDBOX_MANIFEST} or one an earlier k3d-gpu release +# installed before the label (SANDBOX_UNLABELLED_IMAGES), and no other installer marked +# it (another managed-by value, or a Helm release annotation). +SANDBOX_OWNER_LABEL="app.kubernetes.io/managed-by=k3d-gpu" +SANDBOX_UNLABELLED_IMAGES=( + registry.k8s.io/agent-sandbox/agent-sandbox-controller:v1.0.7@sha256:f254384c24605436ce35d7077427559e24bda8f5691cff5a54a4c2e030d81083 +) + +# sandbox_controller: "||" of the controller +# Deployment, nothing when there is none. Fails when kubectl cannot read it. +sandbox_controller() { + kubectl -n agent-sandbox-system get deployment agent-sandbox-controller --ignore-not-found \ + -o jsonpath='{.spec.template.spec.containers[*].image}{"|"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"|"}{.metadata.annotations.meta\.helm\.sh/release-name}' +} + +# sandbox_whose : absent, ours or foreign. +sandbox_whose() { + local images managed helm + [ -n "$1" ] || { echo absent; return; } + IFS='|' read -r images managed helm <<< "$1" + if [ -n "${helm}" ]; then echo foreign + elif [ "${managed}" = k3d-gpu ]; then echo ours + elif [ -n "${managed}" ]; then echo foreign + elif printf '%s\n' "$(sandbox_pinned_image)" "${SANDBOX_UNLABELLED_IMAGES[@]}" | grep -Fx -- "${images}" >/dev/null; then echo ours + else echo foreign + fi +} + +# sandbox_pinned_image: the controller image ${SANDBOX_MANIFEST} installs. +sandbox_pinned_image() { + sed -n 's/^[[:space:]]*image:[[:space:]]*\([^[:space:]]*\/agent-sandbox-controller[:@][^[:space:]]*\).*$/\1/p' "${SANDBOX_MANIFEST}" 2>/dev/null | head -1 +} + +# sandbox_marks : "managed-by M, Helm release R", the set ones. +sandbox_marks() { + local marks="" + [ -z "$1" ] || marks="managed-by $1" + [ -z "$2" ] || marks="${marks:+${marks}, }Helm release $2" + echo "${marks}" +} + +# sandbox_describe : " (managed-by M, Helm release R)". +sandbox_describe() { + local images managed helm marks + IFS='|' read -r images managed helm <<< "$1" + marks=$(sandbox_marks "${managed}" "${helm}") + echo "${images:-no image}${marks:+ (${marks})}" +} + +# sandbox_crds: "|||" per installed CRD. +sandbox_crds() { + kubectl get crd "${SANDBOX_CRDS[@]#crd/}" --ignore-not-found \ + -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.spec.versions[?(@.served==true)].name}{"|"}{.metadata.labels.app\.kubernetes\.io/managed-by}{"|"}{.metadata.annotations.meta\.helm\.sh/release-name}{"\n"}{end}' +} + +# sandbox_foreign_crds : "CRD is another installer's ()" +# for each CRD with a Helm release or a managed-by other than k3d-gpu. +sandbox_foreign_crds() { + local name managed helm + while IFS='|' read -r name _ managed helm; do + [ -n "${name}" ] || continue + if [ -n "${helm}" ] || { [ -n "${managed}" ] && [ "${managed}" != k3d-gpu ]; }; then + echo "CRD ${name} is another installer's ($(sandbox_marks "${managed}" "${helm}"))" + fi + done <<< "$1" +} sandbox_up() { require kubectl - local f ready="" deadline profile fields base model origin + local f ready="" deadline profile fields base model origin ctl whose crds theirs found crd served for f in "${SANDBOX_MANIFEST}" "${SANDBOX_PROFILE_MANIFEST}"; do [ -f "${f}" ] || err "${f} not found — set K3D_GPU_SHARE, K3D_GPU_SANDBOX or K3D_GPU_SANDBOX_PROFILE" done + ctl=$(sandbox_controller) || err "could not read deployment agent-sandbox-system/agent-sandbox-controller; nothing applied" + whose=$(sandbox_whose "${ctl}") + crds=$(sandbox_crds) || err "could not read the agent-sandbox CRDs; nothing applied" + theirs=$(sandbox_foreign_crds "${crds}") + if [ -n "${SANDBOX_ADOPT}" ]; then + [ "${whose}" != absent ] || err "no agent-sandbox controller to adopt in agent-sandbox-system; 'k3d-gpu sandbox up' installs agent-sandbox ${SANDBOX_VERSION}" + # The profile is extensions.agents.x-k8s.io/v1beta1. + for crd in sandboxtemplates.extensions.agents.x-k8s.io sandboxwarmpools.extensions.agents.x-k8s.io; do + served=$(awk -F'|' -v n="${crd}" '$1 == n {print $2}' <<< "${crds}") + [[ " ${served} " == *" v1beta1 "* ]] \ + || err "CRD ${crd} does not serve extensions.agents.x-k8s.io/v1beta1 (served: ${served:-none}); the profile needs it, so nothing applied" + done + elif [ "${whose}" = foreign ] || [ -n "${theirs}" ]; then + # Applying the manifest would replace that installer's controller or CRDs, and + # the label would claim them for k3d-gpu. + found="" + [ "${whose}" != foreign ] || found="agent-sandbox-system/agent-sandbox-controller runs $(sandbox_describe "${ctl}"), not k3d-gpu's ${SANDBOX_VERSION}" + if [ -n "${found}" ] && [ -n "${theirs}" ]; then found+=$'\n'; fi + found+="${theirs}" + [ "${whose}" != absent ] || err "${found} +another installer put agent-sandbox there, so nothing was applied: remove it with that installer, then run 'k3d-gpu sandbox up'" + err "${found} +another installer put agent-sandbox there, so nothing was applied. Either: + k3d-gpu sandbox up --adopt keep that controller and add only the E-core profile (its CRDs must serve v1beta1) + or remove that installer's agent-sandbox with that installer, then run 'k3d-gpu sandbox up'" + fi if [ -n "${REMOTE}" ]; then # Remote: egress is the endpoint's host:port only (no DNS; hostAliases pin # the name), and the sandbox reads the endpoint from its own Secret copy. @@ -6648,11 +6753,20 @@ sandbox_up() { || err "could not pin the sandbox egress to ${base}" info "remote endpoint ${base} (from ${origin}): sandbox egress is ${base#*://} only, no DNS" fi - info "Installing agent-sandbox ${SANDBOX_VERSION} (${SANDBOX_MANIFEST})..." - kubectl apply -f "${SANDBOX_MANIFEST}" + if [ -n "${SANDBOX_ADOPT}" ]; then + info "adopting agent-sandbox-system/agent-sandbox-controller ($(sandbox_describe "${ctl}")): no controller manifest applied" + else + info "Installing agent-sandbox ${SANDBOX_VERSION} (${SANDBOX_MANIFEST})..." + kubectl apply -f "${SANDBOX_MANIFEST}" + fi # The profile's SandboxTemplate and SandboxWarmPool need their CRDs served first. kubectl wait --for=condition=Established "${SANDBOX_CRDS[@]}" --timeout=60s kubectl -n agent-sandbox-system rollout status deployment/agent-sandbox-controller --timeout=180s + if [ -z "${SANDBOX_ADOPT}" ]; then + kubectl -n agent-sandbox-system label --overwrite deployment/agent-sandbox-controller \ + namespace/agent-sandbox-system "${SANDBOX_CRDS[@]}" "${SANDBOX_OWNER_LABEL}" >/dev/null \ + || warn "could not label agent-sandbox ${SANDBOX_OWNER_LABEL}; k3d-gpu still knows its controller by the pinned image" + fi # The Secret exists before the warm pool's pods, which read it at start. [ -z "${REMOTE}" ] || remote_secret_apply "${SANDBOX_NS}" "${base}" "${model}" info "Applying the E-core sandbox profile (${SANDBOX_PROFILE_MANIFEST})..." @@ -6716,27 +6830,66 @@ sandbox_status() { [ "${fail}" -eq 0 ] || err "sandbox pods are not pinned to E-cores, or the endpoint moved" } +# Deletes the profile, then the namespace agent-sandbox-system, the CRDs and the +# controller, unless another installer owns or uses them: deleting a CRD deletes every +# object of its kind in every namespace. sandbox_down() { require kubectl - info "Removing the sandbox profile and agent-sandbox ${SANDBOX_VERSION}..." + local ctl crds kinds="" crd name line objs pods kept=() + info "Removing the sandbox profile (${SANDBOX_PROFILE_MANIFEST})..." # Profile first, while its CRDs still exist; deleting the CRDs removes every sandbox. kubectl delete -f "${SANDBOX_PROFILE_MANIFEST}" --ignore-not-found \ || warn "profile objects not removed (CRDs already gone?)" + ctl=$(sandbox_controller) \ + || err "could not read deployment agent-sandbox-system/agent-sandbox-controller: kept namespace agent-sandbox-system, the agent-sandbox CRDs and the controller" + [ "$(sandbox_whose "${ctl}")" != foreign ] || kept+=("the controller runs $(sandbox_describe "${ctl}"), not k3d-gpu's ${SANDBOX_VERSION}") + crds=$(sandbox_crds) || err "could not read the agent-sandbox CRDs: kept namespace agent-sandbox-system, the CRDs and the controller" + while IFS='|' read -r name _; do + [ -z "${name}" ] || kinds="${kinds:+${kinds},}${name}" + done <<< "${crds}" + while read -r line; do + [ -z "${line}" ] || kept+=("${line}") + done <<< "$(sandbox_foreign_crds "${crds}")" + if [ -n "${kinds}" ]; then + objs=$(kubectl get "${kinds}" -A -o jsonpath='{range .items[*]}{.kind}{" "}{.metadata.namespace}{"/"}{.metadata.name}{"\n"}{end}') \ + || err "could not list the sandbox objects: kept namespace agent-sandbox-system, the agent-sandbox CRDs and the controller" + while read -r crd name; do + if [ -n "${name}" ] && [ "${name%%/*}" != "${SANDBOX_NS}" ]; then kept+=("${crd} ${name} uses the CRDs"); fi + done <<< "${objs}" + fi + pods=$(kubectl -n agent-sandbox-system get pods -l 'app!=agent-sandbox-controller' -o name) \ + || err "could not list the pods in agent-sandbox-system: kept the namespace, the agent-sandbox CRDs and the controller" + while read -r name; do + [ -z "${name}" ] || kept+=("${name} in agent-sandbox-system is not k3d-gpu's") + done <<< "${pods}" + if [ "${#kept[@]}" -gt 0 ]; then + info "Kept namespace agent-sandbox-system, the agent-sandbox CRDs and the controller:" + printf ' %s\n' "${kept[@]}" + echo " remove them with the installer that owns them" + return 0 + fi + info "Removing agent-sandbox ${SANDBOX_VERSION}: namespace agent-sandbox-system, the CRDs and the controller (${SANDBOX_MANIFEST})..." kubectl delete -f "${SANDBOX_MANIFEST}" --ignore-not-found } cmd_sandbox() { - local sub=${1:-} + local sub=${1:-} a [ $# -eq 0 ] || shift remote_parse "$@" - [ "${#REMOTE_REST[@]}" -eq 0 ] || err "unknown sandbox option: ${REMOTE_REST[0]}" + for a in ${REMOTE_REST[@]+"${REMOTE_REST[@]}"}; do + case "${a}" in + --adopt) SANDBOX_ADOPT=1 ;; + *) err "unknown sandbox option: ${a}" ;; + esac + done + [ -z "${SANDBOX_ADOPT}" ] || [ "${sub}" = up ] || err "--adopt goes with 'sandbox up'" [ -z "${REMOTE}" ] || [ "${sub}" = up ] || err "--remote goes with 'sandbox up'" [ -n "${REMOTE}" ] || [ "${#REMOTE_ARGS[@]}" -eq 0 ] || err "--model and --api-key-file go with --remote" case "${sub}" in up) sandbox_up ;; down) sandbox_down ;; status) sandbox_status ;; - *) err "usage: k3d-gpu sandbox up|down|status (up --remote [URL] [--api-key-file F]: egress only to that endpoint)" ;; + *) err "usage: k3d-gpu sandbox up|down|status (up --adopt: keep another installer's agent-sandbox controller, add only the profile; up --remote [URL] [--api-key-file F]: egress only to that endpoint)" ;; esac } diff --git a/tests/detect_test.sh b/tests/detect_test.sh index 94554468..316bea0d 100755 --- a/tests/detect_test.sh +++ b/tests/detect_test.sh @@ -4551,17 +4551,35 @@ fi # readyReplicas), STUB_SANDBOX_PODS (running sandbox pods) and STUB_SANDBOX_CPUS # (the Cpus_allowed_list a sandbox pod reports). A call whose arguments match the # ERE in STUB_FAIL prints nothing and exits 1. +# The installed agent-sandbox: STUB_CONTROLLER_IMAGE is the controller Deployment's +# image (empty: no Deployment), STUB_CONTROLLER_LABEL its app.kubernetes.io/managed-by +# label and STUB_CONTROLLER_HELM its meta.helm.sh/release-name annotation. The four +# CRDs exist unless STUB_NO_CRDS is set; each serves STUB_CRD_SERVED (default v1beta1) +# and carries STUB_CRD_LABEL and STUB_CRD_HELM. STUB_FOREIGN lists sandbox objects +# cluster-wide as "Kind namespace/name" entries split by ';', STUB_SYSTEM_PODS the pods +# in agent-sandbox-system other than the controller's. SSTUB="${WORK}/sbin" mkdir -p "${SSTUB}" cat > "${SSTUB}/kubectl" <<'EOF' #!/bin/sh while [ "${1:-}" = --context ]; do shift 2; done -echo "$*" >> "${STUB_LOG}" -if [ -n "${STUB_FAIL:-}" ] && echo "$*" | grep -Eq -- "${STUB_FAIL}"; then +# printf, not echo: dash's echo (/bin/sh on Ubuntu) turns a jsonpath's \n into a line break. +printf '%s\n' "$*" >> "${STUB_LOG}" +if [ -n "${STUB_FAIL:-}" ] && printf '%s\n' "$*" | grep -Eq -- "${STUB_FAIL}"; then echo "error: stub failure" >&2 exit 1 fi case "$*" in + *"get deployment agent-sandbox-controller"*) + [ -z "${STUB_CONTROLLER_IMAGE:-}" ] \ + || printf '%s|%s|%s' "${STUB_CONTROLLER_IMAGE}" "${STUB_CONTROLLER_LABEL:-}" "${STUB_CONTROLLER_HELM:-}" ;; + *"get crd "*) + [ -n "${STUB_NO_CRDS:-}" ] || for c in sandboxes.agents.x-k8s.io sandboxclaims.extensions.agents.x-k8s.io \ + sandboxtemplates.extensions.agents.x-k8s.io sandboxwarmpools.extensions.agents.x-k8s.io; do + printf '%s|%s|%s|%s\n' "${c}" "${STUB_CRD_SERVED-v1beta1}" "${STUB_CRD_LABEL:-}" "${STUB_CRD_HELM:-}" + done ;; + *" -A -o "*) [ -z "${STUB_FOREIGN:-}" ] || printf '%s\n' "${STUB_FOREIGN}" | tr ';' '\n' ;; + *"get pods -l app!=agent-sandbox-controller"*) for p in ${STUB_SYSTEM_PODS:-}; do echo "pod/${p}"; done ;; *"get nodes -l"*) for n in ${STUB_HYBRID_NODES:-}; do echo "node/${n}"; done ;; *"get sandboxwarmpool "*) printf %s "${STUB_POOL_READY-1}" ;; *"get pods -l k3d-gpu.io/cores=efficient"*) for p in ${STUB_SANDBOX_PODS:-}; do echo "${p}"; done ;; @@ -4574,14 +4592,15 @@ SHARE="$(cd "$(dirname "$0")/.." && pwd)/share" # sandbox_says [negate] [env assignments...]: # output of `k3d-gpu sandbox ` must match , or must not after "negate". -# The kubectl call log is in ${WORK}/kubectl.log. +# is split on spaces ("up --adopt"). The kubectl call log is in ${WORK}/kubectl.log. sandbox_says() { - local name=$1 root=$2 want_rc=$3 re=$4 action=$5 negate="" out rc hit=0 + local name=$1 root=$2 want_rc=$3 re=$4 action=$5 negate="" out rc hit=0 acts=() shift 5 if [ "${1:-}" = negate ]; then negate=1; shift; fi + read -ra acts <<< "${action}" : > "${WORK}/kubectl.log" out=$(env K3D_GPU_SYSROOT="${root}" K3D_GPU_SHARE="${SHARE}" STUB_LOG="${WORK}/kubectl.log" \ - PATH="${SSTUB}:${PATH}" "$@" bash "${K3D_GPU}" sandbox "${action}" 2>&1) + PATH="${SSTUB}:${PATH}" "$@" bash "${K3D_GPU}" sandbox ${acts[@]+"${acts[@]}"} 2>&1) rc=$? grep -E -- "${re}" <<< "${out}" >/dev/null && hit=1 [ -z "${negate}" ] || hit=$((1 - hit)) @@ -4675,6 +4694,154 @@ log_order "sandbox down deletes the profile before the CRDs" \ 'delete -f .*/agent-sandbox-k3d\.yml --ignore-not-found' 'delete -f .*/agent-sandbox\.yml --ignore-not-found' sandbox_says "sandbox with no action prints usage and fails" "${r}" fail 'usage: k3d-gpu sandbox up\|down\|status' "" +# Ownership. The controller is k3d-gpu's when it runs the image pinned in +# share/agent-sandbox.yml or carries app.kubernetes.io/managed-by=k3d-gpu; another +# installer's (Intel's EAT installs v0.5.0 into agent-sandbox-system) is left alone. +PINNED=$(sed -n 's/^ *image: \(.*\/agent-sandbox-controller[:@].*\)$/\1/p' "${SHARE}/agent-sandbox.yml") +THEIRS=registry.k8s.io/agent-sandbox/agent-sandbox-controller:v0.5.0 +UP_OK=(STUB_SANDBOX_PODS=p1 STUB_SANDBOX_CPUS=16) +# first_calls ...: the first kubectl calls match these, one each, in order. +first_calls() { + local name=$1 i=1 re + shift + for re in "$@"; do + if ! sed -n "${i}p" "${WORK}/kubectl.log" | grep -Eq -- "${re}"; then + FAILS=$((FAILS + 1)); echo "FAIL ${name} (call ${i} is not /${re}/)" + sed 's/^/ | /' "${WORK}/kubectl.log" + return + fi + i=$((i + 1)) + done + PASS=$((PASS + 1)); echo "ok ${name}" +} +[ -n "${PINNED}" ] || { FAILS=$((FAILS + 1)); echo "FAIL the controller image is readable from share/agent-sandbox.yml"; } + +sandbox_says "sandbox up on a fresh cluster installs agent-sandbox" "${r}" 0 'Installing agent-sandbox' up "${UP_OK[@]}" +first_calls "sandbox up on a fresh cluster adds two reads (controller, CRDs) before today's apply" \ + '^-n agent-sandbox-system get deployment agent-sandbox-controller --ignore-not-found -o jsonpath=' \ + '^get crd sandboxes\.agents\.x-k8s\.io .* --ignore-not-found -o jsonpath=' \ + 'apply -f .*/agent-sandbox\.yml$' +log_order "sandbox up labels what it installed once the controller is rolled out" \ + 'rollout status deployment/agent-sandbox-controller' \ + 'label --overwrite deployment/agent-sandbox-controller namespace/agent-sandbox-system crd/sandboxes\.agents\.x-k8s\.io .* app\.kubernetes\.io/managed-by=k3d-gpu$' \ + 'apply -f .*/agent-sandbox-k3d\.yml$' +sandbox_says "sandbox up re-applies its own controller (pinned image)" "${r}" 0 'Installing agent-sandbox' up "${UP_OK[@]}" \ + STUB_CONTROLLER_IMAGE="${PINNED}" +log_order "sandbox up on its own controller re-applies the manifest and the profile" \ + 'apply -f .*/agent-sandbox\.yml$' 'apply -f .*/agent-sandbox-k3d\.yml$' +# k3d-gpu releases before the label installed v1.0.7 with no managed-by label. V107_TAG +# is the same tag with no digest, a controller that only the label can make k3d-gpu's. +V107=registry.k8s.io/agent-sandbox/agent-sandbox-controller:v1.0.7@sha256:f254384c24605436ce35d7077427559e24bda8f5691cff5a54a4c2e030d81083 +V107_TAG=${V107%@*} +sandbox_says "sandbox up upgrades a controller labelled managed-by=k3d-gpu" "${r}" 0 'Installing agent-sandbox' up "${UP_OK[@]}" \ + STUB_CONTROLLER_IMAGE="${V107_TAG}" STUB_CONTROLLER_LABEL=k3d-gpu +log_order "sandbox up on a labelled older controller applies the manifest" 'apply -f .*/agent-sandbox\.yml$' +sandbox_says "sandbox up upgrades the unlabelled v1.0.7 controller an earlier k3d-gpu installed" "${r}" 0 'Installing agent-sandbox' up "${UP_OK[@]}" \ + STUB_CONTROLLER_IMAGE="${V107}" +log_order "sandbox up on k3d-gpu's unlabelled v1.0.7 applies the manifest, then labels it" \ + 'apply -f .*/agent-sandbox\.yml$' 'label --overwrite deployment/agent-sandbox-controller' +sandbox_says "sandbox up treats v1.0.7 by tag alone as another installer's" "${r}" fail 'not k3d-gpu' up \ + STUB_CONTROLLER_IMAGE="${V107_TAG}" +sandbox_says "sandbox up refuses another installer's controller and names its image" "${r}" fail \ + "agent-sandbox-controller runs ${THEIRS}" up "${UP_OK[@]}" STUB_CONTROLLER_IMAGE="${THEIRS}" +no_call "sandbox up applies no controller manifest over another installer's" 'apply -f .*/agent-sandbox\.yml$' +no_call "sandbox up applies no profile over another installer's controller" 'apply -f .*/agent-sandbox-k3d\.yml$' +no_call "sandbox up labels nothing it did not install" 'label --overwrite' +sandbox_says "sandbox up offers --adopt for another installer's controller" "${r}" fail \ + "k3d-gpu sandbox up --adopt" up STUB_CONTROLLER_IMAGE="${THEIRS}" +sandbox_says "sandbox up treats a Helm release as another installer's, even on the pinned image" "${r}" fail \ + 'managed-by Helm, Helm release agent-sandbox' up STUB_CONTROLLER_IMAGE="${PINNED}" STUB_CONTROLLER_LABEL=Helm STUB_CONTROLLER_HELM=agent-sandbox +no_call "sandbox up applies nothing over a Helm release" 'apply -f' +sandbox_says "sandbox up stops when it cannot read the controller" "${r}" fail 'could not read' up \ + STUB_FAIL='get deployment agent-sandbox-controller' +no_call "sandbox up applies nothing when it cannot read the controller" 'apply -f' +sandbox_says "sandbox up stops when it cannot read the CRDs" "${r}" fail 'could not read the agent-sandbox CRDs' up \ + STUB_FAIL='get crd ' +no_call "sandbox up applies nothing when it cannot read the CRDs" 'apply -f' +# CRDs another installer marked, with no controller (a Helm release whose controller is +# gone): installing over them would relabel them k3d-gpu's. +sandbox_says "sandbox up refuses CRDs from a Helm release and names them" "${r}" fail \ + "CRD sandboxes\.agents\.x-k8s\.io is another installer's \(managed-by Helm, Helm release agent-sandbox\)" up \ + STUB_CRD_LABEL=Helm STUB_CRD_HELM=agent-sandbox +no_call "sandbox up applies nothing over another installer's CRDs" 'apply -f' +no_call "sandbox up labels no CRDs another installer marked" 'label --overwrite' +sandbox_says "sandbox up refuses CRDs labelled managed-by another installer" "${r}" fail \ + "is another installer's \(managed-by agentic-ai-stack\)" up STUB_CRD_LABEL=agentic-ai-stack +sandbox_says "sandbox up re-applies CRDs labelled managed-by=k3d-gpu" "${r}" 0 'Installing agent-sandbox' up "${UP_OK[@]}" \ + STUB_CONTROLLER_IMAGE="${PINNED}" STUB_CONTROLLER_LABEL=k3d-gpu STUB_CRD_LABEL=k3d-gpu +sandbox_says "sandbox up --adopt leaves Helm's CRDs to Helm" "${r}" 0 'adopting' "up --adopt" "${UP_OK[@]}" \ + STUB_CONTROLLER_IMAGE="${THEIRS}" STUB_CRD_LABEL=Helm STUB_CRD_HELM=agent-sandbox +no_call "sandbox up --adopt applies no CRDs over Helm's" 'apply -f .*/agent-sandbox\.yml$' + +sandbox_says "sandbox up --adopt keeps another installer's controller" "${r}" 0 "adopting .*${THEIRS}" "up --adopt" "${UP_OK[@]}" \ + STUB_CONTROLLER_IMAGE="${THEIRS}" +log_order "sandbox up --adopt checks the CRDs, waits for them and the controller, then applies the profile" \ + 'get deployment agent-sandbox-controller' \ + 'get crd .*sandboxtemplates\.extensions\.agents\.x-k8s\.io.*sandboxwarmpools\.extensions\.agents\.x-k8s\.io' \ + 'wait --for=condition=Established' \ + 'rollout status deployment/agent-sandbox-controller' \ + 'apply -f .*/agent-sandbox-k3d\.yml$' \ + 'get sandboxwarmpool python-ecores' +no_call "sandbox up --adopt applies no controller manifest" 'apply -f .*/agent-sandbox\.yml$' +no_call "sandbox up --adopt labels nothing" 'label --overwrite' +sandbox_says "sandbox up --adopt fails when the CRDs do not serve v1beta1" "${r}" fail \ + 'sandboxtemplates\.extensions\.agents\.x-k8s\.io does not serve extensions\.agents\.x-k8s\.io/v1beta1 \(served: v1alpha1\)' \ + "up --adopt" STUB_CONTROLLER_IMAGE="${THEIRS}" STUB_CRD_SERVED=v1alpha1 +no_call "sandbox up --adopt applies no profile on CRDs without v1beta1" 'apply -f' +sandbox_says "sandbox up --adopt fails when the CRDs are missing" "${r}" fail 'does not serve .*\(served: none\)' \ + "up --adopt" STUB_CONTROLLER_IMAGE="${THEIRS}" STUB_NO_CRDS=1 +no_call "sandbox up --adopt applies no profile without CRDs" 'apply -f' +sandbox_says "sandbox up --adopt with no controller fails and installs nothing" "${r}" fail 'no agent-sandbox controller to adopt' "up --adopt" +no_call "sandbox up --adopt with no controller applies nothing" 'apply -f' +sandbox_says "sandbox up --adopt on its own controller applies only the profile" "${r}" 0 'adopting' "up --adopt" "${UP_OK[@]}" \ + STUB_CONTROLLER_IMAGE="${PINNED}" +no_call "sandbox up --adopt on its own controller applies no manifest" 'apply -f .*/agent-sandbox\.yml$' +log_order "sandbox up --adopt on its own controller applies the profile" 'apply -f .*/agent-sandbox-k3d\.yml$' +sandbox_says "sandbox down --adopt is a usage error" "${r}" fail "--adopt goes with 'sandbox up'" "down --adopt" +no_call "sandbox down --adopt deletes nothing" 'delete' +sandbox_says "sandbox status --adopt is a usage error" "${r}" fail "--adopt goes with 'sandbox up'" "status --adopt" +sandbox_says "sandbox up with an unknown option fails" "${r}" fail 'unknown sandbox option: --bogus' "up --bogus" +sandbox_says "sandbox usage names --adopt" "${r}" fail 'up --adopt: keep another installer' "" +if bash "${K3D_GPU}" help 2>&1 | grep -E '^ sandbox up \[--adopt\]\|down\|status$' >/dev/null; then + PASS=$((PASS + 1)); echo "ok help lists sandbox up [--adopt]" +else + FAILS=$((FAILS + 1)); echo "FAIL help lists sandbox up [--adopt]" +fi + +# down: the profile always goes; the namespace, CRDs and controller only when k3d-gpu +# owns the controller and nothing else lives on them. +sandbox_says "sandbox down on its own controller removes agent-sandbox" "${r}" 0 'Removing agent-sandbox' down \ + STUB_CONTROLLER_IMAGE="${PINNED}" STUB_FOREIGN='SandboxWarmPool agent-sandbox/python-ecores;Sandbox agent-sandbox/python-ecores-x1' +log_order "sandbox down on its own controller deletes the profile, then the manifest" \ + 'delete -f .*/agent-sandbox-k3d\.yml --ignore-not-found' 'delete -f .*/agent-sandbox\.yml --ignore-not-found' +sandbox_says "sandbox down removes the unlabelled v1.0.7 controller an earlier k3d-gpu installed" "${r}" 0 'Removing agent-sandbox' down \ + STUB_CONTROLLER_IMAGE="${V107}" +log_order "sandbox down on k3d-gpu's v1.0.7 deletes the manifest" 'delete -f .*/agent-sandbox\.yml --ignore-not-found' +sandbox_says "sandbox down keeps another installer's controller and names its image" "${r}" 0 \ + "Kept namespace agent-sandbox-system, the agent-sandbox CRDs and the controller" down STUB_CONTROLLER_IMAGE="${THEIRS}" +sandbox_says "sandbox down names the other installer's image" "${r}" 0 "controller runs ${THEIRS}" down STUB_CONTROLLER_IMAGE="${THEIRS}" +log_order "sandbox down still deletes the profile next to another installer's controller" \ + 'delete -f .*/agent-sandbox-k3d\.yml --ignore-not-found' +no_call "sandbox down deletes no manifest under another installer's controller" 'delete -f .*/agent-sandbox\.yml' +sandbox_says "sandbox down keeps agent-sandbox while another installer's warm pool uses it" "${r}" 0 \ + 'SandboxWarmPool agent-sandbox-system/python-pool' down \ + STUB_CONTROLLER_IMAGE="${PINNED}" STUB_FOREIGN='SandboxWarmPool agent-sandbox/python-ecores;SandboxWarmPool agent-sandbox-system/python-pool' +no_call "sandbox down deletes no manifest under another installer's sandboxes" 'delete -f .*/agent-sandbox\.yml' +sandbox_says "sandbox down keeps agent-sandbox-system while other pods run in it" "${r}" 0 \ + 'pod/sandbox-router-7d9f in agent-sandbox-system' down STUB_CONTROLLER_IMAGE="${PINNED}" STUB_SYSTEM_PODS=sandbox-router-7d9f +no_call "sandbox down deletes no namespace other pods run in" 'delete -f .*/agent-sandbox\.yml' +sandbox_says "sandbox down keeps CRDs from a Helm release" "${r}" 0 'Helm release agent-sandbox' down \ + STUB_CRD_LABEL=Helm STUB_CRD_HELM=agent-sandbox +no_call "sandbox down deletes no Helm-owned CRDs" 'delete -f .*/agent-sandbox\.yml' +sandbox_says "sandbox down stops when it cannot read the controller" "${r}" fail 'could not read' down \ + STUB_FAIL='get deployment agent-sandbox-controller' +no_call "sandbox down deletes no manifest when it cannot read the controller" 'delete -f .*/agent-sandbox\.yml' +sandbox_says "sandbox down stops when it cannot list sandbox objects" "${r}" fail 'could not list' down \ + STUB_CONTROLLER_IMAGE="${PINNED}" STUB_FAIL=' -A -o ' +no_call "sandbox down deletes no manifest when it cannot list sandbox objects" 'delete -f .*/agent-sandbox\.yml' +sandbox_says "sandbox down with no CRDs lists no sandbox objects" "${r}" 0 'Removing agent-sandbox' down STUB_NO_CRDS=1 +no_call "sandbox down with no CRDs runs no -A list" ' -A ' + # The shipped profile: sandboxes are labelled for E-cores, request CPU (requests size # the balloon) and every image is pinned by digest. manifest_has() { diff --git a/tests/remote_test.sh b/tests/remote_test.sh index e3f7d59e..2f3ce381 100755 --- a/tests/remote_test.sh +++ b/tests/remote_test.sh @@ -61,6 +61,8 @@ case "$*" in *"get secret k3d-gpu-llm -o jsonpath"*) ns=$2; [ -f "${STUB_STATE}/secret-key-${ns}" ] || exit 1; cat "${STUB_STATE}/secret-key-${ns}" ;; *"get sandboxtemplate"*) [ -f "${STUB_STATE}/pin" ] || exit 1; cat "${STUB_STATE}/pin" ;; + *"get deployment agent-sandbox-controller"*) [ ! -f "${STUB_STATE}/controller" ] || cat "${STUB_STATE}/controller" ;; + *"get crd "*) for c in sandboxtemplates sandboxwarmpools; do printf '%s.extensions.agents.x-k8s.io|v1beta1||\n' "${c}"; done ;; *"get secret open-webui-admin"*) exit 1 ;; *"delete secret"*) rm -f "${STUB_STATE}/secret-${2}" "${STUB_STATE}/secret-key-${2}" ;; esac @@ -202,6 +204,16 @@ has "sandbox up --remote: sandbox Secret copy" "$(cat "${S}/kubectl.log")" "stdi grep -n 'stdin-secret\|apply -f -' "${S}/kubectl.log" | grep -q 'stdin-secret' && \ [ "$(grep -n 'stdin-secret' "${S}/kubectl.log" | head -1 | cut -d: -f1)" -lt "$(grep -n '^kubectl apply -f -$' "${S}/kubectl.log" | tail -1 | cut -d: -f1)" ] \ && ok "sandbox up --remote: Secret before the profile" || fail "sandbox up --remote: Secret before the profile" +# --adopt next to another installer's controller: the same pinned profile, no controller manifest. +: > "${S}/kubectl.log"; : > "${S}/applied" +printf 'registry.k8s.io/agent-sandbox/agent-sandbox-controller:v0.5.0||' > "${S}/controller" +out=$(K3D_GPU_REMOTE_ADDRS=104.21.77.178,172.67.210.235 OPENAI_BASE_URL="https://llm.example.net/v1" OPENAI_API_KEY="${KEY}" \ + run eval 'remote_resolve() { printf "https://llm.example.net/v1\tzai-org/GLM-5.3\tOPENAI_BASE_URL\n"; }; sandbox_status() { :; }; SANDBOX_TIMEOUT=0; kubectl() { command kubectl "$@"; [ "$*" != "-n agent-sandbox get sandboxwarmpool python-ecores -o jsonpath={.status.readyReplicas}" ] || echo 1; }; cmd_sandbox up --adopt --remote') +rm -f "${S}/controller" +has "sandbox up --adopt --remote: adopts the other controller" "${out}" "adopting agent-sandbox-system/agent-sandbox-controller (registry.k8s.io/agent-sandbox/agent-sandbox-controller:v0.5.0)" +has "sandbox up --adopt --remote: pinned profile applied" "$(cat "${S}/applied")" "cidr: 104.21.77.178/32" +has "sandbox up --adopt --remote: sandbox Secret copy" "$(cat "${S}/kubectl.log")" "stdin-secret apply --server-side --force-conflicts --field-manager=k3d-gpu -f -" +lacks "sandbox up --adopt --remote: no controller manifest" "$(cat "${S}/kubectl.log")" "share/agent-sandbox.yml" out=$(run cmd_sandbox status --remote) has "sandbox: --remote only with up" "${out}" "--remote goes with 'sandbox up'" printf 'llm.example.net 104.21.77.178,172.67.210.235' > "${S}/pin"