diff --git a/README.md b/README.md index d629b9a1..b3eef075 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Use the complete installation guide for your environment: After installation, forward the AI Factory Operations Agent UI service: ```bash -kubectl -n mosaic port-forward svc/mosaic-ui 3000:3000 +kubectl -n aifoa port-forward svc/mosaic-ui 3000:3000 ``` Open `http://localhost:3000`. diff --git a/docs/_static/install-builder.mjs b/docs/_static/install-builder.mjs index 8540a745..b3ecf524 100644 --- a/docs/_static/install-builder.mjs +++ b/docs/_static/install-builder.mjs @@ -10,7 +10,7 @@ export const modules = { }; const observabilityServices = ['prometheus', 'alertmanager', 'loki']; const fields = { - namespace: ['Namespace', 'mosaic'], pullSecret: ['NGC pull Secret', 'nvcr-image-pull-secret'], + namespace: ['Namespace', 'aifoa'], pullSecret: ['NGC pull Secret', 'nvcr-image-pull-secret'], version: ['Pinned chart version (optional)', '', 'e.g. 0.0.1'], baseUrl: ['Inference API base URL', '', 'https://inference.example.com/v1'], model: ['Inference model', '', 'Your served model name'], llmSecret: ['Inference Secret (apiKey)', 'mosaic-external-llm'], @@ -90,7 +90,7 @@ export function buildCommand(state) { args.unshift(`-f "$MOSAIC_CHART_WORKDIR/mosaic-stack/profiles/${profile}"`); } args.push('--atomic', '--wait', `--timeout ${state.inference === 'external' ? '12m' : '30m'}`); - return prefix + `helm upgrade --install mosaic ${target} \\\n ${args.join(' \\\n ')}`; + return prefix + `helm upgrade --install aifoa ${target} \\\n ${args.join(' \\\n ')}`; } const root = typeof document === 'undefined' ? null : document.querySelector('#install-command-builder'); diff --git a/docs/bcm.md b/docs/bcm.md index de341761..2b892aa7 100644 --- a/docs/bcm.md +++ b/docs/bcm.md @@ -180,11 +180,11 @@ printf '%s' "$NGC_API_KEY" | helm registry login nvcr.io \ --username '$oauthtoken' \ --password-stdin -kubectl create namespace mosaic --dry-run=client -o yaml | kubectl apply -f - -kubectl -n mosaic create secret generic mosaic-external-llm \ +kubectl create namespace aifoa --dry-run=client -o yaml | kubectl apply -f - +kubectl -n aifoa create secret generic mosaic-external-llm \ --from-literal=apiKey="$EXTERNAL_LLM_API_KEY" \ --dry-run=client -o yaml | kubectl apply -f - -kubectl -n mosaic create secret generic bcm-host-ssh-key \ +kubectl -n aifoa create secret generic bcm-host-ssh-key \ --from-file=id_ecdsa="$CLUSTER_MANAGER_SSH_KEY_PATH" \ --dry-run=client -o yaml | kubectl apply -f - ``` @@ -199,9 +199,9 @@ Base Command Manager inspection. Other cluster integrations remain disabled. ```bash # Remove --devel for the latest stable release, or replace it with --version 0.0.1 to pin that release. -helm upgrade --install mosaic "$MOSAIC_CHART" \ +helm upgrade --install aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --create-namespace \ --set global.registryCredentials.create=true \ --set-string global.registryCredentials.password="$NGC_API_KEY" \ @@ -238,10 +238,10 @@ read-only Base Command Manager request. The first request creates or updates `ai ```bash kubectl get nodes -kubectl -n mosaic get pods +kubectl -n aifoa get pods MANAGER_TEST_PROMPT='/cluster-management Use Base Command Manager to run "device; list" and summarize the first five devices.' -kubectl -n mosaic exec deploy/mosaic-ui -- \ +kubectl -n aifoa exec deploy/mosaic-ui -- \ mosaic --session bcm-install-validation --timeout-ms 180000 \ "$MANAGER_TEST_PROMPT" @@ -250,7 +250,7 @@ cmsh -c 'user; use aichatbotuser; get profile' if systemctl is-active --quiet shorewall; then systemctl restart shorewall kubectl wait --for=condition=Ready node/mosaic-control-plane --timeout=120s - kubectl -n mosaic exec deploy/mosaic-ui -- \ + kubectl -n aifoa exec deploy/mosaic-ui -- \ mosaic --session bcm-restart-validation --timeout-ms 180000 \ "$MANAGER_TEST_PROMPT" fi @@ -264,7 +264,7 @@ remain disabled. Keep this port-forward running on the Base Command Manager head node: ```bash -kubectl -n mosaic port-forward svc/mosaic-ui 3210:3000 +kubectl -n aifoa port-forward svc/mosaic-ui 3210:3000 ``` From the operator's workstation, keep an SSH tunnel to the head node running: @@ -276,9 +276,9 @@ ssh -L 3210:127.0.0.1:3210 root@bcm-head.example.com Retrieve the generated UI credentials on the head node: ```bash -kubectl -n mosaic get secret mosaic-ui-auth \ +kubectl -n aifoa get secret mosaic-ui-auth \ -o jsonpath='{.data.username}' | base64 --decode; echo -kubectl -n mosaic get secret mosaic-ui-auth \ +kubectl -n aifoa get secret mosaic-ui-auth \ -o jsonpath='{.data.password}' | base64 --decode; echo ``` diff --git a/docs/conf.py b/docs/conf.py index 31300b2b..618ee3c7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,7 @@ def render_footprint(state): command = json.loads(subprocess.check_output(["node", "--input-type=module", "-e", script, json.dumps(state)], cwd=ROOT, text=True)) shell = 'trap \'[ -z "${MOSAIC_CHART_WORKDIR:-}" ] || rm -rf "$MOSAIC_CHART_WORKDIR"\' EXIT; helm() { if [ "$1" = pull ]; then ln -s "$CHART" "$MOSAIC_CHART_WORKDIR/mosaic-stack"; else printf \'%s\\0\' "$@"; fi; }; ' arguments = subprocess.check_output(["bash", "-c", shell + command], env={**os.environ, "CHART": str(CHART)}, text=True).split("\0")[4:-1] - rendered = ["template", "mosaic", str(CHART)] + rendered = ["template", "aifoa", str(CHART)] arguments = iter(arguments) for argument in arguments: if argument in {"--atomic", "--wait", "--devel"}: @@ -79,7 +79,7 @@ def render_footprint(state): kind = document.get("kind") spec = document.get("spec", {}) name = document["metadata"]["name"] - key = f"{kind}/{document['metadata'].get('namespace', 'mosaic')}/{name}" + key = f"{kind}/{document['metadata'].get('namespace', 'aifoa')}/{name}" if kind == "PersistentVolumeClaim": components[key] = {"name": name, "kind": kind, "storage": spec["resources"]["requests"]["storage"], "count": 1} elif kind in {"Deployment", "StatefulSet", "DaemonSet", "Job", "Sandbox"}: diff --git a/docs/installation.md b/docs/installation.md index 459f0600..442de267 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -12,6 +12,8 @@ Optionally, use the [installation builder](installation_builder.md) to select mo ## Custom Installation +New installations use `aifoa` for both the Helm release and namespace. For existing installations, retain their current release name and namespace in all commands; changing either targets a different installation and does not migrate its data. Custom names remain supported. + ### 1. Create The Namespace And Registry Access Ask your NVIDIA contact to invite the email address you will use with NGC to @@ -30,8 +32,8 @@ printf '%s' "$NGC_API_KEY" | helm registry login nvcr.io \ --username '$oauthtoken' \ --password-stdin -kubectl create namespace mosaic --dry-run=client -o yaml | kubectl apply -f - -kubectl -n mosaic create secret docker-registry nvcr-image-pull-secret \ +kubectl create namespace aifoa --dry-run=client -o yaml | kubectl apply -f - +kubectl -n aifoa create secret docker-registry nvcr-image-pull-secret \ --docker-server=nvcr.io \ --docker-username='$oauthtoken' \ --docker-password="$NGC_API_KEY" \ @@ -58,13 +60,13 @@ export EXTERNAL_LLM_BASE_URL='https://integrate.api.nvidia.com/v1' export EXTERNAL_LLM_MODEL='nvidia/nemotron-3.5-super-vl-120b-a12b' read -rsp 'External LLM API key: ' EXTERNAL_LLM_API_KEY; echo -kubectl -n mosaic create secret generic mosaic-external-llm \ +kubectl -n aifoa create secret generic mosaic-external-llm \ --from-literal=apiKey="$EXTERNAL_LLM_API_KEY" \ --dry-run=client -o yaml | kubectl apply -f - -helm upgrade --install mosaic "$MOSAIC_CHART" \ +helm upgrade --install aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --set 'global.imagePullSecrets[0].name=nvcr-image-pull-secret' \ --set llm.mode=external \ --set-string llm.external.baseUrl="$EXTERNAL_LLM_BASE_URL" \ @@ -122,8 +124,8 @@ The chart defaults to Nemotron 3.5 Super VL NVFP4 on one compatible GPU, with 64 To install AI Factory Operations Agent with Nemotron 3.5 Super VL running on 1 GPU in vLLM, run: ```bash -helm upgrade --install mosaic "$MOSAIC_CHART_PATH" \ - -n mosaic \ +helm upgrade --install aifoa "$MOSAIC_CHART_PATH" \ + -n aifoa \ -f "$MOSAIC_CHART_PATH/profiles/vllm-super-1gpu.yaml" \ --set 'global.imagePullSecrets[0].name=nvcr-image-pull-secret' \ --set modules.ui.enabled=true \ @@ -155,8 +157,8 @@ The [Ultra model card](https://build.nvidia.com/nvidia/nemotron-3-ultra-550b-a55 When upgrading an existing release, preserve its `llm.vllm.modelCache.size` override. Kubernetes does not allow changing the StatefulSet volume claim template in place, and the smaller Super default does not shrink an existing cache. ```bash -helm upgrade --install mosaic "$MOSAIC_CHART_PATH" \ - -n mosaic \ +helm upgrade --install aifoa "$MOSAIC_CHART_PATH" \ + -n aifoa \ -f "$MOSAIC_CHART_PATH/profiles/vllm-ultra-4gpu.yaml" \ --set 'global.imagePullSecrets[0].name=nvcr-image-pull-secret' \ --set modules.ui.enabled=true \ @@ -242,7 +244,7 @@ The minimal installation includes the UI, headless interfaces, OpenClaw, OpenShe After installing with either inference path, open the UI by running: ```bash -kubectl -n mosaic port-forward svc/mosaic-ui 3000:3000 +kubectl -n aifoa port-forward svc/mosaic-ui 3000:3000 ``` By default, the chart creates the `mosaic-ui-auth` Secret and generates both the UI password and an internal machine token. Leave `mosaicUi.auth.password` and `mosaicUi.auth.machineToken` empty; users do not need to provide either value. The generated credentials are preserved across upgrades. @@ -250,8 +252,8 @@ By default, the chart creates the `mosaic-ui-auth` Secret and generates both the Retrieve the browser login with: ```bash -kubectl -n mosaic get secret mosaic-ui-auth -o jsonpath='{.data.username}' | base64 --decode; echo -kubectl -n mosaic get secret mosaic-ui-auth -o jsonpath='{.data.password}' | base64 --decode; echo +kubectl -n aifoa get secret mosaic-ui-auth -o jsonpath='{.data.username}' | base64 --decode; echo +kubectl -n aifoa get secret mosaic-ui-auth -o jsonpath='{.data.password}' | base64 --decode; echo ``` `username` and `password` are browser credentials. `machineToken` is used automatically by OpenClaw when it calls protected AI Factory Operations Agent UI APIs; it is never entered in the browser. @@ -280,9 +282,9 @@ prefix and omit `/api/v1`. Set `PROMETHEUS_URL` to your endpoint: ```bash export PROMETHEUS_URL='http://kube-prometheus-stack-prometheus.prometheus.svc.cluster.local:9090' -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.observability.enabled=true \ --set-string observability.prometheusUrl="$PROMETHEUS_URL" \ @@ -296,9 +298,9 @@ To connect Mosaic to an existing Loki service, run: ```bash export LOKI_URL='https://logs.example.com' -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.observability.enabled=true \ --set-string observability.loki.url="$LOKI_URL" \ @@ -316,9 +318,9 @@ AI Factory Operations Agent defaults to the Alertmanager service installed by NM ```bash export ALERTMANAGER_URL='https://alerts.example.com/alertmanager' -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set-string observability.alertmanagerUrl="$ALERTMANAGER_URL" \ --wait \ @@ -338,14 +340,14 @@ export GRAFANA_DATASOURCE_TYPE='prometheus' export GRAFANA_USERNAME='admin' read -rsp 'Grafana password: ' GRAFANA_PASSWORD; echo -kubectl -n mosaic create secret generic mosaic-grafana-auth \ +kubectl -n aifoa create secret generic mosaic-grafana-auth \ --from-literal=username="$GRAFANA_USERNAME" \ --from-literal=password="$GRAFANA_PASSWORD" \ --dry-run=client -o yaml | kubectl apply -f - -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.grafana.enabled=true \ --set-string observability.grafanaUrl="$GRAFANA_URL" \ @@ -370,9 +372,9 @@ For a Prometheus datasource, keep `prometheus`. To enable the optional browser terminal service, run: ```bash -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.terminal.enabled=true \ --wait \ @@ -512,13 +514,13 @@ export CLUSTER_MANAGER_HEAD_HOST='bcm-head.example.com' export CLUSTER_MANAGER_SSH_KEY_PATH='/root/.ssh/id_ecdsa' test -r "$CLUSTER_MANAGER_SSH_KEY_PATH" -kubectl -n mosaic create secret generic bcm-host-ssh-key \ +kubectl -n aifoa create secret generic bcm-host-ssh-key \ --from-file=id_ecdsa="$CLUSTER_MANAGER_SSH_KEY_PATH" \ --dry-run=client -o yaml | kubectl apply -f - -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.bcm.enabled=true \ --set bcmMcp.enabled=true \ @@ -552,9 +554,9 @@ Prerequisites: Enable it: ```bash -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.bcm.redfish.enabled=true \ --wait \ @@ -601,15 +603,15 @@ export IRAOP_API_KEY="$(openssl rand -hex 32)" export MOSAIC_CHAT_MODEL='nvidia/nemotron-3.5-super-vl-120b-a12b' export IRA_CORPUS_PATH='/cm/shared/iraop-corpus' -kubectl -n mosaic create secret generic iraop-secrets \ +kubectl -n aifoa create secret generic iraop-secrets \ --from-literal=NVIDIA_API_KEY="$NVIDIA_API_KEY" \ --from-literal=NVIDIA_CHAT_API_KEY=EMPTY \ --from-literal=IRAOP_API_KEY="$IRAOP_API_KEY" \ --dry-run=client -o yaml | kubectl apply -f - -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.research.enabled=true \ --set modules.research.secrets.create=false \ @@ -626,9 +628,9 @@ unset NVIDIA_API_KEY IRAOP_API_KEY The Hardware Agent reuses the Base Command Manager head address and `bcm-host-ssh-key` configured above. The chart exposes the required Base Command Manager lookup inside the cluster and generates its internal credentials. It uses NVDebug evidence and generic analysis. ```bash -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.diagnostics.enabled=true \ --wait \ @@ -649,9 +651,9 @@ DuckDuckGo support in OpenClaw is experimental: it reads DuckDuckGo's HTML resul For air-gapped installations, or to keep the agent from making external requests, disable the module. The tools are then not registered at all. ```bash -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.webSearch.enabled=false \ --wait \ @@ -668,7 +670,7 @@ Brave Search is an API-backed provider. It needs a paid Brave account; at the ti ```bash read -rsp 'Brave Search API key: ' WEB_SEARCH_KEY; echo - kubectl -n mosaic create secret generic mosaic-web-search \ + kubectl -n aifoa create secret generic mosaic-web-search \ --from-literal=WEB_SEARCH_KEY="$WEB_SEARCH_KEY" \ --dry-run=client -o yaml | kubectl apply -f - unset WEB_SEARCH_KEY @@ -677,9 +679,9 @@ Brave Search is an API-backed provider. It needs a paid Brave account; at the ti 4. Select the provider: ```bash - helm upgrade mosaic "$MOSAIC_CHART" \ + helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.webSearch.enabled=true \ --set modules.webSearch.provider=brave \ @@ -688,7 +690,7 @@ Brave Search is an API-backed provider. It needs a paid Brave account; at the ti --timeout 12m ``` -OpenClaw `2026.6.10` does not bundle the Brave provider. When `provider=brave`, an init container downloads `@openclaw/brave-plugin` at `modules.webSearch.brave.pluginVersion` from `registry.npmjs.org`, verifies it against `modules.webSearch.brave.pluginSha256`, and installs it before the gateway starts. The `openclaw` pod then also needs HTTPS egress to `registry.npmjs.org` and `api.search.brave.com`. To rotate the key, update the Secret and run `kubectl -n mosaic rollout restart deployment/openclaw`. Updating only the Secret does not restart the pod. +OpenClaw `2026.6.10` does not bundle the Brave provider. When `provider=brave`, an init container downloads `@openclaw/brave-plugin` at `modules.webSearch.brave.pluginVersion` from `registry.npmjs.org`, verifies it against `modules.webSearch.brave.pluginSha256`, and installs it before the gateway starts. The `openclaw` pod then also needs HTTPS egress to `registry.npmjs.org` and `api.search.brave.com`. To rotate the key, update the Secret and run `kubectl -n aifoa rollout restart deployment/openclaw`. Updating only the Secret does not restart the pod. After enabling the required plugins, use the earlier port-forward to open the UI. @@ -699,9 +701,9 @@ Retain the working site configuration with: ```bash MOSAIC_CHART=oci://nvcr.io/0948643769302270/afoa-release/mosaic-stack # Remove --devel for the latest stable release, or replace it with --version 0.0.1 to pin that release. -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --atomic \ --wait \ @@ -716,7 +718,7 @@ The AI Factory Operations Agent UI service also exposes a headless API: ```bash export MOSAIC_URL=http://127.0.0.1:3000 -export MOSAIC_UI_MACHINE_TOKEN="$(kubectl -n mosaic get secret mosaic-ui-auth \ +export MOSAIC_UI_MACHINE_TOKEN="$(kubectl -n aifoa get secret mosaic-ui-auth \ -o jsonpath='{.data.machineToken}' | base64 --decode)" curl -sS "$MOSAIC_URL/api/headless/chat" \ --header @<(printf 'Authorization: Bearer %s\n' "$MOSAIC_UI_MACHINE_TOKEN") \ @@ -727,7 +729,7 @@ curl -sS "$MOSAIC_URL/api/headless/chat" \ Invoke the packaged CLI inside the `mosaic-ui` pod with: ```bash -kubectl -n mosaic exec deploy/mosaic-ui -- \ +kubectl -n aifoa exec deploy/mosaic-ui -- \ mosaic --session headless "summarize whether the cluster is healthy" ``` @@ -780,9 +782,9 @@ To use the vanilla collector with a site-specific evidence root, run: ```bash export SLURM_EVIDENCE_ROOT='/shared/slurm' -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.slurm.enabled=true \ --set modules.slurm.backend=vanilla \ @@ -796,9 +798,9 @@ The Slurm backend defaults to `auto`. When Base Command Manager and Slurm are bo After enabling the Base Command Manager extension, enable Slurm through it with: ```bash -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.slurm.enabled=true \ --set modules.slurm.backend=auto \ @@ -815,9 +817,9 @@ The local cluster is registered by default using the `openclaw` ServiceAccount. Enable read-only access to the local cluster with: ```bash -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.kubernetes.enabled=true \ --wait \ @@ -827,16 +829,16 @@ helm upgrade mosaic "$MOSAIC_CHART" \ To register another cluster, first create a kubeconfig for a read-only identity on that cluster. Store it as a Secret in the AI Factory Operations Agent namespace: ```bash -kubectl -n mosaic create secret generic production-west-kubeconfig \ +kubectl -n aifoa create secret generic production-west-kubeconfig \ --from-file=config=/path/to/read-only-kubeconfig ``` Register that Secret with AI Factory Operations Agent by running: ```bash -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set modules.kubernetes.enabled=true \ --set-json 'kubernetes.clusters=[{"name":"production-west","kubeconfigSecretRef":{"name":"production-west-kubeconfig","key":"config"}}]' \ @@ -856,9 +858,9 @@ The published chart includes the pinned official OpenShell OCI dependency from ` If the cluster already provides a compatible `agent-sandbox` installation, run: ```bash -helm upgrade mosaic "$MOSAIC_CHART" \ +helm upgrade aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --reuse-values \ --set agentSandbox.install=false \ --wait \ diff --git a/docs/kind_installation.md b/docs/kind_installation.md index 8957cd4f..68ad9446 100644 --- a/docs/kind_installation.md +++ b/docs/kind_installation.md @@ -42,8 +42,8 @@ printf '%s' "$NGC_API_KEY" | helm registry login nvcr.io \ --username '$oauthtoken' \ --password-stdin -kubectl create namespace mosaic --dry-run=client -o yaml | kubectl apply -f - -kubectl -n mosaic create secret generic mosaic-external-llm \ +kubectl create namespace aifoa --dry-run=client -o yaml | kubectl apply -f - +kubectl -n aifoa create secret generic mosaic-external-llm \ --from-literal=apiKey="$EXTERNAL_LLM_API_KEY" \ --dry-run=client -o yaml | kubectl apply -f - ``` @@ -53,9 +53,9 @@ kubectl -n mosaic create secret generic mosaic-external-llm \ ```bash # Remove --devel for the latest stable release, or replace it with --version 0.0.1 to pin that release. -helm upgrade --install mosaic "$MOSAIC_CHART" \ +helm upgrade --install aifoa "$MOSAIC_CHART" \ --devel \ - -n mosaic \ + -n aifoa \ --create-namespace \ --set global.registryCredentials.create=true \ --set-string global.registryCredentials.password="$NGC_API_KEY" \ diff --git a/docs/loki.md b/docs/loki.md index 6766b371..f0c299b1 100644 --- a/docs/loki.md +++ b/docs/loki.md @@ -12,7 +12,7 @@ apiVersion: v1 kind: Secret metadata: name: loki-readonly - namespace: mosaic + namespace: aifoa type: Opaque stringData: authorization: "Bearer " diff --git a/docs/nmc_installation.md b/docs/nmc_installation.md index 0bb71979..48cb8fc6 100644 --- a/docs/nmc_installation.md +++ b/docs/nmc_installation.md @@ -4,6 +4,18 @@ This guide installs AI Factory Operations Agent on an NVIDIA Mission Control adm For self-hosted inference on the separate GPU workload cluster, first follow [Self-hosted inference](#self-hosted-inference), then use its endpoint, served model name, and API key below. The bundled vLLM profile requires GPUs in the same cluster as the agent and should not be used on a GPU-free admin cluster. +## Execution environment + +Run the NMC examples on the Base Command Manager head node in a Bash login shell. They assume a root login, which can read `/root/.ssh/id_ecdsa` and load the cluster administrator modules. From a sudo-enabled account, enter that shell **before** setting credentials: + +```bash +sudo -i +``` + +Run the commands in the same Bash login shell as root or a user with Kubernetes administrator access and a readable SSH key specified by `CLUSTER_MANAGER_SSH_KEY_PATH`. + +Both the agent and standalone inference charts are pulled from NGC. The commands work from any directory and do not require Git or a repository checkout. Each `module load` selects a different Kubernetes cluster for subsequent commands in that shell. + ## Install the agent Run this flow as a Kubernetes administrator on the Base Command Manager head node after setting `NGC_API_KEY`, `EXTERNAL_LLM_API_KEY`, `EXTERNAL_LLM_BASE_URL`, and `EXTERNAL_LLM_MODEL`. The endpoint, model, and API key must belong to the same OpenAI-compatible provider. For OpenAI, use `https://api.openai.com/v1` and a model available to that account. @@ -24,7 +36,7 @@ module load kubernetes/k8s-admin : "${EXTERNAL_LLM_API_KEY:?Set EXTERNAL_LLM_API_KEY for the selected provider}" : "${EXTERNAL_LLM_BASE_URL:?Set EXTERNAL_LLM_BASE_URL for the selected provider}" : "${EXTERNAL_LLM_MODEL:?Set EXTERNAL_LLM_MODEL for the selected provider}" -MOSAIC_NAMESPACE=${MOSAIC_NAMESPACE:-mosaic} +MOSAIC_NAMESPACE=${MOSAIC_NAMESPACE:-aifoa} NMC_POLICY_RELEASE=${NMC_POLICY_RELEASE:-net-pol} NMC_POLICY_NAMESPACE=${NMC_POLICY_NAMESPACE:-default} MOSAIC_CHART=oci://nvcr.io/0948643769302270/afoa-release/mosaic-stack @@ -71,7 +83,7 @@ kubectl -n "$MOSAIC_NAMESPACE" create secret generic mosaic-grafana-auth \ --dry-run=client -o yaml | kubectl apply -f - # Remove --devel for the latest stable release, or replace it with --version 0.0.1 to pin that release. -helm upgrade --install mosaic "$MOSAIC_CHART" \ +helm upgrade --install aifoa "$MOSAIC_CHART" \ --devel \ -n "$MOSAIC_NAMESPACE" \ --create-namespace \ @@ -123,7 +135,7 @@ A rule selecting `app.kubernetes.io/name=coredns` does not match pods labeled on If installation times out, check DNS from the running UI pod. If the UI is unavailable, use a diagnostic pod in the same namespace: ```bash -MOSAIC_NAMESPACE=${MOSAIC_NAMESPACE:-mosaic} +MOSAIC_NAMESPACE=${MOSAIC_NAMESPACE:-aifoa} kubectl -n "$MOSAIC_NAMESPACE" exec deployment/mosaic-ui -- getent ahostsv4 kubernetes.default.svc.cluster.local kubectl -n "$MOSAIC_NAMESPACE" exec deployment/mosaic-ui -- getent ahostsv4 dl.k8s.io ``` @@ -145,7 +157,17 @@ Deploy inference in `k8s-user` and connect the agent in `k8s-admin` to its HTTPS ### 1. Configure inference -Run from a checkout of this repository. The example chart is at `helm/afoa-inference`; it is separate from the published agent chart. You need: +In the login shell described above, sign in to NGC using the same invited account described in [Install the agent](#install-the-agent). The standalone inference chart is published alongside the agent chart: + +```bash +: "${NGC_API_KEY:?Set NGC_API_KEY to an NGC API key with private registry access}" +printf '%s' "$NGC_API_KEY" | helm registry login nvcr.io \ + --username '$oauthtoken' --password-stdin +INFERENCE_CHART=oci://nvcr.io/0948643769302270/afoa-release/afoa-inference +INFERENCE_CHART_VERSION=0.0.2-main.20261030003406.gd14c5492 +``` + +This pins a published development version. Select another published version deliberately when upgrading. No checkout or `cd` step is needed. You need: - A compatible GPU in `k8s-user`, the NVIDIA device plugin and a GPU runtime. The default [Nemotron 3.5 Super VL profile](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4) requests one GPU, 8 CPUs, 64 GiB host memory and a 500 GiB persistent cache. Check the model card for GPU memory and driver requirements. - A StorageClass for the cache, plus access to the vLLM image and model weights. @@ -173,7 +195,8 @@ kubectl -n afoa-inference create secret generic afoa-inference-auth \ --from-literal=apiKey="$EXTERNAL_LLM_API_KEY" \ --dry-run=client -o yaml | kubectl apply -f - -helm upgrade --install afoa-inference ./helm/afoa-inference \ +helm upgrade --install afoa-inference "$INFERENCE_CHART" \ + --version "$INFERENCE_CHART_VERSION" \ -n afoa-inference \ -f inference-values.yaml \ --wait \ @@ -186,7 +209,26 @@ When upgrading from Nemotron 3 Super, preserve site settings and cache size, and The first download and GPU kernel compilation can take more than 30 minutes. Wait for Helm to finish; startup checks are below. -### 3. Connect the agent +### 3. Verify inference + +Before installing the agent, test the HTTPS endpoint using the inference API key selected above; replace the example hostname and model if customized: + +```bash +curl --fail-with-body --max-time 120 \ + "https://inference.example.com/v1/chat/completions" \ + -H "Authorization: Bearer $EXTERNAL_LLM_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "nemotron-3.5-super-vl", + "messages": [{"role": "user", "content": "Reply with ready."}], + "max_tokens": 128, + "chat_template_kwargs": {"enable_thinking": false} + }' +``` + +Confirm the response contains generated text in `choices[0].message.content`. This checks inference, authentication, DNS, TLS and ingress from the head node; the check below separately verifies access from the agent pod. If the request fails, inspect the startup and connectivity checks below before continuing. + +### 4. Connect the agent ```bash export EXTERNAL_LLM_BASE_URL='https://inference.example.com/v1' @@ -318,3 +360,43 @@ JS Use your configured served model name in the completion check if you changed the default. HTTP 200 for both requests establishes DNS, routing, TLS trust, authentication, and basic inference from the admin cluster. Then ask a short question through the agent UI to verify the full agent path. TLS errors require correcting certificate trust, timeouts require checking routing or server readiness, and 401 responses indicate mismatched keys. + + +## Uninstall across both clusters + +In the administrator login shell, confirm each release and namespace with `helm list -A`; use their existing names in the commands below. Remove the agent before its dedicated inference server, and keep shared inference running. + +Back up required agent data first: Helm deletes its workspace and audit-log PVCs, and the storage reclaim policy determines whether backing data is deleted. If this release installed the AgentSandbox controller, confirm no other release depends on it. Keep shared CRDs and platform controllers. + +### 1. Remove the agent from `k8s-admin` + +```bash +module load kubernetes/k8s-admin +: "${AGENT_NAMESPACE:?Set the existing agent namespace}" +helm list -A +helm uninstall aifoa --namespace "$AGENT_NAMESPACE" --wait --timeout 12m +``` + +Separately created Secrets remain. Delete the namespace only if it is dedicated to this installation and all remaining resources and data can be discarded: + +```bash +kubectl delete namespace "$AGENT_NAMESPACE" +``` + +Keep NMC's shared `net-pol` release. After the namespace has no remaining consumers, its policy owner can remove its entry from `admin.namespaceList` through the existing policy-management workflow. + +### 2. Remove dedicated inference from `k8s-user` + +Skip this section for existing or shared inference endpoints. + +```bash +module load kubernetes/k8s-user +helm list -A +helm uninstall afoa-inference --namespace afoa-inference --wait --timeout 12m +``` + +The model-cache PVC and separately created API-key and TLS Secrets remain. Delete the namespace only if it is dedicated to this installation and its remaining data can be discarded; this deletes the PVC, with backing-storage deletion determined by its reclaim policy: + +```bash +kubectl delete namespace afoa-inference +``` diff --git a/docs/skills/ai-factory-operations-agent-headless/SKILL.md b/docs/skills/ai-factory-operations-agent-headless/SKILL.md index 41bc7f6f..6e4f9ff5 100644 --- a/docs/skills/ai-factory-operations-agent-headless/SKILL.md +++ b/docs/skills/ai-factory-operations-agent-headless/SKILL.md @@ -26,11 +26,11 @@ curl -fsS "${MOSAIC_URL:-http://127.0.0.1:3000}/api/openclaw/commands" \ For a Kubernetes deployment, port-forward the UI service and set `MOSAIC_URL`: ```bash -kubectl -n mosaic port-forward svc/mosaic-ui 3000:3000 +kubectl -n aifoa port-forward svc/mosaic-ui 3000:3000 export MOSAIC_URL=http://127.0.0.1:3000 ``` -Use the target namespace instead of `mosaic` if the operator installed the service elsewhere. +Use the target namespace instead of `aifoa` if the operator installed the service elsewhere. ## Invoke By CLI @@ -132,9 +132,9 @@ Install from the packaged Helm chart: ```bash # Remove --devel for the latest stable release, or replace it with --version 0.0.1 to pin that release. -helm upgrade --install mosaic oci://nvcr.io/0948643769302270/mosaic-stack \ +helm upgrade --install aifoa oci://nvcr.io/0948643769302270/mosaic-stack \ --devel \ - -n mosaic \ + -n aifoa \ --create-namespace \ --wait \ --timeout 12m @@ -143,8 +143,8 @@ helm upgrade --install mosaic oci://nvcr.io/0948643769302270/mosaic-stack \ For source-based development, deploy the tracked chart from the repository: ```bash -helm upgrade --install mosaic ./helm/mosaic-stack \ - -n mosaic \ +helm upgrade --install aifoa ./helm/mosaic-stack \ + -n aifoa \ --create-namespace \ --reset-values \ --wait \ @@ -156,8 +156,8 @@ Before deploying, set image repositories/tags and LLM mode for the target cluste After install, verify readiness: ```bash -kubectl -n mosaic rollout status deploy/mosaic-ui --timeout=5m -kubectl -n mosaic get pods +kubectl -n aifoa rollout status deploy/mosaic-ui --timeout=5m +kubectl -n aifoa get pods ``` Then port-forward `svc/mosaic-ui`, set `MOSAIC_URL`, and call the service headlessly. diff --git a/helm/mosaic-stack/files/agent-documentation.tar.gz b/helm/mosaic-stack/files/agent-documentation.tar.gz index 4abee1d0..63443125 100644 Binary files a/helm/mosaic-stack/files/agent-documentation.tar.gz and b/helm/mosaic-stack/files/agent-documentation.tar.gz differ diff --git a/tests/docs/install-builder.test.mjs b/tests/docs/install-builder.test.mjs index 3ba30798..00dcbb1c 100644 --- a/tests/docs/install-builder.test.mjs +++ b/tests/docs/install-builder.test.mjs @@ -14,6 +14,7 @@ function argumentsFor(state) { const result = spawnSync('bash', ['-c', shell], {encoding: 'utf8'}); assert.equal(result.status, 0, result.stderr); const args = result.stdout.split('\0'); + assert.deepEqual(args.slice(0, 3), ['upgrade', '--install', 'aifoa']); args.pop(); const workdir = args.pop(); return args.slice(4).map(arg => workdir ? arg.replace(`${workdir}/mosaic-stack`, chart) : arg); @@ -34,7 +35,7 @@ for (const state of [ if (['--timeout','--version'].includes(args[i])) {i++; continue;} renderArgs.push(args[i]); } - const result = spawnSync('helm', ['template','mosaic',chart,...renderArgs], {encoding:'utf8',maxBuffer:8*1024*1024}); + const result = spawnSync('helm', ['template','aifoa',chart,...renderArgs], {encoding:'utf8',maxBuffer:8*1024*1024}); assert.equal(result.status,0,result.stderr); }); test('dependencies are derived without changing requested modules', () => { @@ -170,3 +171,10 @@ for (const grafanaType of [undefined, 'victoriametrics-metrics-datasource']) tes assert.ok(args.includes(`observability.grafanaDatasourceType=${grafanaType || 'prometheus'}`)); assert.ok(args.includes('observability.grafanaDatasourceUid=metrics')); }); + +test('namespace selection defaults to aifoa and preserves custom installations', () => { + for (const namespace of [undefined, 'existing-agent']) { + const args = argumentsFor({...example, modules:[], namespace}); + assert.equal(args[args.indexOf('--namespace') + 1], namespace ?? 'aifoa'); + } +});