diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73009d05..a777fac5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,8 @@ Open an issue before substantial implementation work so maintainers can confirm 4. Open a pull request using the repository template. 5. Address review feedback and ensure required status checks pass. +When adding a heading or installation option to `docs/installation.md`, update `docs/installation_builder.md` and `docs/_static/install-builder.mjs` in the same change. Keep the builder's options, generated Helm flags, prerequisites, and guide links aligned with the installation guide. For sections that do not map to a selectable option, include the relevant guidance or link in the builder page. Validate with `node --test tests/docs/*.test.mjs` and `uv run --project docs --locked sphinx-build -W --keep-going -b html -c docs . docs/_build/html`, then inspect the rendered builder. + ## Developer Certificate Of Origin External contributions require agreement to the [Developer Certificate of Origin 1.1](DCO.md). Every commit must include a `Signed-off-by` line. Add it automatically with the `-s` option: diff --git a/docs/_static/docs.css b/docs/_static/docs.css index 07c43af0..994c35c7 100644 --- a/docs/_static/docs.css +++ b/docs/_static/docs.css @@ -21,14 +21,17 @@ pre, .admonition, .search-button { border-radius: 0; } #install-command-builder .builder-settings { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: .8rem; } #install-command-builder .builder-settings > label { flex-direction: column; align-items: stretch; text-align: left; } #install-command-builder input:not([type=checkbox]), #install-command-builder select { width: 100%; padding: .45rem; border: 1px solid var(--pst-color-border); border-radius: 0; background: var(--pst-color-background); color: var(--pst-color-text-base); } -.builder-modules { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: .5rem; } +.builder-modules { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: .5rem; align-items: start; } +.builder-module { background: var(--pst-color-surface); } +.builder-module:has(.builder-suboptions:not([hidden])) { grid-row: span 2; } +.builder-suboptions { padding-left: 1.5rem; } .builder-modules label { padding: .5rem; background: var(--pst-color-surface); cursor: pointer; } .builder-modules input { accent-color: var(--pst-color-primary); } .builder-prerequisites { font-size: .9rem; } #install-command-builder button { padding: .4rem .8rem; border: 1px solid var(--pst-color-border); border-radius: 0; background: var(--pst-color-surface); color: var(--pst-color-text-base); } #install-command-builder [role=status] { margin-left: .6rem; } #install-command-builder pre { margin-top: .6rem; padding: 1rem; white-space: pre; overflow: auto; } -@media (max-width: 600px) { #install-command-builder .builder-settings, .builder-modules { grid-template-columns: 1fr; } } +@media (max-width: 600px) { #install-command-builder .builder-settings, .builder-modules { grid-template-columns: 1fr; } .builder-module:has(.builder-suboptions:not([hidden])) { grid-row: auto; } } #install-command-builder .builder-settings > label:has(input[type=checkbox]) { flex-direction: row; align-items: center; grid-column: 1 / -1; } .builder-sizing { margin: 2rem 0; } diff --git a/docs/_static/install-builder.mjs b/docs/_static/install-builder.mjs index 4c0753e7..8540a745 100644 --- a/docs/_static/install-builder.mjs +++ b/docs/_static/install-builder.mjs @@ -47,6 +47,7 @@ export function buildCommand(state) { set('llm.external.existingSecret', value('llmSecret'), true); } for (const name of ['ui', 'execution', 'observability', ...Object.keys(modules).filter(name => !observabilityServices.includes(name))]) set(`modules.${name}.enabled`, ['ui', 'execution'].includes(name) || enabled.has(name)); + set('modules.bcm.redfish.enabled', enabled.has('bcm') && !!state.redfish); for (const service of ['prometheus', 'alertmanager']) { set(`observability.${service}Url`, enabled.has(service) ? value(`${service}Url`) : '', true); } @@ -111,10 +112,16 @@ if (root) { const moduleGroup = document.createElement('fieldset'); moduleGroup.innerHTML = 'Modules

UI and execution are always enabled. Required dependencies are selected automatically.

'; const buttons = document.createElement('div'); buttons.className = 'builder-modules'; const inputs = new Map(); + const redfishOptions = document.createElement('div'); redfishOptions.className = 'builder-suboptions'; + const redfishLabel = document.createElement('label'); const redfishInput = document.createElement('input'); redfishInput.type = 'checkbox'; redfishInput.name = 'redfish'; + redfishInput.addEventListener('change', () => {state.redfish = redfishInput.checked; update(false);}); redfishLabel.append(redfishInput, document.createTextNode('Read-only Redfish')); redfishOptions.append(redfishLabel); for (const [name, label] of Object.entries(modules)) { const wrapper = document.createElement('label'); const input = document.createElement('input'); input.type = 'checkbox'; input.name = name; input.addEventListener('change', () => {const requested = new Set(state.modules); input.checked ? requested.add(name) : requested.delete(name); state.modules = [...requested]; update(true);}); - wrapper.append(input, document.createTextNode(label)); buttons.append(wrapper); inputs.set(name, input); + wrapper.append(input, document.createTextNode(label)); inputs.set(name, input); + if (name === 'bcm') { + const block = document.createElement('div'); block.className = 'builder-module'; block.append(wrapper, redfishOptions); buttons.append(block); + } else buttons.append(wrapper); } moduleGroup.append(buttons); form.append(moduleGroup); const settings = document.createElement('fieldset'); settings.innerHTML = 'Connection settings'; @@ -134,6 +141,7 @@ if (root) { }).then(data => {catalog = data; update(false);}).catch(error => {sizing.textContent = `Deployment footprint unavailable: ${error.message}`;}); function update(renderSettings) { const enabled = enabledModules(state); + redfishOptions.hidden = !enabled.has('bcm'); redfishInput.checked = !!state.redfish; for (const [name, input] of inputs) {input.checked = enabled.has(name); input.disabled = enabled.has(name) && !state.modules.includes(name);} if (renderSettings) { settingsBody.replaceChildren(); @@ -167,6 +175,7 @@ if (root) { } if (enabled.has('grafana')) notes.push(`Create ${state.grafanaSecret || fields.grafanaSecret[1]} with keys username and password.`); if (enabled.has('bcm')) notes.push(`Create ${state.bcmSecret || fields.bcmSecret[1]} with key id_ecdsa; its SSH identity must bootstrap the read-only CMSH user.`); + if (enabled.has('bcm') && state.redfish) notes.push('Redfish reads node BMCs through the BCM head using credentials stored in BCM. Resets and configuration changes are unavailable. BMC TLS certificate verification is off by default; see the Redfish installation guide to configure it.'); if (enabled.has('research')) notes.push('Create iraop-secrets with NVIDIA_API_KEY, NVIDIA_CHAT_API_KEY and IRAOP_API_KEY; prepare the corpus directory.'); if (enabled.has('webSearch')) notes.push('Web search uses DuckDuckGo without an API key; the OpenClaw pod needs HTTPS egress to duckduckgo.com. Clear it for air-gapped installations.'); if (state.inference !== 'external') notes.push('Verify GPU, memory and model cache requirements in the on-prem inference section.'); diff --git a/docs/bcm.md b/docs/bcm.md index b0e14b87..de341761 100644 --- a/docs/bcm.md +++ b/docs/bcm.md @@ -284,3 +284,5 @@ kubectl -n mosaic get secret mosaic-ui-auth \ Open `http://127.0.0.1:3210` on the workstation. Both forwarding processes must remain running. + +For optional BMC inventory and monitoring, follow the [Read-only Redfish installation](installation.md#read-only-redfish). diff --git a/docs/installation.md b/docs/installation.md index 681a2283..459f0600 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -531,6 +531,65 @@ helm upgrade mosaic "$MOSAIC_CHART" \ The SSH identity must be allowed to create or update the dedicated read-only CMSH user during startup. +### Read-only Redfish + +Redfish lets the agent read a node's BMC when the node is down or unreachable in-band: power state, +hardware health, temperatures, fans, power supplies, BMC event logs, and firmware inventory. The +agent passes only a Base Command Manager hostname. The Base Command Manager adapter looks up the +BMC address and credentials through CMSH, the same lookup the Hardware Agent uses, and never returns +them to the model. + +Prerequisites: + +- The [Base Command Manager extension](#base-command-manager) is configured in `ssh-adapter` mode. +- The adapter's SSH identity can read `bmcsettings` and BMC interfaces through CMSH on the head node. +- The head node can reach the BMC network over HTTPS and has `curl`. Check one BMC from the head node: + + ```bash + curl -sk https:///redfish/v1 | head -c 300; echo + ``` + +Enable it: + +```bash +helm upgrade mosaic "$MOSAIC_CHART" \ + --devel \ + -n mosaic \ + --reuse-values \ + --set modules.bcm.redfish.enabled=true \ + --wait \ + --timeout 12m +``` + +By default each request runs `curl` on the head node over the adapter's existing SSH session +(`modules.bcm.redfish.viaHeadNode=true`), because the head node is routed to the BMC network. The +BMC credential reaches `curl` on standard input, so it does not appear in process listings. Set +`viaHeadNode=false` only when the adapter pod itself can reach the BMCs. BMCs usually present +self-signed certificates, so `modules.bcm.redfish.verifyTls` defaults to `false`. +`modules.bcm.redfish.timeoutSeconds` (default `60`) bounds each request; some DGX BMCs take about +30 seconds to return the expanded firmware inventory. + +The adapter enforces the following allowlist before it looks up any credential. Only `GET` is +used, and the only query parameters accepted are `$top`, `$skip`, and `$expand=.` limited to one +level, which is refused on the service root: + +| Resource | Paths under `/redfish/v1` | +| --- | --- | +| Service root and collections | `/`, `Systems`, `Chassis`, `Managers` | +| Systems | `Systems/{id}` and its `Processors`, `Memory`, `Storage`, `EthernetInterfaces`, and `LogServices/{id}/Entries` | +| Chassis | `Chassis/{id}` and its `Thermal`, `ThermalSubsystem`, `Power`, `PowerSubsystem`, and `Sensors` | +| Managers | `Managers/{id}` and its `EthernetInterfaces` and `LogServices/{id}/Entries` | +| Firmware and tasks | `UpdateService/FirmwareInventory`, `TaskService/Tasks` | + +Everything else is refused, including any path containing `Actions`, and `AccountService`, +`SessionService`, `CertificateService`, `EventService`, `SecureBoot`, `Bios/Settings`, and +`VirtualMedia`. Paths with `..`, `//`, `%`, or other query parameters are also refused. The adapter +also checks every resource embedded in a response, such as `$expand` results and continuation pages, +against the same allowlist. It reduces any embedded resource outside it to a bare link, so a BMC that +expands more than requested cannot return denied content. Fields named like passwords, tokens, or +secrets are redacted from responses. Resets and other BMC actions are not +available in any session mode. + ### Research To deploy the Research Agent and OpenSearch, set the embedding API key, active diff --git a/docs/installation_builder.md b/docs/installation_builder.md index f19b150f..602765df 100644 --- a/docs/installation_builder.md +++ b/docs/installation_builder.md @@ -4,6 +4,8 @@ Select your inference path and modules to generate an installation command. Comp CPU, RAM, GPUs and disk update automatically with your selections. Values come from this documentation version’s rendered Helm manifests. Different chart versions or site overrides can change the requirements. +Select Base Command Manager to expose the optional **Read-only Redfish** checkbox inside its module block. See the [Redfish installation guide](installation.md#read-only-redfish) for BMC credential resolution and TLS verification settings. + ```{raw} html
``` diff --git a/docs/security_model.md b/docs/security_model.md index 3dffa9cc..06223842 100644 --- a/docs/security_model.md +++ b/docs/security_model.md @@ -39,6 +39,8 @@ The read-only and administrative Base Command Manager paths use separate identit Base Command Manager's configured profile remains authoritative. Enabling Base Command Manager administration requires an operator-provided credential with the intended cluster permissions. +When `modules.bcm.redfish.enabled=true`, the `bcm_redfish_*` tools read node BMCs in every mode, including View and automated sessions. They are never mutation tools. The model supplies only a Base Command Manager hostname and a path or named query. The Base Command Manager adapter resolves the BMC address and credentials through CMSH and does not return them. Its `RedfishPolicy` allowlist is enforced server-side before any credential lookup, and again on every resource embedded in a response, so `$expand` cannot reach a denied resource. It permits only `GET` on inventory, health, sensor, log, firmware, and task resources, and rejects any path containing `Actions`, as well as account, session, certificate, event, Secure Boot, BIOS settings, and virtual media resources. Responses have password, token, and secret fields redacted, and errors omit the BMC address. Like other read-only tools, Redfish is not a confidentiality boundary: anything the allowlist permits is visible to the session. + ## Diagnostic Commands And SSH Local `exec` runs inside the per-session OpenShell sandbox in View, Edit, Auto, and automated sessions. OpenShell enforces this boundary; the session guard does not parse or allowlist sandbox commands. diff --git a/helm/mosaic-stack/files/agent-documentation.tar.gz b/helm/mosaic-stack/files/agent-documentation.tar.gz index 37c5d506..4abee1d0 100644 Binary files a/helm/mosaic-stack/files/agent-documentation.tar.gz and b/helm/mosaic-stack/files/agent-documentation.tar.gz differ diff --git a/helm/mosaic-stack/files/bcm-mcp-ssh-server.py b/helm/mosaic-stack/files/bcm-mcp-ssh-server.py index 37a4a36a..49d8d513 100644 --- a/helm/mosaic-stack/files/bcm-mcp-ssh-server.py +++ b/helm/mosaic-stack/files/bcm-mcp-ssh-server.py @@ -2,14 +2,22 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import base64 import hmac import ipaddress +import json import logging +import os import re +import selectors import shlex +import ssl import subprocess import textwrap import threading +import time +import urllib.error +import urllib.request from bcm_mcp_tools.cli.support_mcp.main import configure_logging from bcm_mcp_tools.cli.support_mcp.options import get_env_bool, get_env_int, get_env_str @@ -67,6 +75,163 @@ def last_cmsh_value(output: str) -> str | None: return lines[-1] if lines else None +class RedfishError(Exception): + def __init__(self, status: int, message: str): + super().__init__(message) + self.status = status + + +class RedfishPolicy: + """Read-only Redfish allowlist enforced before any BMC credential is resolved.""" + + _ID = r"[A-Za-z0-9_-][A-Za-z0-9._-]*" + _SUB = rf"(?:/{_ID})*" + _LOGS = rf"/LogServices(?:/{_ID}(?:/Entries(?:/{_ID})?)?)?" + ALLOWED = tuple(re.compile(f"/redfish/v1{pattern}") for pattern in ( + "", + "/(?:Systems|Chassis|Managers)", + rf"/Systems/{_ID}", + rf"/Systems/{_ID}/(?:Processors|Memory|Storage|EthernetInterfaces){_SUB}", + rf"/Systems/{_ID}{_LOGS}", + rf"/Chassis/{_ID}", + rf"/Chassis/{_ID}/(?:Thermal|ThermalSubsystem|Power|PowerSubsystem|Sensors){_SUB}", + rf"/Managers/{_ID}", + rf"/Managers/{_ID}{_LOGS}", + rf"/Managers/{_ID}/EthernetInterfaces{_SUB}", + rf"/UpdateService/FirmwareInventory{_SUB}", + rf"/TaskService/Tasks{_SUB}", + )) + DENIED_SEGMENTS = frozenset({ + "accountservice", "sessionservice", "certificateservice", "eventservice", + "actions", "secureboot", "virtualmedia", + }) + QUERY = { + "$expand": re.compile(r"\.(?:\(\$levels=1\))?"), + "$top": re.compile(r"[0-9]{1,4}"), + "$skip": re.compile(r"[0-9]{1,7}"), + } + + @classmethod + def normalize(cls, path: str) -> str | None: + if not isinstance(path, str) or len(path) > 512 or any(c in path for c in "%#\\\"' \t\r\n"): + return None + resource, separator, query = path.partition("?") + if separator and not query: + return None + if "//" in resource or ".." in resource or "/bios/settings" in resource.lower(): + return None + if len(resource) > len("/redfish/v1") and resource.endswith("/"): + resource = resource[:-1] + if any(segment.lower() in cls.DENIED_SEGMENTS for segment in resource.split("/")): + return None + if not any(pattern.fullmatch(resource) for pattern in cls.ALLOWED): + return None + params = [] + for item in query.split("&") if query else []: + name, separator, value = item.partition("=") + if not separator or name not in cls.QUERY or not cls.QUERY[name].fullmatch(value): + return None + params.append(item) + if resource == "/redfish/v1" and params: + return None + return resource + ("?" + "&".join(params) if params else "") + + @classmethod + def is_allowed(cls, method: str, path: str) -> bool: + return method == "GET" and cls.normalize(path) is not None + + +REDFISH_SECRET_SUFFIXES = ("password", "token", "secret") +REDFISH_METADATA = ("@odata.context", "@odata.etag", "@odata.type") +REDFISH_MAX_CHARS = 32000 +REDFISH_MAX_BYTES = 8 * 1024 * 1024 +REDFISH_MAX_PAGES = 50 +REDFISH_MAX_LOG_SERVICES = 10 +REDFISH_MAX_UNQUERIED_ENTRIES = 1000 +REDFISH_SKIPPED_LOGS = frozenset({"debugtokenservice", "diagnosticlog", "dump", "fdr", "journal"}) +CURL_ERRORS = {6: "BMC address could not be resolved", 7: "BMC is unreachable", 28: "BMC request timed out", + 35: "BMC TLS handshake failed", 60: "BMC TLS certificate verification failed", + 63: "BMC response exceeds the size limit"} + + +def redfish_secret_key(key: str) -> bool: + key = key.lower().replace("-", "").replace("_", "") + return key.endswith((*REDFISH_SECRET_SUFFIXES, "apikey")) or key in { + "authorization", "proxyauthorization", "cookie", "setcookie", + } + + +def scrub_redfish_header(header) -> str: + if not isinstance(header, str): + return "[redacted]" + name, separator, _ = header.partition(":") + if not separator: + return "[redacted]" + return f"{name}: [redacted]" if redfish_secret_key(name.strip()) else header + + +def scrub_redfish(value): + if isinstance(value, dict): + result = {} + for key, item in value.items(): + if key in REDFISH_METADATA: + continue + if redfish_secret_key(key): + item = "[redacted]" + elif key.lower() == "httpheaders": + item = [scrub_redfish_header(header) for header in item] if isinstance(item, list) else "[redacted]" + elif key.lower() == "jsonbody" and isinstance(item, str) and item: + try: + item = json.dumps(scrub_redfish(json.loads(item))) + except ValueError: + item = "[redacted]" + else: + item = scrub_redfish(item) + result[key] = item + return result + if isinstance(value, list): + return [scrub_redfish(item) for item in value] + return value + + +def confine_redfish(value): + """Reduce embedded resources outside the allowlist, such as $expand results, to bare links.""" + if isinstance(value, list): + return [confine_redfish(item) for item in value] + if not isinstance(value, dict): + return value + link = value.get("@odata.id") + if isinstance(link, str) and set(value) - {"@odata.id"} and RedfishPolicy.normalize(link.split("#", 1)[0]) is None: + return {"@odata.id": link} + return {key: confine_redfish(item) for key, item in value.items()} + + +def curl_user_config(username: str, password: str) -> bytes: + if any(c in username + password for c in "\r\n\0"): + raise RedfishError(502, "BMC credentials in BCM are malformed") + escaped = f"{username}:{password}".replace("\\", "\\\\").replace('"', '\\"') + return f'user = "{escaped}"\n'.encode() + + +def redfish_member_paths(data: dict) -> list[str]: + return [m["@odata.id"] for m in data.get("Members", []) if isinstance(m, dict) and isinstance(m.get("@odata.id"), str)] + + +def health(item: dict) -> str | None: + status = item.get("Status") or {} + return status.get("Health") if isinstance(status, dict) else None + + +def present(item: dict) -> bool: + status = item.get("Status") or {} + return not isinstance(status, dict) or status.get("State") != "Absent" + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *_args, **_kwargs): + return None + + class StaticTokenVerifier: def __init__(self, token: str): self.token = token @@ -89,18 +254,63 @@ def __init__(self, *args, **kwargs): self.cmsh_path = get_env_str("MOSAIC_BCM_CMSH_PATH", "/cm/local/apps/cmd/bin/cmsh") or "/cm/local/apps/cmd/bin/cmsh" self.hardware_compat_enabled = get_env_bool("MOSAIC_BCM_HARDWARE_COMPAT_ENABLED", False) self.bcm_head_host = get_env_str("MOSAIC_BCM_HEAD_HOST", "") or "" + self.redfish_enabled = get_env_bool("MOSAIC_BCM_REDFISH_ENABLED", False) + self.redfish_verify_tls = get_env_bool("MOSAIC_BCM_REDFISH_VERIFY_TLS", False) + self.redfish_timeout = get_env_int("MOSAIC_BCM_REDFISH_TIMEOUT", 60) + self.redfish_via_head_node = get_env_bool("MOSAIC_BCM_REDFISH_VIA_HEAD_NODE", True) + self._redfish_members: dict[tuple[str, str], list[str]] = {} self._cmsh_ready = False self._cmsh_setup_lock = threading.Lock() - def _ssh(self, script: str, timeout: int) -> subprocess.CompletedProcess[bytes]: + def _ssh(self, script: str, timeout: int, stdin: bytes | None = None, + max_output_bytes: int | None = None) -> subprocess.CompletedProcess[bytes]: if not self.bcm_head_host: raise RuntimeError("BCM head host is not configured") - return subprocess.run( - ["ssh", self.bcm_head_host, "sh -lc " + shlex.quote(script)], - capture_output=True, - timeout=timeout, - check=False, - ) + command = ["ssh", self.bcm_head_host, "sh -lc " + shlex.quote(script)] + if max_output_bytes is None: + return subprocess.run(command, capture_output=True, timeout=timeout, check=False, input=stdin) + deadline = time.monotonic() + timeout + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) as process: + try: + output = bytearray() + pending = memoryview(stdin or b"") + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + if pending: + os.set_blocking(process.stdin.fileno(), False) + selector.register(process.stdin, selectors.EVENT_WRITE) + else: + process.stdin.close() + while selector.get_map(): + remaining = deadline - time.monotonic() + events = selector.select(max(0, remaining)) + if remaining <= 0 or not events: + raise subprocess.TimeoutExpired(command, timeout) + for key, _ in events: + if key.fileobj is process.stdin: + try: + pending = pending[os.write(key.fd, pending):] + except BlockingIOError: + continue + except BrokenPipeError: + pending = memoryview(b"") + if not pending: + selector.unregister(process.stdin) + process.stdin.close() + else: + chunk = os.read(key.fd, min(65536, max_output_bytes + 1 - len(output))) + if not chunk: + selector.unregister(process.stdout) + output.extend(chunk) + if len(output) > max_output_bytes: + raise RedfishError(502, "BMC response exceeds the size limit") + code = process.wait(timeout=max(0, deadline - time.monotonic())) + return subprocess.CompletedProcess(command, code, bytes(output), b"") + finally: + if process.poll() is None: + process.kill() + process.wait() def _ensure_cmsh_user(self) -> None: if self._cmsh_ready: @@ -197,6 +407,297 @@ def read(field: str) -> str | None: "host_password": host_password, } + def _redfish_bmc(self, hostname: str) -> dict[str, str]: + bmc = self.hardware_bmc(hostname) + if bmc is None: + raise RedfishError(404, f"no BMC address and credentials are configured in BCM for {hostname}") + return bmc + + def _redfish_fetch(self, bmc: dict[str, str], path: str) -> tuple[int, bytes]: + url = f"https://{bmc['bmc_ip']}{path}" + if self.redfish_via_head_node: + command = ["curl", "-sS", "--proto", "=https", "--max-time", str(self.redfish_timeout), + "--max-filesize", str(REDFISH_MAX_BYTES), + "-H", "Accept: application/json", "-K", "/dev/stdin", "-w", "\n%{http_code}"] + if not self.redfish_verify_tls: + command.append("-k") + try: + result = self._ssh(shlex.join(command + [url]), self.redfish_timeout + 30, + curl_user_config(bmc["bmc_username"], bmc["bmc_password"]), + max_output_bytes=REDFISH_MAX_BYTES + 4) + except subprocess.TimeoutExpired as error: + raise RedfishError(504, "BMC request timed out") from error + if result.returncode != 0: + raise RedfishError(502, CURL_ERRORS.get(result.returncode, "BMC request through the BCM head node failed")) + body, _, code = result.stdout.rpartition(b"\n") + return int(code) if code.isdigit() else 502, body + context = ssl.create_default_context() + if not self.redfish_verify_tls: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + credentials = base64.b64encode(f"{bmc['bmc_username']}:{bmc['bmc_password']}".encode()).decode() + request = urllib.request.Request(url, headers={"Accept": "application/json", "Authorization": f"Basic {credentials}"}) + opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=context), _NoRedirect()) + try: + with opener.open(request, timeout=self.redfish_timeout) as response: + return response.status, response.read(REDFISH_MAX_BYTES + 1) + except urllib.error.HTTPError as error: + return error.code, b"" + except (urllib.error.URLError, OSError) as error: + raise RedfishError(502, "BMC is unreachable from the BCM MCP service") from error + + def _redfish_read(self, bmc: dict[str, str], path: str) -> dict: + allowed = RedfishPolicy.normalize(path) + if allowed is None: + raise RedfishError(403, f"{path} is not permitted by the read-only Redfish policy") + status, body = self._redfish_fetch(bmc, allowed) + if status != 200: + raise RedfishError(status if 400 <= status < 600 else 502, f"BMC returned HTTP {status} for {allowed}") + if len(body) > REDFISH_MAX_BYTES: + raise RedfishError(502, f"BMC response for {allowed} is too large") + try: + data = json.loads(body) + except ValueError as error: + raise RedfishError(502, f"BMC returned non-JSON content for {allowed}") from error + data = scrub_redfish(confine_redfish(data)) + return data if isinstance(data, dict) else {"value": data} + + def _redfish_members_of(self, hostname: str, bmc: dict[str, str], collection: str) -> list[str]: + key = (hostname, collection) + if key not in self._redfish_members: + members = redfish_member_paths(self._redfish_read(bmc, f"/redfish/v1/{collection}")) + self._redfish_members[key] = [path for path in members if RedfishPolicy.normalize(path) == path] + return self._redfish_members[key] + + def _redfish_chassis(self, hostname: str, bmc: dict[str, str], chassis: str) -> str: + if chassis: + path = f"/redfish/v1/Chassis/{chassis}" + if RedfishPolicy.normalize(path) != path: + raise ValueError("invalid chassis id") + return path + for system in self._redfish_members_of(hostname, bmc, "Systems")[:1]: + linked = redfish_member_paths({"Members": (self._redfish_read(bmc, system).get("Links") or {}).get("Chassis", [])}) + if linked: + return linked[0] + members = self._redfish_members_of(hostname, bmc, "Chassis") + if not members: + raise RedfishError(404, "the BMC reports no chassis") + return members[0] + + def redfish_get(self, hostname: str, path: str) -> dict: + if RedfishPolicy.normalize(path) is None: + raise RedfishError(403, f"{path} is not permitted by the read-only Redfish policy") + return self._redfish_read(self._redfish_bmc(hostname), path) + + def redfish_system_summary(self, hostname: str) -> dict: + bmc = self._redfish_bmc(hostname) + systems = [] + for path in self._redfish_members_of(hostname, bmc, "Systems")[:4]: + data = self._redfish_read(bmc, path) + status = data.get("Status") or {} + systems.append({ + "id": data.get("Id"), "powerState": data.get("PowerState"), + "health": status.get("Health"), "healthRollup": status.get("HealthRollup"), "state": status.get("State"), + "manufacturer": data.get("Manufacturer"), "model": data.get("Model"), "serialNumber": data.get("SerialNumber"), + "biosVersion": data.get("BiosVersion"), "lastResetTime": data.get("LastResetTime"), + "bootProgress": (data.get("BootProgress") or {}).get("LastState"), + }) + return {"hostname": hostname, "systems": systems} + + def redfish_thermal(self, hostname: str, chassis: str = "") -> dict: + bmc = self._redfish_bmc(hostname) + base = self._redfish_chassis(hostname, bmc, chassis) + try: + data = self._redfish_read(bmc, f"{base}/Thermal") + temps = [{"name": t.get("Name"), "readingCelsius": t.get("ReadingCelsius"), + "upperCritical": t.get("UpperThresholdCritical"), "upperFatal": t.get("UpperThresholdFatal"), + "health": health(t)} for t in data.get("Temperatures", []) if present(t) and t.get("ReadingCelsius") is not None] + fans = [{"name": f.get("Name"), "reading": f.get("Reading"), "units": f.get("ReadingUnits"), + "lowerCritical": f.get("LowerThresholdCritical"), "health": health(f)} + for f in data.get("Fans", []) if present(f) and f.get("Reading") is not None] + source = "Thermal" + except RedfishError as error: + if error.status not in (404, 405): + raise + metrics = self._redfish_read(bmc, f"{base}/ThermalSubsystem/ThermalMetrics") + temps = [{"name": t.get("DeviceName") or t.get("DataSourceUri"), "readingCelsius": t.get("Reading"), + "upperCritical": None, "upperFatal": None, "health": None} + for t in metrics.get("TemperatureReadingsCelsius", []) if t.get("Reading") is not None] + fan_data = self._redfish_read(bmc, f"{base}/ThermalSubsystem/Fans?$expand=.($levels=1)") + fans = [{"name": f.get("Name") or f.get("Id"), "reading": (f.get("SpeedPercent") or {}).get("Reading"), + "units": "%", "lowerCritical": None, "health": health(f)} + for f in fan_data.get("Members", []) if isinstance(f, dict) and present(f)] + source = "ThermalSubsystem" + + def margin(row): + limit = row["upperCritical"] if row["upperCritical"] is not None else row["upperFatal"] + return (0, limit - row["readingCelsius"]) if limit is not None else (1, -row["readingCelsius"]) + + readings = [f["reading"] for f in fans if isinstance(f["reading"], (int, float))] + return { + "hostname": hostname, "chassis": base.rsplit("/", 1)[1], "source": source, + "temperatureCount": len(temps), + "unhealthyTemperatures": [t for t in temps if t["health"] not in (None, "OK")], + "closestToLimit": sorted(temps, key=margin)[:15], + "fanCount": len(fans), + "unhealthyFans": [f for f in fans if f["health"] not in (None, "OK")], + "fanReadingRange": [min(readings), max(readings)] if readings else None, + "slowestFans": sorted((f for f in fans if isinstance(f["reading"], (int, float))), key=lambda f: f["reading"])[:5], + } + + def redfish_power(self, hostname: str, chassis: str = "") -> dict: + bmc = self._redfish_bmc(hostname) + base = self._redfish_chassis(hostname, bmc, chassis) + result: dict = {"hostname": hostname, "chassis": base.rsplit("/", 1)[1], "powerControl": [], "powerSupplies": []} + try: + data = self._redfish_read(bmc, f"{base}/Power") + for control in data.get("PowerControl", []): + metrics = control.get("PowerMetrics") or {} + result["powerControl"].append({ + "name": control.get("Name"), "consumedWatts": control.get("PowerConsumedWatts"), + "averageWatts": metrics.get("AverageConsumedWatts"), "maxWatts": metrics.get("MaxConsumedWatts"), + "capacityWatts": control.get("PowerCapacityWatts"), "health": health(control), + }) + for supply in data.get("PowerSupplies", []): + result["powerSupplies"].append({ + "name": supply.get("Name"), "health": health(supply), "present": present(supply), + "inputWatts": supply.get("PowerInputWatts"), "outputWatts": supply.get("PowerOutputWatts"), + "capacityWatts": supply.get("PowerCapacityWatts"), + }) + result["unhealthyVoltages"] = [{"name": v.get("Name"), "readingVolts": v.get("ReadingVolts"), "health": health(v)} + for v in data.get("Voltages", []) if present(v) and health(v) not in (None, "OK")] + except RedfishError as error: + if error.status not in (404, 405): + raise + if not result["powerSupplies"]: + try: + supplies = self._redfish_read(bmc, f"{base}/PowerSubsystem/PowerSupplies?$expand=.($levels=1)") + except RedfishError as error: + if error.status not in (404, 405): + raise + else: + result["powerSupplies"] = [{ + "name": s.get("Name") or s.get("Id"), "health": health(s), "present": present(s), + "capacityWatts": s.get("PowerCapacityWatts"), "model": s.get("Model"), + } for s in supplies.get("Members", []) if isinstance(s, dict)] + return result + + def _redfish_entries(self, bmc: dict[str, str], path: str, limit: int) -> tuple[list[dict], object, bool]: + """Read up to limit members from path, following bounded nextLink pages; report whether more exist.""" + members: list[dict] = [] + count = None + page = path + for _ in range(REDFISH_MAX_PAGES): + data = self._redfish_read(bmc, page) + if count is None: + count = data.get("Members@odata.count") + members += [m for m in data.get("Members", []) if isinstance(m, dict)] + link = data.get("Members@odata.nextLink") + if not isinstance(link, str) or not link or len(members) >= limit: + return members[:limit], count, len(members) > limit or bool(isinstance(link, str) and link) + page = link + return members[:limit], count, True + + def redfish_sel(self, hostname: str, limit: int = 50) -> dict: + limit = max(1, min(int(limit), 200)) + bmc = self._redfish_bmc(hostname) + services: list[str] = [] + errors: list[dict] = [] + + def source_of(path: str) -> str: + return path.removeprefix("/redfish/v1/").replace("/LogServices/", "/") + + for collection in ("Managers", "Systems"): + for member in self._redfish_members_of(hostname, bmc, collection): + try: + listed = redfish_member_paths(self._redfish_read(bmc, f"{member}/LogServices")) + except RedfishError as error: + if error.status != 404: + errors.append({"source": source_of(member), "status": error.status, "error": str(error)}) + continue + services += [path for path in listed + if path.rsplit("/", 1)[1].lower() not in REDFISH_SKIPPED_LOGS and path not in services] + if len(services) > REDFISH_MAX_LOG_SERVICES: + errors.append({"source": None, "status": None, + "error": f"{len(services) - REDFISH_MAX_LOG_SERVICES} more log services were not read"}) + entries, sources = [], [] + for service in services[:REDFISH_MAX_LOG_SERVICES]: + source = source_of(service) + try: + try: + members, count, more = self._redfish_entries(bmc, f"{service}/Entries?$top={limit}", limit) + except RedfishError as error: + if error.status != 400: + raise + # Some log services reject every query parameter; read them unqueried instead. + members, more = self._redfish_entries(bmc, f"{service}/Entries", REDFISH_MAX_UNQUERIED_ENTRIES)[::2] + count = None + if isinstance(count, int) and count > limit: + newest, _, _ = self._redfish_entries(bmc, f"{service}/Entries?$skip={count - limit}&$top={limit}", limit) + members += newest + if len(newest) < limit: + errors.append({"source": source, "status": None, + "error": "fewer of the newest entries were returned than requested"}) + elif more: + errors.append({"source": source, "status": None, + "error": "more entries exist than could be read, so the newest may be missing"}) + except RedfishError as error: + errors.append({"source": source, "status": error.status, "error": str(error)}) + continue + sources.append(source) + for entry in members: + entries.append({ + "created": entry.get("Created"), "severity": entry.get("Severity"), + "message": str(entry.get("Message") or "")[:240], "source": source, "id": entry.get("Id"), + }) + failed = [e for e in errors if e["status"] is not None] + if failed and not sources: + raise RedfishError(failed[0]["status"], "no BMC event log could be read: " + + "; ".join(f"{e['source']}: {e['error']}" for e in failed[:5])) + unique = {(e["source"], e["id"]): e for e in entries}.values() + newest = sorted(unique, key=lambda e: str(e["created"] or ""), reverse=True)[:limit] + result = {"hostname": hostname, "sources": sources, "entries": newest, "complete": not errors} + if errors: + result["errors"] = errors + return result + + def redfish_firmware(self, hostname: str) -> dict: + bmc = self._redfish_bmc(hostname) + try: + members = self._redfish_read(bmc, "/redfish/v1/UpdateService/FirmwareInventory?$expand=.($levels=1)").get("Members", []) + if members and set(members[0]) <= {"@odata.id"}: + raise RedfishError(501, "expand not honored") + except RedfishError: + paths = redfish_member_paths(self._redfish_read(bmc, "/redfish/v1/UpdateService/FirmwareInventory")) + members = [self._redfish_read(bmc, path) for path in paths[:100]] + return {"hostname": hostname, "firmware": [ + {"id": m.get("Id"), "name": m.get("Name"), "version": m.get("Version"), + "updateable": m.get("Updateable"), "health": health(m)} + for m in members if isinstance(m, dict) + ]} + + def redfish_tool(self, call, *args) -> str: + try: + payload = call(*args) + except RedfishError as error: + payload = {"error": str(error), "status": error.status} + except ValueError as error: + payload = {"error": str(error), "status": 400} + except RuntimeError: + logging.getLogger("mosaic.bcm").exception("Redfish BCM lookup failed") + payload = {"error": "BCM lookup for the BMC failed", "status": 502} + text = json.dumps(payload, separators=(",", ":"), default=str) + entries = payload.get("entries") if isinstance(payload, dict) else None + while len(text) > REDFISH_MAX_CHARS and isinstance(entries, list) and entries: + dropped = max(1, len(entries) // 10) + del entries[-dropped:] + payload["omittedOlderEntries"] = payload.get("omittedOlderEntries", 0) + dropped + text = json.dumps(payload, separators=(",", ":"), default=str) + if len(text) > REDFISH_MAX_CHARS: + text = (text[:REDFISH_MAX_CHARS] + "\n...(truncated: use a named Redfish tool, a member path, or $top" + " for a smaller response)") + return text + def register_hardware_routes(self, token: str) -> None: def authorized(request: Request) -> bool: return hmac.compare_digest(request.headers.get("authorization", ""), f"Bearer {token}") @@ -263,6 +764,37 @@ async def slurm_job_evidence(job_id: str) -> str: """Return BCM WLM metadata and bounded stdout/stderr for a numeric Slurm job ID.""" return await asyncio.to_thread(self.slurm_job_evidence, job_id) + if self.redfish_enabled: + @self.mcp.tool(name="redfish_get") + async def redfish_get_tool(hostname: str, path: str) -> str: + """Read one allowlisted Redfish resource from a node's BMC with GET.""" + return await asyncio.to_thread(self.redfish_tool, self.redfish_get, hostname, path) + + @self.mcp.tool(name="redfish_system_summary") + async def redfish_system_summary_tool(hostname: str) -> str: + """Return BMC-reported power state, health, model, serial and BIOS version.""" + return await asyncio.to_thread(self.redfish_tool, self.redfish_system_summary, hostname) + + @self.mcp.tool(name="redfish_thermal") + async def redfish_thermal_tool(hostname: str, chassis: str = "") -> str: + """Summarize BMC temperature and fan sensors with thresholds and health.""" + return await asyncio.to_thread(self.redfish_tool, self.redfish_thermal, hostname, chassis) + + @self.mcp.tool(name="redfish_power") + async def redfish_power_tool(hostname: str, chassis: str = "") -> str: + """Return BMC-reported power draw and power supply status.""" + return await asyncio.to_thread(self.redfish_tool, self.redfish_power, hostname, chassis) + + @self.mcp.tool(name="redfish_sel") + async def redfish_sel_tool(hostname: str, limit: int = 50) -> str: + """Return the newest BMC manager and system log entries.""" + return await asyncio.to_thread(self.redfish_tool, self.redfish_sel, hostname, limit) + + @self.mcp.tool(name="redfish_firmware") + async def redfish_firmware_tool(hostname: str) -> str: + """Return the BMC firmware inventory.""" + return await asyncio.to_thread(self.redfish_tool, self.redfish_firmware, hostname) + def main() -> int: host = get_env_str("MOSAIC_BCM_ADAPTER_HOST", "0.0.0.0") or "0.0.0.0" diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts b/helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts index c77afb6e..cc970398 100644 --- a/helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts +++ b/helm/mosaic-stack/files/openclaw-seed/extensions/automation-context.ts @@ -27,6 +27,12 @@ const READ_ONLY_TOOLS = new Set([ "bcm_health", "bcm_list_notes", "bcm_node_health_summary", + "bcm_redfish_firmware", + "bcm_redfish_get", + "bcm_redfish_power", + "bcm_redfish_sel", + "bcm_redfish_system_summary", + "bcm_redfish_thermal", "bcm_search_notes", "bcm_search_tools", "dashboard_create", diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/bcm/index.ts b/helm/mosaic-stack/files/openclaw-seed/extensions/bcm/index.ts index 25610ec7..d10281b9 100644 --- a/helm/mosaic-stack/files/openclaw-seed/extensions/bcm/index.ts +++ b/helm/mosaic-stack/files/openclaw-seed/extensions/bcm/index.ts @@ -29,6 +29,7 @@ type BcmConfig = { hitl: boolean; approvalTimeoutMs: number; slurmEnabled: boolean; + redfishEnabled: boolean; timeoutMs: number; subagentEventsUrl: string; }; @@ -148,6 +149,7 @@ function readConfig(pluginConfig: unknown): BcmConfig { hitl: boolConfig(pluginConfig, "hitl", true), approvalTimeoutMs: numberConfig(pluginConfig, "approvalTimeoutMs", 120_000), slurmEnabled: boolConfig(pluginConfig, "slurmEnabled", false), + redfishEnabled: boolConfig(pluginConfig, "redfishEnabled", false), timeoutMs: numberConfig(pluginConfig, "timeoutMs", DEFAULT_TIMEOUT_MS), subagentEventsUrl: resolveSubagentEventsUrl(pluginConfig), }; @@ -489,6 +491,35 @@ async function callBcmTool( } } +async function redfishTool( + config: BcmConfig, + mcpToolName: string, + args: Record, + options: SubagentOptions, +) { + const result = await callBcmTool(config, mcpToolName, args, options); + const text = textFromResult(result.details); + let payload: unknown; + try { + payload = JSON.parse(text); + } catch { + return result; + } + if (payload && typeof payload === "object" && "error" in payload) { + const { error, status } = payload as { error?: unknown; status?: unknown }; + throw new Error(status === 403 + ? `Redfish request refused: ${String(error)}. Redfish access is read-only; actions and account, session, certificate, and event services are unavailable.` + : `Redfish request failed: ${String(error)}`); + } + return jsonToolResult(payload); +} + +function hostnameParam(value: unknown) { + const hostname = stringParam(value); + if (!/^[A-Za-z0-9][A-Za-z0-9-]{0,62}$/.test(hostname)) throw new Error("hostname must be a BCM node hostname"); + return hostname; +} + function stringParam(value: unknown) { return typeof value === "string" && value.trim() ? value.trim() : ""; } @@ -694,7 +725,7 @@ export default definePluginEntry({ registerTool({ name: "bcm_search_tools", label: "BCM Search Tools", - description: "Search bcm-mcp-tools diagnostic tool IDs and descriptions before choosing a BCM command to run.", + description: "Search the diagnostic command catalog used by bcm_execute_tool. This does not search registered agent tools, including the separate bcm_redfish_* tools. Use the current tool definitions to determine agent capabilities; zero catalog matches do not mean a capability is unavailable.", parameters: { type: "object", additionalProperties: false, @@ -706,7 +737,7 @@ export default definePluginEntry({ }, async execute(toolCallId: string, rawParams: Record) { const query = stringParam(rawParams.query); - return callBcmTool( + const result = await callBcmTool( config, "search_tool", { query, @@ -715,6 +746,13 @@ export default definePluginEntry({ }, subagentOptions(config, toolCallId, "bcm_search_tools", "BCM tool search"), ); + return { + ...result, + content: [ + { type: "text", text: "Search scope: diagnostic commands for bcm_execute_tool only. Registered agent tools are outside this catalog. Check the current tool definitions before concluding that a capability is unavailable." }, + ...result.content, + ], + }; }, }); @@ -814,6 +852,91 @@ export default definePluginEntry({ }, }); + if (config.redfishEnabled) { + const hostname = { type: "string", description: "BCM node hostname, for example dgx-03. The BMC address and credentials are resolved from BCM." }; + const chassis = { type: "string", description: "Optional Redfish chassis id. Defaults to the chassis linked to the node's system." }; + const redfishTools: Array<{ + name: string; + label: string; + description: string; + mcpToolName: string; + properties: Record; + args: (params: Record) => Record; + }> = [ + { + name: "bcm_redfish_system_summary", + label: "BCM Redfish System Summary", + description: "Read a node's power state, health, model, serial number, and BIOS version from its BMC over Redfish. Read-only; works when the node is unreachable in-band.", + mcpToolName: "redfish_system_summary", + properties: {}, + args: () => ({}), + }, + { + name: "bcm_redfish_thermal", + label: "BCM Redfish Thermal", + description: "Summarize a node's BMC temperature and fan sensors with thresholds and health over Redfish. Read-only.", + mcpToolName: "redfish_thermal", + properties: { chassis }, + args: (params) => ({ chassis: stringParam(params.chassis) }), + }, + { + name: "bcm_redfish_power", + label: "BCM Redfish Power", + description: "Read a node's power draw and power supply status from its BMC over Redfish. Read-only.", + mcpToolName: "redfish_power", + properties: { chassis }, + args: (params) => ({ chassis: stringParam(params.chassis) }), + }, + { + name: "bcm_redfish_sel", + label: "BCM Redfish Event Log", + description: "Return the newest BMC system event log and manager log entries for a node over Redfish, newest first. Read-only.", + mcpToolName: "redfish_sel", + properties: { limit: { type: "number", description: "Maximum entries to return, 1-200. Default 50." } }, + args: (params) => ({ limit: numberParam(params.limit, 50, 1, 200) }), + }, + { + name: "bcm_redfish_firmware", + label: "BCM Redfish Firmware", + description: "List a node's firmware inventory from its BMC over Redfish. Read-only.", + mcpToolName: "redfish_firmware", + properties: {}, + args: () => ({}), + }, + { + name: "bcm_redfish_get", + label: "BCM Redfish GET", + description: "Read one Redfish resource from a node's BMC with GET. The server allows only inventory, health, sensor, log, firmware, and task paths under /redfish/v1 and refuses actions and account, session, certificate, and event services. Optional query parameters: $top, $skip, and $expand=. (one level, not on the service root).", + mcpToolName: "redfish_get", + properties: { path: { type: "string", description: "Redfish path, for example /redfish/v1/Chassis/DGX/Sensors." } }, + args: (params) => { + const path = stringParam(params.path); + if (!path.startsWith("/redfish/v1") || path.length > 512) throw new Error("path must be a /redfish/v1 resource path"); + return { path }; + }, + }, + ]; + for (const tool of redfishTools) registerTool({ + name: tool.name, + label: tool.label, + description: tool.description, + parameters: { + type: "object", + additionalProperties: false, + required: ["hostname", ...(tool.name === "bcm_redfish_get" ? ["path"] : [])], + properties: { hostname, ...tool.properties }, + }, + async execute(toolCallId: string, rawParams: Record) { + return redfishTool( + config, + tool.mcpToolName, + { hostname: hostnameParam(rawParams.hostname), ...tool.args(rawParams) }, + subagentOptions(config, toolCallId, tool.name, tool.label), + ); + }, + }); + } + registerTool({ name: "bcm_list_notes", label: "BCM List Notes", diff --git a/helm/mosaic-stack/files/openclaw-seed/extensions/bcm/openclaw.plugin.json b/helm/mosaic-stack/files/openclaw-seed/extensions/bcm/openclaw.plugin.json index f4714ad4..1535cb56 100644 --- a/helm/mosaic-stack/files/openclaw-seed/extensions/bcm/openclaw.plugin.json +++ b/helm/mosaic-stack/files/openclaw-seed/extensions/bcm/openclaw.plugin.json @@ -17,7 +17,13 @@ "bcm_list_notes", "bcm_search_notes", "bcm_add_note", - "bcm_remove_note" + "bcm_remove_note", + "bcm_redfish_system_summary", + "bcm_redfish_thermal", + "bcm_redfish_power", + "bcm_redfish_sel", + "bcm_redfish_firmware", + "bcm_redfish_get" ], "trustedToolPolicies": ["bcm-access"] }, @@ -74,6 +80,10 @@ "type": "boolean", "description": "Expose BCM WLM-backed Slurm job evidence." }, + "redfishEnabled": { + "type": "boolean", + "description": "Expose read-only Redfish tools that read node BMCs through the BCM MCP service." + }, "timeoutMs": { "type": "number", "minimum": 1000, diff --git a/helm/mosaic-stack/files/openclaw-seed/nemoclaw.json b/helm/mosaic-stack/files/openclaw-seed/nemoclaw.json index 094a90a0..dfcd85f7 100644 --- a/helm/mosaic-stack/files/openclaw-seed/nemoclaw.json +++ b/helm/mosaic-stack/files/openclaw-seed/nemoclaw.json @@ -327,7 +327,8 @@ "editEnabled": {{ and .Values.modules.edit.enabled .Values.modules.edit.bcm.enabled (eq (include "mosaic-stack.bcmEnabled" .) "true") }}, "hitl": {{ .Values.modules.edit.hitl }}, "approvalTimeoutMs": {{ .Values.modules.edit.approvalTimeoutMs }}, - "slurmEnabled": {{ eq (include "mosaic-stack.slurmBcmEnabled" .) "true" }} + "slurmEnabled": {{ eq (include "mosaic-stack.slurmBcmEnabled" .) "true" }}, + "redfishEnabled": {{ and .Values.modules.bcm.redfish.enabled (eq (include "mosaic-stack.bcmEnabled" .) "true") }} } }, "remote-ssh": { diff --git a/helm/mosaic-stack/files/openclaw-seed/skills/bcm/SKILL.md b/helm/mosaic-stack/files/openclaw-seed/skills/bcm/SKILL.md index e3846f47..4db6a4d8 100644 --- a/helm/mosaic-stack/files/openclaw-seed/skills/bcm/SKILL.md +++ b/helm/mosaic-stack/files/openclaw-seed/skills/bcm/SKILL.md @@ -44,3 +44,18 @@ Pass CMSH commands as the same semicolon-separated command string used by `cmsh - `kubernetes; foreach * (show)` For broad cluster summaries, combine `bcm_get_info` with `bcm_execute_cmsh` command strings such as `kubernetes; list`, `category; list`, and `device; list`. For a short inventory, summarize the first rows returned by `device; list`; do not use shell pipes or `limit` inside CMSH. + +## Redfish (out-of-band) + +Determine available capabilities from the tools registered in the current session. `bcm_search_tools` searches only the diagnostic command catalog for `bcm_execute_tool`; it does not enumerate agent tools. Empty diagnostic search results provide no evidence that another registered capability is absent. + +When the `bcm_redfish_*` tools are registered, they read a node's BMC over Redfish through the Base Command Manager service. Prefer them over CMSH when the node is unreachable in-band or down, for power state, hardware health, temperature, fan, power supply, or system event log questions, and for firmware inventory. Pass only the Base Command Manager hostname; the service resolves the BMC address and credentials from Base Command Manager. Never ask the user for BMC addresses, usernames, or passwords, and never put them in a reply. + +Redfish access is read-only in every session mode. Resets, power actions, BMC configuration, accounts, sessions, certificates, event subscriptions, and virtual media are unavailable; if the user asks for one, say that Redfish actions are not available through this agent rather than retrying through another tool. A refusal from `bcm_redfish_get` means the path is outside the read-only allowlist. An `Actions` object in a returned resource is descriptive metadata, not an executable tool. When reporting it, distinguish the actions the BMC advertises from the read-only operations available through the registered Redfish tools. Do not offer to execute advertised actions or ask for confirmation to perform them. + +Use the named tools first: `bcm_redfish_system_summary` (power state, health, model, BIOS version), `bcm_redfish_thermal` (temperatures closest to their thresholds and fan readings), `bcm_redfish_power` (power draw and power supplies), `bcm_redfish_sel` (newest BMC event log entries across log services), and `bcm_redfish_firmware`. When `bcm_redfish_sel` returns `complete: false`, report the failed sources from `errors`; missing entries from a failed source are unknown, not an empty log. Systems such as DGX expose several chassis; pass `chassis` to the thermal or power tool, for example a GPU baseboard chassis, when the default chassis does not cover the question. Use `bcm_redfish_get` with an allowlisted `/redfish/v1/...` path only for detail the named tools omit, starting from a collection such as `/redfish/v1/Chassis` to discover member ids. + +Examples: + +- "What's the power state and health of dgx-03 according to its BMC?" → `bcm_redfish_system_summary` with `hostname: "dgx-03"`. +- "Show the last 20 SEL entries on dgx-03" → `bcm_redfish_sel` with `hostname: "dgx-03", limit: 20`; report timestamp, severity, source, and message for each entry. diff --git a/helm/mosaic-stack/files/openclaw-seed/workspace/AGENTS.md b/helm/mosaic-stack/files/openclaw-seed/workspace/AGENTS.md index cfb96770..ff8ba00d 100644 --- a/helm/mosaic-stack/files/openclaw-seed/workspace/AGENTS.md +++ b/helm/mosaic-stack/files/openclaw-seed/workspace/AGENTS.md @@ -75,7 +75,7 @@ For GPU networking measurement or diagnosis, read the gpu-networking skill and i A Kubernetes cluster contains only its registered nodes. The physical fleet can also include other Kubernetes clusters, scheduler-managed nodes, bare-metal services, and management hosts. An empty GPU inventory in one Kubernetes context says nothing about GPUs elsewhere in the fleet. -Establish capabilities from this session's registered tools and configured targets. Pods, labels, or services belonging to another deployment do not establish that this agent has that integration, credentials, or access. For broader inventory, use configured BCM inventory tools, additional registered Kubernetes clusters, Slurm inventory, or monitoring targets as appropriate. Correlate hostnames and addresses across sources and state each source's coverage. If no broader inventory is available, report that boundary without concluding the hosts do not exist. Inventory discovery does not authorize access to newly found hosts. +Establish capabilities from this session's registered tools and configured targets. Before offering an operation, verify that a registered tool supports it in the current session. Actions advertised by a remote API describe the target's capabilities, not this agent's executable capabilities. Describe unsupported operations as unavailable; requesting confirmation does not make an unsupported operation executable. Pods, labels, or services belonging to another deployment do not establish that this agent has that integration, credentials, or access. For broader inventory, use configured BCM inventory tools, additional registered Kubernetes clusters, Slurm inventory, or monitoring targets as appropriate. Correlate hostnames and addresses across sources and state each source's coverage. If no broader inventory is available, report that boundary without concluding the hosts do not exist. Inventory discovery does not authorize access to newly found hosts. For service-placement questions, distinguish the agent runtime, proxies, and backend service. Inspect the current deployment's non-secret configuration and follow service references and upstream endpoints before identifying where the backend runs. An agent pod's node or GPU allocation does not establish the inference server's placement or GPU count. Confirm those from the backend's deployment or host evidence. Chart source may not be mounted; rendered workloads and ConfigMaps remain useful read-only evidence. Never retrieve Secrets or dump credential-bearing configuration for this purpose. diff --git a/helm/mosaic-stack/templates/bcm-mcp-tools.yaml b/helm/mosaic-stack/templates/bcm-mcp-tools.yaml index 60f088c6..6b745fd0 100644 --- a/helm/mosaic-stack/templates/bcm-mcp-tools.yaml +++ b/helm/mosaic-stack/templates/bcm-mcp-tools.yaml @@ -127,6 +127,16 @@ spec: value: {{ .Values.bcmMcp.config.cmshExecTimeout | default "120" | quote }} - name: MOSAIC_BCM_CMSH_PATH value: {{ .Values.bcmMcp.config.cmshPath | default "/cm/local/apps/cmd/bin/cmsh" | quote }} + {{- if .Values.modules.bcm.redfish.enabled }} + - name: MOSAIC_BCM_REDFISH_ENABLED + value: "true" + - name: MOSAIC_BCM_REDFISH_VERIFY_TLS + value: {{ .Values.modules.bcm.redfish.verifyTls | quote }} + - name: MOSAIC_BCM_REDFISH_TIMEOUT + value: {{ .Values.modules.bcm.redfish.timeoutSeconds | quote }} + - name: MOSAIC_BCM_REDFISH_VIA_HEAD_NODE + value: {{ .Values.modules.bcm.redfish.viaHeadNode | quote }} + {{- end }} {{- end }} ports: - containerPort: {{ .Values.bcmMcp.service.port }} diff --git a/helm/mosaic-stack/templates/validation.yaml b/helm/mosaic-stack/templates/validation.yaml index df312868..e3365208 100644 --- a/helm/mosaic-stack/templates/validation.yaml +++ b/helm/mosaic-stack/templates/validation.yaml @@ -24,6 +24,14 @@ {{- fail (printf "Slack edit user %q must also appear in openclaw.slack.allowedUserIds" .) -}} {{- end -}} {{- end -}} +{{- if .Values.modules.bcm.redfish.enabled -}} +{{- if not (and .Values.modules.bcm.enabled .Values.bcmMcp.enabled (eq .Values.bcmMcp.mode "ssh-adapter")) -}} +{{- fail "modules.bcm.redfish.enabled requires modules.bcm.enabled=true, bcmMcp.enabled=true and bcmMcp.mode=ssh-adapter" -}} +{{- end -}} +{{- if or (lt (int .Values.modules.bcm.redfish.timeoutSeconds) 1) (gt (int .Values.modules.bcm.redfish.timeoutSeconds) 300) -}} +{{- fail "modules.bcm.redfish.timeoutSeconds must be between 1 and 300" -}} +{{- end -}} +{{- end -}} {{- if .Values.modules.webSearch.enabled -}} {{- if not (has .Values.modules.webSearch.provider (list "duckduckgo" "brave")) -}} {{- fail "modules.webSearch.provider must be one of: duckduckgo, brave" -}} diff --git a/helm/mosaic-stack/tests/bcm-tool-boundary.test.mjs b/helm/mosaic-stack/tests/bcm-tool-boundary.test.mjs index 3ca0e416..7d183882 100644 --- a/helm/mosaic-stack/tests/bcm-tool-boundary.test.mjs +++ b/helm/mosaic-stack/tests/bcm-tool-boundary.test.mjs @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; +import { toolAccess, toolRequiresEdit } from "../files/openclaw-seed/extensions/automation-context.ts"; const plugin = readFileSync( new URL("../files/openclaw-seed/extensions/bcm/index.ts", import.meta.url), @@ -56,3 +57,131 @@ test("retains the authenticated admin backend without readonly fallback", () => assert.match(adapter, /StaticTokenVerifier/); assert.doesNotMatch(adapter, /_run_cmsh_admin[\s\S]*?su\s+-/); }); + +const redfishToolNames = [ + "bcm_redfish_system_summary", + "bcm_redfish_thermal", + "bcm_redfish_power", + "bcm_redfish_sel", + "bcm_redfish_firmware", + "bcm_redfish_get", +]; + +test("classifies every registered Redfish tool as read-only in the global session policy", () => { + for (const name of redfishToolNames) { + assert.equal(toolAccess(name), "read", name); + assert.equal(toolRequiresEdit(name), false, name); + } +}); + +async function loadBcmPlugin() { + const { registerHooks } = await import("node:module"); + const hooks = registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier === "openclaw/plugin-sdk/plugin-entry") return { + url: "data:text/javascript,export const definePluginEntry = entry => entry;", shortCircuit: true, + }; + return nextResolve(specifier, context); + }, + }); + try { + return (await import("../files/openclaw-seed/extensions/bcm/index.ts")).default; + } finally { + hooks.deregister(); + } +} + +function registerBcm(plugin, pluginConfig) { + const tools = new Map(); + let policy; + plugin.register({ + pluginConfig: { cmshEnabled: true, ...pluginConfig }, + session: { state: { registerSessionExtension() {} } }, + registerTrustedToolPolicy: value => { policy = value; }, + registerTool: tool => tools.set(tool.name, tool), + }); + return { tools, policy }; +} + +test("registers read-only Redfish tools only when enabled", async () => { + const plugin = await loadBcmPlugin(); + const disabled = registerBcm(plugin, {}); + assert.deepEqual(redfishToolNames.filter(name => disabled.tools.has(name)), []); + const enabled = registerBcm(plugin, { redfishEnabled: true }); + assert.deepEqual(redfishToolNames.filter(name => enabled.tools.has(name)), redfishToolNames); + const manifest = JSON.parse(readFileSync( + new URL("../files/openclaw-seed/extensions/bcm/openclaw.plugin.json", import.meta.url), + )); + for (const name of redfishToolNames) assert.ok(manifest.contracts.tools.includes(name), name); + assert.equal(manifest.configSchema.properties.redfishEnabled.type, "boolean"); +}); + +test("never treats Redfish tools as mutations", async () => { + const plugin = await loadBcmPlugin(); + const { policy } = registerBcm(plugin, { redfishEnabled: true, editEnabled: true }); + const automation = "agent:default:mosaic-automation-alert-node-down"; + assert.equal((await policy.evaluate({ toolName: "bcm_add_note", toolCallId: "call-0", params: {} }, { sessionKey: automation })).block, true); + for (const toolName of redfishToolNames) { + for (const sessionKey of ["agent:default:main", automation]) { + assert.equal(await policy.evaluate({ toolName, toolCallId: "call-1", params: {} }, { sessionKey }), undefined); + } + } +}); + +test("diagnostic searches retain catalog results and explain their capability discovery scope", async () => { + const plugin = await loadBcmPlugin(); + const { tools } = registerBcm(plugin, { redfishEnabled: true, mcpUrl: "http://catalog-scope.test/mcp" }); + const originalFetch = globalThis.fetch; + let payload; + globalThis.fetch = async (url, init) => { + if (String(url).includes("/api/subagents/events")) return new Response("{}"); + const request = JSON.parse(init.body); + assert.equal(request.params.name, "search_tool"); + return Response.json({ jsonrpc: "2.0", id: request.id, result: payload }); + }; + try { + for (const [query, text] of [["inventory", "Matches: 0"], ["sensors", "Matches: 1\nexample_diagnostic"]]) { + payload = { content: [{ type: "text", text }] }; + const result = await tools.get("bcm_search_tools").execute("scope-check", { query }); + assert.deepEqual(result.details, payload); + assert.deepEqual(result.content.slice(1), payload.content); + assert.match(result.content[0].text, /diagnostic commands for bcm_execute_tool only/); + assert.match(result.content[0].text, /Registered agent tools are outside this catalog/); + } + for (const name of redfishToolNames) assert.ok(tools.has(name)); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("surfaces Redfish policy refusals and validates hostnames before calling the backend", async () => { + const plugin = await loadBcmPlugin(); + const { tools } = registerBcm(plugin, { redfishEnabled: true, mcpUrl: "http://bcm.test/mcp", authToken: "token" }); + const calls = []; + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url, init) => { + if (String(url).includes("/api/subagents/events")) return new Response("{}"); + const request = JSON.parse(init.body); + calls.push(request.params); + const payload = request.params.arguments.path === "/redfish/v1/AccountService" + ? { error: "/redfish/v1/AccountService is not permitted by the read-only Redfish policy", status: 403 } + : { hostname: request.params.arguments.hostname, systems: [{ id: "DGX", powerState: "On", health: "OK" }] }; + return new Response(JSON.stringify({ jsonrpc: "2.0", id: request.id, result: { content: [{ type: "text", text: JSON.stringify(payload) }] } })); + }; + try { + await assert.rejects( + tools.get("bcm_redfish_get").execute("call-1", { hostname: "dgx-03", path: "/redfish/v1/AccountService" }), + /Redfish request refused: .*read-only/, + ); + const summary = await tools.get("bcm_redfish_system_summary").execute("call-2", { hostname: "dgx-03" }); + assert.equal(summary.details.systems[0].powerState, "On"); + assert.deepEqual(calls.map(call => call.name), ["redfish_get", "redfish_system_summary"]); + for (const hostname of ["dgx-03;reboot", "../dgx", "", "10.0.0.1"]) { + await assert.rejects(tools.get("bcm_redfish_thermal").execute("call-3", { hostname }), /hostname must be a BCM node hostname/); + } + await assert.rejects(tools.get("bcm_redfish_get").execute("call-4", { hostname: "dgx-03", path: "https://10.0.0.1/redfish/v1" }), /path must be/); + assert.equal(calls.length, 2); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/helm/mosaic-stack/tests/chart-render.test.mjs b/helm/mosaic-stack/tests/chart-render.test.mjs index 26363ba4..6052459c 100644 --- a/helm/mosaic-stack/tests/chart-render.test.mjs +++ b/helm/mosaic-stack/tests/chart-render.test.mjs @@ -916,6 +916,44 @@ test("renders an authenticated BCM admin path only for BCM edit mode", () => { assert.doesNotMatch(output, /"authToken": "[A-Za-z0-9]{48}"/); }); +const bcmAdapterArgs = [ + "--set", "bcmMcp.enabled=true", + "--set", "bcmMcp.mode=ssh-adapter", + "--set", "bcmMcp.headHost=bcm-head", + "--set", "bcmMcp.hostSshKeySecretName=bcm-host-ssh", +]; + +test("renders read-only Redfish only when enabled", () => { + const disabled = render(...bcmAdapterArgs); + assert.doesNotMatch(disabled, /name: MOSAIC_BCM_REDFISH_/); + assert.match(disabled, /"redfishEnabled": false/); + + const enabled = render(...bcmAdapterArgs, "--set", "modules.bcm.redfish.enabled=true"); + assert.match(enabled, /name: MOSAIC_BCM_REDFISH_ENABLED\s+value: "true"/); + assert.match(enabled, /name: MOSAIC_BCM_REDFISH_VERIFY_TLS\s+value: "false"/); + assert.match(enabled, /name: MOSAIC_BCM_REDFISH_TIMEOUT\s+value: "60"/); + assert.match(enabled, /name: MOSAIC_BCM_REDFISH_VIA_HEAD_NODE\s+value: "true"/); + assert.match(enabled, /"redfishEnabled": true/); + assert.doesNotMatch(enabled, /name: MOSAIC_BCM_REDFISH_(?:USER|PASS|IP)/); + + const direct = render(...bcmAdapterArgs, "--set", "modules.bcm.redfish.enabled=true", + "--set", "modules.bcm.redfish.viaHeadNode=false", "--set", "modules.bcm.redfish.verifyTls=true"); + assert.match(direct, /name: MOSAIC_BCM_REDFISH_VERIFY_TLS\s+value: "true"/); + assert.match(direct, /name: MOSAIC_BCM_REDFISH_VIA_HEAD_NODE\s+value: "false"/); +}); + +test("rejects Redfish without the BCM SSH adapter", () => { + for (const [args, expected] of [ + [["--set", "modules.bcm.redfish.enabled=true"], /modules\.bcm\.redfish\.enabled requires/], + [["--set", "modules.bcm.redfish.enabled=true", "--set", "bcmMcp.enabled=true"], /requires|ssh-adapter/], + [[...bcmAdapterArgs, "--set", "modules.bcm.redfish.enabled=true", "--set", "modules.bcm.redfish.timeoutSeconds=0"], /timeoutSeconds must be between 1 and 300/], + ]) { + const result = spawnSync("helm", ["template", "mosaic", chart, "--namespace", "mosaic-test", ...args], { encoding: "utf8" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, expected); + } +}); + test("keeps the managed BCM adapter read-only when edit mode is disabled", () => { const output = render( "--set", diff --git a/helm/mosaic-stack/tests/test_bcm_server.py b/helm/mosaic-stack/tests/test_bcm_server.py index 108e7c79..b92976df 100644 --- a/helm/mosaic-stack/tests/test_bcm_server.py +++ b/helm/mosaic-stack/tests/test_bcm_server.py @@ -4,6 +4,7 @@ import asyncio import importlib.util import inspect +import json import subprocess import sys import threading @@ -97,6 +98,11 @@ def server(self): server.cmsh_admin_enabled = False server.hardware_compat_enabled = True server.bcm_head_host = "head.example" + server.redfish_enabled = False + server.redfish_verify_tls = False + server.redfish_timeout = 20 + server.redfish_via_head_node = True + server._redfish_members = {} server._cmsh_ready = False server._cmsh_setup_lock = threading.Lock() return server @@ -109,6 +115,43 @@ def test_ssh_always_targets_the_configured_head(self): self.assertEqual(run.call_args.args[0][:2], ["ssh", "head.example"]) self.assertNotIn("pip", run.call_args.args[0][-1]) + def test_bounded_ssh_returns_small_output_and_stops_oversized_output(self): + actual_popen = subprocess.Popen + for content, limit in ((b"small response", 128), (b"x" * 8192, 128)): + with self.subTest(size=len(content), limit=limit): + processes = [] + + def launch(_command, **kwargs): + program = f"import os; os.write(1, {content!r})" + process = actual_popen([sys.executable, "-c", program], **kwargs) + processes.append(process) + return process + + with patch.object(self.module.subprocess, "Popen", side_effect=launch): + if len(content) <= limit: + result = self.server()._ssh("diagnostic", 5, max_output_bytes=limit) + self.assertEqual(result.stdout, content) + self.assertEqual(result.returncode, 0) + else: + with self.assertRaises(self.module.RedfishError) as raised: + self.server()._ssh("diagnostic", 5, max_output_bytes=limit) + self.assertEqual(raised.exception.status, 502) + self.assertIsNotNone(processes[0].poll()) + + def test_bounded_ssh_preserves_timeout_and_reaps_the_child(self): + actual_popen = subprocess.Popen + processes = [] + + def launch(_command, **kwargs): + process = actual_popen([sys.executable, "-c", "import time; time.sleep(30)"], **kwargs) + processes.append(process) + return process + + with patch.object(self.module.subprocess, "Popen", side_effect=launch): + with self.assertRaises(subprocess.TimeoutExpired): + self.server()._ssh("diagnostic", 0.1, stdin=b"x" * 131072, max_output_bytes=128) + self.assertIsNotNone(processes[0].poll()) + def test_cmsh_identity_is_checked_once(self): server = self.server() server._ssh = Mock(return_value=subprocess.CompletedProcess([], 0, b"", b"")) @@ -241,6 +284,280 @@ def __init__(self, authorization=""): self.assertEqual((denied.status_code, denied.content), (401, {"detail": "unauthorized"})) self.assertEqual((allowed.status_code, allowed.content), (200, {"devices": [], "count": 0})) + def test_redfish_policy_allows_read_only_inventory(self): + policy = self.module.RedfishPolicy + for path in [ + "/redfish/v1", + "/redfish/v1/", + "/redfish/v1/Systems", + "/redfish/v1/Systems/DGX", + "/redfish/v1/Systems/System.Embedded.1/Processors/CPU.1", + "/redfish/v1/Systems/DGX/LogServices/BIOS/Entries?$top=5", + "/redfish/v1/Chassis/DGX/ThermalSubsystem/ThermalMetrics", + "/redfish/v1/Chassis/DGX/PowerSubsystem/PowerSupplies?$expand=.($levels=1)", + "/redfish/v1/Managers/BMC/LogServices/SEL/Entries?$skip=3467&$top=75", + "/redfish/v1/Managers/BMC/EthernetInterfaces/eth0", + "/redfish/v1/UpdateService/FirmwareInventory/CX7_0", + "/redfish/v1/TaskService/Tasks/10", + ]: + self.assertTrue(policy.is_allowed("GET", path), path) + + def test_redfish_policy_rejects_writes_and_sensitive_resources(self): + policy = self.module.RedfishPolicy + self.assertFalse(policy.is_allowed("POST", "/redfish/v1/Systems/DGX")) + self.assertFalse(policy.is_allowed("PATCH", "/redfish/v1/Systems/DGX")) + for path in [ + "/redfish/v1/AccountService", + "/redfish/v1/AccountService/Accounts/2", + "/redfish/v1/SessionService/Sessions", + "/redfish/v1/CertificateService", + "/redfish/v1/EventService/Subscriptions", + "/redfish/v1/Systems/DGX/Actions/ComputerSystem.Reset", + "/redfish/v1/Managers/BMC/Actions/Manager.Reset", + "/redfish/v1/Systems/DGX/LogServices/SEL/Actions/LogService.ClearLog", + "/redfish/v1/Systems/DGX/SecureBoot", + "/redfish/v1/Systems/DGX/Bios/Settings", + "/redfish/v1/Managers/BMC/VirtualMedia/CD1", + "/redfish/v1/Systems/../AccountService", + "/redfish/v1/Systems/DGX/..", + "/redfish/v1//Systems", + "/redfish/v1/Systems/DGX%2F..", + "/redfish/v1/Systems?$filter=Id", + "/redfish/v1/Systems?$expand=*&$select=Id", + "/redfish/v1/Systems?$top=1;rm", + "/redfish/v1/Systems/DGX?", + "/redfish/v1/Systems/DGX#x", + "/redfish/v1/Systems/DGX/Oem", + "/redfish/v1/Managers/BMC/NodeManager", + "/redfish/v1/UpdateService", + "/redfish/v1/Systems/DGX/LogServices/SEL/Entries/1/attachment", + "/redfish/v2", + "https://10.0.0.1/redfish/v1", + "", + ]: + self.assertFalse(policy.is_allowed("GET", path), path) + + def test_redfish_denied_paths_never_resolve_bmc_credentials(self): + server = self.server() + server.hardware_bmc = Mock() + with self.assertRaises(self.module.RedfishError) as raised: + server.redfish_get("atlas-01", "/redfish/v1/AccountService") + self.assertEqual(raised.exception.status, 403) + server.hardware_bmc.assert_not_called() + payload = json.loads(server.redfish_tool(server.redfish_get, "atlas-01", "/redfish/v1/Systems/DGX/Actions/ComputerSystem.Reset")) + self.assertEqual(payload["status"], 403) + + def test_redfish_head_node_transport_keeps_credentials_off_the_command_line(self): + server = self.server() + server.hardware_bmc = Mock(return_value={ + "hostname": "atlas-01", "bmc_ip": "10.30.0.11", "bmc_username": "admin", "bmc_password": 'p"a\\ss', + }) + body = b'{"@odata.type":"#ComputerSystem","Id":"DGX","PowerState":"On","Oem":{"UserPassword":"x","AuthToken":"y"}}' + server._ssh = Mock(return_value=subprocess.CompletedProcess([], 0, body + b"\n200", b"")) + result = server.redfish_get("atlas-01", "/redfish/v1/Systems/DGX") + script, timeout, stdin = server._ssh.call_args.args + self.assertNotIn("p\"a", script) + self.assertIn("-K /dev/stdin", script) + self.assertIn("https://10.30.0.11/redfish/v1/Systems/DGX", script) + self.assertEqual(stdin, b'user = "admin:p\\"a\\\\ss"\n') + self.assertGreater(timeout, server.redfish_timeout) + self.assertEqual(result, {"Id": "DGX", "PowerState": "On", "Oem": {"UserPassword": "[redacted]", "AuthToken": "[redacted]"}}) + + def test_redfish_errors_omit_the_bmc_address(self): + server = self.server() + server.hardware_bmc = Mock(return_value={ + "hostname": "atlas-01", "bmc_ip": "10.30.0.11", "bmc_username": "admin", "bmc_password": "secret", + }) + server._ssh = Mock(return_value=subprocess.CompletedProcess([], 7, b"", b"curl: (7) Failed to connect to 10.30.0.11 port 443")) + text = server.redfish_tool(server.redfish_system_summary, "atlas-01") + self.assertEqual(json.loads(text), {"error": "BMC is unreachable", "status": 502}) + server._ssh = Mock(return_value=subprocess.CompletedProcess([], 0, b"\n401", b"")) + text = server.redfish_tool(server.redfish_get, "atlas-01", "/redfish/v1/Systems") + self.assertEqual(json.loads(text)["status"], 401) + self.assertNotIn("10.30.0.11", text) + self.assertNotIn("secret", text) + + def test_redfish_task_payload_redacts_secrets_without_losing_diagnostics(self): + for secret_key, header in (("Password", "Authorization"), ("ClientSecret", "X-Api-Key"), ("AccessToken", "Cookie")): + with self.subTest(secret_key=secret_key, header=header): + server = self.server() + server.hardware_bmc = Mock(return_value={}) + payload = { + "@odata.id": "/redfish/v1/TaskService/Tasks/42", + "TaskState": "Completed", + "Payload": { + "HttpOperation": "POST", + "HttpHeaders": [f"{header}: private-value", "Content-Type: application/json"], + "JsonBody": json.dumps({"nested": {secret_key: "private-value", "Mode": "diagnostic"}}), + }, + } + server._redfish_fetch = Mock(return_value=(200, json.dumps(payload).encode())) + result = server.redfish_get("atlas-01", payload["@odata.id"]) + self.assertNotIn("private-value", json.dumps(result)) + self.assertEqual(result["TaskState"], "Completed") + self.assertEqual(result["Payload"]["HttpOperation"], "POST") + self.assertIn("Content-Type: application/json", result["Payload"]["HttpHeaders"]) + body = json.loads(result["Payload"]["JsonBody"]) + self.assertEqual(body["nested"]["Mode"], "diagnostic") + self.assertEqual(body["nested"][secret_key], "[redacted]") + self.assertEqual(self.module.scrub_redfish({"JsonBody": "not JSON"})["JsonBody"], "[redacted]") + self.assertEqual(self.module.scrub_redfish({"HttpHeaders": "Authorization: private-value"}), + {"HttpHeaders": "[redacted]"}) + + def test_redfish_tools_register_only_when_enabled(self): + names = {"redfish_get", "redfish_system_summary", "redfish_thermal", "redfish_power", "redfish_sel", "redfish_firmware"} + for enabled in (False, True): + server = self.server() + server.redfish_enabled = enabled + server.register_mosaic_tools() + registered = set(server.mcp._tool_manager._tools) & names + self.assertEqual(registered, names if enabled else set()) + + def test_redfish_sel_merges_newest_entries_across_log_services(self): + server = self.server() + server.hardware_bmc = Mock(return_value={"bmc_ip": "10.30.0.11", "bmc_username": "u", "bmc_password": "p"}) + entry = lambda n, day: {"Id": str(n), "Created": f"2026-09-{day:02d}T00:00:00+00:00", "Severity": "OK", "Message": f"event {n}"} + responses = { + "/redfish/v1/Managers": {"Members": [{"@odata.id": "/redfish/v1/Managers/BMC"}]}, + "/redfish/v1/Systems": {"Members": [{"@odata.id": "/redfish/v1/Systems/DGX"}]}, + "/redfish/v1/Managers/BMC/LogServices": {"Members": [ + {"@odata.id": "/redfish/v1/Managers/BMC/LogServices/SEL"}, + {"@odata.id": "/redfish/v1/Managers/BMC/LogServices/Journal"}, + ]}, + "/redfish/v1/Systems/DGX/LogServices": {"Members": [{"@odata.id": "/redfish/v1/Systems/DGX/LogServices/BIOS"}]}, + "/redfish/v1/Managers/BMC/LogServices/SEL/Entries?$top=2": {"Members@odata.count": 5, "Members": [entry(1, 1), entry(2, 2)]}, + "/redfish/v1/Managers/BMC/LogServices/SEL/Entries?$skip=3&$top=2": {"Members": [entry(4, 4), entry(5, 5)]}, + "/redfish/v1/Systems/DGX/LogServices/BIOS/Entries?$top=2": {"Members@odata.count": 1, "Members": [entry(9, 3)]}, + } + requested = [] + + def fetch(_bmc, path): + requested.append(path) + return 200, json.dumps(responses[path]).encode() + + server._redfish_fetch = fetch + result = server.redfish_sel("atlas-01", 2) + self.assertEqual(result["sources"], ["Managers/BMC/SEL", "Systems/DGX/BIOS"]) + self.assertEqual([(e["source"], e["id"]) for e in result["entries"]], [("Managers/BMC/SEL", "5"), ("Managers/BMC/SEL", "4")]) + self.assertNotIn("/redfish/v1/Managers/BMC/LogServices/Journal/Entries?$top=2", requested) + + def test_redfish_expand_cannot_reach_denied_resources(self): + policy = self.module.RedfishPolicy + for path in [ + "/redfish/v1?$expand=*($levels=5)", + "/redfish/v1?$expand=.", + "/redfish/v1?$top=1", + "/redfish/v1/Systems?$expand=*", + "/redfish/v1/Systems?$expand=~", + "/redfish/v1/Systems?$expand=.($levels=2)", + ]: + self.assertFalse(policy.is_allowed("GET", path), path) + self.assertTrue(policy.is_allowed("GET", "/redfish/v1/Managers?$expand=.($levels=1)")) + + server = self.server() + server.hardware_bmc = Mock(return_value={"bmc_ip": "10.30.0.11", "bmc_username": "u", "bmc_password": "p"}) + expanded = {"@odata.id": "/redfish/v1/Managers", "Members": [ + {"@odata.id": "/redfish/v1/Managers/BMC", "FirmwareVersion": "25.06", "Links": {"ManagerForServers": [{"@odata.id": "/redfish/v1/Systems/DGX"}]}}, + {"@odata.id": "/redfish/v1/Managers/BMC/NodeManager", "NvidiaPSUPolicies": {"Mode": "x"}}, + {"@odata.id": "/redfish/v1/AccountService", "Accounts": {"Members": [{"@odata.id": "/redfish/v1/AccountService/Accounts/2", "UserName": "admin"}]}}, + ]} + server._redfish_fetch = Mock(return_value=(200, json.dumps(expanded).encode())) + result = server.redfish_get("atlas-01", "/redfish/v1/Managers?$expand=.($levels=1)") + self.assertEqual(result["Members"], [ + {"@odata.id": "/redfish/v1/Managers/BMC", "FirmwareVersion": "25.06", "Links": {"ManagerForServers": [{"@odata.id": "/redfish/v1/Systems/DGX"}]}}, + {"@odata.id": "/redfish/v1/Managers/BMC/NodeManager"}, + {"@odata.id": "/redfish/v1/AccountService"}, + ]) + self.assertNotIn("admin", json.dumps(result)) + + def test_redfish_sel_reports_failed_log_reads(self): + server = self.server() + server.hardware_bmc = Mock(return_value={"bmc_ip": "10.30.0.11", "bmc_username": "u", "bmc_password": "p"}) + entry = {"Id": "1", "Created": "2026-09-01T00:00:00+00:00", "Severity": "Critical", "Message": "PSU fault"} + statuses = {} + base = { + "/redfish/v1/Managers": {"Members": [{"@odata.id": "/redfish/v1/Managers/BMC"}]}, + "/redfish/v1/Systems": {"Members": [{"@odata.id": "/redfish/v1/Systems/DGX"}]}, + "/redfish/v1/Managers/BMC/LogServices": {"Members": [{"@odata.id": "/redfish/v1/Managers/BMC/LogServices/SEL"}]}, + "/redfish/v1/Systems/DGX/LogServices": {"Members": [{"@odata.id": "/redfish/v1/Systems/DGX/LogServices/EventLog"}]}, + "/redfish/v1/Managers/BMC/LogServices/SEL/Entries?$top=50": {"Members@odata.count": 1, "Members": [entry]}, + "/redfish/v1/Systems/DGX/LogServices/EventLog/Entries?$top=50": {"Members@odata.count": 1, "Members": [entry]}, + } + + def fetch(_bmc, path): + if path in statuses: + return statuses[path], b"" + return 200, json.dumps(base[path]).encode() + + server._redfish_fetch = fetch + statuses["/redfish/v1/Systems/DGX/LogServices/EventLog/Entries?$top=50"] = 503 + partial = server.redfish_sel("atlas-01") + self.assertFalse(partial["complete"]) + self.assertEqual(partial["sources"], ["Managers/BMC/SEL"]) + self.assertEqual([(e["source"], e["status"]) for e in partial["errors"]], [("Systems/DGX/EventLog", 503)]) + self.assertEqual(len(partial["entries"]), 1) + + for status in (403, 503, 504): + server._redfish_members = {} + statuses.clear() + statuses["/redfish/v1/Managers/BMC/LogServices/SEL/Entries?$top=50"] = status + statuses["/redfish/v1/Systems/DGX/LogServices"] = status + payload = json.loads(server.redfish_tool(server.redfish_sel, "atlas-01")) + self.assertEqual(payload["status"], status) + self.assertIn("no BMC event log could be read", payload["error"]) + self.assertNotIn("entries", payload) + + def test_redfish_sel_follows_next_links_for_newest_entries(self): + server = self.server() + server.hardware_bmc = Mock(return_value={"bmc_ip": "10.30.0.11", "bmc_username": "u", "bmc_password": "p"}) + events = [{"Id": str(n), "Created": f"2026-09-{n:02d}T00:00:00+00:00", "Severity": "OK", "Message": f"event {n}"} + for n in range(1, 11)] + service = "/redfish/v1/Managers/BMC/LogServices/SEL" + + def fetch(_bmc, path): + if path == "/redfish/v1/Managers": + return 200, json.dumps({"Members": [{"@odata.id": "/redfish/v1/Managers/BMC"}]}).encode() + if path == "/redfish/v1/Systems": + return 200, json.dumps({"Members": []}).encode() + if path == "/redfish/v1/Managers/BMC/LogServices": + return 200, json.dumps({"Members": [{"@odata.id": service}]}).encode() + params = dict(item.split("=") for item in path.partition("?")[2].split("&") if item) + skip, top = int(params.get("$skip", 0)), int(params.get("$top", len(events))) + page = events[skip:skip + min(top, 2)] + body = {"Members@odata.count": len(events), "Members": page} + if skip + len(page) < min(skip + top, len(events)): + body["Members@odata.nextLink"] = f"{service}/Entries?$skip={skip + 2}&$top={top - 2}" + return 200, json.dumps(body).encode() + + server._redfish_fetch = fetch + result = server.redfish_sel("atlas-01", 4) + self.assertEqual([e["id"] for e in result["entries"]], ["10", "9", "8", "7"]) + self.assertTrue(result["complete"]) + + def unqueried(_bmc, path): + if path.endswith("/Entries"): + return 200, json.dumps({"Members": events[:5], "Members@odata.nextLink": f"{service}/Entries?$skip=5"}).encode() + if path.endswith("/Entries?$skip=5"): + return 200, json.dumps({"Members": events[5:]}).encode() + if "/Entries?" in path: + return 400, b"" + return fetch(_bmc, path) + + server._redfish_fetch = unqueried + server._redfish_members = {} + result = server.redfish_sel("atlas-01", 4) + self.assertEqual([e["id"] for e in result["entries"]], ["10", "9", "8", "7"]) + + def test_redfish_sel_output_stays_valid_json_within_budget(self): + server = self.server() + entries = [{"created": f"2026-09-01T00:{n // 60:02d}:{n % 60:02d}", "severity": "OK", "message": "x" * 240, + "source": "Managers/BMC/SEL", "id": str(n)} for n in range(200, 0, -1)] + text = server.redfish_tool(lambda: {"hostname": "atlas-01", "sources": [], "entries": entries, "complete": True}) + payload = json.loads(text) + self.assertLessEqual(len(text), self.module.REDFISH_MAX_CHARS) + self.assertEqual(payload["entries"][0]["id"], "200") + self.assertEqual(len(payload["entries"]) + payload["omittedOlderEntries"], 200) + if __name__ == "__main__": unittest.main() diff --git a/helm/mosaic-stack/values.yaml b/helm/mosaic-stack/values.yaml index 7c905e07..3b505fd2 100644 --- a/helm/mosaic-stack/values.yaml +++ b/helm/mosaic-stack/values.yaml @@ -50,6 +50,14 @@ modules: enabled: true bcm: enabled: true + # Read-only Redfish through the BCM MCP SSH adapter; BMCs are resolved by hostname from BCM. + redfish: + enabled: false + # BMCs usually present self-signed certificates. + verifyTls: false + timeoutSeconds: 60 + # Run each GET with curl on the BCM head node, which is routed to the BMC network. + viaHeadNode: true slurm: enabled: true backend: auto diff --git a/tests/docs/install-builder.test.mjs b/tests/docs/install-builder.test.mjs index 35730bbe..3ba30798 100644 --- a/tests/docs/install-builder.test.mjs +++ b/tests/docs/install-builder.test.mjs @@ -23,6 +23,7 @@ const example = {inference:'external', baseUrl:'https://inference.example.com/v1 corpus:'/data/corpus', evidenceRoot:'/data/jobs', evidenceNode:'login-node'}; for (const state of [ {modules:[]}, {modules:['terminal']}, {modules:['slurm']}, {modules:['slurm'], slurmBackend:'vanilla'}, + {modules:['bcm'],redfish:true}, {modules:['kubernetes','prometheus','alertmanager','loki','grafana','bcm','slurm','diagnostics','research','terminal','edit','clusters']}, {inference:'super',modules:['terminal']}, {inference:'ultra',modules:['grafana']}, ]) test(`generated selection renders in Helm: ${JSON.stringify(state)}`, () => { @@ -41,6 +42,14 @@ test('dependencies are derived without changing requested modules', () => { assert.deepEqual([...enabledModules(state)].sort(),['bcm','diagnostics','kubernetes','terminal']); assert.deepEqual(state.modules,['diagnostics','terminal']); }); +test('Redfish is opt-in and requires BCM, including when BCM is selected by a dependency', () => { + for (const [selected, redfish, expected] of [ + [['bcm'],true,true], [['diagnostics'],true,true], [['bcm'],false,false], [[],true,false], + ]) { + const args = argumentsFor({...example, modules:selected, redfish}); + assert.ok(args.includes(`modules.bcm.redfish.enabled=${expected}`)); + } +}); test('shell metacharacters remain literal arguments and Helm list separators are escaped', () => { for(const model of ["quoted' name,revision", 'model $(printf unsafe); $HOME']) { const args=argumentsFor({...example,modules:[],model});