diff --git a/bench/local/container/run-in-container.sh b/bench/local/container/run-in-container.sh index c41c38c..5912310 100755 --- a/bench/local/container/run-in-container.sh +++ b/bench/local/container/run-in-container.sh @@ -56,7 +56,11 @@ url=${BENCH_URL:-https://openrouter.ai/api/v1} # The NUC tunnel is the Mac's localhost; from the container it is host.docker.internal. url=${url//127.0.0.1/host.docker.internal}; url=${url//localhost/host.docker.internal} # The key goes in through a file only this user can read, removed when the -# run ends: on the command line (-e KEY=...) it showed in every `ps` listing. +# run ends: on the command line (-e KEY=...) it showed in every `ps` listing, +# and as --env-file it sat in the container's environment (docker inspect, and +# every process of the run). It is mounted read-only under /root (mode 700, so +# the agent user cannot reach it); run.py reads it into memory and hands it to +# Maat on a pipe (MAAT_KEYS_FD), so it is in no environment at all. envf=$(mktemp); chmod 600 "$envf"; trap 'rm -f "$envf"' EXIT printf 'OPENROUTER_API_KEY=%s\n' "$key" > "$envf" # The graders and reference solutions are mounted where only root can reach (/root is 700), @@ -83,7 +87,7 @@ fi echo "maat $img → $out/${RESULTS:-results.jsonl}" docker run --rm --name "maat-bench-$name" \ -v "$local_dir":/root/bench-src:ro -v "$out":/results -v "$work":/work ${=credmount} \ - --env-file "$envf" -e REFERENCE="${REFERENCE:-0}" -e BENCH_MODEL="${BENCH_MODEL:-}" \ + -v "$envf":/root/bench-keys/keys:ro -e BENCH_KEYS_FILE=/root/bench-keys/keys -e REFERENCE="${REFERENCE:-0}" -e BENCH_MODEL="${BENCH_MODEL:-}" \ -e ARMS="${ARMS:-}" -e BENCH_TASKS="${BENCH_TASKS:-}" -e BENCH_URL="$url" -e BENCH_LIMIT="${BENCH_LIMIT:-600}" -e BENCH_REASONING="${BENCH_REASONING:-}" -e BENCH_GATE="${BENCH_GATE:-}" -e RESULTS="${RESULTS:-results.jsonl}" \ -e MAAT_BUILD="$sha" -e PYTHONDONTWRITEBYTECODE=1 \ "$img" sh -c "$startcmd" run "$@" diff --git a/bench/local/run.py b/bench/local/run.py index a3e5e1b..2bbf4f5 100644 --- a/bench/local/run.py +++ b/bench/local/run.py @@ -16,6 +16,7 @@ import json import os +import re import shutil import subprocess import sys @@ -38,8 +39,28 @@ URL = os.environ.get("BENCH_URL") or "https://openrouter.ai/api/v1" +def load_keys_file() -> dict: + """BENCH_KEYS_FILE: NAME=value lines in a root-only file (the containers mount it). Read into + memory here, never put in an environment: `--env-file` put the key in the container's own + environment, in `docker inspect`, and in every process of the run.""" + path = os.environ.get("BENCH_KEYS_FILE") + if not path: + return {} + keys = {} + for line in Path(path).read_text().splitlines(): + name, sep, value = line.strip().partition("=") + if sep and name and not name.startswith("#"): + keys[name] = value + return keys + + +KEYS = load_keys_file() + + def openrouter_key() -> str: - if os.environ.get("OPENROUTER_API_KEY"): # the container is given the key, not the Mac's auth file + if KEYS.get("OPENROUTER_API_KEY"): # the container is given the key, not the Mac's auth file + return KEYS["OPENROUTER_API_KEY"] + if os.environ.get("OPENROUTER_API_KEY"): return os.environ["OPENROUTER_API_KEY"] return subprocess.run( ["node", "-e", "import('%s').then(m=>process.stdout.write(m.readAuth().openrouter||''))" % (Path.home() / "Documents/molt-desktop/dist/providers.js")], @@ -66,6 +87,33 @@ def as_agent(cmd: list, env: dict | None = None) -> tuple[list, dict | None]: return ["runuser", "-u", AGENT_USER, "--", "env", f"HOME={home}", *cmd], env +# The same shapes Maat treats as credentials (src/secrets.ts). +SECRET_SUFFIX = re.compile(r"(?:^|_)(?:API_?KEY|ACCESS_?KEY|SECRET(?:_ACCESS)?_?KEY|PRIVATE_?KEY|SECRET|TOKEN|PASSWORD|PASSWD|CREDENTIALS?)$", re.I) +SECRET_PREFIX = re.compile(r"^(?:XAI|GROK|TOGETHER)_", re.I) +SECRET_NAMES = {"MAAT_API_KEY", "MOLT_API_KEY", "MAAT_JUDGE_KEY", "MOLT_JUDGE_KEY", "MAAT_KEYS_FD", "MAAT_KEYS_FILE"} + + +def is_secret(name: str) -> bool: + return name.upper() in SECRET_NAMES or bool(SECRET_SUFFIX.search(name)) or bool(SECRET_PREFIX.match(name)) + + +def keys_by_fd(env: dict) -> tuple[dict, dict, int]: + """Take every credential out of the agent's environment and hand it over on a pipe instead + (MAAT_KEYS_FD). In the environment it stayed in Maat's /proc//environ for the whole run, + and a worker of Maat's own user read it back with `cat /proc/$PPID/environ`; deleting it inside + Maat does not change that copy. Returns (clean env, Popen kwargs, read end to close after Popen).""" + secrets = {k: v for k, v in env.items() if is_secret(k)} + clean = {k: v for k, v in env.items() if not is_secret(k)} + payload = json.dumps(secrets).encode() + if len(payload) > 60_000: # one write into an empty pipe must not block + raise SystemExit("keys too large for the handover pipe") + r, w = os.pipe() + os.write(w, payload) + os.close(w) + clean["MAAT_KEYS_FD"] = str(r) + return clean, {"pass_fds": (r,)}, r + + def kill_tree(proc: subprocess.Popen) -> None: """Kill the agent's whole process group, and in a container every process of the unprivileged agent user (a backend CLI may start its own session).""" @@ -94,7 +142,7 @@ def provider_capped(out: str, steps: int) -> bool: def run_molt(d: Path, prompt: str, log: Path) -> dict: key = openrouter_key() if "openrouter.ai" in URL else "local" - env = os.environ | {"MOLT_API_KEY": key, "MOLT_JUDGMENT": "0"} # nobody rules on a benchmark run + env = os.environ | KEYS | {"MOLT_API_KEY": key, "MOLT_JUDGMENT": "0"} # nobody rules on a benchmark run cmd = [ "node", str(MOLT), "run", "--url", URL, "--model", os.environ.get("BENCH_MODEL") or MODEL, "--reasoning", os.environ.get("BENCH_REASONING") or "low", *(["--yes"] if os.environ.get("BENCH_GATE") == "yes" else ["--sandbox"]), "--json", "--criteria", "auto", "--batch", "--review", "3", "--steps", "200", @@ -105,11 +153,15 @@ def run_molt(d: Path, prompt: str, log: Path) -> dict: ] t0 = time.time() cmd, env = as_agent(cmd, env) + env, fds, keys_fd = keys_by_fd(env) # Own session, so the backstop kills the whole tree. subprocess.run's timeout killed only # the top process (runuser in the containers) and then waited on the pipe Maat and its # backend still held: one hung Grok turn ran 3570 s against a 600 s limit (2026-10-07). - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env, - start_new_session=True) + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env, + start_new_session=True, **fds) + finally: + os.close(keys_fd) try: out, err = proc.communicate(timeout=LIMIT) timed_out = False diff --git a/bench/local/tests/test_keys_handover.py b/bench/local/tests/test_keys_handover.py new file mode 100644 index 0000000..deb2f47 --- /dev/null +++ b/bench/local/tests/test_keys_handover.py @@ -0,0 +1,32 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import run # noqa: E402 + + +def test_keys_leave_the_environment_and_arrive_on_the_descriptor(): + env = {"PATH": os.environ.get("PATH", "/bin"), "MOLT_API_KEY": "k1", "OPENROUTER_API_KEY": "k2", + "MAAT_JUDGE_KEY": "k3", "XAI_BASE": "k4", "BENCH_MODEL": "m"} + clean, fds, r = run.keys_by_fd(env) + try: + child = subprocess.run( + [sys.executable, "-c", "import os,sys;print(os.read(int(os.environ['MAAT_KEYS_FD']),65536).decode());print(sorted(os.environ))"], + env=clean, capture_output=True, text=True, **fds) + finally: + os.close(r) + keys_line, env_line = child.stdout.strip().split("\n") + assert json.loads(keys_line) == {"MOLT_API_KEY": "k1", "OPENROUTER_API_KEY": "k2", "MAAT_JUDGE_KEY": "k3", "XAI_BASE": "k4"} + names = eval(env_line) + assert "BENCH_MODEL" in names and "MAAT_KEYS_FD" in names + assert not any(n in names for n in ("MOLT_API_KEY", "OPENROUTER_API_KEY", "MAAT_JUDGE_KEY", "XAI_BASE")) + + +def test_keys_file_is_read_into_memory(tmp_path, monkeypatch): + f = tmp_path / "keys" + f.write_text("OPENROUTER_API_KEY=sk-x\n# c\n") + monkeypatch.setenv("BENCH_KEYS_FILE", str(f)) + assert run.load_keys_file() == {"OPENROUTER_API_KEY": "sk-x"} diff --git a/docs/secrets.md b/docs/secrets.md new file mode 100644 index 0000000..ba0d419 --- /dev/null +++ b/docs/secrets.md @@ -0,0 +1,127 @@ +# Keeping keys and hidden checks from the worker + +The worker's commands, and the deliverables a check runs, are model-written +code. When they run as Maat's own user, they can read anything that user can: +other processes' command lines (`/proc//cmdline`, `ps`), the environment +a process was started with (`/proc//environ`, `ps -E` on macOS), files, +and, where the kernel allows ptrace between processes of one user, memory. +This page says what Maat does about each, and where that stops. + +**Same-user isolation is best effort.** The boundary is running the worker as +a different user (`--worker-user`, docs/privilege-separation.md once that +lands), so that Maat's `/proc` entries, state and memory belong to someone +else. For unattended and benchmark runs, use it. Everything below is defence +in depth for runs that do not. + +## Hidden check commands + +A check never appears in an argv. Every check run (bar runs, mutation runs, +drafting trial runs, reference checks, mission assertions; anything that goes +through `runCheck` or `preflightCriteria`) starts its shell with one fixed +script, `HIDDEN_WRAPPER` in `src/run.ts`: + +```sh +__maat_c=$(cat <&3) || exit 125; exec 3<&-; eval "unset __maat_c; $__maat_c" +``` + +The command arrives on fd 3, a socket (which another process cannot reopen +through `/proc//fd/3`). The wrapper reads it to the end, closes fd 3 so +nothing the check starts inherits it, and evals it with the holding variable +unset first; the variable is never exported. So a deliverable that reads +`/proc/$PPID/cmdline` gets the wrapper, not the check, and a process scanning +`/proc/*/cmdline` while the check runs finds nothing. + +What it does not cover: + +- A check whose own command line contains its logic, such as + `python3 -c "import sol; assert sol.f(2) == 4"`, runs the deliverable inside + that python process, whose argv is the check. The wrapper hides the shell + line, not argv words the shell then passes to a program. Prefer checks that + run the deliverable as its own process and compare its output. +- Maat's own memory holds every check. A worker of the same user, where + `kernel.yama.ptrace_scope` is 0 (common in containers), can read it. +- A failing check's output goes back to the model, masked only for verbatim + copies of the command. + +## Provider keys + +At startup (`captureSecrets()`, `src/secrets.ts`, called first thing by the +CLI and the desktop app) Maat moves every credential-shaped variable out of +its environment into memory. Nothing it starts afterwards inherits one, +whatever environment the spawn site builds. A credential is: + +- any name ending in `_API_KEY`, `_APIKEY`, `_ACCESS_KEY`, `_SECRET_KEY`, + `_SECRET_ACCESS_KEY`, `_PRIVATE_KEY`, `_SECRET`, `_TOKEN`, `_PASSWORD`, + `_PASSWD`, `_CREDENTIAL(S)`; +- any `XAI_*`, `GROK_*`, `TOGETHER_*`; +- `MAAT_API_KEY`, `MOLT_API_KEY`, `MAAT_JUDGE_KEY`, `MOLT_JUDGE_KEY`. + +`MAAT_KEEP_ENV=NAME,NAME` lets named variables through when a task needs its +own token (`HF_TOKEN` for a download, say). The model's own client processes +(the Grok Build and OpenCode backends) are given back the keys they need; the +worker's shell, background jobs and checks never are. + +Moving a key out of `process.env` does not remove it from the copy of the +environment the kernel kept when Maat was started. On Linux, +`/proc//environ` still returns it to any process of the same user, +including the worker's bash, which is Maat's child. On macOS, `ps -E` does the +same. So keys should not be in Maat's exec environment at all: + +| Way in | Where the key is visible to a same-user worker | +| --- | --- | +| Environment variable (`OPENROUTER_API_KEY=… maat run`) | Maat's `/proc//environ` for the whole run | +| `MAAT_KEYS_FD=` | nowhere: read to the end and closed at startup | +| `MAAT_KEYS_FILE=` | the file, for as long as it exists | + +Both carry `NAME=value` lines (`export` and quotes allowed, `#` comments) or a +JSON object. `MAAT_KEYS_FILE` must be a regular file of Maat's user with no +group or other permissions (mode 0600), or Maat refuses to start. Remove it +once Maat has started, or prefer the descriptor. + +From a shell, without the key passing through an argv or the environment: + +```sh +MAAT_KEYS_FD=3 maat run --sandbox … 3< ~/.config/maat/keys +``` + +From Python (what `bench/local/run.py` does): + +```python +r, w = os.pipe(); os.write(w, json.dumps(keys).encode()); os.close(w) +subprocess.Popen(cmd, env=clean | {"MAAT_KEYS_FD": str(r)}, pass_fds=(r,)) +os.close(r) +``` + +The bench container runner mounts the key file read-only under `/root` +(mode 700, out of the agent user's reach) instead of `--env-file`, which had +put it in the container's environment and in `docker inspect`; `run.py` +reads it into memory and hands it to Maat on a pipe. + +What it does not cover, with a worker of Maat's own user: + +- Maat's stored login (`auth.json` in its config directory) is a file that + user can read. +- Maat's memory holds the keys (see ptrace above). +- The model's own backend process (Grok Build, OpenCode) is given its keys in + its environment; if that backend runs the worker's commands itself, they + are its children. +- `bench/harbor` still passes `MOLT_API_KEY` through the harness's exec + environment. + +All of these are closed by a different user for the worker. + +## Tests + +`test/secret-hygiene.test.ts` runs a long check carrying a marker and scans +every process's argv and environment for it (`/proc` on Linux, `ps -E` on +macOS), with an unhidden control run that the scan must find; has a check run +a child that reads every ancestor's argv and environment; and runs the real +CLI with a stub provider whose bash call does the same for keys given by fd, +by file and by environment. The Linux half is meant for a container: + +```sh +docker run --rm -v "$PWD":/src:ro node:22-bookworm bash -c ' + mkdir /work && cd /src && tar --exclude=./node_modules -cf - . | tar -xf - -C /work + cp -a /src/node_modules /work/ && chown -R node:node /work + su node -c "cd /work && node --test dist-test/test/secret-hygiene.test.js"' +``` diff --git a/electron/main.ts b/electron/main.ts index 24aa0f4..070f421 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -40,6 +40,7 @@ import { desktopSurfaces } from "./theme-surfaces.js"; import { barInitText, parseJournal, mutatesSession } from "./limits.js"; import { resolvePath, PATH_PROBE, type PathFixReport } from "./login-path.js"; import { Engine, MAX_STEPS } from "../src/engine.js"; +import { captureSecrets } from "../src/secrets.js"; import { Archive } from "../src/archive.js"; import { Receipts, receiptName } from "../src/receipts.js"; import { Journal } from "../src/journal.js"; @@ -1063,6 +1064,11 @@ function fixPath(): PathFixReport { return report; } +// Credentials out of the environment and into memory before any session, +// check or backend is spawned (src/secrets.ts). A refused keys source is +// reported, not fatal: the window still opens and keys can be entered there. +for (const p of captureSecrets().problems) console.error(`maat: ${p}`); + const pathFix = fixPath(); // A driven window (self-check, self-drive, e2e) gets a profile of its own. diff --git a/src/acp.ts b/src/acp.ts index 16de030..8e06a60 100644 --- a/src/acp.ts +++ b/src/acp.ts @@ -71,6 +71,7 @@ * `unaccountedTools()` reports anything that got through, and the session * surfaces it as an `info` event rather than discovering it in a receipt. */ +import { withSecrets } from "./secrets.js"; import { spawn, type ChildProcess } from "node:child_process"; import { readFileSync } from "node:fs"; import { createServer, type Server } from "node:http"; @@ -530,7 +531,9 @@ export class AcpConnection { // `electron/login-path.ts` has already repaired process.env.PATH by the // time anything gets here. env: { - ...(this.spec.childEnv ? this.spec.childEnv(process.env) : process.env), + // The backend is the model's own client and needs its credentials, + // which captureSecrets() took out of process.env (src/secrets.ts). + ...(this.spec.childEnv ? this.spec.childEnv(withSecrets(process.env)) : withSecrets(process.env)), ...this.spec.env, ...(this.spec.env ? { PWD: this.opts.cwd ?? process.cwd() } : {}), }, diff --git a/src/bar.ts b/src/bar.ts index 8425eb1..f72b3ae 100644 --- a/src/bar.ts +++ b/src/bar.ts @@ -1006,7 +1006,7 @@ async function mutationCheck( }; } if (!prior) { - const baseline = await runCommand(run, { cwd: ctx.cwd, timeoutMs, signal: ctx.signal }); + const baseline = await runCommand(run, { cwd: ctx.cwd, timeoutMs, signal: ctx.signal, hideCommand: true }); if (baseline.code !== 0) { return { ok: false, @@ -1039,6 +1039,7 @@ async function mutationCheck( cwd: ctx.cwd, timeoutMs, signal: ctx.signal, + hideCommand: true, }); // A mutation the command still passes is a line nothing checks — unless // it was a boundary nudge and negating the same condition is caught. @@ -1052,7 +1053,7 @@ async function mutationCheck( let negKilled = false; if (neg && negText !== null && negText !== file.text) { writeFileSync(file.abs, negText, "utf8"); - const rn = await runCommand(run, { cwd: ctx.cwd, timeoutMs, signal: ctx.signal }); + const rn = await runCommand(run, { cwd: ctx.cwd, timeoutMs, signal: ctx.signal, hideCommand: true }); negKilled = rn.code !== 0; } if (negKilled) boundaryOnly.push(`${m.path}:${m.line} (${m.operator}) — ${m.before.trim()}`); @@ -2193,6 +2194,9 @@ export async function runCheck(check: Check, ctx: BarContext): Promise