diff --git a/README.md b/README.md index 00b78ef94..96771e3a9 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,11 @@ environment-owned, workspace-local endpoint. The endpoint is available only to `bwrap` workspaces with network isolation and is bound to the exact CLI process selected by the harness. Platform credentials stay in the environment-owned relay rather than the CLI environment, workspace manifest, or child processes. +Codex therefore uses the environment sandbox instead of starting a nested +Codex sandbox for process-bound hosted execution. +The workspace probes its substrate and uses either seccomp notification or a +ptrace-backed seccomp guard; it does not advertise process-bound connections +when neither enforcement backend is available. → [Run & deploy](https://docs.hud.ai/v6/reference/runtime) diff --git a/hud/agents/codex/agent.py b/hud/agents/codex/agent.py index c26447a65..26e4eddf6 100644 --- a/hud/agents/codex/agent.py +++ b/hud/agents/codex/agent.py @@ -215,6 +215,7 @@ def codex_command( connection: Connection | None = None, ) -> str: env: dict[str, str] = {} + sandbox = "danger-full-access" if connection is not None else config.sandbox args = [ executable, "exec", @@ -224,7 +225,7 @@ def codex_command( "--color", "never", "--sandbox", - config.sandbox, + sandbox, "--model", config.model, ] diff --git a/hud/agents/tests/test_codex_cli_agent.py b/hud/agents/tests/test_codex_cli_agent.py index b896425a6..395e3422e 100644 --- a/hud/agents/tests/test_codex_cli_agent.py +++ b/hud/agents/tests/test_codex_cli_agent.py @@ -106,7 +106,10 @@ def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatc assert command.endswith(" -") -def test_command_uses_process_bound_connection_without_its_credential() -> None: +@pytest.mark.parametrize("sandbox", ["read-only", "workspace-write", "danger-full-access"]) +def test_command_uses_process_bound_connection_without_its_credential( + sandbox: str, +) -> None: connection = Connection( name="inference", capability="ssh", @@ -114,7 +117,11 @@ def test_command_uses_process_bound_connection_without_its_credential() -> None: headers={"Authorization": "Bearer scoped-runtime-token"}, ) - command = codex_command(CodexCLIConfig(use_hud_gateway=True), "bash", connection=connection) + command = codex_command( + CodexCLIConfig.model_validate({"use_hud_gateway": True, "sandbox": sandbox}), + "bash", + connection=connection, + ) assert "scoped-runtime-token" not in command assert "HUD_CONNECTION_CREDENTIAL=hud-process-bound" in command @@ -123,6 +130,9 @@ def test_command_uses_process_bound_connection_without_its_credential() -> None: assert f'model_providers.hud.base_url="{connection.client_url}"' in command assert "Trace-Id" not in command assert "exec env" in command + assert "--sandbox danger-full-access" in command + assert "--sandbox workspace-write" not in command + assert "--sandbox read-only" not in command @pytest.mark.parametrize("shell", ["bash", "powershell"]) diff --git a/hud/agents/types.py b/hud/agents/types.py index be1956597..6e55fa815 100644 --- a/hud/agents/types.py +++ b/hud/agents/types.py @@ -186,6 +186,8 @@ class CodexCLIConfig(AgentConfig): Without an explicit inference connection or API key, the agent leaves ``CODEX_HOME`` unchanged so a login in that execution environment can apply. + A process-bound inference connection runs Codex without its inner sandbox; + the connection is available only inside the environment's isolated workspace. """ model_name: str = "Codex CLI" diff --git a/hud/environment/process_guard.py b/hud/environment/process_guard.py index 1e6e89e9b..fdc1a429f 100644 --- a/hud/environment/process_guard.py +++ b/hud/environment/process_guard.py @@ -1,5 +1,7 @@ """Linux process-bound network connection enforcement.""" +# ruff: noqa: UP045 + from __future__ import annotations import argparse @@ -10,16 +12,19 @@ import errno import fcntl import ipaddress +import json import os import platform import select +import shutil +import signal import socket import struct import subprocess import sys import threading from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal, NoReturn, Optional, cast if TYPE_CHECKING: from collections.abc import Collection, Sequence @@ -29,6 +34,7 @@ _SECCOMP_FILTER_FLAG_NEW_LISTENER = 8 _SECCOMP_RET_KILL_PROCESS = 0x80000000 _SECCOMP_RET_USER_NOTIF = 0x7FC00000 +_SECCOMP_RET_TRACE = 0x7FF00000 _SECCOMP_RET_ERRNO = 0x00050000 _SECCOMP_RET_ALLOW = 0x7FFF0000 @@ -38,13 +44,33 @@ _SECCOMP_IOCTL_NOTIF_RECV = 0xC0502100 _SECCOMP_IOCTL_NOTIF_SEND = 0xC0182101 +_PIDFD_OPEN_SYSCALL = 434 _PIDFD_GETFD_SYSCALL = 438 _REGISTER_ADDRESS = b"\0hud-process-connection-register" _SANDBOX_SOCKET = "/tmp/.hud-process-connection/control.sock" # noqa: S108 +_SANDBOX_HELPER = "/tmp/.hud-process-connection/process_guard.py" # noqa: S108 _READY_TIMEOUT_SECONDS = 10.0 +_PTRACE_TRACEME = 0 +_PTRACE_CONT = 7 +_PTRACE_GETREGS = 12 +_PTRACE_SETREGS = 13 +_PTRACE_SETOPTIONS = 0x4200 +_PTRACE_O_TRACEFORK = 0x00000002 +_PTRACE_O_TRACEVFORK = 0x00000004 +_PTRACE_O_TRACECLONE = 0x00000008 +_PTRACE_O_TRACEEXEC = 0x00000010 +_PTRACE_O_TRACESECCOMP = 0x00000080 +_PTRACE_O_EXITKILL = 0x00100000 +_PTRACE_EVENT_SECCOMP = 7 +_WAIT_ALL = getattr(os, "WALL", 0x40000000) +_MAX_SOCKADDR_BYTES = 128 + _LIBC = ctypes.CDLL(None, use_errno=True) -_supported: bool | None = None +_LIBC.ptrace.restype = ctypes.c_long +GuardBackend = Literal["notify", "ptrace"] +_backend: Optional[GuardBackend] = None +_backend_probed = False class _SockFilter(ctypes.Structure): @@ -87,6 +113,38 @@ class _SeccompNotifResp(ctypes.Structure): ] +class _UserRegsStruct(ctypes.Structure): + _fields_ = [ + ("r15", ctypes.c_ulonglong), + ("r14", ctypes.c_ulonglong), + ("r13", ctypes.c_ulonglong), + ("r12", ctypes.c_ulonglong), + ("rbp", ctypes.c_ulonglong), + ("rbx", ctypes.c_ulonglong), + ("r11", ctypes.c_ulonglong), + ("r10", ctypes.c_ulonglong), + ("r9", ctypes.c_ulonglong), + ("r8", ctypes.c_ulonglong), + ("rax", ctypes.c_ulonglong), + ("rcx", ctypes.c_ulonglong), + ("rdx", ctypes.c_ulonglong), + ("rsi", ctypes.c_ulonglong), + ("rdi", ctypes.c_ulonglong), + ("orig_rax", ctypes.c_ulonglong), + ("rip", ctypes.c_ulonglong), + ("cs", ctypes.c_ulonglong), + ("eflags", ctypes.c_ulonglong), + ("rsp", ctypes.c_ulonglong), + ("ss", ctypes.c_ulonglong), + ("fs_base", ctypes.c_ulonglong), + ("gs_base", ctypes.c_ulonglong), + ("ds", ctypes.c_ulonglong), + ("es", ctypes.c_ulonglong), + ("fs", ctypes.c_ulonglong), + ("gs", ctypes.c_ulonglong), + ] + + def _architecture() -> tuple[int, int, int, int, int]: machine = platform.machine() if machine == "x86_64": @@ -96,22 +154,27 @@ def _architecture() -> tuple[int, int, int, int, int]: raise RuntimeError(f"process-bound connections do not support {machine!r}") -def _install_connect_listener() -> int: - audit_arch, seccomp_syscall, connect_syscall, io_uring_setup, io_uring_enter = _architecture() +def _connect_filter(connect_action: int) -> tuple[_SockFprog, object]: + audit_arch, _, connect_syscall, io_uring_setup, io_uring_enter = _architecture() instructions = (_SockFilter * 11)( _SockFilter(_BPF_LD_W_ABS, 0, 0, 4), _SockFilter(_BPF_JMP_JEQ_K, 1, 0, audit_arch), _SockFilter(_BPF_RET_K, 0, 0, _SECCOMP_RET_KILL_PROCESS), _SockFilter(_BPF_LD_W_ABS, 0, 0, 0), _SockFilter(_BPF_JMP_JEQ_K, 0, 1, connect_syscall), - _SockFilter(_BPF_RET_K, 0, 0, _SECCOMP_RET_USER_NOTIF), + _SockFilter(_BPF_RET_K, 0, 0, connect_action), _SockFilter(_BPF_JMP_JEQ_K, 0, 1, io_uring_setup), _SockFilter(_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM), _SockFilter(_BPF_JMP_JEQ_K, 0, 1, io_uring_enter), _SockFilter(_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM), _SockFilter(_BPF_RET_K, 0, 0, _SECCOMP_RET_ALLOW), ) - program = _SockFprog(len(instructions), instructions) + return _SockFprog(len(instructions), instructions), instructions + + +def _install_connect_listener() -> int: + _, seccomp_syscall, _, _, _ = _architecture() + program, instructions = _connect_filter(_SECCOMP_RET_USER_NOTIF) if _LIBC.prctl(_PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0: raise OSError(ctypes.get_errno(), "prctl(PR_SET_NO_NEW_PRIVS)") listener = _LIBC.syscall( @@ -120,28 +183,54 @@ def _install_connect_listener() -> int: _SECCOMP_FILTER_FLAG_NEW_LISTENER, ctypes.byref(program), ) + del instructions if listener < 0: raise OSError(ctypes.get_errno(), "seccomp(NEW_LISTENER)") return int(listener) -def process_connections_supported() -> bool: - """Whether this substrate can install and broker seccomp notifications.""" - global _supported - if _supported is not None: - return _supported - if sys.platform != "linux" or not hasattr(os, "pidfd_open"): - _supported = False - return False - probe = subprocess.run( - [sys.executable, "-m", __name__, "--probe"], - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=10, +def _install_connect_trace() -> None: + _, seccomp_syscall, _, _, _ = _architecture() + program, instructions = _connect_filter(_SECCOMP_RET_TRACE) + if _LIBC.prctl(_PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "prctl(PR_SET_NO_NEW_PRIVS)") + result = _LIBC.syscall( + seccomp_syscall, + _SECCOMP_SET_MODE_FILTER, + 0, + ctypes.byref(program), ) - _supported = probe.returncode == 0 - return _supported + del instructions + if result != 0: + raise OSError(ctypes.get_errno(), "seccomp(TRACE)") + + +def _detected_backend() -> Optional[GuardBackend]: + global _backend, _backend_probed + if _backend_probed: + return _backend + _backend_probed = True + if sys.platform != "linux": + return None + try: + probe = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--probe"], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + candidate = probe.stdout.strip() + if probe.returncode == 0 and candidate in {"notify", "ptrace"}: + _backend = cast("GuardBackend", candidate) + return _backend + + +def process_connections_supported() -> bool: + """Whether this substrate has a race-free process connection guard.""" + return _detected_backend() is not None def _send_fd(channel: socket.socket, descriptor: int) -> None: @@ -178,7 +267,7 @@ def _read_process(pid: int, address: int, length: int) -> bytes: return data -def _destination(raw: bytes) -> tuple[str, int] | None: +def _destination(raw: bytes) -> Optional[tuple[str, int]]: if len(raw) < 2: return None family = struct.unpack_from("H", raw)[0] @@ -191,11 +280,10 @@ def _destination(raw: bytes) -> tuple[str, int] | None: return None -def _emulate_connect(pid: int, descriptor: int, address: bytes) -> int: - pidfd_open = getattr(os, "pidfd_open", None) - if pidfd_open is None: - return -errno.ENOSYS - pidfd = int(pidfd_open(pid)) +def _emulate_connect(tgid: int, descriptor: int, address: bytes) -> int: + pidfd = _LIBC.syscall(_PIDFD_OPEN_SYSCALL, tgid, 0) + if pidfd < 0: + return -ctypes.get_errno() try: duplicate = _LIBC.syscall(_PIDFD_GETFD_SYSCALL, pidfd, descriptor, 0) if duplicate < 0: @@ -211,6 +299,197 @@ def _emulate_connect(pid: int, descriptor: int, address: bytes) -> int: os.close(duplicate) +def _ptrace(request: int, pid: int, address: object = 0, data: object = 0) -> int: + ctypes.set_errno(0) + result = _LIBC.ptrace(request, pid, address, data) + if result == -1: + error = ctypes.get_errno() + if error: + raise OSError(error, f"ptrace({request})") + return int(result) + + +def _trace_options(pid: int) -> None: + options = ( + _PTRACE_O_TRACEFORK + | _PTRACE_O_TRACEVFORK + | _PTRACE_O_TRACECLONE + | _PTRACE_O_TRACEEXEC + | _PTRACE_O_TRACESECCOMP + | _PTRACE_O_EXITKILL + ) + _ptrace(_PTRACE_SETOPTIONS, pid, 0, options) + + +def _trace_registers(pid: int) -> _UserRegsStruct: + registers = _UserRegsStruct() + _ptrace(_PTRACE_GETREGS, pid, 0, ctypes.byref(registers)) + return registers + + +def _complete_traced_connect( + pid: int, + trusted_tgid: int, + protected: Collection[tuple[str, int]], + allowed: Collection[tuple[str, int]], +) -> None: + registers = _trace_registers(pid) + process_tgid = _tgid(pid) + length = int(registers.rdx) + if length < 0 or length > _MAX_SOCKADDR_BYTES: + result = -errno.EFAULT + else: + try: + address = _read_process(pid, int(registers.rsi), length) + target = _destination(address) + if target in protected and (process_tgid != trusted_tgid or target not in allowed): + result = -errno.EPERM + else: + result = _emulate_connect(process_tgid, int(registers.rdi), address) + except (OSError, RuntimeError): + result = -errno.EPERM + registers.orig_rax = ctypes.c_ulonglong(-1).value + registers.rax = ctypes.c_ulonglong(result).value + _ptrace(_PTRACE_SETREGS, pid, 0, ctypes.byref(registers)) + + +def _trace_loop( + original: int, + protected: Collection[tuple[str, int]], + allowed: Collection[tuple[str, int]], +) -> int: + while True: + try: + pid, status = os.waitpid(-1, _WAIT_ALL) + except InterruptedError: + continue + if os.WIFEXITED(status) or os.WIFSIGNALED(status): + if pid == original: + return status + continue + if not os.WIFSTOPPED(status): + continue + event = status >> 16 + stop_signal = os.WSTOPSIG(status) + if event == _PTRACE_EVENT_SECCOMP: + _complete_traced_connect(pid, original, protected, allowed) + deliver = 0 + elif event or stop_signal in {signal.SIGSTOP, signal.SIGTRAP}: + deliver = 0 + else: + deliver = stop_signal + try: + _ptrace(_PTRACE_CONT, pid, 0, deliver) + except OSError as exc: + if exc.errno != errno.ESRCH: + raise + + +def _forward_signals(original: int) -> None: + def forward(signum: int, _frame: object) -> None: + with contextlib.suppress(ProcessLookupError): + os.kill(original, signum) + + for name in ("SIGHUP", "SIGINT", "SIGQUIT", "SIGTERM", "SIGTSTP", "SIGCONT", "SIGWINCH"): + if signum := getattr(signal, name, None): + signal.signal(signum, forward) + + +def _exit_from_wait_status(status: int) -> NoReturn: + if os.WIFEXITED(status): + raise SystemExit(os.WEXITSTATUS(status)) + signum = os.WTERMSIG(status) + signal.signal(signum, signal.SIG_DFL) + os.kill(os.getpid(), signum) + raise RuntimeError("failed to reproduce traced process signal exit") + + +def _trace_exec( + channel: socket.socket, + protected: Collection[tuple[str, int]], + allowed: Collection[tuple[str, int]], + argv: Sequence[str], +) -> NoReturn: + if platform.machine() != "x86_64": + raise RuntimeError("ptrace process connections currently require x86_64") + original = os.fork() + if original == 0: + channel.close() + try: + _ptrace(_PTRACE_TRACEME, 0) + os.kill(os.getpid(), signal.SIGSTOP) + _install_connect_trace() + os.execvp(argv[0], list(argv)) # noqa: S606 - exact controller-built argv + except BaseException: + os._exit(127) + try: + _, status = os.waitpid(original, 0) + if not os.WIFSTOPPED(status): + raise RuntimeError("guarded process did not stop for ptrace") + _trace_options(original) + _forward_signals(original) + channel.sendall(b"R") + channel.close() + _ptrace(_PTRACE_CONT, original) + _exit_from_wait_status(_trace_loop(original, protected, allowed)) + except BaseException: + with contextlib.suppress(ProcessLookupError): + os.kill(original, signal.SIGKILL) + with contextlib.suppress(ChildProcessError): + os.waitpid(original, 0) + raise + + +def _receive_policy(channel: socket.socket) -> tuple[frozenset[tuple[str, int]], ...]: + raw = bytearray() + while not raw.endswith(b"\n"): + chunk = channel.recv(65536 - len(raw)) + if not chunk: + raise RuntimeError("guard broker closed before sending its policy") + raw.extend(chunk) + if len(raw) >= 65536: + raise RuntimeError("guard policy exceeded 64 KiB") + document = json.loads(raw) + protected = frozenset((str(host), int(port)) for host, port in document["protected"]) + allowed = frozenset((str(host), int(port)) for host, port in document["allowed"]) + if not allowed <= protected: + raise RuntimeError("guard policy allowed an unprotected destination") + return protected, allowed + + +def _probe_ptrace() -> bool: + if platform.machine() != "x86_64": + return False + original = os.fork() + if original == 0: + try: + _ptrace(_PTRACE_TRACEME, 0) + os.kill(os.getpid(), signal.SIGSTOP) + _install_connect_trace() + descriptor = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + descriptor.connect(("127.0.0.1", 9)) + except OSError as exc: + os._exit(0 if exc.errno == errno.EPERM else 1) + os._exit(1) + except BaseException: + os._exit(1) + try: + _, status = os.waitpid(original, 0) + if not os.WIFSTOPPED(status): + return False + _trace_options(original) + _ptrace(_PTRACE_CONT, original) + status = _trace_loop(original, {("127.0.0.1", 9)}, set()) + return os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0 + except (OSError, RuntimeError): + with contextlib.suppress(ProcessLookupError): + os.kill(original, signal.SIGKILL) + with contextlib.suppress(ChildProcessError): + os.waitpid(original, 0) + return False + + class ProcessConnectionGuard: """Broker connects for one trusted process while constraining descendants.""" @@ -219,26 +498,39 @@ def __init__( directory: Path, protected: Collection[tuple[str, int]], allowed: Collection[tuple[str, int]], + *, + backend: Optional[GuardBackend] = None, ) -> None: self.directory = directory self.socket_path = directory / "control.sock" + self.helper_path = directory / "process_guard.py" self.protected = frozenset(protected) self.allowed = frozenset(allowed) if not self.allowed <= self.protected: raise ValueError("allowed process connections must be protected destinations") - self._server: socket.socket | None = None - self._listener: int | None = None + selected_backend = backend or _detected_backend() + if selected_backend is None: + raise RuntimeError("process connection guards are unavailable on this substrate") + self.backend: GuardBackend = selected_backend + self._server: Optional[socket.socket] = None + self._listener: Optional[int] = None self._stop_read, self._stop_write = os.pipe() self._ready = threading.Event() - self._error: BaseException | None = None - self._thread: threading.Thread | None = None + self._error: Optional[BaseException] = None + self._thread: Optional[threading.Thread] = None @property def sandbox_socket(self) -> str: return _SANDBOX_SOCKET + @property + def sandbox_helper(self) -> str: + return _SANDBOX_HELPER + def start(self) -> None: self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(Path(__file__), self.helper_path) + self.helper_path.chmod(0o500) self._server = socket.socket(socket.AF_UNIX) self._server.bind(str(self.socket_path)) self._server.listen(1) @@ -277,6 +569,8 @@ def close(self) -> None: os.close(descriptor) with contextlib.suppress(FileNotFoundError): self.socket_path.unlink() + with contextlib.suppress(FileNotFoundError): + self.helper_path.unlink() with contextlib.suppress(OSError): self.directory.rmdir() @@ -293,10 +587,22 @@ def _serve(self) -> None: self._server.close() self._server = None with channel: - self._listener = _receive_fd(channel) + if self.backend == "notify": + self._listener = _receive_fd(channel) + else: + policy = { + "protected": sorted([host, port] for host, port in self.protected), + "allowed": sorted([host, port] for host, port in self.allowed), + } + channel.sendall(json.dumps(policy, separators=(",", ":")).encode() + b"\n") + if channel.recv(1) != b"R": + raise RuntimeError("ptrace guard did not acknowledge its policy") with contextlib.suppress(FileNotFoundError): self.socket_path.unlink() - self._broker() + if self.backend == "notify": + self._broker() + else: + self._ready.set() except BaseException as exc: self._error = exc self._ready.set() @@ -306,7 +612,7 @@ def _broker(self) -> None: poller = select.poll() poller.register(self._listener, select.POLLIN) poller.register(self._stop_read, select.POLLIN) - trusted_tgid: int | None = None + trusted_tgid: Optional[int] = None while True: ready = {descriptor for descriptor, _ in poller.poll()} if self._stop_read in ready: @@ -326,10 +632,13 @@ def _broker(self) -> None: response.error = -errno.ECONNREFUSED self._ready.set() else: + address_length = int(notification.data.args[2]) + if address_length < 0 or address_length > _MAX_SOCKADDR_BYTES: + raise OSError(errno.EFAULT, "invalid sockaddr length") address = _read_process( notification.pid, notification.data.args[1], - notification.data.args[2], + address_length, ) target = _destination(address) if target in self.protected and ( @@ -338,7 +647,7 @@ def _broker(self) -> None: response.error = -errno.EPERM else: result = _emulate_connect( - notification.pid, + process_tgid, notification.data.args[0], address, ) @@ -353,12 +662,15 @@ def _broker(self) -> None: raise -def guarded_exec(socket_path: str, argv: Sequence[str]) -> None: +def guarded_exec(backend: GuardBackend, socket_path: str, argv: Sequence[str]) -> None: """Install the guard, register this process, and replace it with ``argv``.""" if not argv: raise ValueError("guarded execution requires a command") channel = socket.socket(socket.AF_UNIX) channel.connect(socket_path) + if backend == "ptrace": + protected, allowed = _receive_policy(channel) + _trace_exec(channel, protected, allowed, argv) listener = _install_connect_listener() try: _send_fd(channel, listener) @@ -377,18 +689,32 @@ def guarded_exec(socket_path: str, argv: Sequence[str]) -> None: def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("--probe", action="store_true") + parser.add_argument("--probe", choices=("auto", "notify", "ptrace"), nargs="?", const="auto") + parser.add_argument("--backend", choices=("notify", "ptrace")) parser.add_argument("socket", nargs="?") parser.add_argument("argv", nargs=argparse.REMAINDER) args = parser.parse_args() if args.probe: - listener = _install_connect_listener() - os.close(listener) - return + if args.probe in {"auto", "notify"}: + try: + listener = _install_connect_listener() + except OSError: + if args.probe == "notify": + raise SystemExit(1) from None + else: + os.close(listener) + sys.stdout.write("notify\n") + return + if args.probe in {"auto", "ptrace"} and _probe_ptrace(): + sys.stdout.write("ptrace\n") + return + raise SystemExit(1) + if args.backend is None: + parser.error("--backend is required") if args.socket is None: parser.error("socket is required") argv = args.argv[1:] if args.argv[:1] == ["--"] else args.argv - guarded_exec(args.socket, argv) + guarded_exec(args.backend, args.socket, argv) if __name__ == "__main__": diff --git a/hud/environment/tests/test_process_connection.py b/hud/environment/tests/test_process_connection.py index 7523fba1e..a072e5573 100644 --- a/hud/environment/tests/test_process_connection.py +++ b/hud/environment/tests/test_process_connection.py @@ -2,11 +2,14 @@ from __future__ import annotations +import asyncio import shlex +import subprocess import sys import threading from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import TYPE_CHECKING, Any, cast +from pathlib import Path +from typing import Any, cast import pytest @@ -14,15 +17,24 @@ from hud.clients import connect from hud.environment import Environment, Peer from hud.environment.egress import ANY_HOST, BRIDGE_PORT -from hud.environment.process_guard import process_connections_supported +from hud.environment.process_guard import ProcessConnectionGuard, process_connections_supported from hud.eval import LocalRuntime, Task -if TYPE_CHECKING: - from pathlib import Path - pytestmark = pytest.mark.skipif( not process_connections_supported(), - reason="seccomp user notification is unavailable", + reason="process connection guards are unavailable", +) + +_GUARD_PATH = Path(__file__).parents[1] / "process_guard.py" +_PTRACE_SUPPORTED = ( + sys.platform == "linux" + and subprocess.run( + [sys.executable, str(_GUARD_PATH), "--probe", "ptrace"], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode + == 0 ) @@ -64,6 +76,17 @@ def _fetch_script(url: str) -> str: return f"exec {shlex.quote(sys.executable)} -c {shlex.quote(_fetch_source(url))}" +def _threaded_fetch_source(url: str) -> str: + return ( + "import threading,urllib.request;" + "result=[];" + f"request=urllib.request.Request({url!r});" + "thread=threading.Thread(target=lambda:result.append(" + "urllib.request.urlopen(request,timeout=5).read().decode()));" + "thread.start();thread.join();print(result[0])" + ) + + def _proxy_fetch_source(url: str) -> str: return ( "import http.client,sys;" @@ -83,6 +106,75 @@ def _direct_fetch_source(host: str, port: int) -> str: ) +def test_guard_projects_a_standalone_helper(tmp_path: Path) -> None: + guard = ProcessConnectionGuard(tmp_path / "guard", set(), set(), backend="notify") + try: + guard.start() + assert guard.helper_path.read_bytes() == _GUARD_PATH.read_bytes() + assert guard.helper_path.stat().st_mode & 0o777 == 0o500 + finally: + guard.close() + + +@pytest.mark.skipif(not _PTRACE_SUPPORTED, reason="ptrace guard backend is unavailable") +@pytest.mark.asyncio +async def test_ptrace_backend_emulates_connects_and_blocks_descendants(tmp_path: Path) -> None: + protected, protected_thread = _server(_ProtectedUpstream) + ordinary, ordinary_thread = _server(_OrdinaryUpstream) + protected_url = f"http://127.0.0.1:{protected.server_address[1]}" + ordinary_url = f"http://127.0.0.1:{ordinary.server_address[1]}" + child = ( + "import subprocess,sys,threading,urllib.request;" + f"protected={_fetch_source(protected_url)!r};" + f"ordinary={_fetch_source(ordinary_url)!r};" + f"print(urllib.request.urlopen({protected_url!r},timeout=5).read().decode());" + "threaded=[];" + f"thread=threading.Thread(target=lambda:threaded.append(urllib.request.urlopen({protected_url!r},timeout=5).read().decode()));" + "thread.start();thread.join();print(threaded[0]);" + "blocked=subprocess.run([sys.executable,'-c',protected]);" + "print(f'blocked={blocked.returncode}');" + "permitted=subprocess.run([sys.executable,'-c',ordinary]);" + "print(f'ordinary={permitted.returncode}')" + ) + target = ("127.0.0.1", protected.server_address[1]) + guard = ProcessConnectionGuard(tmp_path / "guard", {target}, {target}, backend="ptrace") + process = None + try: + guard.start() + process = await asyncio.create_subprocess_exec( + sys.executable, + str(_GUARD_PATH), + "--backend", + "ptrace", + str(guard.socket_path), + "--", + sys.executable, + "-c", + child, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await guard.wait_ready() + stdout, stderr = await asyncio.wait_for(process.communicate(), 20) + assert process.returncode == 0, stderr.decode(errors="replace") + lines = stdout.decode().splitlines() + assert lines[:2] == ["ok", "ok"] + assert lines[2] != "blocked=0" + assert lines[3:] == ["ok", "ordinary=0"] + finally: + if process is not None and process.returncode is None: + process.kill() + await process.wait() + guard.close() + for server, thread in ( + (protected, protected_thread), + (ordinary, ordinary_thread), + ): + server.shutdown() + server.server_close() + thread.join() + + async def test_only_bound_process_reaches_controller_connection(tmp_path: Path) -> None: protected, protected_thread = _server(_ProtectedUpstream) ordinary, ordinary_thread = _server(_OrdinaryUpstream) @@ -126,6 +218,15 @@ async def test_only_bound_process_reaches_controller_connection(tmp_path: Path) assert completed.returncode == 0 assert completed.stdout == b"ok\n" + threaded = await ssh.create_process( + f"exec {shlex.quote(sys.executable)} -c " + f"{shlex.quote(_threaded_fetch_source(connection.client_url))}", + connections=(connection,), + ) + threaded_result = await threaded.wait() + assert threaded_result.returncode == 0 + assert threaded_result.stdout == b"ok\n" + child_source = ( "import subprocess,sys;" f"result=subprocess.run([sys.executable,'-c',{_fetch_source(connection.client_url)!r}]);" diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py index 3c929808e..51d801e9f 100644 --- a/hud/environment/tests/test_workspace.py +++ b/hud/environment/tests/test_workspace.py @@ -29,6 +29,7 @@ from hud.capabilities import Connection, SSHClient from hud.environment import namespace as namespace_mod +from hud.environment import process_guard as process_guard_mod from hud.environment import workspace as workspace_mod from hud.environment.egress import ( ConnectionRelay, @@ -910,6 +911,54 @@ async def test_namespace_management_does_not_share_process_connections( management.wait_closed.assert_awaited_once_with() +def test_process_guard_executes_projected_file_without_module_reentry() -> None: + guard = cast( + "Any", + SimpleNamespace( + backend="ptrace", + sandbox_helper="/tmp/.hud-process-connection/process_guard.py", + sandbox_socket="/tmp/.hud-process-connection/control.sock", + ), + ) + argv = workspace_mod._guarded_process_argv( + "/usr/bin/python3", + guard, + ["bash", "-lc", "true"], + ) + + assert argv == [ + "/usr/bin/python3", + "/tmp/.hud-process-connection/process_guard.py", + "--backend", + "ptrace", + "/tmp/.hud-process-connection/control.sock", + "--", + "bash", + "-lc", + "true", + ] + result = subprocess.run( + [sys.executable, str(Path(workspace_mod.__file__).with_name("process_guard.py")), "--help"], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert result.stderr == "" + + +def test_process_guard_selects_probed_ptrace_backend(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(process_guard_mod, "_backend", None) + monkeypatch.setattr(process_guard_mod, "_backend_probed", False) + monkeypatch.setattr(process_guard_mod.sys, "platform", "linux") + run = Mock(return_value=SimpleNamespace(returncode=0, stdout="ptrace\n")) + monkeypatch.setattr(process_guard_mod.subprocess, "run", run) + + assert process_guard_mod._detected_backend() == "ptrace" + assert process_guard_mod._detected_backend() == "ptrace" + run.assert_called_once() + + @pytest.mark.asyncio async def test_namespace_host_only_terminates_a_used_session_holder( tmp_path: Path, diff --git a/hud/environment/workspace.py b/hud/environment/workspace.py index 2a174166f..f90e06ace 100644 --- a/hud/environment/workspace.py +++ b/hud/environment/workspace.py @@ -57,6 +57,40 @@ _INVALID_DWORD = 0xFFFFFFFF +def _process_guard_interpreter() -> str | None: + candidates = ( + Path("/usr/bin/python3"), + Path("/usr/local/bin/python3"), + Path(sys.executable).resolve(), + ) + return next( + ( + str(candidate) + for candidate in dict.fromkeys(candidates) + if candidate.is_file() + and os.access(candidate, os.X_OK) + and candidate.is_relative_to("/usr") + ), + None, + ) + + +def _guarded_process_argv( + interpreter: str, + guard: ProcessConnectionGuard, + argv: Sequence[str], +) -> list[str]: + return [ + interpreter, + guard.sandbox_helper, + "--backend", + guard.backend, + guard.sandbox_socket, + "--", + *argv, + ] + + class _WindowsJob: """Windows Job Object which owns a subprocess and all of its descendants.""" @@ -661,7 +695,12 @@ def remove_peer(self, peer: Peer) -> None: @property def supports_process_connections(self) -> bool: - return self.bwrap_available and self.owns_netns and process_connections_supported() + return ( + self.bwrap_available + and self.owns_netns + and _process_guard_interpreter() is not None + and process_connections_supported() + ) def add_process_connection(self, name: str, peer: Peer) -> None: if name in self._process_connections: @@ -1679,7 +1718,12 @@ async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> No session_env = {"TERM": term_type} if term_type else None requested_connections = self._requested_process_connections(process) if self._process_connections: - if pid is None or not self.supports_process_connections: + guard_interpreter = _process_guard_interpreter() + if ( + pid is None + or not self.supports_process_connections + or guard_interpreter is None + ): raise RuntimeError("process-bound connections require an isolated workspace") guard_directory = Path( tempfile.mkdtemp(prefix="process-connection-", dir=self._credentials_dir()) @@ -1696,15 +1740,14 @@ async def _handle_process(self, process: asyncssh.SSHServerProcess[bytes]) -> No if process.command is not None else ["bash", "-l"] ) - guarded_command = [ - sys.executable, - "-m", - "hud.environment.process_guard", - guard.sandbox_socket, - "--", - *self._drop_argv(), - *shell_command, - ] + guarded_command = _guarded_process_argv( + guard_interpreter, + guard, + [ + *self._drop_argv(), + *shell_command, + ], + ) argv = self.bwrap_argv( guarded_command, env=session_env, diff --git a/hud/eval/runtime/__init__.py b/hud/eval/runtime/__init__.py index 46d8ab207..ba815d4a1 100644 --- a/hud/eval/runtime/__init__.py +++ b/hud/eval/runtime/__init__.py @@ -1,6 +1,6 @@ """Runtime placement and provider configuration.""" -from .compose import ComposeProject +from .compose import ComposeProject, DockerBindMount from .core import ( Provider, Runtime, @@ -21,6 +21,7 @@ __all__ = [ "ComposeProject", "DaytonaRuntime", + "DockerBindMount", "DockerRuntime", "HUDRuntime", "HostedRuntime", diff --git a/hud/eval/runtime/compose.py b/hud/eval/runtime/compose.py index e1ec7e4c1..8506c76f0 100644 --- a/hud/eval/runtime/compose.py +++ b/hud/eval/runtime/compose.py @@ -11,7 +11,7 @@ import tarfile import tempfile from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any import yaml @@ -29,7 +29,7 @@ from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode if TYPE_CHECKING: - from collections.abc import Iterator, Mapping + from collections.abc import Iterator, Mapping, Sequence _COMPOSE_VARIABLE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") @@ -443,6 +443,47 @@ class ComposeLaunchFiles: archive: Path | None +@dataclass(frozen=True, slots=True) +class DockerBindMount: + """A provider-owned bind mount injected into a Docker environment.""" + + source: Path + target: PurePosixPath + read_only: bool = True + + def __init__( + self, + source: str | Path, + target: str | PurePosixPath, + read_only: bool = True, + ) -> None: + resolved_source = Path(source) + resolved_target = PurePosixPath(target) + if not resolved_source.is_absolute(): + raise ValueError("Docker bind mount source must be absolute") + if not resolved_target.is_absolute(): + raise ValueError("Docker bind mount target must be absolute") + if "," in str(resolved_source) or "," in str(resolved_target): + raise ValueError("Docker bind mount paths cannot contain commas") + object.__setattr__(self, "source", resolved_source) + object.__setattr__(self, "target", resolved_target) + object.__setattr__(self, "read_only", read_only) + + def docker_argument(self) -> str: + argument = f"type=bind,source={self.source},target={self.target}" + return f"{argument},readonly" if self.read_only else argument + + def compose_volume(self) -> dict[str, str | bool]: + volume: dict[str, str | bool] = { + "type": "bind", + "source": str(self.source), + "target": str(self.target), + } + if self.read_only: + volume["read_only"] = True + return volume + + class ComposeProject(BaseModel): """A Compose recipe and the project data it may need at runtime.""" @@ -512,6 +553,7 @@ def stage( port_service: str = "main", seccomp: str | Path, service_socket: str | None = None, + bind_mounts: Sequence[DockerBindMount] = (), env_vars: Mapping[str, str] | None = None, cpu: float | None = None, memory_mb: int | None = None, @@ -528,14 +570,17 @@ def stage( "apparmor=unconfined", ], } + volumes = [mount.compose_volume() for mount in bind_mounts] if service_socket is not None: - main["volumes"] = [ + volumes.append( { "type": "bind", "source": service_socket, "target": "/media/hud/docker.sock", } - ] + ) + if volumes: + main["volumes"] = volumes if env_vars: main["environment"] = dict(env_vars) if cpu is not None: diff --git a/hud/eval/runtime/docker.py b/hud/eval/runtime/docker.py index 4582e2df4..75c92ee88 100644 --- a/hud/eval/runtime/docker.py +++ b/hud/eval/runtime/docker.py @@ -18,7 +18,7 @@ from hud.utils.docker import docker as _docker from hud.utils.process import create_process_group_exec, finish_output, stream_output -from .compose import ComposeConfig +from .compose import ComposeConfig, DockerBindMount from .core import Runtime, RuntimeConfig, validate_session_id if TYPE_CHECKING: @@ -134,12 +134,14 @@ def __init__( *, port: int = 8765, run_args: Sequence[str] = (), + bind_mounts: Sequence[DockerBindMount] = (), compose_service_socket: str | Path | None = None, runtime_config: RuntimeConfig | dict[str, Any] | None = None, env_vars: Mapping[str, str] | None = None, ) -> None: self.port = port self.run_args = tuple(run_args) + self.bind_mounts = tuple(bind_mounts) self.env_vars = dict(env_vars or {}) self.compose_service_socket = ( str(Path(compose_service_socket)) if compose_service_socket is not None else None @@ -204,6 +206,7 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: port_service=port_service, seccomp=_DOCKER_SECCOMP_PROFILE, service_socket=service_socket, + bind_mounts=self.bind_mounts, env_vars=self.env_vars, cpu=resources.cpu if resources is not None else None, memory_mb=resources.memory_mb if resources is not None else None, @@ -296,12 +299,16 @@ async def __call__(self, task: Task) -> AsyncIterator[Runtime]: env_args: list[str] = [] for key, value in self.env_vars.items(): env_args.extend(("--env", f"{key}={value}")) + mount_args: list[str] = [] + for mount in self.bind_mounts: + mount_args.extend(("--mount", mount.docker_argument())) out, _ = await _docker( "run", "--detach", *self.run_args, *env_args, *resource_args, + *mount_args, *_DOCKER_SECURITY_ARGS, "--publish", f"127.0.0.1::{self.port}", diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py index a366bde2c..931f428b5 100644 --- a/hud/eval/tests/test_docker_provider.py +++ b/hud/eval/tests/test_docker_provider.py @@ -26,6 +26,7 @@ import hud.utils.process as process_module from hud.eval.runtime import ( DaytonaRuntime, + DockerBindMount, DockerRuntime, ModalRuntime, RuntimeConfig, @@ -602,6 +603,46 @@ async def test_acquisition_publishes_ephemeral_port_and_removes_container( assert capsys.readouterr().out == "ImportError: boom\n" +async def test_docker_runtime_injects_provider_bind_mount( + tmp_path: Path, + docker_log: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) + bundle = tmp_path / "agents" / "codex" + bundle.mkdir(parents=True) + + provider = DockerRuntime( + "img:tag", + bind_mounts=(DockerBindMount(bundle, "/usr/local/lib/agents/codex"),), + ) + async with provider(_row()): + pass + + assert (await _docker_calls(docker_log))[0] == ( + f"run --detach --mount type=bind,source={bundle}," + "target=/usr/local/lib/agents/codex,readonly " + f"{_docker_security_args()} --publish 127.0.0.1::8765 img:tag" + ) + + +@pytest.mark.parametrize( + ("source", "target", "message"), + [ + ("relative", "/opt/agents", "source must be absolute"), + ("/opt/agents", "relative", "target must be absolute"), + ("/opt/agents,old", "/opt/agents", "paths cannot contain commas"), + ], +) +def test_docker_bind_mount_rejects_ambiguous_paths( + source: str, + target: str, + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + DockerBindMount(source, target) + + async def test_docker_session_archives_inside_the_container( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -994,6 +1035,53 @@ async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: ] +async def test_docker_runtime_stages_provider_mount_with_service_socket( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + compose = tmp_path / "compose.yaml" + compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") + bundle = tmp_path / "agents" / "claude" + bundle.mkdir(parents=True) + rendered: dict[str, Any] = {} + + async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: + if args[-4:] == ("up", "--detach", "--build", "--remove-orphans"): + files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] + rendered.update(json.loads(files[1].read_text("utf-8"))) + if args[-3:] == ("port", "main", "8765"): + return "127.0.0.1:43210\n", "" + return "", "" + + monkeypatch.setattr(runtime_module, "_docker", fake_docker) + task = Task( + env="any-env", + id="t", + runtime_config=RuntimeConfig(compose=ComposeProject(document=compose, service_access=True)), + ) + provider = DockerRuntime( + compose_service_socket="/vm/run/docker.sock", + bind_mounts=(DockerBindMount(bundle, "/usr/local/lib/agents/claude"),), + ) + + async with provider(task): + pass + + assert rendered["services"]["main"]["volumes"] == [ + { + "type": "bind", + "source": str(bundle), + "target": "/usr/local/lib/agents/claude", + "read_only": True, + }, + { + "type": "bind", + "source": "/vm/run/docker.sock", + "target": "/media/hud/docker.sock", + }, + ] + + def test_docker_runtime_accepts_only_one_environment_definition(tmp_path: Path) -> None: with pytest.raises(ValueError, match="either image or compose"): RuntimeConfig(