diff --git a/anvil/entrypoint.sh b/anvil/entrypoint.sh index d192676..954ec0c 100644 --- a/anvil/entrypoint.sh +++ b/anvil/entrypoint.sh @@ -183,8 +183,9 @@ copy_shared_assets() { # Unified definitions are markdown files whose YAML frontmatter is a superset # of the OpenCode agent schema (description, mode, model, temperature, tools) # plus optional per-harness override blocks (claude:, codex:, opencode:). -# One translator (swarmforge.agents.translate) emits each harness's dialect, so -# adding a new harness means adding an emitter there plus a case arm here. +# One translator (swarmforge.agents.translate) drives each harness's registered +# emitter, so adding a new harness means an emitter in its module under +# swarmforge/harness/ plus a case arm here. # # Unified Swarmforge agent definitions live under /agents in the # harness-neutral .swarmforge asset layers, mounted read-only via diff --git a/pyproject.toml b/pyproject.toml index eda0689..93bb70d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,5 +47,10 @@ packages = [ "swarmforge.agents", "swarmforge.anvil", "swarmforge.config", + "swarmforge.harness", + "swarmforge.harness.claude", + "swarmforge.harness.codex", + "swarmforge.harness.grok", + "swarmforge.harness.opencode", "swarmforge.tongs", ] diff --git a/swarmforge/agents/__init__.py b/swarmforge/agents/__init__.py index 00c7dbe..817b327 100644 --- a/swarmforge/agents/__init__.py +++ b/swarmforge/agents/__init__.py @@ -1,6 +1,8 @@ """Agent-definition tooling. -The unified agent format is the repo's own; every harness gets it translated -into its native dialect here, so a new harness means a new emitter in -`translate` rather than hand-written dialects scattered across the tree. +The unified agent format is the repo's own. `translate` drives one harness's +registered emitter over the unified definitions, and `emit` holds the +rendering and frontmatter helpers the emitters share; each harness's emitter +lives with its harness module under `swarmforge.harness`, so no hand-written +dialects are scattered across the tree. """ diff --git a/swarmforge/agents/emit.py b/swarmforge/agents/emit.py new file mode 100644 index 0000000..8980a7c --- /dev/null +++ b/swarmforge/agents/emit.py @@ -0,0 +1,113 @@ +"""Rendering and frontmatter helpers for the unified agent format. + +The translator CLI and the harness modules share these: splitting a unified +definition into frontmatter and body, rendering frontmatter back as YAML, and +rendering a mapping as TOML for the harnesses whose native agents are TOML. +""" + +import json +import re +import sys + +from swarmforge.yamlite import parse_map, parse_scalar + +# Unified-schema fields only OpenCode consumes; other harnesses drop them. +OPENCODE_ONLY_FIELDS = { + "mode", + "temperature", + "top_p", + "steps", + "permission", + "hidden", + "disable", + "tools", +} + + +# The prefix names the CLI entry point a user invokes, not this module. +def warn(message): + print("swarmforge.agents.translate: %s" % message, file=sys.stderr) + + +PLAIN_SCALAR_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 _.,()/+-]*$") + + +def emit_scalar(value): + if value is True: + return "true" + if value is False: + return "false" + if value is None: + return "null" + if isinstance(value, (int, float)): + return repr(value) + text = str(value) + if PLAIN_SCALAR_RE.match(text) and not text.endswith(" "): + if parse_scalar(text) == text: + return text + return json.dumps(text) + + +def emit_map(mapping, indent=0): + lines = [] + pad = " " * indent + for key, value in mapping.items(): + if isinstance(value, dict): + lines.append("%s%s:" % (pad, key)) + lines.extend(emit_map(value, indent + 2)) + elif isinstance(value, list): + lines.append("%s%s:" % (pad, key)) + for item in value: + lines.append("%s - %s" % (pad, emit_scalar(item))) + else: + lines.append("%s%s: %s" % (pad, key, emit_scalar(value))) + return lines + + +def split_frontmatter(text): + if not text.startswith("---\n"): + return {}, text + lines = text.split("\n") + for end in range(1, len(lines)): + if lines[end].strip() == "---": + meta, _ = parse_map(lines[1:end], 0, 0) + body = "\n".join(lines[end + 1 :]).lstrip("\n") + return meta, body + raise ValueError("unterminated frontmatter") + + +def render(meta, body): + return "---\n%s\n---\n\n%s" % ("\n".join(emit_map(meta)), body) + + +TOML_BARE_KEY_RE = re.compile(r"^[A-Za-z0-9_-]+$") + + +def emit_toml_key(value): + text = str(value) + return text if TOML_BARE_KEY_RE.fullmatch(text) else emit_toml_string(text) + + +def emit_toml_string(value): + return json.dumps(str(value), ensure_ascii=False) + + +def emit_toml_multiline(value): + text = str(value).replace("\\", "\\\\").replace('"', '\\"') + return chr(34) * 3 + "\n" + text + chr(34) * 3 + + +def emit_toml_value(value): + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return repr(value) + if isinstance(value, list): + return "[%s]" % ", ".join(emit_toml_value(item) for item in value) + if isinstance(value, dict): + pairs = ( + "%s = %s" % (emit_toml_key(key), emit_toml_value(item)) + for key, item in value.items() + ) + return "{ %s }" % ", ".join(pairs) + return emit_toml_string(value) diff --git a/swarmforge/agents/translate.py b/swarmforge/agents/translate.py index 323512f..7ec0d6d 100644 --- a/swarmforge/agents/translate.py +++ b/swarmforge/agents/translate.py @@ -42,258 +42,73 @@ empty source paths are skipped. Only top-level *.md files are read. """ -import json import os -import re import sys +from swarmforge import harness +from swarmforge.agents.emit import ( + OPENCODE_ONLY_FIELDS, + PLAIN_SCALAR_RE, + TOML_BARE_KEY_RE, + emit_map, + emit_scalar, + emit_toml_key, + emit_toml_multiline, + emit_toml_string, + emit_toml_value, + render, + split_frontmatter, + warn, +) +from swarmforge.harness.claude import CLAUDE_TOOL_NAMES, to_claude +from swarmforge.harness.codex import ( + CODEX_AGENT_TABLE_FIELDS, + normalize_codex_name, + render_codex, + to_codex, +) +from swarmforge.harness.opencode import to_opencode +from swarmforge.harness.spec import provided from swarmforge.yamlite import parse_map, parse_scalar -HARNESS_OVERRIDE_KEYS = {"claude", "codex", "opencode"} - -# OpenCode tool id -> Claude Code tool name. Ids mapping to None have no -# Claude equivalent and are dropped. -CLAUDE_TOOL_NAMES = { - "bash": "Bash", - "edit": "Edit", - "write": "Write", - "read": "Read", - "grep": "Grep", - "glob": "Glob", - "list": None, - "patch": None, - "skill": "Skill", - "task": "Task", - "todoread": None, - "todowrite": "TodoWrite", - "webfetch": "WebFetch", - "websearch": "WebSearch", -} - -OPENCODE_ONLY_FIELDS = { - "mode", - "temperature", - "top_p", - "steps", - "permission", - "hidden", - "disable", - "tools", -} - - -def warn(message): - print("swarmforge.agents.translate: %s" % message, file=sys.stderr) - - -PLAIN_SCALAR_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 _.,()/+-]*$") - - -def emit_scalar(value): - if value is True: - return "true" - if value is False: - return "false" - if value is None: - return "null" - if isinstance(value, (int, float)): - return repr(value) - text = str(value) - if PLAIN_SCALAR_RE.match(text) and not text.endswith(" "): - if parse_scalar(text) == text: - return text - return json.dumps(text) - - -def emit_map(mapping, indent=0): - lines = [] - pad = " " * indent - for key, value in mapping.items(): - if isinstance(value, dict): - lines.append("%s%s:" % (pad, key)) - lines.extend(emit_map(value, indent + 2)) - elif isinstance(value, list): - lines.append("%s%s:" % (pad, key)) - for item in value: - lines.append("%s - %s" % (pad, emit_scalar(item))) - else: - lines.append("%s%s: %s" % (pad, key, emit_scalar(value))) - return lines - - -def split_frontmatter(text): - if not text.startswith("---\n"): - return {}, text - lines = text.split("\n") - for end in range(1, len(lines)): - if lines[end].strip() == "---": - meta, _ = parse_map(lines[1:end], 0, 0) - body = "\n".join(lines[end + 1 :]).lstrip("\n") - return meta, body - raise ValueError("unterminated frontmatter") - - -def render(meta, body): - return "---\n%s\n---\n\n%s" % ("\n".join(emit_map(meta)), body) - - -TOML_BARE_KEY_RE = re.compile(r"^[A-Za-z0-9_-]+$") - - -def emit_toml_key(value): - text = str(value) - return text if TOML_BARE_KEY_RE.fullmatch(text) else emit_toml_string(text) - - -def emit_toml_string(value): - return json.dumps(str(value), ensure_ascii=False) - - -def emit_toml_multiline(value): - text = str(value).replace("\\", "\\\\").replace('"', '\\"') - return chr(34) * 3 + "\n" + text + chr(34) * 3 - - -def emit_toml_value(value): - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, (int, float)): - return repr(value) - if isinstance(value, list): - return "[%s]" % ", ".join(emit_toml_value(item) for item in value) - if isinstance(value, dict): - pairs = ( - "%s = %s" % (emit_toml_key(key), emit_toml_value(item)) - for key, item in value.items() - ) - return "{ %s }" % ", ".join(pairs) - return emit_toml_string(value) - - -def render_codex(meta): - lines = [] - for key, value in meta.items(): - rendered = ( - emit_toml_multiline(value) - if key == "developer_instructions" - else emit_toml_value(value) - ) - lines.append("%s = %s" % (emit_toml_key(key), rendered)) - return "\n".join(lines) + "\n" - - -# --- Per-harness emitters --------------------------------------------------- - - -def to_opencode(name, meta): - out = {k: v for k, v in meta.items() if k not in HARNESS_OVERRIDE_KEYS and k != "name"} - model = out.get("model") - if model is not None and "/" not in str(model): - del out["model"] - overrides = meta.get("opencode") - if isinstance(overrides, dict): - out.update(overrides) - return out - - -def to_claude(name, meta): - if meta.get("disable") is True: - return None - out = {"name": meta.get("name", name)} - if "description" in meta: - out["description"] = meta["description"] - else: - warn("agent '%s' has no description" % name) - - model = meta.get("model") - if model is not None: - provider, sep, model_id = str(model).partition("/") - if not sep: - out["model"] = model - elif provider == "anthropic": - out["model"] = model_id - - tools = meta.get("tools") - if isinstance(tools, dict): - disallowed = [] - for tool, enabled in tools.items(): - if enabled is not False: - continue - mapped = CLAUDE_TOOL_NAMES.get(tool) - if mapped is None: - if tool not in CLAUDE_TOOL_NAMES: - warn("agent '%s': unknown tool '%s' skipped" % (name, tool)) - continue - disallowed.append(mapped) - if disallowed: - out["disallowedTools"] = ", ".join(disallowed) - elif tools is not None: - warn("agent '%s': 'tools' must be a map of tool -> bool" % name) - - skipped = OPENCODE_ONLY_FIELDS | HARNESS_OVERRIDE_KEYS | {"name", "description", "model"} - for key, value in meta.items(): - if key not in skipped: - out[key] = value - - overrides = meta.get("claude") - if isinstance(overrides, dict): - out.update(overrides) - return out - - -CODEX_AGENT_TABLE_FIELDS = { - "default_subagent_model", - "enabled", - "max_depth", -} - - -def normalize_codex_name(name): - normalized = re.sub(r"[^A-Za-z0-9 _-]+", "-", str(name)) - normalized = normalized.strip(" _-") or "agent" - if normalized in CODEX_AGENT_TABLE_FIELDS: - normalized = "agent-" + normalized - return normalized - - -def to_codex(name, meta, body): - if meta.get("disable") is True: - return None - requested_name = meta.get("name", name) - codex_name = normalize_codex_name(requested_name) - if codex_name != requested_name: - warn("agent '%s': Codex name normalized to '%s'" % (name, codex_name)) - out = {"name": codex_name, "developer_instructions": body} - if "description" in meta: - out["description"] = meta["description"] - else: - warn("agent '%s' has no description" % name) - - model = meta.get("model") - if model is not None: - provider, sep, model_id = str(model).partition("/") - if not sep: - out["model"] = model - elif provider == "openai": - out["model"] = model_id - - if "tools" in meta: - warn( - "agent '%s': tool restrictions are not translated for Codex; " - "use codex sandbox/MCP settings" % name - ) - - overrides = meta.get("codex") - if isinstance(overrides, dict): - out.update(overrides) - out["name"] = normalize_codex_name(out["name"]) - return out - +# This module's public surface: the rendering helpers and the per-harness +# emitters stay importable from the CLI's own module name. +__all__ = [ + "CLAUDE_TOOL_NAMES", + "CODEX_AGENT_TABLE_FIELDS", + "EMITTERS", + "HARNESS_OVERRIDE_KEYS", + "OPENCODE_ONLY_FIELDS", + "PLAIN_SCALAR_RE", + "TOML_BARE_KEY_RE", + "emit_map", + "emit_scalar", + "emit_toml_key", + "emit_toml_multiline", + "emit_toml_string", + "emit_toml_value", + "load_agents", + "main", + "normalize_codex_name", + "parse_map", + "parse_scalar", + "render", + "render_codex", + "split_frontmatter", + "to_claude", + "to_codex", + "to_opencode", + "warn", +] + +# Frontmatter keys that are per-harness override blocks, and the emitter +# for each harness that defines one, both read off the harness registry. +HARNESS_OVERRIDE_KEYS = harness.agent_override_keys() EMITTERS = { - "opencode": to_opencode, - "claude": to_claude, - "codex": to_codex, + name: harness.get(name).SPEC.agent_emitter + for name in harness.names() + if provided(harness.get(name).SPEC.agent_emitter) } @@ -318,6 +133,11 @@ def load_agents(src_dirs): def main(argv): + """Run the target harness's emitter over the loaded agents. + + Each emitted file is written under `dest_dir`, then the harness's + finalize hook runs over everything that was emitted. + """ if len(argv) < 3: print(__doc__.strip(), file=sys.stderr) return 2 @@ -332,29 +152,19 @@ def main(argv): return 0 os.makedirs(dest_dir, exist_ok=True) - codex_registrations = {} + emitted = [] for filename, (meta, body) in agents.items(): name = filename[: -len(".md")] - out_meta = emitter(name, meta, body) if target == "codex" else emitter(name, meta) - if out_meta is None: + result = emitter(name, meta, body) + if result is None: continue - out_filename = ( - "%s.toml" % normalize_codex_name(name) if target == "codex" else filename - ) + out_filename, text = result out_path = os.path.join(dest_dir, out_filename) with open(out_path, "w", encoding="utf-8") as handle: - if target == "codex": - handle.write(render_codex(out_meta)) - codex_registrations[out_meta["name"]] = { - "config_file": os.path.abspath(out_path) - } - else: - handle.write(render(out_meta, body)) + handle.write(text) + emitted.append((name, meta, out_path)) - if codex_registrations: - config_path = os.path.join(dest_dir, "config.toml") - with open(config_path, "w", encoding="utf-8") as handle: - handle.write(render_codex({"agents": codex_registrations})) + harness.get(target).SPEC.finalize_agents(dest_dir, emitted) return 0 diff --git a/swarmforge/anvil/orchestrate.py b/swarmforge/anvil/orchestrate.py index 3de0a5b..5f57d60 100644 --- a/swarmforge/anvil/orchestrate.py +++ b/swarmforge/anvil/orchestrate.py @@ -30,6 +30,11 @@ # same git-dir mounts (and the same read-only guards) the anvil does. from swarmforge import gitguard +# The per-harness registry: MCP fragment shape and delivery are read off each +# harness's spec. +from swarmforge import harness as harnesses +from swarmforge.harness.spec import provided + from .errors import OrchestrationError from .readiness import wait_ready from .secretchan import SecretChannel, make_secret_resolver @@ -122,9 +127,13 @@ def ensure_mcp_harness_supported(merged, harness): name for name in sorted(merged) if (merged[name]["definition"].get("interface") or {}).get("kind") == "mcp" ] - if not mcp_names or harness in tongs.MCP_EMITTERS: + module = harnesses.get(harness) + if not mcp_names or (module is not None and provided(module.SPEC.mcp_fragment)): return - supported = ", ".join(sorted(tongs.MCP_EMITTERS)) + supported = ", ".join( + name for name in harnesses.names() + if provided(harnesses.get(name).SPEC.mcp_fragment) + ) got = harness if harness else "none" raise OrchestrationError( "mcp tong(s) %s require --harness to be one of: %s (got %s)" @@ -253,9 +262,10 @@ def _injection_pre_image_args(injection): return args -# Where the generated MCP config is mounted in the anvil, and the env var the -# entrypoint reads to merge it into the harness's own config file. Claude Code -# is pointed at the same in-container path with `--mcp-config` instead. +# Where the generated MCP config is mounted in the anvil. A harness whose spec +# delivers by env var is pointed at that path through the variable below, which +# the entrypoint reads; one that delivers by flag is pointed at it on its own +# command line instead. MCP_CONFIG_CONTAINER_PATH = "/tmp/swarmforge-tong-mcp.json" MCP_FILE_ENV = "SWARMFORGE_TONG_MCP_FILE" @@ -265,12 +275,13 @@ def _mcp_injection(mcp_config, harness, mcp_dir): `mcp_config` is the per-harness fragment from `tongs.plan_injection` (already shaped for the harness). It is written into `mcp_dir` on the host and mounted - read-only into the anvil. For Claude Code the mount is paired with - `--mcp-config ` (a harness arg, so it appends after the image); for - every other harness the mount is paired with `SWARMFORGE_TONG_MCP_FILE=`, - which the entrypoint reads to merge the fragment into that harness's config. - With an empty fragment nothing is written, mounted, or appended, so the - anvil argv is unchanged. + read-only into the anvil; the harness spec's `mcp_delivery` decides how the + harness is told where it landed. A `("flag", FLAG)` harness gets `FLAG ` + appended as a harness arg after the image; an `("env", VAR)` harness gets the + mount paired with `VAR=`, which the entrypoint reads to merge the + fragment into that harness's config. An unregistered harness falls back to + the env-var delivery. With an empty fragment nothing is written, mounted, or + appended, so the anvil argv is unchanged. """ if not mcp_config: return [], [] @@ -278,9 +289,14 @@ def _mcp_injection(mcp_config, harness, mcp_dir): with open(host_path, "w", encoding="utf-8") as handle: json.dump(mcp_config, handle) mount = ["-v", "%s:%s:ro" % (host_path, MCP_CONFIG_CONTAINER_PATH)] - if harness == "claude": - return mount, ["--mcp-config", MCP_CONFIG_CONTAINER_PATH] - return mount + ["-e", "%s=%s" % (MCP_FILE_ENV, MCP_CONFIG_CONTAINER_PATH)], [] + delivery = ("env", MCP_FILE_ENV) + module = harnesses.get(harness) + if module is not None: + delivery = module.SPEC.mcp_delivery + kind, name = delivery + if kind == "flag": + return mount, [name, MCP_CONFIG_CONTAINER_PATH] + return mount + ["-e", "%s=%s" % (name, MCP_CONFIG_CONTAINER_PATH)], [] def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, @@ -438,10 +454,10 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, raise OrchestrationError("tong '%s' did not become ready in time" % name) # `port`/`volume` reachability splices in before the image; the MCP - # config adds a read-only mount (and, for OpenCode, the env var the - # entrypoint reads) before the image, plus Claude's `--mcp-config` as a - # harness arg after it. With no `mcp` tongs the fragment is empty and - # nothing is written or appended. + # config adds a read-only mount before the image, paired with either the + # env var the entrypoint reads or a harness arg after the image, + # whichever the harness spec's delivery names. With no `mcp` tongs the + # fragment is empty and nothing is written or appended. pre_image_args = _injection_pre_image_args(injection) post_image_args = [] if injection["mcp"]: diff --git a/swarmforge/commands/translate.py b/swarmforge/commands/translate.py index a98950c..d41b55d 100644 --- a/swarmforge/commands/translate.py +++ b/swarmforge/commands/translate.py @@ -2,90 +2,16 @@ """Translate portable slash commands into Codex skill packages. Usage: python3 -m swarmforge.commands.translate + +The implementation lives in swarmforge.harness.codex.commands; this module is +the stable invocation path the container entrypoint runs. """ -import os -import re -import shutil import sys -from swarmforge.agents.translate import split_frontmatter - - -SHELL_INTERPOLATION_RE = re.compile(r"!`([^`\n]+)`") -POSITIONAL_RE = re.compile(r"\$(\d+)") - - -def warn(message): - print("swarmforge.commands.translate: %s" % message, file=sys.stderr) - - -def describe_positionals(command): - positions = sorted({int(value) for value in POSITIONAL_RE.findall(command)}) - if not positions: - return "" - labels = ", ".join("$%d" % position for position in positions) - return ", replacing %s with the corresponding positional invocation argument%s" % ( - labels, - "" if len(positions) == 1 else "s", - ) - - -def translate_body(body): - body = body.replace( - "$ARGUMENTS", "the arguments supplied with this skill invocation" - ) - - def shell_instruction(match): - command = match.group(1) - return "Run `%s`%s and use its output." % ( - command, - describe_positionals(command), - ) - - return SHELL_INTERPOLATION_RE.sub(shell_instruction, body) - - -def translate_file(path, dest_dir): - filename = os.path.basename(path) - name = filename[:-3] - with open(path, "r", encoding="utf-8") as handle: - meta, body = split_frontmatter(handle.read()) - description = meta.get("description") - if not description: - warn("skipping %s: command has no description" % path) - return - skill_dir = os.path.join(dest_dir, name) - if os.path.lexists(skill_dir): - if os.path.isdir(skill_dir) and not os.path.islink(skill_dir): - shutil.rmtree(skill_dir) - else: - os.unlink(skill_dir) - os.makedirs(skill_dir) - with open(os.path.join(skill_dir, "SKILL.md"), "w", encoding="utf-8") as handle: - handle.write( - "---\nname: %s\ndescription: %s\n---\n\n%s" - % (name, description, translate_body(body)) - ) - - -def main(argv): - if len(argv) != 2: - print(__doc__.strip(), file=sys.stderr) - return 2 - dest_dir, src_dir = argv - if not src_dir or not os.path.isdir(src_dir): - return 0 - os.makedirs(dest_dir, exist_ok=True) - for filename in sorted(os.listdir(src_dir)): - path = os.path.join(src_dir, filename) - if filename.endswith(".md") and os.path.isfile(path): - try: - translate_file(path, dest_dir) - except ValueError as exc: - warn("skipping %s: %s" % (path, exc)) - return 0 +from swarmforge.harness.codex.commands import main +__all__ = ["main"] if __name__ == "__main__": sys.exit(main(sys.argv[1:])) diff --git a/swarmforge/harness/__init__.py b/swarmforge/harness/__init__.py new file mode 100644 index 0000000..7361fa8 --- /dev/null +++ b/swarmforge/harness/__init__.py @@ -0,0 +1,39 @@ +"""Registry of the harnesses Swarmforge can drive. + +One module per harness, each declaring a `HarnessSpec` (see `spec`) plus the +functions the spec points at. The registry is a static dict rather than +directory discovery: greppable, import-order explicit, and closed at image +build time. +""" + +from swarmforge.harness import claude, codex, grok, opencode +from swarmforge.harness.spec import provided + +_REGISTRY = { + "claude": claude, + "codex": codex, + "grok": grok, + "opencode": opencode, +} + + +def get(name): + """The module for the harness registered under `name`, or None.""" + return _REGISTRY.get(name) + + +def names(): + """Every registered harness name, sorted.""" + return sorted(_REGISTRY) + + +def agent_override_keys(): + """Frontmatter keys that name per-harness override blocks. + + A harness claims its own name as an override key exactly when it defines + an agent emitter to consume the block. + """ + return { + name for name, module in _REGISTRY.items() + if provided(module.SPEC.agent_emitter) + } diff --git a/swarmforge/harness/claude/__init__.py b/swarmforge/harness/claude/__init__.py new file mode 100644 index 0000000..591ef0f --- /dev/null +++ b/swarmforge/harness/claude/__init__.py @@ -0,0 +1,122 @@ +"""The Claude Code harness.""" + +from swarmforge.agents.emit import OPENCODE_ONLY_FIELDS, render, warn +from swarmforge.harness.spec import HarnessSpec, Waiver + +# OpenCode tool id -> Claude Code tool name. Ids mapping to None have no +# Claude equivalent and are dropped. +CLAUDE_TOOL_NAMES = { + "bash": "Bash", + "edit": "Edit", + "write": "Write", + "read": "Read", + "grep": "Grep", + "glob": "Glob", + "list": None, + "patch": None, + "skill": "Skill", + "task": "Task", + "todoread": None, + "todowrite": "TodoWrite", + "webfetch": "WebFetch", + "websearch": "WebSearch", +} + + +def _override_keys(): + # Imported at call time: the registry package imports this module while + # building the harness table, so the table does not exist yet at import. + from swarmforge import harness + + return harness.agent_override_keys() + + +def to_claude(name, meta): + if meta.get("disable") is True: + return None + out = {"name": meta.get("name", name)} + if "description" in meta: + out["description"] = meta["description"] + else: + warn("agent '%s' has no description" % name) + + model = meta.get("model") + if model is not None: + provider, sep, model_id = str(model).partition("/") + if not sep: + out["model"] = model + elif provider == "anthropic": + out["model"] = model_id + + tools = meta.get("tools") + if isinstance(tools, dict): + disallowed = [] + for tool, enabled in tools.items(): + if enabled is not False: + continue + mapped = CLAUDE_TOOL_NAMES.get(tool) + if mapped is None: + if tool not in CLAUDE_TOOL_NAMES: + warn("agent '%s': unknown tool '%s' skipped" % (name, tool)) + continue + disallowed.append(mapped) + if disallowed: + out["disallowedTools"] = ", ".join(disallowed) + elif tools is not None: + warn("agent '%s': 'tools' must be a map of tool -> bool" % name) + + skipped = OPENCODE_ONLY_FIELDS | _override_keys() | {"name", "description", "model"} + for key, value in meta.items(): + if key not in skipped: + out[key] = value + + overrides = meta.get("claude") + if isinstance(overrides, dict): + out.update(overrides) + return out + + +def agent_emitter(name, meta, body): + """The native filename and full file text for one agent, or None to skip it.""" + out = to_claude(name, meta) + if out is None: + return None + return "%s.md" % name, render(out, body) + + +def mcp_fragment(servers): + """Claude Code `--mcp-config` document for the given servers. + + HTTP MCP servers keyed by canonical alias under `mcpServers`, the shape + Claude reads from the file passed as `claude --mcp-config `. Returns + `{}` when `servers` is empty. + """ + out = {alias: {"type": "http", "url": url} for alias, url in servers.items()} + return {"mcpServers": out} if out else {} + + +SPEC = HarnessSpec( + name="claude", + binary="claude", + config_dest="/run/swarmforge/claude-config", + config_reset=False, + layer_excludes=( + "./skills", + "./commands", + "./agents", + "./settings.json", + "./.credentials.json", + ), + keyed_files=("opencode.json",), + skills_dest="{config}/skills", + commands_dest="{config}/commands", + agents_dest="{config}/agents", + mcp_fragment=mcp_fragment, + mcp_delivery=("flag", "--mcp-config"), + mcp_merge=Waiver( + "the fragment reaches claude on its command line; nothing merges it " + "into a config file" + ), + agent_emitter=agent_emitter, + extra_chown_paths=("/run/swarmforge/claude-config",), +) diff --git a/swarmforge/harness/codex/__init__.py b/swarmforge/harness/codex/__init__.py new file mode 100644 index 0000000..0266587 --- /dev/null +++ b/swarmforge/harness/codex/__init__.py @@ -0,0 +1,136 @@ +"""The Codex CLI harness.""" + +import os +import re + +from swarmforge.agents.emit import ( + emit_toml_key, + emit_toml_multiline, + emit_toml_value, + warn, +) +from swarmforge.harness.spec import HarnessSpec, Waiver, toml_mcp_fragment + +CODEX_AGENT_TABLE_FIELDS = { + "default_subagent_model", + "enabled", + "max_depth", +} + + +def render_codex(meta): + lines = [] + for key, value in meta.items(): + rendered = ( + emit_toml_multiline(value) + if key == "developer_instructions" + else emit_toml_value(value) + ) + lines.append("%s = %s" % (emit_toml_key(key), rendered)) + return "\n".join(lines) + "\n" + + +def normalize_codex_name(name): + normalized = re.sub(r"[^A-Za-z0-9 _-]+", "-", str(name)) + normalized = normalized.strip(" _-") or "agent" + if normalized in CODEX_AGENT_TABLE_FIELDS: + normalized = "agent-" + normalized + return normalized + + +def registered_name(name, meta): + """The name an emitted agent registers under in config.toml. + + The declared `name` (falling back to the source filename) after codex + normalization, with a `codex:` override block's own `name` winning. + """ + requested = meta.get("name", name) + overrides = meta.get("codex") + if isinstance(overrides, dict) and "name" in overrides: + requested = overrides["name"] + return normalize_codex_name(requested) + + +def to_codex(name, meta, body): + if meta.get("disable") is True: + return None + requested_name = meta.get("name", name) + codex_name = normalize_codex_name(requested_name) + if codex_name != requested_name: + warn("agent '%s': Codex name normalized to '%s'" % (name, codex_name)) + out = {"name": codex_name, "developer_instructions": body} + if "description" in meta: + out["description"] = meta["description"] + else: + warn("agent '%s' has no description" % name) + + model = meta.get("model") + if model is not None: + provider, sep, model_id = str(model).partition("/") + if not sep: + out["model"] = model + elif provider == "openai": + out["model"] = model_id + + if "tools" in meta: + warn( + "agent '%s': tool restrictions are not translated for Codex; " + "use codex sandbox/MCP settings" % name + ) + + overrides = meta.get("codex") + if isinstance(overrides, dict): + out.update(overrides) + out["name"] = registered_name(name, meta) + return out + + +def agent_emitter(name, meta, body): + """The native filename and full file text for one agent, or None to skip it.""" + out = to_codex(name, meta, body) + if out is None: + return None + return "%s.toml" % normalize_codex_name(name), render_codex(out) + + +def finalize_agents(dest_dir, emitted): + """Register every emitted agent in a config.toml beside the agent files.""" + registrations = {} + for name, meta, path in emitted: + registrations[registered_name(name, meta)] = { + "config_file": os.path.abspath(path) + } + if not registrations: + return + config_path = os.path.join(dest_dir, "config.toml") + with open(config_path, "w", encoding="utf-8") as handle: + handle.write(render_codex({"agents": registrations})) + + +SPEC = HarnessSpec( + name="codex", + binary="codex", + config_dest="/run/swarmforge/codex-config", + config_reset=True, + layer_excludes=( + "./skills", + "./packages", + "./sessions", + "./history.jsonl", + "./log", + "./config.toml", + ), + keyed_files=("opencode.json",), + skills_dest="{home}/.agents/skills", + commands_dest=Waiver( + "portable commands become skill packages under the skills destination " + "instead of a commands directory" + ), + agents_dest="/run/swarmforge/codex-agents", + mcp_fragment=toml_mcp_fragment, + mcp_delivery=("env", "SWARMFORGE_TONG_MCP_FILE"), + mcp_merge="toml-managed-block", + agent_emitter=agent_emitter, + finalize_agents=finalize_agents, + extra_chown_paths=("/run/swarmforge/codex-agents",), +) diff --git a/swarmforge/harness/codex/commands.py b/swarmforge/harness/codex/commands.py new file mode 100644 index 0000000..a51c630 --- /dev/null +++ b/swarmforge/harness/codex/commands.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Translate portable slash commands into Codex skill packages. + +Usage: python3 -m swarmforge.commands.translate +""" + +import os +import re +import shutil +import sys + +from swarmforge.agents.emit import split_frontmatter + + +SHELL_INTERPOLATION_RE = re.compile(r"!`([^`\n]+)`") +POSITIONAL_RE = re.compile(r"\$(\d+)") + + +def warn(message): + print("swarmforge.commands.translate: %s" % message, file=sys.stderr) + + +def describe_positionals(command): + positions = sorted({int(value) for value in POSITIONAL_RE.findall(command)}) + if not positions: + return "" + labels = ", ".join("$%d" % position for position in positions) + return ", replacing %s with the corresponding positional invocation argument%s" % ( + labels, + "" if len(positions) == 1 else "s", + ) + + +def translate_body(body): + body = body.replace( + "$ARGUMENTS", "the arguments supplied with this skill invocation" + ) + + def shell_instruction(match): + command = match.group(1) + return "Run `%s`%s and use its output." % ( + command, + describe_positionals(command), + ) + + return SHELL_INTERPOLATION_RE.sub(shell_instruction, body) + + +def translate_file(path, dest_dir): + filename = os.path.basename(path) + name = filename[:-3] + with open(path, "r", encoding="utf-8") as handle: + meta, body = split_frontmatter(handle.read()) + description = meta.get("description") + if not description: + warn("skipping %s: command has no description" % path) + return + skill_dir = os.path.join(dest_dir, name) + if os.path.lexists(skill_dir): + if os.path.isdir(skill_dir) and not os.path.islink(skill_dir): + shutil.rmtree(skill_dir) + else: + os.unlink(skill_dir) + os.makedirs(skill_dir) + with open(os.path.join(skill_dir, "SKILL.md"), "w", encoding="utf-8") as handle: + handle.write( + "---\nname: %s\ndescription: %s\n---\n\n%s" + % (name, description, translate_body(body)) + ) + + +def main(argv): + if len(argv) != 2: + print(__doc__.strip(), file=sys.stderr) + return 2 + dest_dir, src_dir = argv + if not src_dir or not os.path.isdir(src_dir): + return 0 + os.makedirs(dest_dir, exist_ok=True) + for filename in sorted(os.listdir(src_dir)): + path = os.path.join(src_dir, filename) + if filename.endswith(".md") and os.path.isfile(path): + try: + translate_file(path, dest_dir) + except ValueError as exc: + warn("skipping %s: %s" % (path, exc)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/swarmforge/harness/grok/__init__.py b/swarmforge/harness/grok/__init__.py new file mode 100644 index 0000000..a2dddda --- /dev/null +++ b/swarmforge/harness/grok/__init__.py @@ -0,0 +1,28 @@ +"""The Grok Build harness.""" + +from swarmforge.harness.spec import HarnessSpec, Waiver, toml_mcp_fragment + +SPEC = HarnessSpec( + name="grok", + binary="grok", + config_dest=Waiver( + "the run's SWARMFORGE_CONFIG_DEST names the destination, and an unset " + "variable skips the config phase" + ), + config_reset=False, + layer_excludes=("./skills", "./commands", "./bin", "./downloads", "./completions"), + keyed_files=("opencode.json",), + skills_dest="{home}/.grok/skills", + commands_dest="{home}/.grok/commands", + agents_dest=Waiver( + "no destination is declared and unified agent definitions are not " + "delivered to grok" + ), + mcp_fragment=toml_mcp_fragment, + mcp_delivery=("env", "SWARMFORGE_TONG_MCP_FILE"), + mcp_merge="toml-managed-block", + agent_emitter=Waiver( + "no emitter is defined, so the translator rejects grok as a target" + ), + extra_chown_paths=(), +) diff --git a/swarmforge/harness/opencode/__init__.py b/swarmforge/harness/opencode/__init__.py new file mode 100644 index 0000000..507824f --- /dev/null +++ b/swarmforge/harness/opencode/__init__.py @@ -0,0 +1,64 @@ +"""The OpenCode harness.""" + +from swarmforge.agents.emit import render +from swarmforge.harness.spec import HarnessSpec, Waiver + + +def _override_keys(): + # Imported at call time: the registry package imports this module while + # building the harness table, so the table does not exist yet at import. + from swarmforge import harness + + return harness.agent_override_keys() + + +def to_opencode(name, meta): + override_keys = _override_keys() + out = {k: v for k, v in meta.items() if k not in override_keys and k != "name"} + model = out.get("model") + if model is not None and "/" not in str(model): + del out["model"] + overrides = meta.get("opencode") + if isinstance(overrides, dict): + out.update(overrides) + return out + + +def agent_emitter(name, meta, body): + """The native filename and full file text for one agent, or None to skip it.""" + return "%s.md" % name, render(to_opencode(name, meta), body) + + +def mcp_fragment(servers): + """OpenCode `mcp` fragment for the given servers. + + Remote (HTTP) MCP servers keyed by canonical alias, shaped for merging + into `opencode.json`. Returns `{}` when `servers` is empty, so the + fragment is omitted entirely. + """ + out = { + alias: {"type": "remote", "url": url, "enabled": True} + for alias, url in servers.items() + } + return {"mcp": out} if out else {} + + +SPEC = HarnessSpec( + name="opencode", + binary="opencode", + config_dest=Waiver( + "the run's SWARMFORGE_CONFIG_DEST names the destination, and an unset " + "variable skips the config phase" + ), + config_reset=False, + layer_excludes=("./skills", "./command"), + keyed_files=("opencode.json",), + skills_dest="{config}/skills", + commands_dest="{config}/command", + agents_dest="{config}/agents", + mcp_fragment=mcp_fragment, + mcp_delivery=("env", "SWARMFORGE_TONG_MCP_FILE"), + mcp_merge="json-replace-mcp", + agent_emitter=agent_emitter, + extra_chown_paths=(), +) diff --git a/swarmforge/harness/spec.py b/swarmforge/harness/spec.py new file mode 100644 index 0000000..15fd5c6 --- /dev/null +++ b/swarmforge/harness/spec.py @@ -0,0 +1,107 @@ +"""The contract a harness module declares to the rest of Swarmforge.""" + +import dataclasses + + +@dataclasses.dataclass(frozen=True) +class Waiver: + """An explicit opt-out of a contract field. + + A field holding a Waiver is declared unimplemented, with the reason on + record -- distinct from a field nobody filled in, which the mandatory + constructor arguments make impossible to express. + """ + + reason: str + + +def provided(value): + """True when a contract value is a real declaration rather than an opt-out.""" + return value is not None and not isinstance(value, Waiver) + + +def finalize_agents(dest_dir, emitted): + """Default finalize-agents hook: nothing follows the emitted files.""" + + +def toml_mcp_fragment(servers): + """`mcp_servers` fragment for the given servers, TOML-shaped. + + HTTP MCP servers keyed by canonical alias, in the shape Grok Build and + Codex CLI share: TOML `[mcp_servers.]` tables, where a `url` key is + what selects the remote transport -- there is no type key. The fragment + stays JSON here; swarmforge.config.merge_toml_mcp renders it. Returns + `{}` when `servers` is empty. + """ + out = {alias: {"url": url} for alias, url in servers.items()} + return {"mcp_servers": out} if out else {} + + +@dataclasses.dataclass(frozen=True) +class HarnessSpec: + """Everything Swarmforge needs to know about one harness. + + The fields record the facts the container entrypoint acts on per harness: + where its config and assets live, how it learns about MCP servers, and how + unified agent definitions reach it. A field a harness does not implement + holds a `Waiver` naming the reason. + """ + + # The name the harness is registered and selected by. + name: str + + # The executable under /usr/local/bin the entrypoint execs. + binary: str + + # Container path the layered config is merged into when the harness forces + # one; a Waiver when the run's SWARMFORGE_CONFIG_DEST decides instead, and + # an unset variable skips the config phase. + config_dest: object + + # True when the harness always rebuilds the config destination from + # scratch; False when the run's SWARMFORGE_CONFIG_RESET decides. + config_reset: bool + + # Additions to the shared config-layer tar excludes ("./opencode.json", + # "./.swarmforge"), applied when a config layer is merged. + layer_excludes: tuple + + # Files merged key-by-key per layer rather than overlaid whole. + keyed_files: tuple + + # Where portable skills and commands land. A string may hold the + # placeholders "{home}" (the anvil user's home) and "{config}" (the merged + # config destination); a Waiver opts the harness out. + skills_dest: object + commands_dest: object + + # Where translated native agents land, under the same placeholder rules. + agents_dest: object + + # Callable `(servers) -> dict` shaping `{alias: url}` into the harness's + # MCP config fragment, `{}` for no servers. + mcp_fragment: object + + # How the anvil learns the generated MCP config path: ("flag", FLAG) + # appends `FLAG ` to the harness argv, ("env", VAR) sets + # `VAR=` for the entrypoint to merge. + mcp_delivery: tuple + + # How the delivered fragment merges into the harness config: + # "json-replace-mcp" (opencode.json key merge, whole MCP entries replaced) + # or "toml-managed-block" (a rewritten managed block in config.toml); a + # Waiver when nothing merges it into a file. + mcp_merge: object + + # Callable `(name, meta, body) -> (filename, text) | None` producing one + # native agent file, or None to skip that agent; a Waiver when the harness + # has no emitter and unified agents are not translated for it. + agent_emitter: object + + # Container paths outside the home handed to the anvil uid before + # privileges drop. + extra_chown_paths: tuple + + # Hook `(dest_dir, emitted)` run after every agent file is written, where + # `emitted` lists `(name, meta, path)` for the agents actually emitted. + finalize_agents: object = finalize_agents diff --git a/swarmforge/tongs/mcp.py b/swarmforge/tongs/mcp.py index d366f69..0bf3108 100644 --- a/swarmforge/tongs/mcp.py +++ b/swarmforge/tongs/mcp.py @@ -2,6 +2,10 @@ import re +from swarmforge import harness as harnesses +from swarmforge.harness import claude as _claude, grok as _grok, opencode as _opencode +from swarmforge.harness.spec import provided + from .model import ENV_PREFIX, warn @@ -179,56 +183,45 @@ def mcp_tongs(merged): return out -def mcp_config_opencode(merged): - """OpenCode `mcp` fragment for the discovered `mcp` tongs. - - Remote (HTTP) MCP servers keyed by canonical alias, shaped for merging into - `opencode.json` through the entrypoint's existing merge path. Returns `{}` - when no `mcp` tongs exist, so the fragment is omitted entirely. - """ - servers = {} - for alias, defn in mcp_tongs(merged).items(): - servers[alias] = {"type": "remote", "url": mcp_url(defn, alias), "enabled": True} - return {"mcp": servers} if servers else {} +def _mcp_servers(merged): + """Canonical alias -> endpoint URL for every mcp tong in the merged set.""" + return {alias: mcp_url(defn, alias) for alias, defn in mcp_tongs(merged).items()} -def mcp_config_claude(merged): - """Claude Code `--mcp-config` document for the discovered `mcp` tongs. +_MERGED_EMITTERS = {} - HTTP MCP servers keyed by canonical alias under `mcpServers`, the shape - Claude reads from the file passed as `claude --mcp-config `. Returns - `{}` when no `mcp` tongs exist. - """ - servers = {} - for alias, defn in mcp_tongs(merged).items(): - servers[alias] = {"type": "http", "url": mcp_url(defn, alias)} - return {"mcpServers": servers} if servers else {} +def _merged_emitter(fragment): + """The merged-set emitter for one harness `mcp_fragment`. -def mcp_config_toml(merged): - """`mcp_servers` fragment for the discovered `mcp` tongs, TOML-shaped. - - HTTP MCP servers keyed by canonical alias, in the shape Grok Build and - Codex CLI share: TOML `[mcp_servers.]` tables, where a `url` key is - what selects the remote transport -- there is no type key. The fragment - stays JSON here; swarmforge.config.merge_toml_mcp renders it. Returns - `{}` when no `mcp` tongs exist. + One emitter per distinct fragment function: harnesses that share a + fragment (the TOML-config pair) share the emitter object too. """ - servers = {} - for alias, defn in mcp_tongs(merged).items(): - servers[alias] = {"url": mcp_url(defn, alias)} - return {"mcp_servers": servers} if servers else {} + if fragment not in _MERGED_EMITTERS: + def emitter(merged): + """The harness's `mcp_fragment` over the mcp tongs in `merged`.""" + return fragment(_mcp_servers(merged)) + + _MERGED_EMITTERS[fragment] = emitter + return _MERGED_EMITTERS[fragment] -# Per-harness MCP emitters, dispatched by harness name, mirroring the EMITTERS -# table in swarmforge/agents/translate.py. +# Per-harness MCP emitters keyed by harness name, read off the harness +# registry: every harness that declares an `mcp_fragment` gets the merged-set +# emitter for it, so adding a harness needs no table here. MCP_EMITTERS = { - "opencode": mcp_config_opencode, - "claude": mcp_config_claude, - "grok": mcp_config_toml, - "codex": mcp_config_toml, + name: _merged_emitter(harnesses.get(name).SPEC.mcp_fragment) + for name in harnesses.names() + if provided(harnesses.get(name).SPEC.mcp_fragment) } +# Named emitters for the shapes callers ask for directly: the fragment merged +# into `opencode.json`, the `--mcp-config` document, and the `mcp_servers` +# tables Grok Build and Codex CLI share. +mcp_config_opencode = _merged_emitter(_opencode.SPEC.mcp_fragment) +mcp_config_claude = _merged_emitter(_claude.SPEC.mcp_fragment) +mcp_config_toml = _merged_emitter(_grok.SPEC.mcp_fragment) + def plan_injection(merged, harness): """Everything the discovered tongs contribute to one anvil launch. @@ -252,6 +245,7 @@ def plan_injection(merged, harness): continue env[key] = value mounts.extend(anvil_mounts(name, defn)) - emit = MCP_EMITTERS.get(harness) - mcp = emit(merged) if emit else {} + module = harnesses.get(harness) + fragment = module.SPEC.mcp_fragment if module is not None else None + mcp = _merged_emitter(fragment)(merged) if provided(fragment) else {} return {"env": env, "mounts": mounts, "mcp": mcp} diff --git a/tests/make_argv_fixtures.py b/tests/make_argv_fixtures.py new file mode 100644 index 0000000..c4ed4c3 --- /dev/null +++ b/tests/make_argv_fixtures.py @@ -0,0 +1,486 @@ +"""Recorded make-target argv, pinned word for word. + +RUN_ARGV holds the complete argv each `run_*` target hands PYTHON: the +launcher path, the launcher's own flags, and the `docker run` command +after the `--` separator. BUILD_ARGV holds the argv each `build_*` +recipe hands docker. Both are recorded against the throwaway checkout +the argv tests lay out, with machine-specific words replaced by +placeholders: + + {SWARMFORGE} this repository's root + {TMP} the test's temporary directory (its HOME and the + project checkout both live under it) + {UID} {GID} the invoking user's ids + +A mismatch means the make interface changed. Fix the recipe, or -- for +a deliberate change -- update the recording in the same diff: print the +normalized argv from the failing test and paste it here. Nothing +rewrites this file automatically. +""" + +import os + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def normalize(argv, tmp): + """Replace machine-specific words with the placeholders above.""" + uid, gid = str(os.getuid()), str(os.getgid()) + normalized = [] + for arg in argv: + arg = arg.replace(tmp, "{TMP}").replace(REPO_ROOT, "{SWARMFORGE}") + if arg == "SWARMFORGE_UID=" + uid: + arg = "SWARMFORGE_UID={UID}" + elif arg == "SWARMFORGE_GID=" + gid: + arg = "SWARMFORGE_GID={GID}" + normalized.append(arg) + return normalized + + +RUN_ARGV = {'run_claude': ['{SWARMFORGE}/bin/run-anvil', + '--repo-tongs', + '{SWARMFORGE}/tongs', + '--workspace-tongs', + '{TMP}/proj/.swarmforge/tongs', + '--workspace', + '{TMP}/proj', + '--approvals', + '{TMP}/home/.swarmforge/approvals.json', + '--providers', + '{TMP}/home/.swarmforge/secret-providers.yaml', + '--harness', + 'claude', + '--anvil-image', + 'claude-code:local', + '--', + 'docker', + 'run', + '-it', + '--rm', + '--name', + 'claude-proj', + '--network', + 'opencode-net', + '-e', + 'SWARMFORGE_UID={UID}', + '-e', + 'SWARMFORGE_GID={GID}', + '-e', + 'TZ=Etc/UTC', + '-e', + 'TERM', + '-e', + 'COLORTERM', + '-e', + 'SWARMFORGE_AGENT_BIN=claude', + '-e', + 'SWARMFORGE_CONFIG_USER_DIR=/tmp/swarmforge-config/user', + '-e', + 'SWARMFORGE_CONFIG_ORG_DIR=/tmp/swarmforge-config/org', + '-e', + 'SWARMFORGE_CONFIG_REPO_DIR=/tmp/swarmforge-config/repo', + '-e', + 'SWARMFORGE_ASSETS_USER_DIR=/tmp/swarmforge-assets/user', + '-e', + 'SWARMFORGE_ASSETS_ORG_DIR=/tmp/swarmforge-assets/org', + '-e', + 'SWARMFORGE_ASSETS_REPO_DIR=/tmp/swarmforge-assets/repo', + '-e', + 'SWARMFORGE_DOTAGENTS_USER_DIR=/tmp/swarmforge-dotagents/user', + '-e', + 'SWARMFORGE_DOTAGENTS_ORG_DIR=/tmp/swarmforge-dotagents/org', + '-e', + 'SWARMFORGE_CONFIG_DEST=', + '-e', + 'SWARMFORGE_CONFIG_RESET=0', + '-e', + 'SWARMFORGE_SKILLS_DIR=/home/anvil/.swarmforge/skills', + '-e', + 'SWARMFORGE_COMMAND_DIR=/home/anvil/.swarmforge/command', + '-v', + '{TMP}/proj:/workspace', + '-v', + '{TMP}/proj:/repos/proj', + '-v', + '{TMP}/home/.local/share/claude/home:/home/anvil', + '-v', + '{TMP}/home/.local/share/claude/home/.claude/plugins:/home/anvil/.claude/plugins:ro', + '-v', + '{TMP}/home/.claude:/tmp/swarmforge-config/user:ro', + '-v', + '{SWARMFORGE}/skills:/home/anvil/.swarmforge/skills:ro', + '-v', + '{SWARMFORGE}/commands:/home/anvil/.swarmforge/command:ro', + '-v', + '{TMP}/proj/.git:/workspace/.git', + '-v', + '{TMP}/proj/.git:/repos/proj/.git', + '-v', + '{TMP}/proj/.git/config:/workspace/.git/config:ro', + '-v', + '{TMP}/proj/.git/config:/repos/proj/.git/config:ro', + '-v', + '{TMP}/proj/.git/commondir:/workspace/.git/commondir:ro', + '-v', + '{TMP}/proj/.git/commondir:/repos/proj/.git/commondir:ro', + '-v', + '{TMP}/proj/.git/hooks:/workspace/.git/hooks:ro', + '-v', + '{TMP}/proj/.git/hooks:/repos/proj/.git/hooks:ro', + '-v', + '{TMP}/proj/.git/remotes:/workspace/.git/remotes:ro', + '-v', + '{TMP}/proj/.git/remotes:/repos/proj/.git/remotes:ro', + '-v', + '{TMP}/proj/.git/branches:/workspace/.git/branches:ro', + '-v', + '{TMP}/proj/.git/branches:/repos/proj/.git/branches:ro', + '-w', + '/repos/proj', + 'claude-code:local'], + 'run_codex': ['{SWARMFORGE}/bin/run-anvil', + '--repo-tongs', + '{SWARMFORGE}/tongs', + '--workspace-tongs', + '{TMP}/proj/.swarmforge/tongs', + '--workspace', + '{TMP}/proj', + '--approvals', + '{TMP}/home/.swarmforge/approvals.json', + '--providers', + '{TMP}/home/.swarmforge/secret-providers.yaml', + '--harness', + 'codex', + '--anvil-image', + 'codex-cli:local', + '--', + 'docker', + 'run', + '-it', + '--rm', + '--name', + 'codex-proj', + '--network', + 'opencode-net', + '-e', + 'SWARMFORGE_UID={UID}', + '-e', + 'SWARMFORGE_GID={GID}', + '-e', + 'TZ=Etc/UTC', + '-e', + 'TERM', + '-e', + 'COLORTERM', + '-e', + 'SWARMFORGE_AGENT_BIN=codex', + '-e', + 'SWARMFORGE_CONFIG_USER_DIR=/tmp/swarmforge-config/user', + '-e', + 'SWARMFORGE_CONFIG_ORG_DIR=/tmp/swarmforge-config/org', + '-e', + 'SWARMFORGE_CONFIG_REPO_DIR=/tmp/swarmforge-config/repo', + '-e', + 'SWARMFORGE_ASSETS_USER_DIR=/tmp/swarmforge-assets/user', + '-e', + 'SWARMFORGE_ASSETS_ORG_DIR=/tmp/swarmforge-assets/org', + '-e', + 'SWARMFORGE_ASSETS_REPO_DIR=/tmp/swarmforge-assets/repo', + '-e', + 'SWARMFORGE_DOTAGENTS_USER_DIR=/tmp/swarmforge-dotagents/user', + '-e', + 'SWARMFORGE_DOTAGENTS_ORG_DIR=/tmp/swarmforge-dotagents/org', + '-e', + 'SWARMFORGE_CONFIG_DEST=', + '-e', + 'SWARMFORGE_CONFIG_RESET=0', + '-e', + 'SWARMFORGE_SKILLS_DIR=/home/anvil/.swarmforge/skills', + '-e', + 'SWARMFORGE_COMMAND_DIR=/home/anvil/.swarmforge/command', + '-v', + '{TMP}/proj:/workspace', + '-v', + '{TMP}/proj:/repos/proj', + '-v', + '{TMP}/home/.local/share/codex/home:/home/anvil', + '--tmpfs', + '/home/anvil/.agents/skills:exec', + '-v', + '{TMP}/home/.codex:/tmp/swarmforge-config/user:ro', + '-v', + '{SWARMFORGE}/skills:/home/anvil/.swarmforge/skills:ro', + '-v', + '{SWARMFORGE}/commands:/home/anvil/.swarmforge/command:ro', + '-v', + '{TMP}/proj/.git:/workspace/.git', + '-v', + '{TMP}/proj/.git:/repos/proj/.git', + '-v', + '{TMP}/proj/.git/config:/workspace/.git/config:ro', + '-v', + '{TMP}/proj/.git/config:/repos/proj/.git/config:ro', + '-v', + '{TMP}/proj/.git/commondir:/workspace/.git/commondir:ro', + '-v', + '{TMP}/proj/.git/commondir:/repos/proj/.git/commondir:ro', + '-v', + '{TMP}/proj/.git/hooks:/workspace/.git/hooks:ro', + '-v', + '{TMP}/proj/.git/hooks:/repos/proj/.git/hooks:ro', + '-v', + '{TMP}/proj/.git/remotes:/workspace/.git/remotes:ro', + '-v', + '{TMP}/proj/.git/remotes:/repos/proj/.git/remotes:ro', + '-v', + '{TMP}/proj/.git/branches:/workspace/.git/branches:ro', + '-v', + '{TMP}/proj/.git/branches:/repos/proj/.git/branches:ro', + '-w', + '/repos/proj', + 'codex-cli:local'], + 'run_grok': ['{SWARMFORGE}/bin/run-anvil', + '--repo-tongs', + '{SWARMFORGE}/tongs', + '--workspace-tongs', + '{TMP}/proj/.swarmforge/tongs', + '--workspace', + '{TMP}/proj', + '--approvals', + '{TMP}/home/.swarmforge/approvals.json', + '--providers', + '{TMP}/home/.swarmforge/secret-providers.yaml', + '--harness', + 'grok', + '--anvil-image', + 'grok-build:local', + '--', + 'docker', + 'run', + '-it', + '--rm', + '--name', + 'grok-proj', + '--network', + 'opencode-net', + '-e', + 'SWARMFORGE_UID={UID}', + '-e', + 'SWARMFORGE_GID={GID}', + '-e', + 'TZ=Etc/UTC', + '-e', + 'TERM', + '-e', + 'COLORTERM', + '-e', + 'SWARMFORGE_AGENT_BIN=grok', + '-e', + 'SWARMFORGE_CONFIG_USER_DIR=/tmp/swarmforge-config/user', + '-e', + 'SWARMFORGE_CONFIG_ORG_DIR=/tmp/swarmforge-config/org', + '-e', + 'SWARMFORGE_CONFIG_REPO_DIR=/tmp/swarmforge-config/repo', + '-e', + 'SWARMFORGE_ASSETS_USER_DIR=/tmp/swarmforge-assets/user', + '-e', + 'SWARMFORGE_ASSETS_ORG_DIR=/tmp/swarmforge-assets/org', + '-e', + 'SWARMFORGE_ASSETS_REPO_DIR=/tmp/swarmforge-assets/repo', + '-e', + 'SWARMFORGE_DOTAGENTS_USER_DIR=/tmp/swarmforge-dotagents/user', + '-e', + 'SWARMFORGE_DOTAGENTS_ORG_DIR=/tmp/swarmforge-dotagents/org', + '-e', + 'SWARMFORGE_CONFIG_DEST=/home/anvil/.grok', + '-e', + 'SWARMFORGE_CONFIG_RESET=0', + '-e', + 'SWARMFORGE_SKILLS_DIR=/home/anvil/.swarmforge/skills', + '-e', + 'SWARMFORGE_COMMAND_DIR=/home/anvil/.swarmforge/command', + '-v', + '{TMP}/proj:/workspace', + '-v', + '{TMP}/proj:/repos/proj', + '-v', + '{TMP}/home/.local/share/grok/home:/home/anvil', + '--tmpfs', + '/home/anvil/.grok/skills:exec', + '--tmpfs', + '/home/anvil/.grok/commands', + '-v', + '{TMP}/home/.grok:/tmp/swarmforge-config/user:ro', + '-v', + '{SWARMFORGE}/skills:/home/anvil/.swarmforge/skills:ro', + '-v', + '{SWARMFORGE}/commands:/home/anvil/.swarmforge/command:ro', + '-v', + '{TMP}/proj/.git:/workspace/.git', + '-v', + '{TMP}/proj/.git:/repos/proj/.git', + '-v', + '{TMP}/proj/.git/config:/workspace/.git/config:ro', + '-v', + '{TMP}/proj/.git/config:/repos/proj/.git/config:ro', + '-v', + '{TMP}/proj/.git/commondir:/workspace/.git/commondir:ro', + '-v', + '{TMP}/proj/.git/commondir:/repos/proj/.git/commondir:ro', + '-v', + '{TMP}/proj/.git/hooks:/workspace/.git/hooks:ro', + '-v', + '{TMP}/proj/.git/hooks:/repos/proj/.git/hooks:ro', + '-v', + '{TMP}/proj/.git/remotes:/workspace/.git/remotes:ro', + '-v', + '{TMP}/proj/.git/remotes:/repos/proj/.git/remotes:ro', + '-v', + '{TMP}/proj/.git/branches:/workspace/.git/branches:ro', + '-v', + '{TMP}/proj/.git/branches:/repos/proj/.git/branches:ro', + '-w', + '/repos/proj', + 'grok-build:local'], + 'run_opencode': ['{SWARMFORGE}/bin/run-anvil', + '--repo-tongs', + '{SWARMFORGE}/tongs', + '--workspace-tongs', + '{TMP}/proj/.swarmforge/tongs', + '--workspace', + '{TMP}/proj', + '--approvals', + '{TMP}/home/.swarmforge/approvals.json', + '--providers', + '{TMP}/home/.swarmforge/secret-providers.yaml', + '--harness', + 'opencode', + '--anvil-image', + 'opencode:local', + '--', + 'docker', + 'run', + '-it', + '--rm', + '--name', + 'opencode-proj', + '--network', + 'opencode-net', + '-e', + 'SWARMFORGE_UID={UID}', + '-e', + 'SWARMFORGE_GID={GID}', + '-e', + 'TZ=Etc/UTC', + '-e', + 'TERM', + '-e', + 'COLORTERM', + '-e', + 'SWARMFORGE_CONFIG_USER_DIR=/tmp/swarmforge-config/user', + '-e', + 'SWARMFORGE_CONFIG_ORG_DIR=/tmp/swarmforge-config/org', + '-e', + 'SWARMFORGE_CONFIG_REPO_DIR=/tmp/swarmforge-config/repo', + '-e', + 'SWARMFORGE_ASSETS_USER_DIR=/tmp/swarmforge-assets/user', + '-e', + 'SWARMFORGE_ASSETS_ORG_DIR=/tmp/swarmforge-assets/org', + '-e', + 'SWARMFORGE_ASSETS_REPO_DIR=/tmp/swarmforge-assets/repo', + '-e', + 'SWARMFORGE_DOTAGENTS_USER_DIR=/tmp/swarmforge-dotagents/user', + '-e', + 'SWARMFORGE_DOTAGENTS_ORG_DIR=/tmp/swarmforge-dotagents/org', + '-e', + 'SWARMFORGE_CONFIG_DEST=/home/anvil/.config/opencode', + '-e', + 'SWARMFORGE_CONFIG_RESET=1', + '-e', + 'SWARMFORGE_SKILLS_DIR=/home/anvil/.swarmforge/skills', + '-e', + 'SWARMFORGE_COMMAND_DIR=/home/anvil/.swarmforge/command', + '-v', + '{TMP}/proj:/workspace', + '-v', + '{TMP}/home/.config/opencode:/tmp/swarmforge-config/user:ro', + '-v', + '{SWARMFORGE}/opencode:/tmp/swarmforge-config/repo:ro', + '-v', + '{SWARMFORGE}/skills:/home/anvil/.swarmforge/skills:ro', + '-v', + '{SWARMFORGE}/commands:/home/anvil/.swarmforge/command:ro', + '-v', + '{TMP}/home/.local/share/opencode:/home/anvil/.local/share/opencode', + '-v', + '{TMP}/proj/.git:/workspace/.git', + '-v', + '{TMP}/proj/.git/config:/workspace/.git/config:ro', + '-v', + '{TMP}/proj/.git/commondir:/workspace/.git/commondir:ro', + '-v', + '{TMP}/proj/.git/hooks:/workspace/.git/hooks:ro', + '-v', + '{TMP}/proj/.git/remotes:/workspace/.git/remotes:ro', + '-v', + '{TMP}/proj/.git/branches:/workspace/.git/branches:ro', + 'opencode:local']} + +BUILD_ARGV = {'build_claude': ['build', + '--target', + 'claude-runtime', + '--build-arg', + 'AGENT=claude', + '--build-arg', + 'DEBIAN_TAG=trixie-slim', + '--build-arg', + 'SWARMFORGE_HARNESS_INSTALL_BUST=0', + '-f', + '{SWARMFORGE}/anvil/Dockerfile', + '-t', + 'claude-code:local', + '{SWARMFORGE}'], + 'build_codex': ['build', + '--target', + 'codex-runtime', + '--build-arg', + 'AGENT=codex', + '--build-arg', + 'DEBIAN_TAG=trixie-slim', + '--build-arg', + 'SWARMFORGE_HARNESS_INSTALL_BUST=0', + '-f', + '{SWARMFORGE}/anvil/Dockerfile', + '-t', + 'codex-cli:local', + '{SWARMFORGE}'], + 'build_grok': ['build', + '--target', + 'grok-runtime', + '--build-arg', + 'AGENT=grok', + '--build-arg', + 'DEBIAN_TAG=trixie-slim', + '--build-arg', + 'SWARMFORGE_HARNESS_INSTALL_BUST=0', + '-f', + '{SWARMFORGE}/anvil/Dockerfile', + '-t', + 'grok-build:local', + '{SWARMFORGE}'], + 'build_opencode': ['build', + '--target', + 'opencode-runtime', + '--build-arg', + 'AGENT=opencode', + '--build-arg', + 'OPENCODE_VERSION=', + '--build-arg', + 'DEBIAN_TAG=trixie-slim', + '--build-arg', + 'SWARMFORGE_HARNESS_INSTALL_BUST=0', + '-f', + '{SWARMFORGE}/anvil/Dockerfile', + '-t', + 'opencode:local', + '{SWARMFORGE}']} diff --git a/tests/test_image_layout.py b/tests/test_image_layout.py index 5e96b06..b57bc79 100644 --- a/tests/test_image_layout.py +++ b/tests/test_image_layout.py @@ -21,6 +21,8 @@ import tempfile import unittest +import make_argv_fixtures + HERE = os.path.dirname(os.path.abspath(__file__)) REPO_ROOT = os.path.dirname(HERE) MAKEFILE = os.path.join(REPO_ROOT, "Makefile") @@ -533,13 +535,8 @@ def test_entrypoint_reads_the_defaults_where_the_dockerfile_installs_them(self): ) -class BuildRecipeArgv(unittest.TestCase): - """`make build_*` pairs an explicit Dockerfile with a repo-root context. - - Building from `anvil/` again would leave the package outside the context - and the translator unable to import it, and the failure would not surface - until an agent went missing at runtime. - """ +class BuildRecipeCase(unittest.TestCase): + """Runs a build_* target and exposes the docker argv it assembled.""" def setUp(self): self.tmp = os.path.realpath(tempfile.mkdtemp(prefix="swarmforge-build-")) @@ -569,6 +566,15 @@ def build_argv(self, target): with open(self.capture_path) as handle: return handle.read().split("\0")[:-1] + +class BuildRecipeArgv(BuildRecipeCase): + """`make build_*` pairs an explicit Dockerfile with a repo-root context. + + Building from `anvil/` again would leave the package outside the context + and the translator unable to import it, and the failure would not surface + until an agent went missing at runtime. + """ + def assert_builds_from_repo_root(self, target): argv = self.build_argv(target) self.assertEqual(argv[0], "build") @@ -585,6 +591,43 @@ def test_opencode_image_builds_from_repo_root(self): def test_claude_image_builds_from_repo_root(self): self.assert_builds_from_repo_root("build_claude") + def test_grok_image_builds_from_repo_root(self): + self.assert_builds_from_repo_root("build_grok") + + def test_codex_image_builds_from_repo_root(self): + self.assert_builds_from_repo_root("build_codex") + + +class BuildArgvBaseline(BuildRecipeCase): + """Every word of every build_* recipe's docker argv, against a recording. + + The shape assertions above explain the Dockerfile/context pairing; this + pins the rest -- target stage, build args and their defaults, image tag -- + so a drifted recipe fails against `make_argv_fixtures.BUILD_ARGV` instead + of building something subtly different. + """ + + maxDiff = None + + def assert_argv_matches_recording(self, target): + argv = self.build_argv(target) + self.assertEqual( + make_argv_fixtures.normalize(argv, self.tmp), + make_argv_fixtures.BUILD_ARGV[target], + ) + + def test_build_opencode_argv_matches_recording(self): + self.assert_argv_matches_recording("build_opencode") + + def test_build_claude_argv_matches_recording(self): + self.assert_argv_matches_recording("build_claude") + + def test_build_grok_argv_matches_recording(self): + self.assert_argv_matches_recording("build_grok") + + def test_build_codex_argv_matches_recording(self): + self.assert_argv_matches_recording("build_codex") + class ContainerImportLayout(unittest.TestCase): """The container-side modules run the way the image runs them. diff --git a/tests/test_package_layering.py b/tests/test_package_layering.py index f92e92d..1294cfc 100644 --- a/tests/test_package_layering.py +++ b/tests/test_package_layering.py @@ -40,6 +40,12 @@ os.path.join(REPO_ROOT, ".opencode-test-data"), } +# The harness package and the launcher layers it sits under, as the prefixes +# the one-way rule matches on. A module counts as inside one of these when it +# is the package itself or anything below it. +HARNESS_ROOT = "swarmforge.harness" +LAUNCHER_ROOTS = ("swarmforge.tongs", "swarmforge.anvil") + # importlib's load-a-module-from-a-file-path helper. PATH_LOADER = "spec_from_file_location" @@ -195,15 +201,22 @@ def reached_by(target, importer, modules): return reached -def imported_modules(tree, importer, package, modules): +def all_nodes(tree): + """Every node of `tree`, the ones only a call ever runs included.""" + return ast.walk(tree) + + +def imported_modules(tree, importer, package, modules, nodes=import_time_nodes): """The modules in `modules` that this syntax tree imports. `from pkg import name` reaches `pkg.name` when that name is a submodule and `pkg` itself when it is a function or a constant, so only the more - specific of the two is taken as the target. + specific of the two is taken as the target. `nodes` picks which part of + the tree counts: the import-time statements by default, or every node + for a rule that must also catch an import deferred into a call. """ found = set() - for node in import_time_nodes(tree): + for node in nodes(tree): if isinstance(node, ast.Import): for alias in node.names: if alias.name in modules: @@ -242,6 +255,15 @@ def import_graph(): return graph +def is_within(name, package): + """Whether the module `name` is `package` itself or something inside it. + + The dot matters: `swarmforge.tongsmith` starts with `swarmforge.tongs` + as text and is a different package entirely. + """ + return name == package or name.startswith(package + ".") + + def path_loading_sites(path): """The lines in `path` that reach the path loader, however it is spelled. @@ -391,6 +413,113 @@ def test_no_module_in_the_package_imports_in_a_circle(self): self.assertIsNone(cycle, "import cycle: %s" % " -> ".join(cycle or [])) +class HarnessModulesStayBelowTheLaunchers(unittest.TestCase): + """The harness modules describe a harness; they do not drive one. + + Each of them covers a single harness and ships into the container image, + so the only package code they reach for is leaf helpers -- the agent + emitters, the config readers, `yamlite`. The launcher layers go the other + way and dispatch through them by name. An edge from a harness module back + into tongs or anvil would leave the two halves of the package depending on + each other, and would drag launcher-only code onto the path the image + imports. + """ + + def test_the_scan_sees_the_harness_modules_and_the_edge_into_them(self): + """A rule matching no modules would report no violation either. + + These are the three pieces the rule is built out of: a harness module + in the graph with the leaf import it is allowed, the registry edge + that gathers the harnesses up, and the launcher edge that gives the + rule a direction to be one-way in. + """ + graph = import_graph() + self.assertIn( + "swarmforge.agents.emit", graph["swarmforge.harness.claude"]) + self.assertIn("swarmforge.harness.claude", graph["swarmforge.harness"]) + self.assertIn("swarmforge.harness", graph["swarmforge.tongs.mcp"]) + + def test_no_harness_module_imports_tongs_or_anvil(self): + offenders = [] + for name, edges in sorted(import_graph().items()): + if not is_within(name, HARNESS_ROOT): + continue + for edge in sorted(edges): + if any(is_within(edge, root) for root in LAUNCHER_ROOTS): + offenders.append("%s -> %s" % (name, edge)) + self.assertEqual( + offenders, [], + "harness modules import launcher code at import time: %s" + % ", ".join(offenders), + ) + + def test_the_lazy_scan_sees_what_the_import_time_scan_skips(self): + """The two scans differ on exactly the deferred-import shape. + + The call-time rule below exists because a harness module already + defers one import into a function body on purpose; a scan that + skipped those bodies would wave the same trick through for tongs + and anvil. + """ + module = ast.parse("def fn():\n from deferred import x\n") + lazy = { + node.module for node in all_nodes(module) + if isinstance(node, ast.ImportFrom) + } + eager = { + node.module for node in import_time_nodes(module) + if isinstance(node, ast.ImportFrom) + } + self.assertIn("deferred", lazy) + self.assertNotIn("deferred", eager) + # And on the real files: the registry lookup the harness modules + # defer into their functions is visible to the lazy scan. + paths = { + module_name(path): path + for path in python_files(PACKAGE_ROOT) + if path.endswith(".py") + } + claude = paths["swarmforge.harness.claude"] + with open(claude, encoding="utf-8") as handle: + tree = ast.parse(handle.read(), claude) + edges = imported_modules( + tree, "swarmforge.harness.claude", package_of(claude), paths, + nodes=all_nodes) + self.assertIn("swarmforge.harness", edges) + + def test_no_harness_module_reaches_the_launchers_even_from_a_call(self): + """Deferring the import must not smuggle the dependency in. + + The one-way rule above reads import-time edges, which is what the + acyclic check needs -- but a `from swarmforge import tongs` inside a + function body would pass it while still tying the harness half to + the launcher half the first time the function ran. Harness modules + get the stricter reading: no import of tongs or anvil anywhere in + the file. + """ + paths = { + module_name(path): path + for path in python_files(PACKAGE_ROOT) + if path.endswith(".py") + } + offenders = [] + for name, path in sorted(paths.items()): + if not is_within(name, HARNESS_ROOT): + continue + with open(path, encoding="utf-8") as handle: + tree = ast.parse(handle.read(), path) + edges = imported_modules( + tree, name, package_of(path), paths, nodes=all_nodes) + for edge in sorted(edges): + if any(is_within(edge, root) for root in LAUNCHER_ROOTS): + offenders.append("%s -> %s" % (name, edge)) + self.assertEqual( + offenders, [], + "harness modules reach launcher code from a call: %s" + % ", ".join(offenders), + ) + + class PathLoadingStaysInTheShims(unittest.TestCase): """Only the entry-point shims may load python out of a file path. diff --git a/tests/test_run_agent_container.py b/tests/test_run_agent_container.py index 877c54c..81eb49b 100644 --- a/tests/test_run_agent_container.py +++ b/tests/test_run_agent_container.py @@ -16,6 +16,8 @@ import tempfile import unittest +import make_argv_fixtures + HERE = os.path.dirname(os.path.abspath(__file__)) REPO_ROOT = os.path.dirname(HERE) MAKEFILE = os.path.join(REPO_ROOT, "Makefile") @@ -302,6 +304,37 @@ def test_term_and_colorterm_are_passthrough_flags(self): any(flag.startswith("COLORTERM=") for flag in flags), target) +class RunArgvBaseline(MakeRecipeCase): + """Every word of every run_* target's launcher argv, against a recording. + + The classes above pin the properties that carry a rationale; this one + pins everything else -- flag order, env values, mount list, image name -- + so an accidental change to any recipe or shared block surfaces as a diff + against `make_argv_fixtures.RUN_ARGV` rather than passing silently. + """ + + maxDiff = None + + def assert_argv_matches_recording(self, target): + argv = self.launcher_argv(target, self.make_repo()) + self.assertEqual( + make_argv_fixtures.normalize(argv, self.tmp), + make_argv_fixtures.RUN_ARGV[target], + ) + + def test_run_opencode_argv_matches_recording(self): + self.assert_argv_matches_recording("run_opencode") + + def test_run_claude_argv_matches_recording(self): + self.assert_argv_matches_recording("run_claude") + + def test_run_grok_argv_matches_recording(self): + self.assert_argv_matches_recording("run_grok") + + def test_run_codex_argv_matches_recording(self): + self.assert_argv_matches_recording("run_codex") + + if __name__ == "__main__": if shutil.which("make") is None or shutil.which("git") is None: sys.stderr.write("make and git are required for these tests\n") diff --git a/tests/test_translate_agents.py b/tests/test_translate_agents.py index 8c08d62..e44ac53 100644 --- a/tests/test_translate_agents.py +++ b/tests/test_translate_agents.py @@ -1,13 +1,17 @@ #!/usr/bin/env python3 """Unit tests for swarmforge.agents.translate. Run: python3 tests/test_translate_agents.py""" +import contextlib +import io import os +import shutil import sys import tempfile import tomllib import unittest REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +FIXTURE_DIR = os.path.join(REPO_ROOT, "tests", "translate_fixtures") # The image puts the swarmforge package on PYTHONPATH; standing in for that # here keeps this file runnable on its own, not just under a discovery run @@ -441,6 +445,36 @@ def test_codex_writes_normalized_toml_filename(self): }, ) + def test_codex_name_override_keys_the_registration(self): + # The emitted file keeps the source stem; the registration and the + # file's own name follow the `codex:` block's declared name. + with tempfile.TemporaryDirectory() as tmp: + src = os.path.join(tmp, "src") + dest = os.path.join(tmp, "dest") + os.makedirs(src) + with open(os.path.join(src, "reviewer.md"), "w") as f: + f.write( + "---\ndescription: Reviews code.\ncodex:\n" + " name: review bot\n---\n\nReview carefully.\n" + ) + + rc = ta.main(["codex", dest, src]) + self.assertEqual(rc, 0) + self.assertEqual(set(os.listdir(dest)), {"reviewer.toml", "config.toml"}) + role_path = os.path.join(dest, "reviewer.toml") + with open(role_path, "rb") as f: + self.assertEqual(tomllib.load(f)["name"], "review bot") + with open(os.path.join(dest, "config.toml"), "rb") as f: + config = tomllib.load(f) + self.assertEqual( + config, + { + "agents": { + "review bot": {"config_file": os.path.abspath(role_path)} + } + }, + ) + def test_overlay_precedence_and_in_place(self): with tempfile.TemporaryDirectory() as tmp: shared = os.path.join(tmp, "shared") @@ -473,5 +507,47 @@ def test_overlay_precedence_and_in_place(self): self.assertEqual(meta["tools"], {"write": False, "edit": False, "bash": False}) +class RecordedFixtureTests(unittest.TestCase): + """Every target's output over one source tree, byte for byte. + + The classes above pin individual rules; this pins the whole rendering -- + field order, quoting, filenames, the codex registration file -- against + recordings under tests/translate_fixtures/. The sources exercise every + emitter and every per-harness override block, plus the branches that skip + an agent, drop a model, or normalize a name. `{DEST}` in an expected file + stands for the destination directory, which the codex registration embeds + as an absolute path. A deliberate output change is re-recorded by running + main() over src/ and substituting `{DEST}` back; nothing rewrites the + recordings automatically. + """ + + maxDiff = None + + def assert_output_matches_recording(self, target): + src = os.path.join(FIXTURE_DIR, "src") + expected_dir = os.path.join(FIXTURE_DIR, "expected", target) + dest = tempfile.mkdtemp(prefix="translate-recorded-") + self.addCleanup(shutil.rmtree, dest, True) + with contextlib.redirect_stderr(io.StringIO()): + rc = ta.main([target, dest, src]) + self.assertEqual(rc, 0) + self.assertEqual(sorted(os.listdir(dest)), sorted(os.listdir(expected_dir))) + for filename in sorted(os.listdir(expected_dir)): + with open(os.path.join(dest, filename), encoding="utf-8") as handle: + actual = handle.read().replace(dest, "{DEST}") + with open(os.path.join(expected_dir, filename), encoding="utf-8") as handle: + expected = handle.read() + self.assertEqual(actual, expected, filename) + + def test_opencode_output_matches_recording(self): + self.assert_output_matches_recording("opencode") + + def test_claude_output_matches_recording(self): + self.assert_output_matches_recording("claude") + + def test_codex_output_matches_recording(self): + self.assert_output_matches_recording("codex") + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/translate_fixtures/expected/claude/code.reviewer.md b/tests/translate_fixtures/expected/claude/code.reviewer.md new file mode 100644 index 0000000..22cfaed --- /dev/null +++ b/tests/translate_fixtures/expected/claude/code.reviewer.md @@ -0,0 +1,6 @@ +--- +name: code reviewer +description: Careful second reviewer. +--- + +Review it again. diff --git a/tests/translate_fixtures/expected/claude/coder.md b/tests/translate_fixtures/expected/claude/coder.md new file mode 100644 index 0000000..605ba6b --- /dev/null +++ b/tests/translate_fixtures/expected/claude/coder.md @@ -0,0 +1,7 @@ +--- +name: coder +description: Writes code. +disallowedTools: Write +--- + +Write the code. diff --git a/tests/translate_fixtures/expected/claude/enabled.md b/tests/translate_fixtures/expected/claude/enabled.md new file mode 100644 index 0000000..93001e4 --- /dev/null +++ b/tests/translate_fixtures/expected/claude/enabled.md @@ -0,0 +1,6 @@ +--- +name: enabled +description: Toggles features on. +--- + +Toggle the feature. diff --git a/tests/translate_fixtures/expected/claude/helper.md b/tests/translate_fixtures/expected/claude/helper.md new file mode 100644 index 0000000..170c73a --- /dev/null +++ b/tests/translate_fixtures/expected/claude/helper.md @@ -0,0 +1,7 @@ +--- +name: helper +description: Answers quick questions. +model: haiku +--- + +Help briefly. diff --git a/tests/translate_fixtures/expected/claude/local-model.md b/tests/translate_fixtures/expected/claude/local-model.md new file mode 100644 index 0000000..e1b5b5b --- /dev/null +++ b/tests/translate_fixtures/expected/claude/local-model.md @@ -0,0 +1,6 @@ +--- +name: local-model +description: Runs against a local model. +--- + +Answer locally. diff --git a/tests/translate_fixtures/expected/claude/nodesc.md b/tests/translate_fixtures/expected/claude/nodesc.md new file mode 100644 index 0000000..0c74231 --- /dev/null +++ b/tests/translate_fixtures/expected/claude/nodesc.md @@ -0,0 +1,5 @@ +--- +name: nodesc +--- + +No description here. diff --git a/tests/translate_fixtures/expected/claude/prompt-quotes.md b/tests/translate_fixtures/expected/claude/prompt-quotes.md new file mode 100644 index 0000000..9475672 --- /dev/null +++ b/tests/translate_fixtures/expected/claude/prompt-quotes.md @@ -0,0 +1,7 @@ +--- +name: prompt-quotes +description: "Formats: strings and paths" +--- + +Line one with "quotes" and a backslash \ here. +Line two. diff --git a/tests/translate_fixtures/expected/claude/reviewer.md b/tests/translate_fixtures/expected/claude/reviewer.md new file mode 100644 index 0000000..ef9a0b4 --- /dev/null +++ b/tests/translate_fixtures/expected/claude/reviewer.md @@ -0,0 +1,14 @@ +--- +name: reviewer +description: Reviews code for defects. +model: claude-sonnet-4-6 +disallowedTools: Write, Edit, Bash +metadata: + team: search + tags: + - review + - python +maxTurns: 12 +--- + +You are the reviewer agent. diff --git a/tests/translate_fixtures/expected/codex/agent-enabled.toml b/tests/translate_fixtures/expected/codex/agent-enabled.toml new file mode 100644 index 0000000..198e010 --- /dev/null +++ b/tests/translate_fixtures/expected/codex/agent-enabled.toml @@ -0,0 +1,5 @@ +name = "agent-enabled" +developer_instructions = """ +Toggle the feature. +""" +description = "Toggles features on." diff --git a/tests/translate_fixtures/expected/codex/code-reviewer.toml b/tests/translate_fixtures/expected/codex/code-reviewer.toml new file mode 100644 index 0000000..9431561 --- /dev/null +++ b/tests/translate_fixtures/expected/codex/code-reviewer.toml @@ -0,0 +1,5 @@ +name = "code reviewer" +developer_instructions = """ +Review it again. +""" +description = "Careful second reviewer." diff --git a/tests/translate_fixtures/expected/codex/coder.toml b/tests/translate_fixtures/expected/codex/coder.toml new file mode 100644 index 0000000..b443309 --- /dev/null +++ b/tests/translate_fixtures/expected/codex/coder.toml @@ -0,0 +1,6 @@ +name = "coder" +developer_instructions = """ +Write the code. +""" +description = "Writes code." +model = "gpt-5.3-codex" diff --git a/tests/translate_fixtures/expected/codex/config.toml b/tests/translate_fixtures/expected/codex/config.toml new file mode 100644 index 0000000..566f7e2 --- /dev/null +++ b/tests/translate_fixtures/expected/codex/config.toml @@ -0,0 +1 @@ +agents = { "code reviewer" = { config_file = "{DEST}/code-reviewer.toml" }, coder = { config_file = "{DEST}/coder.toml" }, agent-enabled = { config_file = "{DEST}/agent-enabled.toml" }, helper = { config_file = "{DEST}/helper.toml" }, local-model = { config_file = "{DEST}/local-model.toml" }, nodesc = { config_file = "{DEST}/nodesc.toml" }, prompt-quotes = { config_file = "{DEST}/prompt-quotes.toml" }, reviewer = { config_file = "{DEST}/reviewer.toml" } } diff --git a/tests/translate_fixtures/expected/codex/helper.toml b/tests/translate_fixtures/expected/codex/helper.toml new file mode 100644 index 0000000..3e84e94 --- /dev/null +++ b/tests/translate_fixtures/expected/codex/helper.toml @@ -0,0 +1,6 @@ +name = "helper" +developer_instructions = """ +Help briefly. +""" +description = "Answers quick questions." +model = "haiku" diff --git a/tests/translate_fixtures/expected/codex/local-model.toml b/tests/translate_fixtures/expected/codex/local-model.toml new file mode 100644 index 0000000..8020981 --- /dev/null +++ b/tests/translate_fixtures/expected/codex/local-model.toml @@ -0,0 +1,5 @@ +name = "local-model" +developer_instructions = """ +Answer locally. +""" +description = "Runs against a local model." diff --git a/tests/translate_fixtures/expected/codex/nodesc.toml b/tests/translate_fixtures/expected/codex/nodesc.toml new file mode 100644 index 0000000..de0a5bf --- /dev/null +++ b/tests/translate_fixtures/expected/codex/nodesc.toml @@ -0,0 +1,4 @@ +name = "nodesc" +developer_instructions = """ +No description here. +""" diff --git a/tests/translate_fixtures/expected/codex/prompt-quotes.toml b/tests/translate_fixtures/expected/codex/prompt-quotes.toml new file mode 100644 index 0000000..69ff0bb --- /dev/null +++ b/tests/translate_fixtures/expected/codex/prompt-quotes.toml @@ -0,0 +1,7 @@ +name = "prompt-quotes" +developer_instructions = """ +Line one with \"quotes\" and a backslash \\ here. +Line two. +""" +description = "Formats: strings and paths" +extras = { nested = true, items = ["a", "b"] } diff --git a/tests/translate_fixtures/expected/codex/reviewer.toml b/tests/translate_fixtures/expected/codex/reviewer.toml new file mode 100644 index 0000000..067bd81 --- /dev/null +++ b/tests/translate_fixtures/expected/codex/reviewer.toml @@ -0,0 +1,7 @@ +name = "reviewer" +developer_instructions = """ +You are the reviewer agent. +""" +description = "Reviews code for defects." +model_reasoning_effort = "high" +sandbox_mode = "read-only" diff --git a/tests/translate_fixtures/expected/opencode/code.reviewer.md b/tests/translate_fixtures/expected/opencode/code.reviewer.md new file mode 100644 index 0000000..2a9fe96 --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/code.reviewer.md @@ -0,0 +1,5 @@ +--- +description: Careful second reviewer. +--- + +Review it again. diff --git a/tests/translate_fixtures/expected/opencode/coder.md b/tests/translate_fixtures/expected/opencode/coder.md new file mode 100644 index 0000000..6daa93c --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/coder.md @@ -0,0 +1,9 @@ +--- +description: Writes code. +model: openai/gpt-5.3-codex +tools: + foo: false + write: false +--- + +Write the code. diff --git a/tests/translate_fixtures/expected/opencode/disabled.md b/tests/translate_fixtures/expected/opencode/disabled.md new file mode 100644 index 0000000..cdd5e21 --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/disabled.md @@ -0,0 +1,6 @@ +--- +description: Retired agent. +disable: true +--- + +No longer used. diff --git a/tests/translate_fixtures/expected/opencode/enabled.md b/tests/translate_fixtures/expected/opencode/enabled.md new file mode 100644 index 0000000..6b2642a --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/enabled.md @@ -0,0 +1,5 @@ +--- +description: Toggles features on. +--- + +Toggle the feature. diff --git a/tests/translate_fixtures/expected/opencode/helper.md b/tests/translate_fixtures/expected/opencode/helper.md new file mode 100644 index 0000000..5a618a2 --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/helper.md @@ -0,0 +1,5 @@ +--- +description: Answers quick questions. +--- + +Help briefly. diff --git a/tests/translate_fixtures/expected/opencode/local-model.md b/tests/translate_fixtures/expected/opencode/local-model.md new file mode 100644 index 0000000..e5878af --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/local-model.md @@ -0,0 +1,6 @@ +--- +description: Runs against a local model. +model: ollama/llama3.1 +--- + +Answer locally. diff --git a/tests/translate_fixtures/expected/opencode/nodesc.md b/tests/translate_fixtures/expected/opencode/nodesc.md new file mode 100644 index 0000000..d750e43 --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/nodesc.md @@ -0,0 +1,6 @@ +--- +mode: all +tools: everything +--- + +No description here. diff --git a/tests/translate_fixtures/expected/opencode/prompt-quotes.md b/tests/translate_fixtures/expected/opencode/prompt-quotes.md new file mode 100644 index 0000000..62e2a22 --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/prompt-quotes.md @@ -0,0 +1,6 @@ +--- +description: "Formats: strings and paths" +--- + +Line one with "quotes" and a backslash \ here. +Line two. diff --git a/tests/translate_fixtures/expected/opencode/reviewer.md b/tests/translate_fixtures/expected/opencode/reviewer.md new file mode 100644 index 0000000..54c0234 --- /dev/null +++ b/tests/translate_fixtures/expected/opencode/reviewer.md @@ -0,0 +1,20 @@ +--- +description: Reviews code for defects. +mode: subagent +temperature: 0.1 +model: anthropic/claude-sonnet-4-6 +tools: + write: false + edit: false + bash: false + patch: false + webfetch: true +metadata: + team: search + tags: + - review + - python +steps: 8 +--- + +You are the reviewer agent. diff --git a/tests/translate_fixtures/src/code.reviewer.md b/tests/translate_fixtures/src/code.reviewer.md new file mode 100644 index 0000000..22cfaed --- /dev/null +++ b/tests/translate_fixtures/src/code.reviewer.md @@ -0,0 +1,6 @@ +--- +name: code reviewer +description: Careful second reviewer. +--- + +Review it again. diff --git a/tests/translate_fixtures/src/coder.md b/tests/translate_fixtures/src/coder.md new file mode 100644 index 0000000..6daa93c --- /dev/null +++ b/tests/translate_fixtures/src/coder.md @@ -0,0 +1,9 @@ +--- +description: Writes code. +model: openai/gpt-5.3-codex +tools: + foo: false + write: false +--- + +Write the code. diff --git a/tests/translate_fixtures/src/disabled.md b/tests/translate_fixtures/src/disabled.md new file mode 100644 index 0000000..cdd5e21 --- /dev/null +++ b/tests/translate_fixtures/src/disabled.md @@ -0,0 +1,6 @@ +--- +description: Retired agent. +disable: true +--- + +No longer used. diff --git a/tests/translate_fixtures/src/enabled.md b/tests/translate_fixtures/src/enabled.md new file mode 100644 index 0000000..6b2642a --- /dev/null +++ b/tests/translate_fixtures/src/enabled.md @@ -0,0 +1,5 @@ +--- +description: Toggles features on. +--- + +Toggle the feature. diff --git a/tests/translate_fixtures/src/helper.md b/tests/translate_fixtures/src/helper.md new file mode 100644 index 0000000..2437a60 --- /dev/null +++ b/tests/translate_fixtures/src/helper.md @@ -0,0 +1,6 @@ +--- +description: Answers quick questions. +model: haiku +--- + +Help briefly. diff --git a/tests/translate_fixtures/src/local-model.md b/tests/translate_fixtures/src/local-model.md new file mode 100644 index 0000000..e5878af --- /dev/null +++ b/tests/translate_fixtures/src/local-model.md @@ -0,0 +1,6 @@ +--- +description: Runs against a local model. +model: ollama/llama3.1 +--- + +Answer locally. diff --git a/tests/translate_fixtures/src/nodesc.md b/tests/translate_fixtures/src/nodesc.md new file mode 100644 index 0000000..d750e43 --- /dev/null +++ b/tests/translate_fixtures/src/nodesc.md @@ -0,0 +1,6 @@ +--- +mode: all +tools: everything +--- + +No description here. diff --git a/tests/translate_fixtures/src/prompt-quotes.md b/tests/translate_fixtures/src/prompt-quotes.md new file mode 100644 index 0000000..616735a --- /dev/null +++ b/tests/translate_fixtures/src/prompt-quotes.md @@ -0,0 +1,12 @@ +--- +description: "Formats: strings and paths" +codex: + extras: + nested: true + items: + - a + - b +--- + +Line one with "quotes" and a backslash \ here. +Line two. diff --git a/tests/translate_fixtures/src/reviewer.md b/tests/translate_fixtures/src/reviewer.md new file mode 100644 index 0000000..2638200 --- /dev/null +++ b/tests/translate_fixtures/src/reviewer.md @@ -0,0 +1,26 @@ +--- +description: Reviews code for defects. +mode: subagent +temperature: 0.1 +model: anthropic/claude-sonnet-4-6 +tools: + write: false + edit: false + bash: false + patch: false + webfetch: true +metadata: + team: search + tags: + - review + - python +claude: + maxTurns: 12 +opencode: + steps: 8 +codex: + model_reasoning_effort: high + sandbox_mode: read-only +--- + +You are the reviewer agent.