From 28f3fd6efb94dad9d9cfa44702088247021afb01 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:33:57 +0900 Subject: [PATCH 1/2] feat: add local SkillEval artifact handoff and trusted replay Separate frozen inputs from the writable parser artifact. Preserve author-run provenance, independently replay retained source in a fail-closed namespace, and keep artifact outcomes distinct from skill efficacy and PR acceptance. Agent-assisted-by: Codex (implementation and artifact verifier) Signed-off-by: Codex Reviewed-local-source-commit: c43e76f56ec0f62753e7977a0bd0e8ce389d8fd4 Published via GitHub Git Data API with identical tree bytes; connector commit metadata may differ. --- docs/local-skill-evaluation.md | 189 ++++++++++++ .../harness/skill_eval_engineering.py | 284 ++++++++++++++++++ src/inferrsi/harness/skill_evaluation.py | 70 ++++- .../skill_eval_engineering/contract.md | 74 +++++ .../fixtures/skill_eval_engineering/parser.py | 2 + tests/test_skill_eval_engineering.py | 256 ++++++++++++++++ tests/test_skill_eval_local.py | 249 +++++++++++++++ tools/skill_eval_compare.py | 236 +++++++++++++-- tools/skill_eval_local.py | 220 ++++++++++++++ 9 files changed, 1544 insertions(+), 36 deletions(-) create mode 100644 docs/local-skill-evaluation.md create mode 100644 src/inferrsi/harness/skill_eval_engineering.py create mode 100644 tests/fixtures/skill_eval_engineering/contract.md create mode 100644 tests/fixtures/skill_eval_engineering/parser.py create mode 100644 tests/test_skill_eval_engineering.py create mode 100644 tests/test_skill_eval_local.py create mode 100644 tools/skill_eval_local.py diff --git a/docs/local-skill-evaluation.md b/docs/local-skill-evaluation.md new file mode 100644 index 000000000..cd0571837 --- /dev/null +++ b/docs/local-skill-evaluation.md @@ -0,0 +1,189 @@ +# Local engineering evidence for SkillEval + +SkillEval can check a **code artifact**, rather than ask a model to grade its own +answer. The first adapter is one public CPU response-parser task reconstructed +from the error-stream observation in [PR 90](https://github.com/InferMatrixLab/InferRSI/pull/90). +It is not the original server, a GPU experiment, a hidden benchmark, or acceptance +of that PR. The [frozen task contract](../tests/fixtures/skill_eval_engineering/contract.md) +records source revisions, starter hash, exact behavior and exclusions. + +## What changed + +```mermaid +flowchart LR + A[Existing prepare: pinned old/new skill] --> B[Frozen task plus parser starter] + B --> C[Author's approved local agent] + C --> D[Actual parser plus author logs] + D --> E[Trusted isolated artifact replay] + E --> F[Existing report plus reviewer decision] +``` + +`tools/skill_eval_compare.py` still owns prepare/execute/report and paired +snapshots. `tools/skill_eval_local.py` adds manual stage/capture, without launching +an agent. `src/inferrsi/harness/skill_eval_engineering.py` owns the one task and +19 public behavior probes for that one task. `skill_evaluation.py` separates the single writable +artifact from frozen inputs. The existing cloud read/submit route is unchanged; +it cannot perform this artifact-editing task and is rejected for this adapter. + +```mermaid +sequenceDiagram + participant R as Reviewer / author + participant S as SkillEval + participant A as Approved local agent + participant V as Isolated verifier + R->>S: prepare pinned plan + R->>S: execute --stage-local (one arm) + S-->>R: Frozen task / skill + writable parser.py + R->>A: Run under separately approved environment and budget + A-->>R: parser.py and retained execution logs + R->>S: execute --capture-local + S->>V: Actual retained source, one input vector at a time + V-->>S: Raw returned values (not self-assessed PASS) + S-->>R: Report, exact checks, limits and artifact links + R->>R: Repeat other arm; independently review evidence and PR scope +``` + +There is no SSH dispatcher, CI result-return protocol, new judge/translator call, +or automatic merge. The author posts an accessible evidence report in the PR; +the reviewer checks its current head and accepted scope before deciding. + +## Prepare a plan + +Start from an existing comparison configuration. Set `adapter` to +`engineering-artifact-v1`, `mode` to `update`, explicit baseline/candidate full +Git SHAs, selected skill ID/root, approved model/settings and existing limits. +Do not add a provider `client`, custom `schema` or `grader`. Generate the case +without silently changing its contract: + +```python +from inferrsi.harness.skill_eval_engineering import REQUEST +from inferrsi.infra.artifacts import write_json + +write_json("cases.json", [{ + "id": "serving-response-v1", "split": "development", + "request": REQUEST, + "provenance": "Reconstructed CPU fixture from PR 90's public error-stream case", +}]) +``` + +```sh +python tools/skill_eval_compare.py prepare --repo /path/to/repo \ + --config /path/to/config.json --output /path/to/evidence +``` + +Keep the returned frozen-plan SHA externally with the acceptance record. This +freezes task, schema, starter, verifier/controllers and both selected bundles. +Before any real agent call separately establish trustworthy **whole-run cost +bounds**, model identity, tool access, isolated environment, run order/repeats, +timeout and treatment of failed/retried attempts. A plan hash is not permission +to spend, and wall-clock/token stop-new-trials is not a hard per-call dollar cap. + +## Author-run handoff + +Use separate fresh workspace paths outside the evidence tree for each arm. Stop +all producers before capture. The CLI's `--current-head` is supplied by the +reviewer after reading GitHub; it is checked against the plan, not fetched or +authenticated by this offline command. + +```sh +python tools/skill_eval_compare.py execute --output /path/to/evidence \ + --authorize PLAN_SHA --stage-local /path/to/baseline-work \ + --task-id serving-response-v1 --condition baseline --repeat 1 --current-head HEAD_SHA +``` + +Only `parser.py` may change. `task.json`, `INSTRUCTIONS.txt` and `knowledge/` stay +frozen. The task needs no imports, dependencies, network or hardware. Author +execution must occur in an already-approved environment; staging does not start +it or imply that the host is safe. Do not expose credentials, the other arm's +work, or private verifier/campaign files to the agent. Tests here are public, so +this is not a blind generalization experiment. The old default Codex client uses +`danger-full-access` and is explicitly refused for automatic engineering runs. +A `workspace-write` label alone also does not establish read confinement. + +Retain agent stdout/stderr and an author execution description in a separate +submission directory. `execution.json` must have exactly these fields: + +```json +{ + "model": {"name": "the-frozen-approved-model", "effort": "the-frozen-effort"}, + "environment": "runtime/image and versions actually used", + "access": "actual tools, filesystem/network/oracle/other-arm access and limits", + "resources": "approved bound, observed cost or unknown, all attempts and retries", + "usage": null, + "elapsed_seconds": 30, + "agent_exit_code": 0, + "summary": "What changed, what remains, and scope limits" +} +``` + +`usage` may contain observed nonnegative `input_tokens`, `output_tokens` and +`cached_input_tokens`; missing usage stays unknown. Logs and metadata are +**author-supplied**, not an authenticated controller trace or proof of spending. +Do not fabricate a model run from deterministic software controls. Preserve failed +attempts; stage/capture refuse to overwrite an existing attempt. New trials need +a new prepared evidence directory, with earlier attempts linked by the author. + +```sh +python tools/skill_eval_compare.py execute --output /path/to/evidence \ + --authorize PLAN_SHA --capture-local /path/to/baseline-work \ + --submission /path/to/baseline-logs --task-id serving-response-v1 \ + --condition baseline --repeat 1 --current-head HEAD_SHA +python tools/skill_eval_compare.py report --output /path/to/evidence +``` + +Repeat for candidate under the same approved controls. Actual source is captured +separately from the summary, hashed and reverified at report time. Changing any +frozen file, missing artifact/log, mismatched head or unavailable sandbox must not +become a clean comparison. Nonzero author exit or exceeding frozen timeout is +reported separately from the artifact's independently checked behavior. + +## Replay environment and interpretation + +Replay requires Linux, bubblewrap namespace creation and system Python. It mounts +only system runtime libraries, a trusted runner and the submitted artifact; +there is no home, repository, other arm or network. The runner has bounded CPU, +wall time, address space, file size and descriptor/process resources. Expected +outputs remain in the trusted parent; each child sees just one input vector. +A capture from an isolation-blocked host can still retain source and logs. Later +`report` on a qualified reviewer host replays that source into separate +`artifact_checks` / `artifact_verdict`; it preserves the original capture error, +INCONCLUSIVE trial and unverified comparability rather than rewriting history. + +No unsafe host fallback exists. Syntax/runtime failures after successful sandbox +preflight are failed task checks. Inability to establish isolation is INCONCLUSIVE. + +The implementation workspace's probe returned +`NETLINK_ROUTE ... Operation not permitted`; no independent artifact replay or +real agent effect is claimed from that host. Use a host where this exact sandbox +preflight succeeds, under independently approved execution/cost conditions. +Do not request privilege escalation or disable isolation to make a test green. + +Manual pairs use `artifact_improved` / `artifact_regressed` / `artifacts_both_pass` +/ `artifacts_both_fail`, **not causal skill fix/regression claims**. Environment, +access and usage comparability remain unverified author reports. A reviewer must +reconcile them; checks cannot authenticate who produced an artifact or prove +logs exhaustive. Negative/no-gain/inconclusive evidence can still be useful. + +The first screen should answer: what artifact behavior changed, exact failures, +coverage limits and what this does/does not imply for the PR. Share report and +readable retained artifacts after privacy review, not local paths alone. +Preserve separate conclusions for knowledge/source correctness, software CI, +artifact checks, skill benefit and whole-PR acceptance. + +## Tracking disposition + +This delivery is a bounded software slice, not closure of the full evolution: + +| Item | Disposition / next evidence | +| --- | --- | +| Risk-based merge evidence | Source-only, operational and hardware claims remain distinct; pilot is not a repository-wide gate | +| First engineering task | Frozen reconstructed CPU parser; source/code/limits linked above | +| Local entrypoint and artifacts | Stage/capture/replay via existing compare flow; real host and execution remain prerequisites | +| Judge applicability/calibration | Deferred: deterministic artifact checks need no judge; any later semantic judge needs fixed rubric, blinded order and known-bad/partial controls before use | +| Real paired run + PR review | Pending approved local environment/cost bounds, actual logs and independently reviewed PR evidence; no fabricated success | +| Negative controls | Software tests cover correct/wrong artifact, missing evidence, stale head, frozen-input changes and unavailable isolation; not model outcomes | +| Later corpus scoring | Separate bounded scope decision; not implemented or accepted by this single-task delivery | + +Benchmark construction and retrieval quality remain separate work. This adapter +neither requires a giant per-skill question bank nor establishes full skill or +knowledge-corpus coverage. diff --git a/src/inferrsi/harness/skill_eval_engineering.py b/src/inferrsi/harness/skill_eval_engineering.py new file mode 100644 index 000000000..93356fcee --- /dev/null +++ b/src/inferrsi/harness/skill_eval_engineering.py @@ -0,0 +1,284 @@ +"""One public, reconstructed serving-parser engineering task. + +The controller supplies retained artifact bytes, not agent-reported checks. Every +artifact invocation is confined by bubblewrap; absence of that boundary is an +infrastructure error, never permission to execute the artifact on the host. +This verifier does not sandbox the agent that produced the artifact. +""" +import hashlib +import json +import os +import re +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + +ADAPTER = "engineering-artifact-v1" +MAX_SOURCE_BYTES = 65536 +STARTER = ( + 'def evaluate(status, frames, requested_tokens):\n' + ' return {"success": status == 200, "reason": "length" if status == 200 else "http_error"}\n' +) +SCHEMA = { + "type": "object", "additionalProperties": False, + "required": ["artifact", "summary"], + "properties": { + "artifact": {"type": "string", "enum": ["parser.py"]}, + "summary": {"type": "string"}, + }, +} +PROMPT = ( + "Read task.json and the assigned knowledge. Repair only parser.py, implementing " + "evaluate(status, frames, requested_tokens) under the public task contract. " + "Do not change frozen inputs or create other files. No network, installs, other agents, " + "commits or publication. Return artifact='parser.py' and a concise summary, not a verdict. " + "The controller retains the actual file and independently checks its behavior." +) +REQUEST = ( + "Repair parser.py: evaluate(status, frames, requested_tokens) returns exactly " + "{success: bool, reason: str}. Inputs are an integer HTTP status, decoded frames, " + "and a positive integer requested count. Content frames have exactly {content: str}; " + "a terminal has exactly {finish_reason: 'length'|'stop', completion_tokens: " + "nonnegative int}. Boolean counts are invalid. At most one optional final '[DONE]' " + "is allowed; DONE alone is not a terminal. First reject HTTP non-200 as http_error. " + "Then inspect frames in order: a dict containing error gives stream_error; malformed " + "fields, duplicate terminals, post-terminal content, misplaced or duplicate DONE " + "give malformed_frame. Require a terminal after scanning, otherwise missing_terminal " + "(also when the final frame is DONE). For length require count == requested; for stop " + "require count <= requested, otherwise token_mismatch. Only valid length or stop " + "returns success=true with that reason; every failure returns success=false. " + "Change only parser.py; all other files are frozen. No imports/dependencies, network " + "or hardware are required. Tests are public, not holdout. This reconstructed CPU " + "fixture derives from the HTTP-200 error-stream observation in " + "https://github.com/InferMatrixLab/InferRSI/pull/90; it does not reproduce the original " + "server, capacity behavior or GPU performance." +) +REASONS = ("length", "stop", "http_error", "stream_error", "malformed_frame", + "missing_terminal", "token_mismatch") +# Public behavior probes, not a hidden benchmark or proof of generalization. +# Expected outputs remain in the trusted parent, never in the artifact subprocess. +VECTORS = ( + ("length", 200, [{"content": "text"}, {"finish_reason": "length", "completion_tokens": 3}, + "[DONE]"], 3, True, "length"), + ("early_stop", 200, [{"finish_reason": "stop", "completion_tokens": 1}], 3, True, "stop"), + ("empty_stop", 200, [{"finish_reason": "stop", "completion_tokens": 0}, "[DONE]"], + 3, True, "stop"), + ("http_failure", 400, [], 3, False, "http_error"), + ("stream_error", 200, [{"content": "partial"}, {"error": "capacity"}], + 3, False, "stream_error"), + ("missing_terminal", 200, [{"content": "partial"}, "[DONE]"], + 3, False, "missing_terminal"), + ("length_mismatch", 200, [{"finish_reason": "length", "completion_tokens": 2}], + 3, False, "token_mismatch"), + ("stop_overflow", 200, [{"finish_reason": "stop", "completion_tokens": 4}], + 3, False, "token_mismatch"), + ("unknown_frame", 200, [{"unexpected": True}], 3, False, "malformed_frame"), + ("boolean_count", 200, [{"finish_reason": "length", "completion_tokens": True}], + 1, False, "malformed_frame"), + ("duplicate_terminal", 200, [{"finish_reason": "stop", "completion_tokens": 1}, + {"finish_reason": "stop", "completion_tokens": 1}], + 3, False, "malformed_frame"), + ("misplaced_done", 200, ["[DONE]", {"content": "late"}], + 3, False, "malformed_frame"), + ("negative_count", 200, [{"finish_reason": "stop", "completion_tokens": -1}], + 3, False, "malformed_frame"), + ("post_terminal_content", 200, [{"finish_reason": "stop", "completion_tokens": 1}, + {"content": "late"}], 3, False, "malformed_frame"), + ("extra_content_field", 200, [{"content": "text", "extra": True}], + 3, False, "malformed_frame"), + ("extra_terminal_field", 200, [{"finish_reason": "stop", "completion_tokens": 1, + "extra": True}], 3, False, "malformed_frame"), + ("unknown_finish", 200, [{"finish_reason": "unknown", "completion_tokens": 1}], + 3, False, "malformed_frame"), + ("duplicate_done", 200, [{"finish_reason": "stop", "completion_tokens": 1}, + "[DONE]", "[DONE]"], 3, False, "malformed_frame"), + ("non_object_frame", 200, [42], 3, False, "malformed_frame"), +) + + +class SandboxUnavailable(RuntimeError): + """The required artifact execution boundary could not be established.""" + + + +def read_regular(path: Path, limit: int) -> bytes: + """Bounded no-follow snapshot after the producer has stopped. + + Reject symlink ancestors and special files; this is not an adversarial + directory-rename sandbox. The actual replay has a separate OS boundary. + """ + path = path.absolute() + if any(parent.is_symlink() for parent in (path, *path.parents)): + raise ValueError("Artifact/evidence paths must not contain symlinks") + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + try: + descriptor = os.open(path, flags) + except OSError as error: + raise ValueError("Cannot read regular evidence file") from error + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > limit: + raise ValueError("Evidence is not a bounded regular file") + data = stream.read(limit + 1) + if len(data) > limit: + raise ValueError("Evidence exceeds byte bound") + return data + + +def input_hashes(workspace: Path) -> dict: + """Frozen engineering inputs only; special nodes are never opened.""" + if any(path.is_symlink() for path in (workspace.absolute(), *workspace.absolute().parents)): + raise ValueError("Workspace paths must not contain symlinks") + hashes = {} + for path in workspace.rglob("*"): + name = path.relative_to(workspace).as_posix() + mode = path.lstat().st_mode + if stat.S_ISLNK(mode) or not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)): + raise ValueError("Workspace contains a symlink or special node") + if stat.S_ISREG(mode) and name != "parser.py": + hashes[name] = hashlib.sha256(read_regular(path, 16 * 1024 * 1024)).hexdigest() + return hashes + +def validate_task(task): + if not isinstance(task, dict) or set(task) != {"id", "split", "request", "provenance"}: + raise ValueError("Engineering task requires id/split/request/provenance only") + if any(type(value) is not str or not value.strip() for value in task.values()): + raise ValueError("Engineering task metadata must be nonempty strings") + if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._-]*", task["id"]): + raise ValueError("Engineering task ID must be a safe path component") + if task["request"] != REQUEST: + raise ValueError("Engineering adapter supports only its frozen serving-parser request") + if task["split"] != "development": + raise ValueError("Engineering probes are public development cases, not holdout/discovery") + + +# Only one vector is visible per process. No checks/expected answers are loaded. +RUNNER = '''import json, resource, sys +resource.setrlimit(resource.RLIMIT_CPU, (2, 2)) +resource.setrlimit(resource.RLIMIT_AS, (268435456, 268435456)) +resource.setrlimit(resource.RLIMIT_FSIZE, (65536, 65536)) +resource.setrlimit(resource.RLIMIT_NOFILE, (32, 32)) +resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +if sys.argv[1:] == ["probe"]: + print('{"ready": true}') +else: + print("INFERRSI_ARTIFACT_READY", file=sys.stderr, flush=True) + arguments = json.loads(sys.stdin.read()) + namespace = {"__name__": "submitted_parser"} + with open("/work/parser.py", encoding="utf-8") as source: + exec(compile(source.read(), "parser.py", "exec"), namespace) + value = namespace["evaluate"](*arguments) + print(json.dumps(value, allow_nan=False)) +''' + + +def sandbox_command(work: Path, *, probe=False): + """Linux system Python only; do not mount the caller's venv/home/repository.""" + bwrap = shutil.which("bwrap") + python = Path("/usr/bin/python3").resolve() + match = re.fullmatch(r"python(3\.\d+)", python.name) + if sys.platform != "linux" or bwrap is None or not match or not python.is_file(): + raise SandboxUnavailable("Artifact replay requires Linux bubblewrap and system Python") + stdlib = Path("/usr/lib") / ("python" + match[1]) + libraries = [path for path in (Path("/usr/lib/x86_64-linux-gnu"), + Path("/usr/lib/aarch64-linux-gnu"), Path("/usr/lib64")) + if path.is_dir()] + if not stdlib.is_dir() or not libraries: + raise SandboxUnavailable("System Python runtime layout is unsupported") + command = [bwrap, "--unshare-all", "--die-with-parent", "--new-session", "--clearenv", + "--ro-bind", str(python), "/usr/bin/python3", + "--ro-bind", str(stdlib), str(stdlib)] + for path in libraries: + command += ["--ro-bind", str(path), str(path)] + command += ["--symlink", "usr/lib", "/lib", "--symlink", "usr/lib64", "/lib64", + "--ro-bind", str(work), "/work", "--proc", "/proc", "--dev", "/dev", + "--chdir", "/work", "--remount-ro", "/", + "--", "/usr/bin/python3", "-I", "-S", "-B", "/work/runner.py"] + if probe: + command.append("probe") + return command + + +def _invoke(work, arguments=None, *, probe=False): + # Files, not PIPE capture: child output cannot grow parent memory. Hard + # RLIMIT_FSIZE in the trusted runner bounds each retained stream to 64 KiB. + with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: + try: + process = subprocess.Popen( + sandbox_command(work, probe=probe), stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, env={"PATH": "/usr/bin:/bin", "LANG": "C"}, + start_new_session=True, + ) + except OSError as error: + raise SandboxUnavailable("Cannot start artifact sandbox") from error + timed_out = False + try: + process.communicate(json.dumps(arguments).encode(), timeout=5) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + timed_out = True + if not probe: + stderr.seek(0) + if not stderr.read(128).startswith(b"INFERRSI_ARTIFACT_READY\n"): + raise SandboxUnavailable("Artifact worker did not establish its trusted boundary") + if timed_out: + return None + stdout.seek(0) + raw = stdout.read(65537) + if process.returncode or len(raw) > 65536: + return None + try: + return json.loads(raw) + except (ValueError, UnicodeDecodeError): + return None + + +def verify(task, output): + validate_task(task) + valid = (isinstance(output, dict) + and set(output) == {"artifact", "summary", "artifact_source"} + and output["artifact"] == "parser.py" + and type(output["summary"]) is str + and type(output["artifact_source"]) is str) + if not valid: + return {"passed": False, "assertions": {"artifact_schema": False}} + try: + source = output["artifact_source"].encode("utf-8") + except UnicodeEncodeError: + source = b"" + if not source or len(source) > MAX_SOURCE_BYTES: + return {"passed": False, "assertions": {"artifact_size": False}} + with tempfile.TemporaryDirectory(prefix="skill-artifact-") as temporary: + work = Path(temporary) + (work / "runner.py").write_text(RUNNER, encoding="utf-8") + (work / "parser.py").write_bytes(source) + if _invoke(work, probe=True) != {"ready": True}: + raise SandboxUnavailable("Artifact sandbox preflight failed; no host fallback") + checks = {} + for name, status, frames, count, success, reason in VECTORS: + actual = _invoke(work, [status, frames, count]) + checks[name] = (isinstance(actual, dict) and set(actual) == {"success", "reason"} + and type(actual["success"]) is bool + and actual == {"success": success, "reason": reason}) + return {"passed": all(checks.values()), "assertions": checks} + + +def main(): + task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + output = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + result = verify(task, output) + Path(os.environ["INFERRSI_RESULT_PATH"]).write_text(json.dumps(result), encoding="utf-8") + raise SystemExit(0 if result["passed"] else 1) + + +if __name__ == "__main__": + main() diff --git a/src/inferrsi/harness/skill_evaluation.py b/src/inferrsi/harness/skill_evaluation.py index 22073e572..4de6960d3 100644 --- a/src/inferrsi/harness/skill_evaluation.py +++ b/src/inferrsi/harness/skill_evaluation.py @@ -7,6 +7,7 @@ from ..infra.artifacts import atomic_write_json, sha256 from ..infra.executors.local import execute_command from ..knowledge_operations import KnowledgeReader +from . import skill_eval_engineering as engineering def global_skills() -> list[str]: @@ -187,11 +188,13 @@ def run_trial( deadline: float, *, timeout_seconds: float | None = None, + engineering_artifact: bool = False, ) -> dict: trial_dir.mkdir(parents=True) final = trial_dir / "answer.json" - before = { - p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file() + before = engineering.input_hashes(workspace) if engineering_artifact else { + p.relative_to(workspace).as_posix(): sha256(p) + for p in workspace.rglob("*") if p.is_file() } command = client.command(workspace, final, schema, prompt) # An injected client may tighten the frozen limit, but cannot extend it. @@ -224,14 +227,51 @@ def run_trial( evaluation_error = "agent_trace_error" output, checks, verification = None, None, None if evaluation_error is None: - if not final.is_file(): + if final.is_symlink(): + evaluation_error = "invalid_output_symlink" + elif not final.is_file(): evaluation_error = "missing_output" else: try: - output = json.loads(final.read_text(encoding="utf-8")) - except ValueError: + answer_bytes = (engineering.read_regular(final, 65536) if engineering_artifact + else final.read_bytes()) + output = json.loads(answer_bytes.decode("utf-8")) + except (ValueError, UnicodeError): evaluation_error = "invalid_output_json" - if evaluation_error is None: + if evaluation_error is None and engineering_artifact: + artifact = workspace / "parser.py" + if not isinstance(output, dict) or set(output) != {"artifact", "summary"}: + evaluation_error = "invalid_artifact_answer" + else: + try: + artifact_bytes = engineering.read_regular(artifact, engineering.MAX_SOURCE_BYTES) + source = artifact_bytes.decode("utf-8") + if any(path.is_symlink() for path in (trial_dir, *trial_dir.absolute().parents)): + raise ValueError("Retained directory contains a symlink") + if any((trial_dir / name).exists() or (trial_dir / name).is_symlink() + for name in ("agent-answer.json", "parser.py", "verification")): + raise ValueError("Agent created reserved evidence paths") + except (ValueError, UnicodeError): + evaluation_error = "missing_or_invalid_artifact" + else: + # A single bounded snapshot supplies both retained bytes and grading. + (trial_dir / "agent-answer.json").write_bytes(answer_bytes) + (trial_dir / "parser.py").write_bytes(artifact_bytes) + output = {**output, "artifact_source": source} + atomic_write_json(final, output) + verification_dir = trial_dir / "verification" + verification_dir.mkdir() + (verification_dir / "stdout.log").write_text("", encoding="utf-8") + (verification_dir / "stderr.log").write_text("", encoding="utf-8") + try: + checks = engineering.verify(task, output) + except engineering.SandboxUnavailable: + evaluation_error = "artifact_sandbox_unavailable" + else: + atomic_write_json(verification_dir / "result.json", checks) + verification = {"status": "completed" if checks["passed"] else "failed", + "returncode": 0 if checks["passed"] else 1} + if evaluation_error is None and not engineering_artifact: verifier_environment = getattr(client, "verifier_environment", None) options = {"environment": verifier_environment()} if verifier_environment else {} verification = execute_command( @@ -259,9 +299,14 @@ def run_trial( checks = {key: raw_checks[key] for key in ("passed", "assertions")} if verification["returncode"] != (0 if checks["passed"] else 1): evaluation_error = "verifier_exit" - after = { - p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file() - } + try: + after = engineering.input_hashes(workspace) if engineering_artifact else { + p.relative_to(workspace).as_posix(): sha256(p) + for p in workspace.rglob("*") if p.is_file() + } + except (ValueError, OSError): + after = None + evaluation_error = evaluation_error or "invalid_workspace_node" verdict = task_verdict(execution, trace, output, checks) if before != after: evaluation_error = evaluation_error or "inputs_changed" @@ -281,7 +326,12 @@ def run_trial( "observations": trace, "inputs_unchanged": before == after, "input_hashes": before, - "output_sha256": sha256(final) if final.is_file() else None, + "output_sha256": sha256(final) if final.is_file() and not final.is_symlink() else None, } + if engineering_artifact: + retained = trial_dir / "parser.py" + record["artifact_sha256"] = (sha256(retained) if retained.is_file() + and not retained.is_symlink() else None) + record["artifact_kind"] = engineering.ADAPTER atomic_write_json(trial_dir / "result.json", record) return record diff --git a/tests/fixtures/skill_eval_engineering/contract.md b/tests/fixtures/skill_eval_engineering/contract.md new file mode 100644 index 000000000..69edc38b6 --- /dev/null +++ b/tests/fixtures/skill_eval_engineering/contract.md @@ -0,0 +1,74 @@ +# Frozen task: serving response classification v1 + +This is a reconstructed CPU regression fixture derived from the public observation +in [PR 90](https://github.com/InferMatrixLab/InferRSI/pull/90), not the original +server code, hardware experiment, or qualification of that whole PR. + +Source: [silent capacity and error failures](https://github.com/InferMatrixLab/InferRSI/blob/86f003364804d116b3202c4d807cc898a3a02ade/src/inferrsi/data/skill_library/methods/inference-optimization-casebook/cases/qwen38-silent-capacity-and-error-failures.md). +Source base: `a1ee3266466194c16f4e2159ab845b758ed9561b`. +Source candidate: `86f003364804d116b3202c4d807cc898a3a02ade`. +Controller implementation baseline: `ab03f3ffb8f2e5bc02c304828c5c41890eef673e`. +Task contract frozen before adapter implementation and before any agent trial. + +## Starter and allowed change + +`parser.py` SHA-256: +`6d9d97fbf9b973c532447ad794a7a87c5c18a4d5d57979d91384f0861836b7c6`. +Only `parser.py` may change. Its API is +`evaluate(status, frames, requested_tokens) -> {"success": bool, "reason": str}`. +The starter incorrectly accepts every HTTP 200 response. Artifacts are bounded +UTF-8 Python source, at most 64 KiB. No dependency installation is needed. +Task JSON, selected skill bundle, schema, validator and all other workspace files +are frozen; additions outside this single allowed file invalidate the trial. + +## Public decision contract + +Inputs are an HTTP status integer, a list of already decoded frames, and a +positive integer requested token count. This is not a wire-format SSE parser. +Content frames have exactly `{"content": }`. A terminal frame has exactly +`{"finish_reason": "length" | "stop", "completion_tokens": }`. +At most one optional final string `[DONE]` is allowed; it is not a terminal frame. +A terminal may not be followed by content or another terminal. Boolean counts +are not integers. All test vectors and this contract are public, not held out. + +Apply HTTP failure first; otherwise inspect frames in order. An error dictionary +(containing `error`) gives `stream_error`; invalid fields, duplicate/post-terminal +frames, or a misplaced/duplicate DONE give `malformed_frame`. Then require a +terminal (`missing_terminal`). For length, completion_tokens must equal requested; +for stop, it may be any integer in [0, requested]. Otherwise `token_mismatch`. +Only a valid length or stop completion returns success=true, with reason `length` +or `stop`. Every failure returns success=false with its specified reason. +HTTP non-200 returns `http_error`. Expected verdict checks both fields, not merely +the boolean. Capacity planning, exception logging and GPU timing are out of scope. + +## Controls, execution and evidence + +Independent public checks cover length, early stop, HTTP error, mid-stream error, +missing terminal, length/stop token mismatch and malformed frames. Software +controls include known-correct and incorrect artifacts, missing artifact, frozen +input tamper, changed/stale PR head and unavailable isolation. These controls are +not model outcomes or evidence that a skill improves an agent. + +The controller retains the actual parser bytes and their hash independently of +the final agent message. Report revalidates those bytes using the trusted in-tree +verifier; agent-reported PASS is not authoritative. Artifact code is executed only +inside a verified bubblewrap namespace with clean environment, no network/home, +read-only runtime and bounded process/time/output resources. Isolation unavailable +means INCONCLUSIVE; there is no unsandboxed fallback. Verifier tests are outside +the writable artifact and are not mounted into its execution view. + +Baseline/candidate get identical starter and task, separate sessions and explicit +selected skill exposure. This does not test retrieval. Existing Codex +`danger-full-access` is not OS confinement and must not be represented as such. +Real agent execution is blocked until its environment/access, oracle and other-arm +visibility, model/settings and trustworthy cost bounds are verified and recorded +in an authorized frozen plan. Local wall-clock/token-stop observations alone are +not a per-call dollar cap. No model run is authorized by this fixture. + +Before a real run freeze exact source/bundle/task/controller/verifier hashes, +execution image/runtime, order/repeats, tool access, timeout and whole-run budget +(including setup, selector, retries and any judge). No silent retries. A later +head change requires explicit revalidation before a PR recommendation. No gain or +negative evidence is valid delivery, not automatic PR rejection; a task PASS is +not whole-PR acceptance. Author posts report and accessible artifacts; reviewer +independently verifies scope/results. No CI result-import or automatic acceptance. diff --git a/tests/fixtures/skill_eval_engineering/parser.py b/tests/fixtures/skill_eval_engineering/parser.py new file mode 100644 index 000000000..fd70069ee --- /dev/null +++ b/tests/fixtures/skill_eval_engineering/parser.py @@ -0,0 +1,2 @@ +def evaluate(status, frames, requested_tokens): + return {"success": status == 200, "reason": "length" if status == 200 else "http_error"} diff --git a/tests/test_skill_eval_engineering.py b/tests/test_skill_eval_engineering.py new file mode 100644 index 000000000..6a734485a --- /dev/null +++ b/tests/test_skill_eval_engineering.py @@ -0,0 +1,256 @@ +import copy +import json +import subprocess +import sys +from pathlib import Path +from unittest import mock + +import pytest + +from inferrsi.harness import skill_eval_engineering as engineering + +TASK = {"id": "serving-parser", "split": "development", "request": engineering.REQUEST, + "provenance": "Reconstructed public regression fixture"} +ANSWER = {"artifact": "parser.py", "summary": "Repair", "artifact_source": engineering.STARTER} + + +def test_starter_matches_frozen_fixture(): + fixture = Path(__file__).parent / "fixtures/skill_eval_engineering/parser.py" + assert fixture.read_text() == engineering.STARTER + + +@pytest.mark.parametrize("change", [{"split": "holdout"}, {"split": "discovery"}, + {"id": "../escape"}, {"expected": True}, + {"request": ""}, {"request": "Grade an unrelated task"}]) +def test_strict_task_boundary(change): + with pytest.raises(ValueError): + engineering.validate_task({**TASK, **change}) + + +@pytest.mark.parametrize("change", [{"artifact": "other.py"}, {"artifact_source": None}, + {"artifact_source": ""}, {"passed": True}, + {"artifact_source": "x" * 65537}]) +def test_missing_wrong_or_oversized_artifact_never_executes(change): + with mock.patch.object(engineering, "_invoke") as invoke: + assert not engineering.verify(TASK, {**ANSWER, **change})["passed"] + invoke.assert_not_called() + + +def test_unavailable_sandbox_is_not_a_failing_skill_or_host_fallback(): + with mock.patch.object(engineering, "_invoke", return_value=None) as invoke: + with pytest.raises(engineering.SandboxUnavailable): + engineering.verify(TASK, ANSWER) + assert invoke.call_count == 1 + assert invoke.call_args.kwargs == {"probe": True} + + +def test_trusted_parent_compares_each_behavior_not_agent_checks(): + expected = [{"success": success, "reason": reason} + for _, _, _, _, success, reason in engineering.VECTORS] + with mock.patch.object(engineering, "_invoke", side_effect=[{"ready": True}, *expected]): + result = engineering.verify(TASK, ANSWER) + assert result["passed"] + assert len(result["assertions"]) == len(engineering.VECTORS) + # These are controller tests with mocked processes, not an artifact PASS. + wrong = copy.deepcopy(expected) + wrong[4] = {"success": True, "reason": "length"} + with mock.patch.object(engineering, "_invoke", side_effect=[{"ready": True}, *wrong]): + result = engineering.verify(TASK, ANSWER) + assert not result["passed"] + assert result["assertions"]["stream_error"] is False + + +def test_child_receives_inputs_without_expected_results(): + calls = [] + + def invoke(work, arguments=None, *, probe=False): + assert (work / "parser.py").read_text() == ANSWER["artifact_source"] + assert sorted(path.name for path in work.iterdir()) == ["parser.py", "runner.py"] + if probe: + return {"ready": True} + calls.append(arguments) + return {"success": False, "reason": "malformed_frame"} + + with mock.patch.object(engineering, "_invoke", side_effect=invoke): + engineering.verify(TASK, ANSWER) + assert calls == [[status, frames, count] for _, status, frames, count, _, _ + in engineering.VECTORS] + assert "VECTORS" not in engineering.RUNNER + + +def test_exact_boolean_and_reason_required(): + for wrong in ({"success": 1, "reason": "length"}, {"success": True, "reason": "anything"}, + {"success": True, "reason": "length", "extra": True}, None): + with mock.patch.object(engineering, "_invoke", + side_effect=[{"ready": True}] + [wrong] * len(engineering.VECTORS)): + assert not engineering.verify(TASK, ANSWER)["passed"] + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux sandbox command layout") +def test_sandbox_command_has_no_host_home_repo_or_network_mount(): + with mock.patch.object(engineering.shutil, "which", return_value="/usr/bin/bwrap"), \ + mock.patch.object(Path, "resolve", return_value=Path("/usr/bin/python3.13")), \ + mock.patch.object(Path, "is_file", return_value=True), \ + mock.patch.object(Path, "is_dir", return_value=True): + command = engineering.sandbox_command(Path("/temporary/test-artifact")) + assert "--unshare-all" in command + assert "--clearenv" in command + assert command[command.index("--remount-ro") + 1] == "/" + bindings = [command[i + 1:i + 3] for i, item in enumerate(command) if item == "--ro-bind"] + assert ["/temporary/test-artifact", "/work"] in bindings + assert all(source not in ("/", "/home", "/tmp", "/workspace") for source, _ in bindings) + assert not any("--share" in item for item in command) + assert command[-5:] == ["/usr/bin/python3", "-I", "-S", "-B", "/work/runner.py"] + + +def test_no_bwrap_is_fail_closed(): + with mock.patch.object(engineering.shutil, "which", return_value=None): + with pytest.raises(engineering.SandboxUnavailable): + engineering.sandbox_command(Path("/unused")) + + +def test_invoke_uses_clean_environment_and_bounded_output(): + process = mock.Mock(returncode=0) + + def start(*args, **kwargs): + assert kwargs["env"] == {"PATH": "/usr/bin:/bin", "LANG": "C"} + assert kwargs["start_new_session"] is True + kwargs["stdout"].write(json.dumps({"ready": True}).encode()) + return process + + with mock.patch.object(engineering, "sandbox_command", return_value=["sandbox"]), \ + mock.patch.object(engineering.subprocess, "Popen", side_effect=start): + assert engineering._invoke(Path("/unused"), probe=True) == {"ready": True} + assert process.communicate.call_args.kwargs["timeout"] == 5 + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux process-group cleanup") +def test_task_timeout_kills_process_group_and_is_failure(): + process = mock.Mock(returncode=-9, pid=123456) + process.communicate.side_effect = [subprocess.TimeoutExpired("sandbox", 5), None] + + def start(*args, **kwargs): + kwargs["stderr"].write(b"INFERRSI_ARTIFACT_READY\n") + return process + + with mock.patch.object(engineering, "sandbox_command", return_value=["sandbox"]), \ + mock.patch.object(engineering.subprocess, "Popen", side_effect=start), \ + mock.patch.object(engineering.os, "killpg") as kill: + assert engineering._invoke(Path("/unused"), []) is None + kill.assert_called_once_with(process.pid, engineering.signal.SIGKILL) + + +def test_real_sandbox_known_bad_starter_or_explicit_unavailable(): + # No model call. This fixture is our published starter, not downloaded code. + try: + result = engineering.verify(TASK, ANSWER) + except engineering.SandboxUnavailable as error: + pytest.skip(str(error)) + assert not result["passed"] + assert result["assertions"]["length"] + assert not result["assertions"]["stream_error"] + + +# Authored software control; never recorded as a model-produced candidate. +GOOD_ARTIFACT = '''def evaluate(status, frames, requested_tokens): + def result(reason): + return {"success": reason in ("length", "stop"), "reason": reason} + if status != 200: + return result("http_error") + terminal = None + done = False + for index, frame in enumerate(frames): + if frame == "[DONE]": + if done or index != len(frames) - 1: + return result("malformed_frame") + done = True + continue + if isinstance(frame, dict) and "error" in frame: + return result("stream_error") + if done or terminal is not None or not isinstance(frame, dict): + return result("malformed_frame") + if set(frame) == {"content"} and type(frame["content"]) is str: + continue + if (set(frame) != {"finish_reason", "completion_tokens"} + or frame["finish_reason"] not in ("length", "stop") + or type(frame["completion_tokens"]) is not int + or frame["completion_tokens"] < 0): + return result("malformed_frame") + terminal = frame + if terminal is None: + return result("missing_terminal") + count = terminal["completion_tokens"] + finish = terminal["finish_reason"] + if count > requested_tokens or (finish == "length" and count != requested_tokens): + return result("token_mismatch") + return result(finish) +''' + + +def test_real_sandbox_known_correct_control_or_explicit_unavailable(): + try: + result = engineering.verify(TASK, {**ANSWER, "artifact_source": GOOD_ARTIFACT}) + except engineering.SandboxUnavailable as error: + pytest.skip(str(error)) + assert result["passed"] + + +def test_regular_snapshot_rejects_symlink_oversize_and_fifo(tmp_path): + source = tmp_path / "parser.py" + source.write_bytes(b"code") + assert engineering.read_regular(source, 4) == b"code" + with pytest.raises(ValueError): + engineering.read_regular(source, 3) + link = tmp_path / "linked.py" + link.symlink_to(source) + with pytest.raises(ValueError): + engineering.read_regular(link, 100) + if hasattr(engineering.os, "mkfifo"): + fifo = tmp_path / "pipe" + engineering.os.mkfifo(fifo) + with pytest.raises(ValueError): + engineering.read_regular(fifo, 100) + + +def test_input_manifest_excludes_only_mutable_artifact_and_rejects_special_nodes(tmp_path): + (tmp_path / "task.json").write_text("{}") + (tmp_path / "parser.py").write_text(engineering.STARTER) + before = engineering.input_hashes(tmp_path) + assert set(before) == {"task.json"} + (tmp_path / "parser.py").write_text("changed") + assert engineering.input_hashes(tmp_path) == before + (tmp_path / "extra.txt").write_text("not allowed") + assert engineering.input_hashes(tmp_path) != before + (tmp_path / "link").symlink_to(tmp_path / "task.json") + with pytest.raises(ValueError): + engineering.input_hashes(tmp_path) + + +def test_negative_count_mutant_is_caught_by_public_oracle(): + # Execute only this repository-authored fixture, not submitted agent code. + # This is an oracle control, not evidence that the real sandbox ran. + mutant = GOOD_ARTIFACT.replace(' or frame["completion_tokens"] < 0', '') + assert mutant != GOOD_ARTIFACT + namespace = {} + exec(compile(mutant, "trusted-mutant-fixture", "exec"), namespace) + name, status, frames, count, success, reason = next( + vector for vector in engineering.VECTORS if vector[0] == "negative_count") + assert namespace["evaluate"](status, frames, count) != {"success": success, "reason": reason} + + +@pytest.mark.parametrize("ready", [False, True]) +def test_worker_startup_failure_differs_from_artifact_crash(ready): + process = mock.Mock(returncode=1) + + def start(*args, **kwargs): + if ready: + kwargs["stderr"].write(b"INFERRSI_ARTIFACT_READY\n") + return process + + with mock.patch.object(engineering, "sandbox_command", return_value=["sandbox"]), \ + mock.patch.object(engineering.subprocess, "Popen", side_effect=start): + if ready: + assert engineering._invoke(Path("/unused"), []) is None + else: + with pytest.raises(engineering.SandboxUnavailable, match="trusted boundary"): + engineering._invoke(Path("/unused"), []) diff --git a/tests/test_skill_eval_local.py b/tests/test_skill_eval_local.py new file mode 100644 index 000000000..ef67aed68 --- /dev/null +++ b/tests/test_skill_eval_local.py @@ -0,0 +1,249 @@ +"""Local handoff software controls; fake grader results are not agent efficacy.""" +import json + +import pytest + +from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.infra.artifacts import sha256, write_json +from tests.fixtures.skill_eval_compare_support import configuration +from tools import skill_eval_compare as compare +from tools import skill_eval_local as local + + +def prepared(tmp_path): + config_path, repo = configuration(tmp_path, ids=("parser",), repeats=1) + config = json.loads(config_path.read_text()) + config["adapter"] = engineering.ADAPTER + write_json(config_path, config) + write_json(tmp_path / "cases.json", [{"id": "parser", "split": "development", + "request": engineering.REQUEST, + "provenance": "public reconstructed test fixture"}]) + output = tmp_path / "run" + digest = compare.prepare(config_path, repo, output) + return output, digest, config["snapshots"]["candidate"]["git_revision"] + + +def submission(tmp_path, output): + root = tmp_path / "author-logs" + root.mkdir() + plan = compare.load_plan(output) + write_json(root / "execution.json", { + "model": plan["model"], "environment": "offline test fixture", + "access": "not an actual agent run", "resources": "no model charges", + "usage": None, "elapsed_seconds": 1.0, "agent_exit_code": 0, + "summary": "software fixture only", + }) + (root / "stdout.log").write_text("author-supplied fixture log\n") + (root / "stderr.log").write_text("") + return root + + +def fake_verify(task, answer): + engineering.validate_task(task) + passed = answer["artifact_source"] == "# authored correct fixture\n" + return {"passed": passed, "assertions": {"software_fixture": passed}} + + +def test_manual_handoff_keeps_artifact_and_author_provenance_separate(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(engineering, "verify", fake_verify) + logs = submission(tmp_path, output) + for role in compare.ROLES: + workspace = tmp_path / role + local.stage(output, digest, "parser", role, 1, workspace, head) + if role == "candidate": + (workspace / "parser.py").write_text("# authored correct fixture\n") + result = local.capture(output, digest, "parser", role, 1, workspace, head, logs) + assert result["counts"]["artifact_improved"] == 1 + assert "do not prove skill causality" in result["claim"] + pair = result["pairs"][0] + assert pair["baseline"]["verdict"] == "FAIL" + assert pair["candidate"]["verdict"] == "PASS" + assert pair["candidate"]["author_reported_execution"] is True + assert pair["candidate"]["grader_trust"] == "reverified" + assert "parser.py" in pair["candidate"]["evidence"] + assert "author-execution.json" in pair["candidate"]["evidence"] + assert pair["candidate"]["observations"]["usage"] is None + assert not (output / "results/start.json").exists() + with pytest.raises(ValueError, match="already started"): + local.capture(output, digest, "parser", "candidate", 1, tmp_path / "candidate", head, logs) + + +@pytest.mark.parametrize("mutation", ["missing", "input", "additional", "symlink"]) +def test_capture_rejects_missing_or_changed_frozen_inputs(tmp_path, monkeypatch, mutation): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(engineering, "verify", fake_verify) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + if mutation == "missing": + (workspace / "parser.py").unlink() + elif mutation == "input": + (workspace / "task.json").write_text("{}") + elif mutation == "additional": + (workspace / "extra").mkdir() + (workspace / "extra/parser.py").write_text("not an allowed edit") + else: + (workspace / "parser.py").unlink() + (workspace / "parser.py").symlink_to(tmp_path / "absent") + result = local.capture(output, digest, "parser", "baseline", 1, workspace, head, + submission(tmp_path, output)) + assert result["pairs"][0]["baseline"]["verdict"] == "INCONCLUSIVE" + + +def test_stale_head_and_missing_logs_are_not_accepted(tmp_path): + output, digest, head = prepared(tmp_path) + workspace = tmp_path / "baseline" + with pytest.raises(ValueError, match="head differs"): + local.stage(output, digest, "parser", "baseline", 1, workspace, "0" * 40) + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + logs = submission(tmp_path, output) + (logs / "stdout.log").unlink() + with pytest.raises(ValueError, match="Supply execution.json"): + local.capture(output, digest, "parser", "baseline", 1, workspace, head, logs) + + +def test_unavailable_sandbox_remains_inconclusive(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + + def unavailable(*_): + raise engineering.SandboxUnavailable("mocked unsupported host") + + monkeypatch.setattr(engineering, "verify", unavailable) + result = local.capture(output, digest, "parser", "baseline", 1, workspace, head, + submission(tmp_path, output)) + trial = result["pairs"][0]["baseline"] + assert trial["verdict"] == "INCONCLUSIVE" + assert trial["evaluation_error"] == "artifact_sandbox_unavailable" + + +def test_no_unsafe_default_agent_launch(tmp_path): + output, digest, _ = prepared(tmp_path) + with pytest.raises(ValueError, match="confined local client"): + compare.execute(output, digest) + + +def test_retained_artifact_tamper_and_missing_author_evidence(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(engineering, "verify", fake_verify) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + result = local.capture(output, digest, "parser", "baseline", 1, workspace, head, + submission(tmp_path, output)) + artifact = output / result["pairs"][0]["baseline"]["evidence"]["parser.py"] + artifact.write_text("# substituted source") + result = compare.read_report(output) + assert result["pairs"][0]["baseline"]["evaluation_error"] == "retained_artifact_hash_mismatch" + + +def test_rewritten_stage_receipt_cannot_authorize_changed_inputs(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(engineering, "verify", fake_verify) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + (workspace / "task.json").write_text("{}") + directory = compare.trial_directory(output, compare.load_plan(output)["schedule"][0]) + receipt = json.loads((directory / "local-stage.json").read_text()) + receipt["input_hashes"]["task.json"] = sha256(workspace / "task.json") + write_json(directory / "local-stage.json", receipt) + result = local.capture(output, digest, "parser", "baseline", 1, workspace, head, + submission(tmp_path, output)) + assert result["pairs"][0]["baseline"]["evaluation_error"] == "inputs_changed" + + +@pytest.mark.parametrize("field,value,error", [ + ("agent_exit_code", 1, "reported_agent_failure"), + ("elapsed_seconds", 61, "reported_run_exceeded_frozen_timeout"), +]) +def test_valid_artifact_does_not_erase_failed_or_overbudget_author_run( + tmp_path, monkeypatch, field, value, error): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(engineering, "verify", fake_verify) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + (workspace / "parser.py").write_text("# authored correct fixture\n") + logs = submission(tmp_path, output) + metadata = json.loads((logs / "execution.json").read_text()) + metadata[field] = value + write_json(logs / "execution.json", metadata) + result = local.capture(output, digest, "parser", "baseline", 1, workspace, head, logs) + trial = result["pairs"][0]["baseline"] + assert trial["verdict"] == "INCONCLUSIVE" + assert trial["artifact_verdict"] == "PASS" + assert trial["evaluation_error"] == error + + +def test_retained_symlink_is_never_read_as_artifact_evidence(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(engineering, "verify", fake_verify) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + result = local.capture(output, digest, "parser", "baseline", 1, workspace, head, + submission(tmp_path, output)) + artifact = output / result["pairs"][0]["baseline"]["evidence"]["parser.py"] + artifact.unlink() + target = tmp_path / "not-evidence.txt" + target.write_text("must not appear in report") + artifact.symlink_to(target) + result = compare.read_report(output) + trial = result["pairs"][0]["baseline"] + assert trial["evaluation_error"] == "unsafe_retained_file" + assert trial["output"] is None + assert "must not appear" not in json.dumps(result) + + +def test_retained_artifact_preserves_crlf_bytes(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(engineering, "verify", fake_verify) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + (workspace / "parser.py").write_bytes(b"# CRLF artifact\r\n") + result = local.capture(output, digest, "parser", "baseline", 1, workspace, head, + submission(tmp_path, output)) + trial = result["pairs"][0]["baseline"] + assert trial["verdict"] == "FAIL" # Fake grader intentionally rejects this source. + assert trial["evaluation_error"] is None + + +@pytest.mark.parametrize("target", ["execution.json", "author-execution.json", "answer.json", + "parser.py", "agent", "verification", "receipt.json"]) +def test_capture_refuses_preexisting_dangling_retained_target(tmp_path, monkeypatch, target): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(engineering, "verify", fake_verify) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + directory = compare.trial_directory(output, compare.load_plan(output)["schedule"][0]) + outside = tmp_path / "must-not-be-created" + (directory / target).symlink_to(outside) + with pytest.raises(ValueError, match="reserved targets"): + local.capture(output, digest, "parser", "baseline", 1, workspace, head, + submission(tmp_path, output)) + assert not outside.exists() + + +def test_report_replays_artifact_after_blocked_capture_without_rewriting_attempt( + tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + workspace = tmp_path / "baseline" + local.stage(output, digest, "parser", "baseline", 1, workspace, head) + (workspace / "parser.py").write_text("# authored correct fixture\n") + + def unavailable(*_): + raise engineering.SandboxUnavailable("capture host blocked") + + monkeypatch.setattr(engineering, "verify", unavailable) + local.capture(output, digest, "parser", "baseline", 1, workspace, head, + submission(tmp_path, output)) + directory = compare.trial_directory(output, compare.load_plan(output)["schedule"][0]) + retained_record = (directory / "result.json").read_bytes() + monkeypatch.setattr(engineering, "verify", fake_verify) + result = compare.read_report(output) + trial = result["pairs"][0]["baseline"] + assert trial["artifact_verdict"] == "PASS" + assert trial["artifact_checks"]["passed"] is True + assert trial["checks"] is None + assert trial["evaluation_error"] == "artifact_sandbox_unavailable" + assert trial["verdict"] == "INCONCLUSIVE" + assert result["pairs"][0]["comparability"] == "unverified_author_reported" + assert (directory / "result.json").read_bytes() == retained_record diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index e07b10f67..07b259f24 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -1,4 +1,5 @@ import argparse +import difflib import hashlib import json import math @@ -12,6 +13,7 @@ from pathlib import Path from inferrsi.harness import skill_eval_diagnostic as diagnostic +from inferrsi.harness import skill_eval_engineering as engineering from inferrsi.harness.skill_evaluation import ( CodexClient, export_bundle, @@ -33,7 +35,8 @@ GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") CONTROLLERS = (GUARD_SOURCE, "tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", "src/inferrsi/harness/skill_evaluation.py", - "src/inferrsi/harness/skill_eval_diagnostic.py", "tools/skill_eval_snapshot.py", + "src/inferrsi/harness/skill_eval_diagnostic.py", + "src/inferrsi/harness/skill_eval_engineering.py", "tools/skill_eval_snapshot.py", "src/inferrsi/infra/executors/local.py", "src/inferrsi/infra/processes.py", "src/inferrsi/infra/artifacts.py") EVIDENCE = ("execution.json", "agent/stdout.log", "agent/stderr.log", @@ -178,13 +181,15 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> "adapter"}: raise ValueError("Unknown configuration field") adapter = config.get("adapter", "audit") - if adapter not in ("audit", diagnostic.ADAPTER): + if adapter not in ("audit", diagnostic.ADAPTER, engineering.ADAPTER): raise ValueError("Unknown experiment adapter") - if adapter == diagnostic.ADAPTER: + if adapter in (diagnostic.ADAPTER, engineering.ADAPTER): if config["mode"] != "update": raise ValueError("Diagnostic comparisons require paired old/new bundles (update mode)") if "grader" in config or "schema" in config: - raise ValueError("Diagnostic adapter owns its frozen schema and verifier") + raise ValueError("Selected adapter owns its frozen schema and verifier") + if adapter == engineering.ADAPTER and "client" in config: + raise ValueError("Engineering artifacts require local execution, not read/submit provider") if config["mode"] not in ("addition", "update"): raise ValueError("mode must be addition or update") if not re.fullmatch(r"[a-z0-9][a-z0-9._-]*", config["skill"]): @@ -218,7 +223,9 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> if not tasks: raise ValueError("cases must be nonempty with unique IDs") for task in tasks: - if adapter == diagnostic.ADAPTER: + if adapter == engineering.ADAPTER: + engineering.validate_task(task) + elif adapter == diagnostic.ADAPTER: diagnostic.validate_task(task) else: validate_input(task, CASE_INPUT) @@ -305,16 +312,17 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: config = read_json(config_path) tasks = read_json(config_path.parent / config["cases"]) is_diagnostic = config.get("adapter") == diagnostic.ADAPTER - grader = Path(diagnostic.__file__) if is_diagnostic else ( - config_path.parent / config["grader"] if "grader" in config else ( - ROOT / "tools/skill_eval_verifier.py" - ) - ) + is_engineering = config.get("adapter") == engineering.ADAPTER + grader = (Path(engineering.__file__) if is_engineering else + Path(diagnostic.__file__) if is_diagnostic else + config_path.parent / config["grader"] if "grader" in config else + ROOT / "tools/skill_eval_verifier.py") validate(config, tasks, default_grader=( grader.read_bytes() == (ROOT / "tools/skill_eval_verifier.py").read_bytes())) require_git_version(subprocess.check_output(("git", "--version"), cwd=repo, text=True)) - schema = diagnostic.SCHEMA if is_diagnostic else ( - read_json(config_path.parent / config["schema"]) if "schema" in config else SCHEMA) + schema = (engineering.SCHEMA if is_engineering else + diagnostic.SCHEMA if is_diagnostic else + read_json(config_path.parent / config["schema"]) if "schema" in config else SCHEMA) if "client" in config: validate_schema(schema) output.mkdir(parents=True, exist_ok=False) @@ -323,11 +331,14 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: write_json(output / "plan/tasks.json", tasks) write_json(output / "plan/schema.json", schema) shutil.copyfile(grader, output / "validator/verify.py") + if is_engineering: + (output / "plan/starter").mkdir() + (output / "plan/starter/parser.py").write_text(engineering.STARTER, encoding="utf-8") arms = {} with tempfile.TemporaryDirectory(prefix="se-") as temporary: for role, short in zip(ROLES, ("b", "c")): selected = config["snapshots"][role] - if is_diagnostic: + if is_diagnostic or is_engineering: closure = selected_snapshot( repo, selected["git_revision"], config["skill_root"], config["skill"], Path(temporary) / short, bundle_revision=selected.get("bundle_revision")) @@ -366,7 +377,8 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: "exam": {"cases": files["plan/tasks.json"], "schema": files["plan/schema.json"], "grader": files["validator/verify.py"]}, "same_bundle": before is not None and before["revision"] == after["revision"], - "prompt": diagnostic.PROMPT if is_diagnostic else PROMPT, "files": files, + "prompt": (engineering.PROMPT if is_engineering else + diagnostic.PROMPT if is_diagnostic else PROMPT), "files": files, "controllers": {name: sha256(ROOT / name) for name in CONTROLLERS}, "retry_policy": "none", "cost": None, "isolation": "Fresh sessions and workspaces; selected bundle only. " @@ -375,6 +387,14 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: } if is_diagnostic: plan["adapter"] = diagnostic.ADAPTER + if is_engineering: + plan["adapter"] = engineering.ADAPTER + name = "src/inferrsi/harness/skill_eval_engineering.py" + plan["controllers"][name] = sha256(ROOT / name) + local_source = "tools/skill_eval_local.py" + plan["controllers"][local_source] = sha256(ROOT / local_source) + plan["isolation"] += (" Artifact replay requires bubblewrap namespaces and fails closed. " + "Public tests, not blind evaluation; reconstructed CPU fixture.") if "client" in config: client = AnthropicAgentClient(config["model"]["name"], config["client"], config["limits"]["timeout_seconds"], @@ -532,6 +552,9 @@ def execute(output: Path, authorization: str, *, client=None, configured.environment() # Missing keys fail before start/trial evidence. client = configured stop_on_unknown_usage = True + if plan.get("adapter") == engineering.ADAPTER and client is None: + raise ValueError("Engineering execution requires a verified confined local client; " + "the default danger-full-access Codex client is not qualified") if client is None: client = CodexClient(plan["model"]["name"], plan["model"]["effort"], plan["limits"]["timeout_seconds"]) @@ -563,6 +586,9 @@ def execute(output: Path, authorization: str, *, client=None, with tempfile.TemporaryDirectory(prefix="se-trial-") as temporary: workspace = Path(temporary) write_json(workspace / "task.json", task) + is_engineering = plan.get("adapter") == engineering.ADAPTER + if is_engineering: + shutil.copyfile(output / "plan/starter/parser.py", workspace / "parser.py") prompt = plan["prompt"] + "\nTask operation: " + task["request"] if bundle is not None: folder = ".agents/skills" if task["split"] == "discovery" else "knowledge" @@ -573,7 +599,8 @@ def execute(output: Path, authorization: str, *, client=None, result = run_trial(client, task, role, entry["repeat"], workspace, directory, output / "plan/schema.json", output / "validator/verify.py", prompt, deadline, - timeout_seconds=plan["limits"]["timeout_seconds"]) + timeout_seconds=plan["limits"]["timeout_seconds"], + engineering_artifact=is_engineering) result["evidence_sha256"] = {name: sha256(directory / name) for name in EVIDENCE if (directory / name).is_file()} atomic_write_json(directory / "result.json", result) @@ -598,7 +625,7 @@ def execute(output: Path, authorization: str, *, client=None, return report(output) -def retained_trial(output: Path, entry: dict) -> dict: +def retained_trial(output: Path, entry: dict, expected_adapter=None) -> dict: directory = trial_directory(output, entry) path = directory / "result.json" if not path.is_file(): @@ -607,6 +634,20 @@ def retained_trial(output: Path, entry: dict) -> dict: "evaluation_error": "interrupted_attempt" if interrupted else None, "missing": not interrupted, "checks": None, "output": None, "evidence": {}, "elapsed_seconds": None, "observations": {"usage": None}} + if expected_adapter == engineering.ADAPTER: + # A retained tree is untrusted input too: reject special/symlink evidence + # before legacy report readers open it. Producers must already be stopped. + names = ("result.json", "receipt.json", "answer.json", "parser.py", + "author-execution.json", *EVIDENCE) + try: + for name in names: + retained_path = directory / name + if retained_path.exists() or retained_path.is_symlink(): + engineering.read_regular(retained_path, 8 * 1024 * 1024) + except ValueError: + return {**entry, "verdict": "INCONCLUSIVE", "evaluation_error": "unsafe_retained_file", + "missing": False, "checks": None, "output": None, "evidence": {}, + "elapsed_seconds": None, "observations": {"usage": None}} raw_record = path.read_bytes() error = receipt_error(raw_record, directory / "receipt.json", "trial") answer = directory / "answer.json" @@ -637,6 +678,13 @@ def retained_trial(output: Path, entry: dict) -> dict: if isinstance(record["observations"].get("commands"), list) else None, } error = record.get("evaluation_error") + if (expected_adapter == engineering.ADAPTER + and record.get("artifact_kind") != engineering.ADAPTER): + error = error or "missing_retained_artifact_identity" + if record.get("execution_state") == "author_reported": + selected["provenance"] = record.get("provenance") + selected["author_reported_execution"] = True + selected["artifact_verdict"] = None # Re-derived below, never trusted from a receipt. if record["observations"].get("client_kind"): selected["observations"].update({ key: record["observations"].get(key) for key in @@ -650,7 +698,9 @@ def retained_trial(output: Path, entry: dict) -> dict: if digests is None: error = error or "missing_retained_evidence_hashes" else: - for name in EVIDENCE: + required_evidence = (*EVIDENCE, "author-execution.json") if ( + record["execution_state"] == "author_reported") else EVIDENCE + for name in required_evidence: if name not in digests: if record["verdict"] in ("PASS", "FAIL"): error = error or "missing_retained_evidence_hashes" @@ -661,7 +711,27 @@ def retained_trial(output: Path, entry: dict) -> dict: error = error or "missing_retained_evidence" elif retained[name] != digests[name]: error = error or "retained_evidence_hash_mismatch" - if error or record["execution_state"] != "completed" or not record["inputs_unchanged"]: + elif name == "author-execution.json": + evidence[name] = artifact.relative_to(output).as_posix() + if record.get("artifact_kind") == engineering.ADAPTER and record.get("artifact_sha256") is None: + error = error or "missing_retained_artifact" + if record.get("artifact_sha256") is not None: + try: + artifact_answer = json.loads(data) if data is not None else None + except ValueError: + artifact_answer = None + artifact = directory / "parser.py" + if artifact.is_symlink() or not artifact.is_file(): + error = error or "missing_retained_artifact" + elif sha256(artifact) != record["artifact_sha256"]: + error = error or "retained_artifact_hash_mismatch" + elif (not isinstance(artifact_answer, dict) + or artifact_answer.get("artifact_source") != artifact.read_bytes().decode("utf-8")): + error = error or "retained_artifact_disagrees_with_answer" + else: + evidence["parser.py"] = artifact.relative_to(output).as_posix() + if (error or record["execution_state"] not in ("completed", "author_reported") + or not record["inputs_unchanged"]): selected["verdict"] = "INCONCLUSIVE" return {**selected, "evaluation_error": error, "missing": False, "output": text, @@ -688,7 +758,7 @@ def condition_summary(trials: list[dict]) -> dict: def read_report(output: Path) -> dict: plan = load_plan(output) - trials = [retained_trial(output, entry) for entry in plan["schedule"]] + trials = [retained_trial(output, entry, plan.get("adapter")) for entry in plan["schedule"]] # A receipt is an unkeyed hash: a rewritten tree can carry a consistent one. # Re-grading retained answers with trusted code is what makes a PASS/FAIL # verdict independent of the artifact copy the agent could reach. This runs @@ -710,7 +780,13 @@ def read_report(output: Path) -> dict: outcome = {("FAIL", "PASS"): "fix", ("PASS", "FAIL"): "regression", ("PASS", "PASS"): "unchanged_correct", ("FAIL", "FAIL"): "unchanged_incorrect"}[before, after] - pairs.append({"case_id": entry["task_id"], "repeat": entry["repeat"], + manual = any(arm.get("author_reported_execution") for arm in arms.values()) + if manual: + outcome = {"fix": "artifact_improved", "regression": "artifact_regressed", + "unchanged_correct": "artifacts_both_pass", + "unchanged_incorrect": "artifacts_both_fail"}.get(outcome, outcome) + pairs.append({"comparability": "unverified_author_reported" if manual else "controller_run", + "case_id": entry["task_id"], "repeat": entry["repeat"], "outcome": outcome, **arms}) discovery = [t for t in trials if t["split"] == "discovery"] status = {"execution_state": "not_completed"} @@ -735,12 +811,27 @@ def read_report(output: Path) -> dict: "pairs": pairs, "counts": {name: sum(p["outcome"] == name for p in pairs) for name in ( "fix", "regression", "unchanged_correct", "unchanged_incorrect", "missing_pair", - "inconclusive")}, + "inconclusive", *( ("artifact_improved", "artifact_regressed", + "artifacts_both_pass", "artifacts_both_fail") + if plan.get("adapter") == engineering.ADAPTER else ()))}, "discovery": discovery, "discovery_conditions": condition_summary(discovery), "status": status, "claim": "Pipeline results only; fixed-output offline clients do not establish " "skill efficacy. Discovery is separate. Unknown usage and cost remain unknown.", } + if plan.get("adapter") == engineering.ADAPTER: + result["adapter"] = engineering.ADAPTER + if any(trial.get("author_reported_execution") for trial in trials): + captured = sum(bool(trial.get("author_reported_execution")) for trial in trials) + result["status"] = {"execution_state": "author_reported_unverified", + "artifact_capture_state": "complete" if captured == len(trials) + else "partial", "recorded_trials": captured, + "expected_trials": len(trials)} + result["claim"] = ( + "Reconstructed CPU parser artifact checks only. Author-run provenance, environment, " + "usage and costs are self-reported, not independently authenticated. Public tests " + "may be overfit. Paired artifact differences do not prove skill causality, original " + "engine behavior, hardware efficacy or whole-PR acceptance. Reviewer decides merge.") if "client" in plan: result["client"] = plan["client"] result["max_api_requests"] = plan["max_api_requests"] @@ -760,6 +851,7 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: trusted_verifier = { sha256(ROOT / "tools/skill_eval_verifier.py"): verify, sha256(Path(diagnostic.__file__)): diagnostic.verify, + sha256(Path(engineering.__file__)): engineering.verify, }.get(plan["exam"]["grader"]) if trusted_verifier is None: for trial in trials: @@ -774,11 +866,32 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: trial["evaluation_error"] = "unverified_custom_grader" return for trial in trials: - if trial["verdict"] not in ("PASS", "FAIL"): + manual_artifact = (trusted_verifier is engineering.verify + and trial.get("author_reported_execution") + and trial.get("evaluation_error") in ( + None, "reported_agent_failure", "artifact_sandbox_unavailable", + "reported_run_exceeded_frozen_timeout")) + if trial["verdict"] not in ("PASS", "FAIL") and not manual_artifact: continue - checks = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) - if checks != trial["checks"] or checks["passed"] != (trial["verdict"] == "PASS"): + try: + checks = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) + except engineering.SandboxUnavailable: + trial["verdict"] = "INCONCLUSIVE" + trial["grader_trust"] = "unverified" + trial["evaluation_error"] = "artifact_sandbox_unavailable" + continue + replay_after_blocked_capture = ( + manual_artifact and trial["checks"] is None + and trial["evaluation_error"] == "artifact_sandbox_unavailable") + if not replay_after_blocked_capture and ( + checks != trial["checks"] or (trial["verdict"] in ("PASS", "FAIL") + and checks["passed"] != (trial["verdict"] == "PASS"))): raise ValueError("Untrusted grading disagrees with the trusted grader") + if manual_artifact: + # Report can recheck a portable artifact on a capable reviewer host, + # without rewriting capture failure or upgrading pair comparability. + trial["artifact_checks"] = checks + trial["artifact_verdict"] = "PASS" if checks["passed"] else "FAIL" trial["grader_trust"] = "reverified" @@ -829,6 +942,9 @@ def markdown(result: dict) -> str: lines += [f"### {trial['task_id']} / {trial['repeat']} / {trial['condition']}", "", f"Verdict: {trial['verdict'] or 'missing'}; error: " f"{trial['evaluation_error'] or 'none'}; split: {trial['split']}", ""] + if trial.get("provenance"): + lines += ["Provenance: " + trial["provenance"], + "Independent artifact outcome: " + str(trial.get("artifact_verdict")), ""] lines += [f"[{name}]({path})" for name, path in trial["evidence"].items()] lines += ["", "Grading assertions:", "", "```json", json.dumps(trial["checks"], ensure_ascii=False, sort_keys=True), "```", "", @@ -839,7 +955,50 @@ def markdown(result: dict) -> str: "### Git source changes", "", "The same frozen exam is used by both arms. " "Source changes below are surfaced, not substituted for that exam.", "", "```text", "\n".join(result["source_changes"]) or "(none)", "```", ""] - return "\n".join(lines) + rendered = "\n".join(lines) + if result.get("adapter") != engineering.ADAPTER: + return rendered + summary = ["# Local engineering artifact evidence", "", result["claim"], "", + "**Merge implication:** artifact checks inform review; they do not authorize merge. " + "Verify current head, source fidelity, author-run controls and remaining PR scope.", + "", + "| Task | Baseline artifact | Candidate artifact | Observed difference |", + "| --- | --- | --- | --- |"] + for pair in result["pairs"]: + outcomes = [pair[role].get("artifact_verdict") or pair[role]["verdict"] or "missing" + for role in ROLES] + summary.append(f"| {pair['case_id']} | {outcomes[0]} | {outcomes[1]} | " + f"{pair['outcome']} |") + summary += ["", "## Failed checks and coverage", "", + "Public reconstructed response-parser checks only; no original server, GPU, " + "retrieval quality or whole-corpus coverage."] + for pair in result["pairs"]: + for role in ROLES: + trial = pair[role] + checks = trial.get("artifact_checks") or trial.get("checks") or {} + failed = [name for name, passed in checks.get("assertions", {}).items() if not passed] + summary.append(f"- {pair['case_id']} / {role}: " + + (", ".join(failed) if failed else "no failed check retained") + + "; limitation/error: " + str(trial.get("evaluation_error") or "none") + + "; comparability: " + pair["comparability"] + ".") + sources = [] + for role in ROLES: + try: + answer = json.loads(pair[role].get("output") or "null") + except ValueError: + answer = None + sources.append(answer.get("artifact_source") if isinstance(answer, dict) else None) + if all(isinstance(source, str) for source in sources): + delta = "".join(difflib.unified_diff(sources[0].splitlines(keepends=True), + sources[1].splitlines(keepends=True), + fromfile="baseline/parser.py", + tofile="candidate/parser.py")) + fence = "`" * max(4, 1 + max((len(run) for run in re.findall(r"`+", delta)), default=0)) + summary += ["", "### Retained artifact difference: " + pair["case_id"], "", + fence + "diff", delta or "(identical artifact bytes)", fence] + summary += ["", "
Revisions, settings, outputs and audit evidence", + "", rendered, "", "
", ""] + return "\n".join(summary) def main(): @@ -849,6 +1008,15 @@ def main(): parser.add_argument("--config", type=Path) parser.add_argument("--repo", type=Path) parser.add_argument("--authorize", help="Explicit execution authorization: frozen plan SHA-256") + handoff = parser.add_mutually_exclusive_group() + handoff.add_argument("--stage-local", type=Path, help="Export one author-run workspace") + handoff.add_argument("--capture-local", type=Path, help="Capture one author-run workspace") + parser.add_argument("--submission", type=Path, help="Author logs and execution.json directory") + parser.add_argument("--task-id") + parser.add_argument("--condition", choices=ROLES) + parser.add_argument("--repeat", type=int, default=1) + parser.add_argument("--current-head", + help="Reviewer-checked current PR head; not fetched by CLI") args = parser.parse_args() output = args.output.resolve() if args.mode == "prepare": @@ -859,7 +1027,23 @@ def main(): elif args.mode == "execute": if args.authorize is None: parser.error("execute requires --authorize ") - execute(output, args.authorize) + if args.stage_local or args.capture_local: + from tools import skill_eval_local + + if not args.task_id or not args.condition or not args.current_head: + parser.error("Local handoff requires --task-id, --condition, --current-head") + common = (output, args.authorize, args.task_id, args.condition, args.repeat) + if args.stage_local: + result = skill_eval_local.stage(*common, args.stage_local.resolve(), + args.current_head) + else: + if args.submission is None: + parser.error("Capture requires --submission with author logs") + result = skill_eval_local.capture(*common, args.capture_local.resolve(), + args.current_head, args.submission.resolve()) + print(json.dumps(result)) + else: + execute(output, args.authorize) else: report(output) diff --git a/tools/skill_eval_local.py b/tools/skill_eval_local.py new file mode 100644 index 000000000..be90f581a --- /dev/null +++ b/tools/skill_eval_local.py @@ -0,0 +1,220 @@ +"""Author-run local artifact handoff; no agent launch or CI result import. + +Stage exports one frozen arm. Capture records author-supplied execution evidence +and independently checks the actual artifact. It does not authenticate who ran an +agent, its access boundaries, its costs, or the completeness of the supplied log. +""" +import hashlib +import json +import math +import os +import shutil +import stat +from pathlib import Path + +from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.harness.skill_evaluation import normalize_usage +from inferrsi.infra.artifacts import sha256, write_json + + +def _selection(output, authorization, task_id, condition, repeat, current_head): + from tools import skill_eval_compare as compare + + if authorization != sha256(output / "plan/freeze.json"): + raise ValueError("Local handoff requires the exact frozen plan SHA-256") + plan = compare.load_plan(output) + if plan.get("adapter") != engineering.ADAPTER: + raise ValueError("Local artifact handoff is only for the engineering adapter") + if current_head != plan["arms"]["candidate"]["git_revision"]: + raise ValueError("Current PR head differs from the frozen candidate; revalidate first") + selected = [entry for entry in plan["schedule"] if entry["task_id"] == task_id + and entry["condition"] == condition and entry["repeat"] == repeat] + if len(selected) != 1: + raise ValueError("Trial is not in the frozen schedule") + task = next(task for task in compare.read_json(output / "plan/tasks.json") + if task["id"] == task_id) + return plan, selected[0], task, compare.trial_directory(output, selected[0]) + + +def _read_regular(path, bound=8 * 1024 * 1024): + if any(parent.is_symlink() for parent in (path, *path.parents)): + raise ValueError("Local evidence must not traverse symlinks") + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + try: + fd = os.open(path, flags) + except OSError as error: + raise ValueError("Local evidence is missing or cannot be safely opened") from error + with os.fdopen(fd, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > bound: + raise ValueError("Local evidence must be a bounded regular file") + data = stream.read(bound + 1) + if len(data) > bound: + raise ValueError("Local evidence grew beyond its bound") + return data + + +def _hashes(workspace): + result = {} + for path in workspace.rglob("*"): + if path.is_symlink(): + raise ValueError("Local workspace must not contain symlinks") + if path.is_dir(): + continue + name = path.relative_to(workspace).as_posix() + if name != "parser.py": + result[name] = hashlib.sha256(_read_regular(path)).hexdigest() + return result + + +def _prompt(plan, task, condition): + prompt = plan["prompt"] + "\nTask operation: " + task["request"] + bundle = plan["arms"][condition]["bundle"] + if bundle: + prompt += "\nUse assigned skill at knowledge/" + bundle["entrypoint"] + return prompt + + +def _expected_inputs(plan, task, condition): + def digest(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + inputs = {"task.json": digest(json.dumps(task, ensure_ascii=False, indent=2, + allow_nan=False) + "\n"), + "INSTRUCTIONS.txt": digest(_prompt(plan, task, condition))} + bundle = plan["arms"][condition]["bundle"] + if bundle: + inputs.update({"knowledge/" + name: value for name, value in bundle["resources"].items()}) + return inputs + + +def stage(output: Path, authorization: str, task_id: str, condition: str, repeat: int, + workspace: Path, current_head: str): + plan, entry, task, directory = _selection( + output, authorization, task_id, condition, repeat, current_head) + if directory.exists() or workspace.exists(): + raise ValueError("Stage requires fresh trial and workspace; never overwrite an attempt") + # Keep writable work separate from retained evidence/validator and the other arm. + if output.resolve() in workspace.resolve().parents: + raise ValueError("Stage workspace outside the evidence tree") + workspace.mkdir(parents=True) + write_json(workspace / "task.json", task) + shutil.copyfile(output / "plan/starter/parser.py", workspace / "parser.py") + bundle = plan["arms"][condition]["bundle"] + prompt = _prompt(plan, task, condition) + if bundle: + shutil.copytree(output / "plan/bundles" / condition, workspace / "knowledge") + (workspace / "INSTRUCTIONS.txt").write_text(prompt, encoding="utf-8") + directory.mkdir(parents=True) + write_json(directory / "local-stage.json", { + "plan_sha256": authorization, "entry": entry, "input_hashes": _hashes(workspace), + "provenance": "author-run; execution not observed by controller", + }) + return {"entry": entry, "plan_sha256": authorization, + "next": "Run approved local agent; only parser.py is writable. Keep logs separately. " + "Capture does not authorize or pay for that run."} + + +def capture(output: Path, authorization: str, task_id: str, condition: str, repeat: int, + workspace: Path, current_head: str, submission: Path): + plan, entry, task, directory = _selection( + output, authorization, task_id, condition, repeat, current_head) + stage_record = json.loads(_read_regular(directory / "local-stage.json")) + if stage_record["plan_sha256"] != authorization or stage_record["entry"] != entry: + raise ValueError("Stage receipt does not match the frozen trial") + if any(path.is_symlink() for path in (directory, *directory.parents)): + raise ValueError("Retained evidence directory must not traverse symlinks") + if {path.name for path in directory.iterdir()} != {"local-stage.json"}: + # Includes dangling symlinks: .exists() alone would let writes escape. + raise ValueError("Capture already started or reserved targets exist; " + "inspect, never overwrite") + meta_path = submission / "execution.json" + required = (meta_path, submission / "stdout.log", submission / "stderr.log") + if any(path.is_symlink() or not path.is_file() for path in required): + raise ValueError("Supply execution.json, stdout.log and stderr.log as regular files") + if any(path.stat().st_size > 8 * 1024 * 1024 for path in required): + raise ValueError("Local evidence exceeds the 8 MiB per-file bound") + submitted = {path.name: _read_regular(path) for path in required} + metadata = json.loads(submitted["execution.json"]) + if set(metadata) != {"model", "environment", "access", "resources", "usage", + "elapsed_seconds", "agent_exit_code", "summary"}: + raise ValueError("Local execution evidence has unknown or missing fields") + if metadata["model"] != plan["model"]: + raise ValueError("Author-stated model differs from frozen model") + if any(not isinstance(metadata[key], str) or not metadata[key].strip() + for key in ("environment", "access", "resources", "summary")): + raise ValueError("Environment, access, resources and summary must be explicit") + elapsed = metadata["elapsed_seconds"] + if (type(elapsed) not in (int, float) or not math.isfinite(elapsed) or elapsed < 0 + or type(metadata["agent_exit_code"]) is not int): + raise ValueError("Invalid author-stated execution outcome") + error = None + try: + expected = _expected_inputs(plan, task, condition) + unchanged = (_hashes(workspace) == expected + and stage_record["input_hashes"] == expected) + except ValueError: + unchanged = False + if not unchanged: + error = "inputs_changed" + artifact = workspace / "parser.py" + source = None + artifact_bytes = None + if artifact.is_symlink() or not artifact.is_file() or artifact.stat().st_size > 65536: + error = error or "missing_or_invalid_artifact" + else: + try: + artifact_bytes = _read_regular(artifact, 65536) + if len(artifact_bytes) > 65536: + raise UnicodeError("artifact grew beyond bound") + source = artifact_bytes.decode("utf-8") + except (UnicodeError, ValueError): + error = error or "invalid_artifact_encoding" + # Evidence is explicitly reported, not an invented controller-run agent process. + execution = {"status": "reported", "agent_exit_code": metadata["agent_exit_code"], + "elapsed_seconds": elapsed, "provenance": "author-supplied"} + write_json(directory / "execution.json", execution) + write_json(directory / "author-execution.json", metadata) + (directory / "agent").mkdir() + for name in ("stdout.log", "stderr.log"): + (directory / "agent" / name).write_bytes(submitted[name]) + answer = {"artifact": "parser.py", "summary": metadata["summary"], + "artifact_source": source} + write_json(directory / "answer.json", answer) + if source is not None: + (directory / "parser.py").write_bytes(artifact_bytes) + checks = None + if error is None: + try: + checks = engineering.verify(task, answer) + except engineering.SandboxUnavailable: + error = "artifact_sandbox_unavailable" + (directory / "verification").mkdir() + for name in ("stdout.log", "stderr.log"): + (directory / "verification" / name).write_text("", encoding="utf-8") + if checks is not None: + write_json(directory / "verification/result.json", checks) + artifact_verdict = None if checks is None else "PASS" if checks["passed"] else "FAIL" + if metadata["agent_exit_code"] != 0: + error = error or "reported_agent_failure" + if elapsed > plan["limits"]["timeout_seconds"]: + error = error or "reported_run_exceeded_frozen_timeout" + verdict = "INCONCLUSIVE" if error else artifact_verdict + from tools import skill_eval_compare as compare + + record = {**entry, "verdict": verdict, "checks": checks, "elapsed_seconds": elapsed, + "observations": {"usage": normalize_usage(metadata["usage"]), "commands": [], + "completed_turns": None, "errors": []}, + "execution_state": "author_reported", "inputs_unchanged": unchanged, + "evaluation_error": error, "input_hashes": stage_record["input_hashes"], + "output_sha256": sha256(directory / "answer.json"), + "artifact_kind": engineering.ADAPTER, + "artifact_sha256": sha256(directory / "parser.py") if source is not None else None, + "provenance": "author-supplied; agent execution and costs not independently verified", + "artifact_verdict": artifact_verdict, + "evidence_sha256": {name: sha256(directory / name) for name in compare.EVIDENCE + if (directory / name).is_file()}} + record["evidence_sha256"]["author-execution.json"] = sha256(directory / "author-execution.json") + write_json(directory / "result.json", record) + write_json(directory / "receipt.json", {"result_sha256": sha256(directory / "result.json")}) + return compare.report(output) From f122b0978920dad78edeac859f332e83291e6bab Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:47:40 +0900 Subject: [PATCH 2/2] fix: reject excessively nested artifact responses Treat decoder recursion exhaustion as an invalid artifact response after the trusted worker boundary, preserving failed checks instead of interrupting evidence capture. Agent-assisted-by: Codex (review follow-up) Signed-off-by: Codex Reviewed-local-source-commit: 3ade25aba495e8896e21bf6edce3281a22c4d67d Published via GitHub Git Data API with identical tree bytes; connector commit metadata may differ. --- src/inferrsi/harness/skill_eval_engineering.py | 2 +- tests/test_skill_eval_engineering.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/inferrsi/harness/skill_eval_engineering.py b/src/inferrsi/harness/skill_eval_engineering.py index 93356fcee..32d99a8a1 100644 --- a/src/inferrsi/harness/skill_eval_engineering.py +++ b/src/inferrsi/harness/skill_eval_engineering.py @@ -238,7 +238,7 @@ def _invoke(work, arguments=None, *, probe=False): return None try: return json.loads(raw) - except (ValueError, UnicodeDecodeError): + except (ValueError, UnicodeDecodeError, RecursionError): return None diff --git a/tests/test_skill_eval_engineering.py b/tests/test_skill_eval_engineering.py index 6a734485a..bd4bc4248 100644 --- a/tests/test_skill_eval_engineering.py +++ b/tests/test_skill_eval_engineering.py @@ -254,3 +254,20 @@ def start(*args, **kwargs): else: with pytest.raises(engineering.SandboxUnavailable, match="trusted boundary"): engineering._invoke(Path("/unused"), []) + + +def test_deeply_nested_artifact_json_is_a_failed_response_not_controller_crash(): + # Bounded, syntactically valid JSON can exceed the host decoder's recursion + # limit. Mock the sandbox output only; do not execute submitted code. + raw = b"[" * 10000 + b"0" + b"]" * 10000 + assert len(raw) < 65536 + process = mock.Mock(returncode=0) + + def start(*args, **kwargs): + kwargs["stderr"].write(b"INFERRSI_ARTIFACT_READY\n") + kwargs["stdout"].write(raw) + return process + + with mock.patch.object(engineering, "sandbox_command", return_value=["sandbox"]), \ + mock.patch.object(engineering.subprocess, "Popen", side_effect=start): + assert engineering._invoke(Path("/unused"), []) is None