template: the verifier's diff shows every edited line, unsafe diffs are refused, and skipped tests count as regressions

src/datasmith/harbor_adapter/template/shared/collect.sh

@@ -4,7 +4,7 @@ mkdir -p /fc_submission
 export GIT_INDEX_FILE="$(mktemp -d)/index"
 git read-tree {{ base_commit }}
 {% include "shared/stage_tree.sh" %}
-git diff --cached --binary {{ base_commit }} > /fc_submission/tree.diff
+GIT_NO_REPLACE_OBJECTS=1 git diff --cached --binary --no-ext-diff --no-textconv {{ base_commit }} > /fc_submission/tree.diff
 # Processes the agent left running would compete for CPU with the timing; kill -1 spares PID 1 and this shell.
 kill -9 -1 2>/dev/null
 true

src/datasmith/harbor_adapter/template/tests/parser.py

@@ -287,12 +287,17 @@ def summarize_snapshots(log_dir: Path) -> dict:
 
 def load_patch_info(log_dir: Path) -> dict:
     path = log_dir / "patch_info.json"
-    if not path.exists():
-        return {"applied": False, "files": 0, "added_lines": 0, "removed_lines": 0}
-    try:
-        return json.loads(path.read_text())
-    except (json.JSONDecodeError, OSError):
-        return {"applied": False, "files": 0, "added_lines": 0, "removed_lines": 0}
+    info = {"applied": False, "files": 0, "added_lines": 0, "removed_lines": 0}
+    if path.exists():
+        try:
+            info = json.loads(path.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+    # prepare.sh writes tree_diff_refused or tree_diff_failed; the host fails such a trial instead of an empty patch.
+    tree = log_dir / "agent_tree.txt"
+    if tree.exists():
+        info["submission_rejected"] = tree.read_text().strip()
+    return info
 
 
 def load_setup_status(log_dir: Path) -> dict:

src/datasmith/harbor_adapter/template/tests/prepare.sh

@@ -144,7 +144,15 @@ echo "[$(ts)] [prepare] LSV init complete."
 SETUP_PHASE="agent_tree"
 # The agent's tree is the base commit plus tree.diff; ignored files (the build) stay as the image has them.
 # A missing or broken tree.diff leaves the starting tree, so the trial has no patch (no_patch).
-if [ -s /fc_submission/tree.diff ]; then
+# Symlinks, *.pth, site/usercustomize.py and package metadata (entry points) run code outside the patched sources.
+_refused="$( { git apply --summary /fc_submission/tree.diff 2>/dev/null | grep -E ' 120000( |$)' || true; \
+  git apply --numstat -z /fc_submission/tree.diff 2>/dev/null | tr '\0' '\n' | sed -E 's/^[0-9-]+\t[0-9-]+\t//' \
+    | grep -E '(^|/)(site|user)customize\.py$|\.pth$|\.(dist|egg)-info/' || true; } )"
+if [ -n "${_refused}" ]; then
+  echo "[$(ts)] [prepare] tree.diff refused: ${_refused}" >&2
+  echo tree_diff_refused > /logs/artifacts/agent_tree.txt
+  printf '%s\n' "${_refused}" > /logs/artifacts/tree_diff_refused.txt
+elif [ -s /fc_submission/tree.diff ]; then
   git read-tree -u --reset {{ base_commit }}
   if ! git apply --binary --whitespace=nowarn /fc_submission/tree.diff; then
     echo "[$(ts)] [prepare] tree.diff does not apply; using the starting tree." >&2

src/datasmith/harbor_adapter/template/tests/test.sh

@@ -65,7 +65,8 @@ FC_BASE="$(cat /opt/fc_baseline_sha 2>/dev/null || echo {{ base_commit }})"
 FC_INDEX="$(mktemp)"
 cp .git/index "${FC_INDEX}" 2>/dev/null || rm -f "${FC_INDEX}"
 GIT_INDEX_FILE="${FC_INDEX}" git add -A -N 2>/dev/null || true
-fc_diff() { GIT_INDEX_FILE="${FC_INDEX}" git diff "${FC_BASE}" "$@" 2>/dev/null; }
+# --text and no drivers: a .gitattributes file in the agent's diff must not hide edited lines from the tamper check.
+fc_diff() { GIT_INDEX_FILE="${FC_INDEX}" GIT_NO_REPLACE_OBJECTS=1 git diff --text --no-ext-diff --no-textconv "${FC_BASE}" "$@" 2>/dev/null; }
 fc_diff > "${LOG_DIR}/patch.diff" || true
 patch_files=$(fc_diff --name-only | wc -l | tr -d ' ')
 patch_numstat=$(fc_diff --numstat || true)

tests/docker/test_pytest_runner_regression.py

@@ -1,4 +1,4 @@
-"""run_base_and_diff: `ran` only when the base side produced results."""
+"""run_base_and_diff and restore_test_edits: the base side runs at the base commit and the agent cannot weaken tests."""
 
 import importlib.util
 import json
@@ -45,10 +45,12 @@ def test_base_results_mean_ran_and_regressions_count(runner, repo: Path) -> None
 
 def test_no_base_results_is_not_ran(runner, repo: Path, monkeypatch) -> None:
     real = runner._run
-    monkeypatch.setattr(runner, "_run", lambda cmd, cwd=None: (1, "", "boom") if cmd[0] == sys.executable else real(cmd, cwd))
+    monkeypatch.setattr(
+        runner, "_run", lambda cmd, cwd=None: (1, "", "boom") if cmd[0] == sys.executable else real(cmd, cwd)
+    )
     out = runner.run_base_and_diff(["test_m.py"], "", str(repo), AGENT)
     assert out["ran"] is False
-    assert out["stashed"] is True
+    assert out["reverted"] is True
     assert "no results" in out["base_error"]
     assert "assert False" in (repo / "test_m.py").read_text()
 
@@ -100,3 +102,182 @@ def test_repo_package_wins_over_a_copy_in_site_packages(tmp_path_factory) -> Non
     proc = subprocess.run([sys.executable, "-c", script], cwd=repo, env=env, capture_output=True, text=True)
     summary = json.loads(proc.stdout.strip().splitlines()[-1])
     assert summary["passed"] == 1, proc.stdout + proc.stderr
+
+
+def _git(cwd: Path, *args) -> str:
+    return subprocess.run(
+        ["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=cwd, check=True, capture_output=True, text=True
+    ).stdout
+
+
+@pytest.fixture
+def src_repo(tmp_path: Path) -> tuple[Path, str]:
+    root = tmp_path / "repo"
+    (root / "pkg").mkdir(parents=True)
+    _git(root, "init", "-q")
+    (root / "pkg" / "m.py").write_text("def f():\n    return 1\n")
+    (root / "pkg" / "test_m.py").write_text("from m import f\n\ndef test_f():\n    assert f() == 1\n")
+    (root / "old.py").write_text("x = 1\n")
+    _git(root, "add", "-A")
+    _git(root, "commit", "-qm", "fc-baseline")
+    return root, _git(root, "rev-parse", "HEAD").strip()
+
+
+def _run_runner(root: Path, base: str, logs: Path) -> tuple[dict, dict]:
+    env = dict(os.environ, T_BENCH_TASK_LOGS_PATH=str(logs))
+    subprocess.run(
+        [sys.executable, str(_RUNNER), "--base", base, "--root", str(root)],
+        cwd=root,
+        env=env,
+        check=True,
+        capture_output=True,
+    )
+    return json.loads((logs / "test_results.json").read_text()), json.loads((logs / "test_edits.json").read_text())
+
+
+def test_weakened_test_is_restored_and_regression_reported(src_repo, tmp_path: Path) -> None:
+    root, base = src_repo
+    (root / "pkg" / "m.py").write_text("def f():\n    return 2\n")
+    (root / "pkg" / "test_m.py").write_text("from m import f\n\ndef test_f():\n    assert f() == 2\n")
+    (root / "pkg" / "test_new.py").write_text("def test_n():\n    assert True\n")
+    results, edits = _run_runner(root, base, tmp_path / "logs")
+    assert edits["modified"] == ["pkg/test_m.py"] and edits["added"] == ["pkg/test_new.py"]
+    assert not (root / "pkg" / "test_new.py").exists()
+    assert results["regression"]["regressed"] == ["pkg/test_m.py::test_f"]
+
+
+def test_committed_agent_change_is_reverted_for_base_run(src_repo, tmp_path: Path) -> None:
+    root, base = src_repo
+    (root / "pkg" / "m.py").write_text("def f():\n    return 2\n")
+    _git(root, "commit", "-qam", "agent")
+    results, _ = _run_runner(root, base, tmp_path / "logs")
+    assert results["regression"]["ran"] is True
+    assert results["regression"]["regressed"] == ["pkg/test_m.py::test_f"]
+
+
+def test_agent_tree_is_restored_byte_for_byte(runner, src_repo, monkeypatch) -> None:
+    root, base = src_repo
+    (root / "pkg" / "m.py").write_text("def f():\n    return 2\n")
+    _git(root, "commit", "-qam", "agent")
+    (root / "pkg" / "test_m.py").write_text("from m import f\n\ndef test_f():\n    assert f() == 3\n")
+    (root / "new").mkdir()
+    (root / "new" / "new.py").write_bytes(b"y = 2\r\n")
+    (root / "old.py").unlink()
+    _git(root, "add", "pkg/test_m.py")
+
+    def snapshot():
+        files = {
+            p.relative_to(root): p.read_bytes()
+            for p in root.rglob("*")
+            if p.is_file() and not str(p.relative_to(root)).startswith(".") and "__pycache__" not in p.parts
+        }
+        return files, _git(root, "status", "--porcelain"), _git(root, "rev-parse", "HEAD")
+
+    before = snapshot()
+    real = runner._run
+    monkeypatch.setattr(
+        runner,
+        "_run",
+        lambda cmd, cwd=None: (_ for _ in ()).throw(KeyboardInterrupt) if cmd[0] == sys.executable else real(cmd, cwd),
+    )
+    with pytest.raises(KeyboardInterrupt):
+        runner.run_base_and_diff(["pkg/test_m.py"], "", str(root), AGENT, base_ref=base)
+    assert snapshot() == before
+    monkeypatch.setattr(runner, "_run", real)
+    assert runner.run_base_and_diff(["pkg/test_m.py"], "", str(root), AGENT, base_ref=base)["ran"] is True
+    assert snapshot() == before
+
+
+def test_failed_restore_sets_restore_ok_false_and_exits_nonzero(src_repo, tmp_path: Path) -> None:
+    root, base = src_repo
+    (root / "pkg" / "m.py").write_text("def f():\n    return 2\n")
+    logs = tmp_path / "logs"
+    env = dict(os.environ, T_BENCH_TASK_LOGS_PATH=str(logs), FC_REBUILD_CMD="grep -q 'return 1' pkg/m.py")
+    proc = subprocess.run(
+        [sys.executable, str(_RUNNER), "--base", base, "--root", str(root)], cwd=root, env=env, capture_output=True
+    )
+    regression = json.loads((logs / "test_results.json").read_text())["regression"]
+    assert proc.returncode == 3
+    assert regression["restore_ok"] is False and "patched rebuild after restore failed" in regression["base_error"]
+
+
+def test_base_pass_test_skipped_by_agent_code_is_a_regression(src_repo, tmp_path: Path) -> None:
+    root, base = src_repo
+    (root / "pkg" / "m.py").write_text("import pytest\n\ndef f():\n    pytest.skip('no')\n")
+    results, _ = _run_runner(root, base, tmp_path / "logs")
+    assert results["regression"]["regressed"] == ["pkg/test_m.py::test_f"]
+
+
+def test_base_pass_test_not_collected_is_a_regression(src_repo, tmp_path: Path) -> None:
+    root, base = src_repo
+    (root / "pkg" / "m.py").write_text("raise ImportError('broken')\n")
+    results, _ = _run_runner(root, base, tmp_path / "logs")
+    assert results["regression"]["regressed"] == ["pkg/test_m.py::test_f"]
+
+
+def test_weakened_testing_helper_is_restored(src_repo, tmp_path: Path) -> None:
+    root, _ = src_repo
+    (root / "pkg" / "testing").mkdir()
+    (root / "pkg" / "testing" / "__init__.py").write_text("")
+    (root / "pkg" / "testing" / "helpers.py").write_text("def check(x):\n    assert x == 1\n")
+    (root / "pkg" / "test_m.py").write_text(
+        "from m import f\nfrom testing.helpers import check\n\ndef test_f():\n    check(f())\n"
+    )
+    _git(root, "add", "-A")
+    _git(root, "commit", "-qm", "helpers")
+    base = _git(root, "rev-parse", "HEAD").strip()
+    (root / "pkg" / "m.py").write_text("def f():\n    return 2\n")
+    (root / "pkg" / "testing" / "helpers.py").write_text("def check(x):\n    pass\n")
+    results, edits = _run_runner(root, base, tmp_path / "logs")
+    assert edits["modified"] == ["pkg/testing/helpers.py"]
+    assert results["regression"]["regressed"] == ["pkg/test_m.py::test_f"]
+
+
+def _base_config_in_setup_cfg(root: Path) -> str:
+    (root / "setup.cfg").write_text("[tool:pytest]\naddopts = -ra\n")
+    _git(root, "add", "-A")
+    _git(root, "commit", "-qm", "config")
+    return _git(root, "rev-parse", "HEAD").strip()
+
+
+def test_agent_pyproject_dotted_key_config_is_not_read(src_repo, tmp_path: Path) -> None:
+    root, _ = src_repo
+    base = _base_config_in_setup_cfg(root)
+    (root / "pyproject.toml").write_text('[tool]\npytest.ini_options.addopts = "--deselect pkg/test_m.py::test_f"\n')
+    (root / "pkg" / "m.py").write_text("def f():\n    return 2\n")
+    results, _ = _run_runner(root, base, tmp_path / "logs")
+    cfg = results["results"]["config_file"]
+    assert cfg.endswith("setup.cfg") and "fc_pytest_cfg_" in cfg
+    assert results["regression"]["regressed"] == ["pkg/test_m.py::test_f"]
+
+
+def test_added_dot_pytest_ini_is_not_read(src_repo, tmp_path: Path) -> None:
+    root, _ = src_repo
+    base = _base_config_in_setup_cfg(root)
+    (root / ".pytest.ini").write_text("[pytest]\naddopts = --deselect pkg/test_m.py::test_f\n")
+    (root / "pkg" / "m.py").write_text("def f():\n    return 2\n")
+    results, _ = _run_runner(root, base, tmp_path / "logs")
+    assert "fc_pytest_cfg_" in results["results"]["config_file"]
+    assert results["regression"]["regressed"] == ["pkg/test_m.py::test_f"]
+
+
+def test_plugin_loaded_only_in_the_agent_run_is_a_regression(src_repo, tmp_path: Path) -> None:
+    root, base = src_repo
+    dist = root / "fcevil-0.dist-info"
+    dist.mkdir()
+    (dist / "METADATA").write_text("Metadata-Version: 2.1\nName: fcevil\nVersion: 0\n")
+    (dist / "entry_points.txt").write_text("[pytest11]\nfcevil = fcevil_mod\n")
+    (root / "fcevil_mod.py").write_text("")
+    (root / "pkg" / "m.py").write_text("def f():\n    return 1  # agent\n")
+    results, _ = _run_runner(root, base, tmp_path / "logs")
+    assert results["regression"]["agent_only_plugins"] == ["fcevil"]
+    assert "plugin:fcevil" in results["regression"]["regressed"]
+
+
+def test_missing_base_config_fails_the_check(runner) -> None:
+    out = {"regression": {"ran": False, "reason": "no tests selected"}}
+    runner.fail_without_base_config(out, {"config_error": "OSError: x"})
+    assert out["regression"]["ran"] is True and out["regression"]["regressed"] == ["config:unavailable"]
+    ok = {"regression": {"ran": True, "regressed": []}}
+    runner.fail_without_base_config(ok, {})
+    assert ok["regression"]["regressed"] == []

tests/docker/test_separate_verifier_tree.py

@@ -107,3 +107,74 @@ def test_broken_tree_diff_leaves_starting_tree(tmp_path):
     bash(prepare_block("agent_tree", "complete", sha, base).replace("/fc_submission", str(sub)), image)
     assert {k: v for k, v in files(image).items() if k not in ("sha", "agent_tree.txt")} == before
     assert (image / "agent_tree.txt").read_text().strip() == "tree_diff_failed"
+
+
+def _collect_and_recreate(tmp_path: Path, edit) -> tuple[Path, Path]:
+    image, base = image_repo(tmp_path)
+    agent, verifier = tmp_path / "agent", tmp_path / "verifier"
+    subprocess.run(["cp", "-a", str(image), str(agent)], check=True)
+    subprocess.run(["cp", "-a", str(image), str(verifier)], check=True)
+    edit(agent)
+    sub = tmp_path / "submission"
+    bash(collect_script(base, sub), agent)
+    sha = verifier / "sha"
+    bash(prepare_block("baseline_commit", "base_copy", sha, base), verifier)
+    bash(prepare_block("agent_tree", "complete", sha, base).replace("/fc_submission", str(sub)), verifier)
+    return verifier, sha
+
+
+def test_gitattributes_cannot_hide_a_benchmark_edit_from_the_verifier_diff(tmp_path):
+    def edit(agent: Path) -> None:
+        (agent / ".gitattributes").write_text("benchmarks/** -diff\n")
+        (agent / "benchmarks/bench.py").write_text("def time_x(): return 0\n")
+
+    verifier, sha = _collect_and_recreate(tmp_path, edit)
+    test_sh = render_template(
+        "tests/test.sh",
+        base_commit="x",
+        run_pytest=False,
+        rounds=None,
+        task_id="t",
+        owner="o",
+        repo="r",
+        issue_number=1,
+    )
+    fc_diff = next(line for line in test_sh.splitlines() if line.startswith("fc_diff()"))
+    setup = "\n".join(
+        line for line in test_sh.splitlines() if line.startswith(("FC_INDEX=", "cp .git/index", "GIT_INDEX_FILE="))
+    )
+    script = f"FC_BASE={sha.read_text().strip()}\n{setup}\n{fc_diff}\nfc_diff"
+    out = subprocess.run(["bash", "-c", script], cwd=verifier, check=True, capture_output=True, text=True).stdout
+    assert "+def time_x(): return 0" in out
+
+
+def test_diff_with_pth_file_or_symlink_is_refused(tmp_path):
+    def edit(agent: Path) -> None:
+        (agent / "pkg.py").write_text("x = 2\n")
+        (agent / "evil.pth").write_text("import os\n")
+        (agent / "link").symlink_to("/etc/passwd")
+
+    verifier, _ = _collect_and_recreate(tmp_path, edit)
+    assert (verifier / "agent_tree.txt").read_text().strip() == "tree_diff_refused"
+    assert not (verifier / "evil.pth").exists() and not (verifier / "link").is_symlink()
+    assert (verifier / "pkg.py").read_text() == "x = 1  # image edit\n"
+
+
+def test_quoted_path_sitecustomize_is_refused(tmp_path):
+    def edit(agent: Path) -> None:
+        (agent / "pkg.py").write_text("x = 2\n")
+        (agent / "é").mkdir()
+        (agent / "é" / "sitecustomize.py").write_text("import os\n")
+
+    verifier, _ = _collect_and_recreate(tmp_path, edit)
+    assert (verifier / "agent_tree.txt").read_text().strip() == "tree_diff_refused"
+    assert not (verifier / "é").exists()
+
+
+def test_diff_with_package_metadata_is_refused(tmp_path):
+    def edit(agent: Path) -> None:
+        (agent / "evil-0.dist-info").mkdir()
+        (agent / "evil-0.dist-info" / "entry_points.txt").write_text("[pytest11]\nevil = evil\n")
+
+    verifier, _ = _collect_and_recreate(tmp_path, edit)
+    assert (verifier / "agent_tree.txt").read_text().strip() == "tree_diff_refused"

tests/docker/test_template_lint.py

@@ -62,3 +62,10 @@ def test_no_undefined_names_in_templates(rel: str) -> None:
         f"These files are excluded from ruff, so nothing else will catch this.\n"
         f"{proc.stdout}\n{proc.stderr}"
     )
+
+
+@pytest.mark.parametrize("path", sorted((_ROOT / "src/datasmith/harbor_adapter/template").rglob("*.py")), ids=lambda p: p.name)
+def test_harbor_template_parses_as_python_3_8(path: Path) -> None:
+    import ast
+
+    ast.parse(path.read_text(), feature_version=(3, 8))