From 06748c77ec79b5e9d63e0794ed2a9a4ff1b2d89e Mon Sep 17 00:00:00 2001 From: Arjun Sharma Date: Thu, 8 Oct 2026 01:17:58 +0000 Subject: [PATCH 1/2] task template: base_copy copies the repo twice instead of moving it, so a huge merged .git does not stop setup --- .../harbor_adapter/template/tests/setup.sh | 12 ++++++--- tests/docker/test_setup_base_copy.py | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 tests/docker/test_setup_base_copy.py diff --git a/src/datasmith/harbor_adapter/template/tests/setup.sh b/src/datasmith/harbor_adapter/template/tests/setup.sh index 30e12f18..633f178c 100644 --- a/src/datasmith/harbor_adapter/template/tests/setup.sh +++ b/src/datasmith/harbor_adapter/template/tests/setup.sh @@ -134,12 +134,16 @@ fi SETUP_PHASE="base_copy" # lsv_measure.py times this unpatched copy (with its build) against the patched repo. Overlayfs cannot rename -# image directories, so the repo is moved out and copied back to make both trees renamable. +# image directories, so both trees are copies. No mv of the image tree: on a merged directory with ~10^5 git objects, +# mv stopped on entries already gone (numpy#12445, pandas#40007); rm -rf skips them. if [ "${FC_LSV_PAIRED:-1}" != "0" ]; then cd / - rm -rf /workspace/.fc_base - mv /workspace/repo /workspace/.fc_base - cp -a /workspace/.fc_base /workspace/repo + rm -rf /workspace/.fc_base /workspace/.fc_new + cp -a /workspace/repo /workspace/.fc_base + cp -a /workspace/repo /workspace/.fc_new + rm -rf /workspace/repo + mv /workspace/.fc_new /workspace/repo + [ "$(git -C /workspace/repo rev-parse HEAD)" = "$(git -C /workspace/.fc_base rev-parse HEAD)" ] cd /workspace/repo fi diff --git a/tests/docker/test_setup_base_copy.py b/tests/docker/test_setup_base_copy.py new file mode 100644 index 00000000..5fccf43b --- /dev/null +++ b/tests/docker/test_setup_base_copy.py @@ -0,0 +1,27 @@ +import subprocess +from pathlib import Path + +SETUP = Path(__file__).resolve().parents[2] / "src/datasmith/harbor_adapter/template/tests/setup.sh" + + +def block(root: Path) -> str: + text = SETUP.read_text() + start, end = text.index('SETUP_PHASE="base_copy"'), text.index('SETUP_PHASE="lsv_init"') + return text[start:end].replace("/workspace", str(root)) + + +def test_base_copy_makes_two_equal_trees_without_moving_the_repo(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + git = lambda *a: subprocess.run(["git", "-C", str(repo), *a], check=True, capture_output=True, text=True).stdout + git("init", "-q") + (repo / "a.py").write_text("x = 1\n") + git("add", "-A") + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "base") + head = git("rev-parse", "HEAD") + subprocess.run(["bash", "-c", "set -euo pipefail\n" + block(tmp_path)], check=True, capture_output=True) + for tree in (repo, tmp_path / ".fc_base"): + assert (tree / "a.py").read_text() == "x = 1\n" + assert subprocess.run(["git", "-C", str(tree), "rev-parse", "HEAD"], capture_output=True, text=True).stdout == head + assert not (tmp_path / ".fc_new").exists() + assert " mv /workspace/repo " not in SETUP.read_text() From 0595e2d972a266c7150447a7f516242e7e7a8105 Mon Sep 17 00:00:00 2001 From: Arjun Sharma Date: Thu, 8 Oct 2026 01:23:12 +0000 Subject: [PATCH 2/2] tests: base_copy and baseline-commit tests read prepare.sh (the separate-verifier template) --- tests/docker/test_setup_base_copy.py | 2 +- tests/docker/test_setup_baseline_commit.py | 33 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/docker/test_setup_baseline_commit.py diff --git a/tests/docker/test_setup_base_copy.py b/tests/docker/test_setup_base_copy.py index 5fccf43b..5fab04f7 100644 --- a/tests/docker/test_setup_base_copy.py +++ b/tests/docker/test_setup_base_copy.py @@ -1,7 +1,7 @@ import subprocess from pathlib import Path -SETUP = Path(__file__).resolve().parents[2] / "src/datasmith/harbor_adapter/template/tests/setup.sh" +SETUP = Path(__file__).resolve().parents[2] / "src/datasmith/harbor_adapter/template/tests/prepare.sh" def block(root: Path) -> str: diff --git a/tests/docker/test_setup_baseline_commit.py b/tests/docker/test_setup_baseline_commit.py new file mode 100644 index 00000000..e52bb27b --- /dev/null +++ b/tests/docker/test_setup_baseline_commit.py @@ -0,0 +1,33 @@ +import re +import subprocess +import time +from pathlib import Path + +SETUP = Path(__file__).resolve().parents[2] / "src/datasmith/harbor_adapter/template/tests/prepare.sh" + + +def baseline_block(sha_file: Path) -> str: + text = SETUP.read_text() + start, end = text.index('SETUP_PHASE="baseline_commit"'), text.index('SETUP_PHASE="base_copy"') + stage = (SETUP.parents[1] / "shared" / "stage_tree.sh").read_text() + block = re.sub(r'\{% filter indent\(2\) %\}\n\{% include "shared/stage_tree.sh" %\}\n\{% endfilter %\}\n', lambda _: stage, text[start:end]) + return block.replace("/opt/fc_baseline_sha", str(sha_file)) + + +def test_baseline_commit_starts_no_background_gc(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + git = lambda *a: subprocess.run(["git", "-C", str(repo), *a], check=True, capture_output=True, text=True).stdout + git("init", "-q") + for i in range(2000): + (repo / f"f{i}.txt").write_text(f"{i}\n") + git("add", "-A") + git("-c", "user.name=t", "-c", "user.email=t@t", "-c", "gc.auto=0", "commit", "-q", "-m", "base") + git("config", "gc.auto", "1") + (repo / "edit.txt").write_text("image edit\n") + subprocess.run(["bash", "-c", "set -euo pipefail\n" + baseline_block(tmp_path / "sha")], cwd=repo, check=True, + capture_output=True) + time.sleep(2) + assert git("log", "-1", "--format=%s").strip() == "fc-baseline" + assert not list((repo / ".git/objects/pack").glob("*.pack")) + assert int(git("count-objects").split()[0]) > 2000