Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 9 additions & 12 deletions src/datasmith/docker/templates/docker_build_run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -106,18 +106,15 @@ lock_repo_to_current_commit() {
# DO NOT run: git gc --prune=now --aggressive
# DO NOT run: git prune --expire=now

# 8) Verification: ensure no ref points to a descendant of HEAD
if while IFS= read -r ref; do
[[ "$ref" == "refs/heads/$BR" ]] && continue
if git merge-base --is-ancestor "$HEAD_SHA" "$(git rev-parse "$ref")"; then
echo "$ref"
exit 0
fi
done < <(git for-each-ref --format='%(refname)') ; then
: # no output means OK
else
echo "Error: some refs still point ahead of HEAD. Aborting."
return 1
# 8) Verification: no ref may point at a descendant of HEAD
local ahead
ahead=$(git for-each-ref --format='%(refname)' | while IFS= read -r ref; do
[[ "$ref" == "refs/heads/$BR" ]] && continue
git merge-base --is-ancestor "$HEAD_SHA" "$(git rev-parse "$ref^{commit}")" 2>/dev/null && echo "$ref"
done || true)
if [[ -n "$ahead" ]]; then
echo "Error: refs ahead of HEAD: $ahead" >&2
return 1
fi

}
Expand Down
3 changes: 2 additions & 1 deletion src/datasmith/harbor_adapter/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ def generate_task(
for d in (env, tests, solution):
d.mkdir(parents=True, exist_ok=True)

copy2(self.template_dir / "environment" / "entrypoint.sh", env / "entrypoint.sh")
for name in ("entrypoint.sh", "scrub_git.sh"):
copy2(self.template_dir / "environment" / name, env / name)
for name in TEST_HELPERS:
copy2(self.template_dir / "tests" / name, tests / name)
# The image build runs these (baseline measurement, agent tools); /tests does not exist in the agent container.
Expand Down
4 changes: 4 additions & 0 deletions src/datasmith/harbor_adapter/template/environment/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -74,5 +74,9 @@ COPY rebuild.sh /usr/local/bin/rebuild-repo
COPY write_asv_conf.py /opt/lsv/
RUN chmod 755 /usr/local/bin/rebuild-repo; cd /workspace/repo && python /opt/lsv/write_asv_conf.py || true

# Last, so no later step brings history back. Base image layers still hold it, but the container cannot see them.
COPY scrub_git.sh /opt/scrub_git.sh
RUN bash /opt/scrub_git.sh /workspace/repo && rm /opt/scrub_git.sh

WORKDIR /workspace/repo
ENTRYPOINT ["/entrypoint.sh"]
27 changes: 27 additions & 0 deletions src/datasmith/harbor_adapter/template/environment/scrub_git.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# Remove git history after HEAD (refs that are not ancestors of HEAD, reflogs, unreachable objects): it holds the upstream fix.
# Keeps ancestor tags and the origin URL; fails the build if HEAD, tree, describe or status change or any later commit remains.
set -euo pipefail
cd "${1:-/workspace/repo}"
cd "$(git rev-parse --show-toplevel)"
state() { printf '%s %s %s %s' "$(git rev-parse HEAD)" "$(git rev-parse 'HEAD^{tree}')" \
"$(git describe --tags --always 2>/dev/null || true)" "$(git status --porcelain --untracked-files=no | sha256sum | cut -c1-16)"; }
before=$(state); head=$(git rev-parse HEAD); abbrev=$(git rev-parse --short HEAD | wc -c)
git for-each-ref --format='%(refname) %(objectname)' | while read -r ref obj; do
c=$(git rev-parse -q --verify "$obj^{commit}" 2>/dev/null) && git merge-base --is-ancestor "$c" "$head" || git update-ref -d "$ref"
done
git stash clear 2>/dev/null || true
rm -rf .git/logs .git/refs/original .git/FETCH_HEAD .git/ORIG_HEAD .git/objects/info/alternates
git config --unset-all gc.pruneExpire 2>/dev/null || true
git config --unset-all gc.worktreePruneExpire 2>/dev/null || true
git config gc.auto 0
git reflog expire --expire=now --expire-unreachable=now --all
git -c gc.pruneExpire=now gc --prune=now --quiet
git config core.abbrev $((abbrev - 1))
after=$(state)
[ "$before" = "$after" ] || { echo "scrub_git: repo state changed: $before -> $after" >&2; exit 1; }
left=$(git fsck --unreachable --no-reflogs --no-progress 2>/dev/null | grep -c '^unreachable commit' || true)
ahead=$(git for-each-ref --format='%(objectname)' | while read -r o; do
c=$(git rev-parse -q --verify "$o^{commit}") || continue; [ "$c" != "$head" ] && git merge-base --is-ancestor "$head" "$c" && echo x; done | wc -l || true)
[ "$left" -eq 0 ] && [ "$ahead" -eq 0 ] || { echo "scrub_git: $left unreachable commits, $ahead refs ahead of HEAD" >&2; exit 1; }
echo "scrub_git: ok ($before)"
57 changes: 57 additions & 0 deletions tests/docker/test_scrub_git.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import shutil
import subprocess
from pathlib import Path

import pytest

SCRIPT = Path(__file__).parents[2] / "src/datasmith/harbor_adapter/template/environment/scrub_git.sh"
DOCKERFILE = SCRIPT.parent / "Dockerfile"

pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="needs git")


def git(repo: Path, *args: str) -> str:
env = {
"GIT_AUTHOR_NAME": "t",
"GIT_AUTHOR_EMAIL": "t@t",
"GIT_COMMITTER_NAME": "t",
"GIT_COMMITTER_EMAIL": "t@t",
"HOME": str(repo),
"PATH": "/usr/bin:/bin",
}
return subprocess.run(
["git", "-C", str(repo), *args], check=True, capture_output=True, text=True, env=env
).stdout.strip()


def commit(repo: Path, name: str) -> str:
(repo / name).write_text(name)
git(repo, "add", name)
git(repo, "commit", "-qm", name)
return git(repo, "rev-parse", "HEAD")


def test_removes_later_history_and_keeps_head(tmp_path):
repo = tmp_path / "repo"
repo.mkdir()
git(repo, "init", "-q", "-b", "main")
commit(repo, "a")
git(repo, "tag", "v1")
base = commit(repo, "b")
fix = commit(repo, "fix")
git(repo, "tag", "v2")
git(repo, "checkout", "-q", "-B", "main", base)
git(repo, "config", "gc.pruneExpire", "never")
describe = git(repo, "describe", "--tags")

subprocess.run(["bash", str(SCRIPT), str(repo)], check=True, capture_output=True)

assert git(repo, "rev-parse", "HEAD") == base
assert git(repo, "describe", "--tags") == describe
assert git(repo, "tag") == "v1"
assert subprocess.run(["git", "-C", str(repo), "cat-file", "-e", fix]).returncode != 0


def test_task_image_runs_it_last():
text = DOCKERFILE.read_text()
assert text.index("scrub_git.sh") > text.rindex("lsv_init.py")
Loading