task template: no background gc from the baseline commit

.github/workflows/main.yml

@@ -44,7 +44,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Run tests
-        run: uv run python -m pytest --cov --cov-config=pyproject.toml --cov-report=xml
+        run: uv run python -m pytest -m "not slow" --cov --cov-config=pyproject.toml --cov-report=xml
 
       - name: Check typing
         run: uv run mypy

.gitignore

@@ -238,3 +238,12 @@ jobs/
 FLOWCHART.md
 WORKFLOW.md
 google*.html
+
+# runtime output and local scratch
+logs/
+rl-prep/
+rl-prep.zip
+*.swp
+
+# tokens.env is ignored above; its timestamped backups carry the same secrets
+tokens.env.bak*

HANDOFF.md

@@ -0,0 +1,6 @@
+# Handoff — see `formula-code/rl-recipe`
+
+The full project handoff lives in the **rl-recipe** repo:
+**`docs/HANDOFF.md`** → https://github.com/formula-code/rl-recipe/blob/formulacode/handoff-2026-09-08/docs/HANDOFF.md
+
+This is one of five repos: `skyrl` (trainer) · `harbor` (trial framework) · `datasmith` (measurement) · `rl-recipe` (recipe + live services) · `rl-experiments` (oracle table / tasks / dashboard / logs). The doc above has the repo map, how-to-run, and the out-of-repo glue.

Makefile

@@ -1,12 +1,16 @@
+##@ Development
+## Everyday targets. `check` and `test` run on every commit; CI invokes
+## `make check` by name (.github/workflows/main.yml), so do not rename it.
+
 .PHONY: install
-install: ## Install the virtual environment and install the pre-commit hooks
+install: ## Install the virtual environment and the pre-commit hooks (once per clone)
 	@echo "Creating virtual environment using uv"
 	@uv sync --all-extras
 	@uv pip install -e .
 	@uv run pre-commit install
 
 .PHONY: check
-check: ## Run code quality tools.
+check: ## Run code quality tools (lock check, pre-commit, mypy, deptry)
 	@echo "Checking lock file consistency with 'pyproject.toml'"
 	@uv lock --locked
 	@echo "Linting code: Running pre-commit"
@@ -17,9 +21,13 @@ check: ## Run code quality tools.
 	@uv run deptry src
 
 .PHONY: test
-test: ## Test the code with pytest
+test: ## Test the code with pytest (excludes -m slow)
 	@echo "Testing code: Running pytest"
-	@uv run python -m pytest --cov --cov-config=pyproject.toml --cov-report=xml
+	@uv run python -m pytest -m "not slow" --cov --cov-config=pyproject.toml --cov-report=xml
+
+##@ Packaging
+## Local wheel builds. CI publishing does not use these targets --
+## .github/workflows/publish.yml runs `uv run python -m build` directly.
 
 .PHONY: build
 build: clean-build ## Build wheel file
@@ -31,11 +39,18 @@ clean-build: ## Clean build artifacts
 	@echo "Removing build artifacts"
 	@uv run python -c "import shutil; import os; shutil.rmtree('dist') if os.path.exists('dist') else None"
 
+##@ Housekeeping
+
 .PHONY: docker-clean
-docker-clean: ## Clean up dangling Docker images and containers
+docker-clean: ## Prune dangling Docker images and containers (reclaim build disk)
 	@echo "Cleaning up dangling Docker images and containers"
 	@docker system prune -f
 
+##@ Database (local Supabase)
+## The tunnel serves both hostnames off the single `datasmith-db` tunnel:
+## db.formulacode.org (CF Access + service-role key) and
+## api.formulacode.org (public anon reads, no Access gate).
+
 .PHONY: supabase-up
 supabase-up: ## Start local Supabase instance
 	@npx supabase start
@@ -48,8 +63,14 @@ supabase-down: ## Stop local Supabase instance
 supabase-status: ## Show Supabase service status and URLs
 	@npx supabase status
 
+.PHONY: db-tunnel
+db-tunnel: ## Expose PostgREST via Cloudflare Tunnel (db + api .formulacode.org)
+	@cloudflared tunnel --config ~/.cloudflared/config-db.yml run datasmith-db
+
+##@ Monitoring (Grafana)
+
 .PHONY: grafana-migrate
-grafana-migrate: ## Apply the grafana_ro read-only database role
+grafana-migrate: ## Apply the grafana_ro read-only role (once per Supabase volume)
 	@docker exec supabase_db_datasmith_new psql -U postgres -d postgres -f /dev/stdin < supabase/migrations/00009_grafana_readonly.sql
 	@echo "grafana_ro role created"
 
@@ -67,17 +88,15 @@ grafana-logs: ## Tail Grafana container logs
 	@docker compose -f grafana/docker-compose.yml logs -f
 
 .PHONY: grafana-tunnel
-grafana-tunnel: ## Expose Grafana publicly via Cloudflare Tunnel
+grafana-tunnel: ## Expose Grafana via Cloudflare Tunnel (datasmith-grafana)
 	@cloudflared tunnel run datasmith-grafana
 
-.PHONY: db-tunnel
-db-tunnel: ## Expose Supabase PostgREST API via Cloudflare Tunnel (db.formulacode.org)
-	@cloudflared tunnel --config ~/.cloudflared/config-db.yml run datasmith-db
+##@ Model proxy (LiteLLM + vLLM)
 
 LITELLM_VENV := .venv-litellm
 
 .PHONY: model-proxy-install
-model-proxy-install: $(LITELLM_VENV)/.installed ## Set up the persistent venv used by model-tunnel
+model-proxy-install: $(LITELLM_VENV)/.installed ## Build the .venv-litellm venv (once per machine; also the stale-venv fix)
 
 $(LITELLM_VENV)/.installed:
 	@echo "Creating $(LITELLM_VENV) and installing litellm[proxy] + prisma"
@@ -90,7 +109,7 @@ $(LITELLM_VENV)/.installed:
 	@touch $@
 
 .PHONY: model-tunnel
-model-tunnel: $(LITELLM_VENV)/.installed ## Start LiteLLM proxy + Cloudflare Tunnel for vLLM (model.formulacode.org)
+model-tunnel: $(LITELLM_VENV)/.installed ## Start LiteLLM + reconciler + Cloudflare Tunnel (model.formulacode.org)
 	@set -eu; \
 	set -a; . ./tokens.env; set +a; \
 	REPO_ROOT=$$(pwd); \
@@ -104,15 +123,25 @@ model-tunnel: $(LITELLM_VENV)/.installed ## Start LiteLLM proxy + Cloudflare Tun
 	    if ! kill -0 $$LITELLM_PID 2>/dev/null; then echo "litellm exited before becoming ready" >&2; exit 1; fi; \
 	    sleep 1; \
 	done; \
-	echo "litellm ready on :4100 (pid $$LITELLM_PID); starting cloudflared"; \
+	echo "litellm ready on :4100 (pid $$LITELLM_PID); starting model reconciler"; \
+	( exec "$$REPO_ROOT/$(LITELLM_VENV)/bin/python" \
+	      "$$REPO_ROOT/infra/refresh_models.py" --watch ) & \
+	REFRESH_PID=$$!; \
+	trap 'kill $$LITELLM_PID $$REFRESH_PID 2>/dev/null || true; wait $$LITELLM_PID $$REFRESH_PID 2>/dev/null || true' EXIT INT TERM; \
+	echo "reconciler running (pid $$REFRESH_PID); starting cloudflared"; \
 	cloudflared tunnel --config ~/.cloudflared/config-model.yml run datasmith-model
 
+.PHONY: model-refresh
+model-refresh: ## Reconcile the LiteLLM registry with live vLLM servers once (ARGS="--dry-run")
+	@set -a; . ./tokens.env; set +a; \
+	python3 infra/refresh_models.py $(ARGS)
 
 .PHONY: help
 help:
-	@uv run python -c "import re; \
-	[[print(f'\033[36m{m[0]:<20}\033[0m {m[1]}') for m in re.findall(r'^([a-zA-Z_-]+):.*?## (.*)$$', open(makefile).read(), re.M)] for makefile in ('$(MAKEFILE_LIST)').strip().split()]"
-
-
+	@awk 'BEGIN {FS = ":.*## "; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} \
+	      /^##@ / {printf "\n\033[1m%s\033[0m\n", substr($$0, 5)} \
+	      /^[a-zA-Z_-]+:.*## / {printf "  \033[36m%-21s\033[0m %s\n", $$1, $$2}' \
+	      $(MAKEFILE_LIST)
+	@echo
 
 .DEFAULT_GOAL := help

README.md

@@ -137,7 +137,7 @@ By default, each operation is cached in Supabase so you don't keep hitting expen
 A pull request is useless if you cannot build a reproducible environment for it. fc-data supports building docker images for any pull request using a three-tier hierarchy:
 
 ```python
-from datasmith.docker import ImageManager, MultiObjVerifier, SmokeVerifier, ProfileVerifier
+from datasmith.docker import ImageManager, read_build_manifest, evaluate_invariants
 
 mgr = ImageManager()
 mgr.build_base_image()                                # formulacode/base:latest (uses the default Dockerfile.base)
@@ -152,29 +152,28 @@ mgr.build_repo_image("pandas-dev", "pandas", context="path/to/custom/context")
 mgr.build_pr_image("pandas-dev", "pandas", 16222, context="path/to/custom/context")
 
 
-# Verify an image with a chain of verifiers — short-circuits on first failure.
-verifier = MultiObjVerifier(verifiers=[
-    SmokeVerifier("pandas"),      # can we import the package?
-    ProfileVerifier(timeout=300), # can we discover and run ASV benchmarks?
-])
-result = verifier.verify("formulacode/pandas-dev-pandas:16222")
-# result.ok, result.rc, result.stdout, result.stderr, result.duration_s
+# Every task image seals a build manifest. Read it back and evaluate the
+# invariants over it — no rebuild, no container run.
+manifest = read_build_manifest("formulacode/pandas-dev-pandas:16222")
+report = evaluate_invariants(manifest)
+# report.ok is True (all fatals held), False (a fatal failed), or None (no
+# manifest — true of any image built before manifests existed).
+# report.fatal / report.warnings / report.skipped list the invariant ids.
 ```
 
 One of the main features of `fc-data` is the ability to automatically synthesize docker containers for a pull request. The synthesizer is a state machine that checks Supabase for cached contexts, tries similar build scripts, then falls back to an installed CLI agent (Claude Code, Codex, or Gemini — auto-detected):
 
 ```python
 from datasmith.agents import Synthesizer
-from datasmith.docker import MultiObjVerifier, SmokeVerifier, ProfileVerifier
 from datasmith.docker.context import DockerContext
 
-# The verifier chain validates each synthesis attempt.
-verifier = MultiObjVerifier(verifiers=[
-    SmokeVerifier("pandas"),      # can we import the package?
-    ProfileVerifier(timeout=300), # can we discover and run ASV benchmarks?
-])
+# Each synthesis attempt is validated by local_ci.py inside the sandbox: it
+# builds the image, runs the test suite, runs the measure step, and gates on
+# the build manifest's FATAL invariants.
 
-# Load a base Docker build context (Dockerfile + shell scripts) to iterate on.
+# A DockerContext holds the Dockerfile + shell scripts. The synthesizer
+# discovers its own starting context (cache -> similar -> default template),
+# so you rarely construct one by hand; this is how you would load one.
 base_context = DockerContext.from_directory("dataset/formulacode_verified/pandas-dev_pandas/abc123")
 
 synth = Synthesizer(max_attempts=3)
@@ -183,11 +182,11 @@ ctx = synth.run(
     repo="pandas",
     issue_number=16222,
     pr_context="This PR optimizes groupby performance by ...",
-    verifier=verifier,
     sha="abc123def456",
-    base_context=base_context,
+    base_sha="def456abc123",   # the pre-optimization commit the image checks out
     env_payload='{"dependencies": ["numpy==1.26.0", "cython==3.0.0"]}',
     python_version="3.10",
+    solution_patch=pr_patch,   # the oracle patch, used only to verify measurability
 )
 # Checking cache for pandas-dev/pandas@abc123def456...             [MISS]
 # Found 4 similar scripts from pandas-dev/pandas
@@ -206,7 +205,7 @@ This can be run asynchronously as well for multiple tasks (WARNING: Might be exp
 ```python
 from datasmith.runners import SynthesizeImagesRunner
 
-runner = SynthesizeImagesRunner(synth, verifier, n_concurrent=8)
+runner = SynthesizeImagesRunner(synth, n_concurrent=8)
 await runner.run(pr_items)
 # Returns None entries for PRs where synthesis failed.
 ```

docs/design/Datasmith - Overview.md

@@ -137,7 +137,7 @@ pr.publish() # Puts the task on huggingface (and the local database)
 One of the main features of `datasmith` is the ability to automatically synthesize docker containers for a pull request. Here is how it works...
 
 ```python
-pr_buildscript = ds.agents.synthesize_image(pr, verifier=ds.docker.MultiObjVerifier)
+pr_buildscript = ds.agents.synthesize_image(pr)  # verification is internal (local_ci.py + build manifest)
 # Checking if base image exists...                   [PASS]
 # Checking if pandas-dev/pandas image exists...      [FAIL]
 # Making pandas-dev/pandas container...
@@ -163,7 +163,7 @@ If ALL attempts fail, `synthesize_image` logs every attempt (stderr, stdout, mod
 
 This can be run asynchronously as well for multiple tasks (WARNING: Might be expensive!)!
 ```python
-pr_build_scripts = ds.runners.synthesize_images(prs=[...], verifier=ds.docker.MultiObjVerifier, n_concurrent=64)
+pr_build_scripts = ds.runners.synthesize_images(prs=[...], n_concurrent=64)
 # Returns list[str | None]. None entries are PRs where synthesis failed.
 ```
 
@@ -199,7 +199,7 @@ Datasmith contains seven high-level modules. FormulaCode-specific logic lives di
 	* `ds.docker.verify.smoke`: A simple smoke test for the build (`import {package_name}`).
 	* `ds.docker.verify.profile`: Collects asv benchmarks and runs the asv profiler with `--quick`.
 	* `ds.docker.verify.pytest`: Collects the pytest suite with `testrunner` without errors. Runs pytest with a 45-second timeout.
-	* `ds.docker.verify.MultiObjVerifier`: Chains `smoke -> profile -> pytest` verifiers.
+	* ~~`ds.docker.verify.MultiObjVerifier`: Chains `smoke -> profile -> pytest` verifiers.~~ **REMOVED** — never reachable from any pipeline path. Superseded by `local_ci.py` + the build manifest (`read_build_manifest` / `evaluate_invariants`).
 * `ds.agents`: Agents for dynamic filtering and automatic build script generation. Simple agents use `dspy`; complex agents use an installed agent (like `codex`). Each module should define its default prompt as a constant.
 	* `ds.agents.dspy.classifier`: Abstract base class for DSPy classifiers.
 	* `ds.agents.codex`: Wrapper for invoking codex in fully autonomous mode (`codex exec --full-auto "..."`). Manages the working directory, prompt construction, and output capture via `--json` streaming.

docs/design/components/datasmith.agents.synthesizer.md

@@ -88,6 +88,16 @@ flowchart TD
 4. The agent iterates internally: reads AGENTS.md, runs verify, reads failure.json, edits build scripts, repeats
 5. On exit, checks for `verification_success.json` and reads back the modified `DockerContext`
 
+**PRODUCE_VERIFY** — The reflexive producer/verifier loop, behind `DATASMITH_PV_ENABLED`. Replaces LLM_GENERATE when enabled; the states before it are untouched, so a repo the stock template already builds never reaches it.
+
+The loop **seeds its first round from the stored context** when `candidate_containers` already holds one for this `(owner, repo, sha)`. Only `docker_build_pkg.sh` and `docker_build_run.sh` carry over — the two files the producer owns — so template fixes to the other scripts stay live. Without this, a `--force` re-run restarted from the stock template and repeated a repair the previous run had already completed: `xdslproject/xdsl#1332` was rejected at round 1 on `pytest_collect` despite an earlier run having solved exactly that.
+
+Seeding changes only where the loop *starts*. The image is rebuilt, the host-side image scan re-run, and the verifier re-grades from scratch, so a seeded round earns its verdict on the same evidence as an unseeded one. Because stored contexts are agent-authored — 128 repositories' stored contexts install a `sitecustomize` shim into site-packages, which is why TRY_SIMILAR can be switched off via `DATASMITH_SKIP_SIMILAR_CONTEXTS` — a stored context is put through `classify_context` before it is used, and a tampered one is discarded in favour of the template. The host image scan remains the gate; the audit exists so a known-bad row costs zero rounds instead of eight.
+
+A **rejected TRY_DEFAULT build falls through to PRODUCE_VERIFY** rather than ending the task. The image built and scanned clean; the verifier simply did not accept it — overwhelmingly on `pytest_pass_ratio` — and that is precisely what the producer exists to repair. On 2026-08-25 `dask/dask#6137`, `dask/dask#6186` and `UXARRAY/uxarray#1118` each produced a clean-scanning container within one hour and were written off as `unverified` rows nothing would revisit.
+
+The context is saved before the fall-through, so the build is never lost if the producer then fails, and the loop is seeded with that context — a build known to work is a far better round-1 starting point than the template. The fall-through applies **only when PRODUCE_VERIFY can actually run** (`DATASMITH_PV_ENABLED` set, an agent available). With the flag off the sole remaining path is LLM_GENERATE, a full sandbox agent, and spending one on every rejected container is not a change this branch should make; the legacy contract that a successful default build ends the task is preserved there.
+
 **FAIL** — All strategies exhausted. Logs a warning and returns `None`.
 
 ### Trace
@@ -184,7 +194,7 @@ class CodexResult:
 | `owner`, `repo`, `issue_number` | Pipeline item | Identifies the PR |
 | `sha` | Pipeline item | Merge commit SHA |
 | `pr_context` | Rendered problem statement | Full PR context for the AGENTS.md template |
-| `verifier` | Caller (typically `MultiObjVerifier`) | Validates the built Docker image (used by TRY_SIMILAR) |
+| ~~`verifier`~~ | *(removed)* | The verifier API was dead and is gone. TRY_SIMILAR validates via `verify_context`, which runs `local_ci.py` — build, tests, measure, then the build manifest's FATAL invariants. |
 | `base_context` | Pipeline item or templates | Base Docker context with all 9 files |
 | `env_payload` | `packages` table (via `ds.resolution`) | JSON array of pinned dependency strings. Populated by the `resolve_packages` pipeline stage, which runs `analyze_commit()` and persists results to the `packages` Supabase table keyed by `(owner, repo, sha)`. See `datasmith.resolution.md`. |
 | `python_version` | `packages` table (via `ds.resolution`) | Target Python version (e.g., "3.10"). Selected by temporal filtering against the commit date to avoid anachronisms (e.g., not using Python 3.12 for a 2019 commit). |
@@ -215,7 +225,7 @@ await runner.run(pr_items)
 
 Each item is dispatched via `asyncio.to_thread()` so multiple PRs are synthesized concurrently without blocking the event loop. Failed items raise `RuntimeError` and are logged to `runner_failures` — the runner never aborts.
 
-Items now include additional fields passed through to the sandbox: `env_payload`, `python_version`, and optionally `base_context`. These are sourced from the `packages` table (populated by the `resolve_packages` pipeline stage) rather than from columns on `pull_requests`. The synthesize stage joins `pull_requests` with `packages` on `(owner, repo, merge_commit_sha = sha)` to obtain resolution data. PRs without a corresponding `packages` row (or where `can_install = FALSE`) are skipped.
+Items now include additional fields passed through to the sandbox: `env_payload`, `python_version`, and optionally `base_context`. These are sourced from the `packages` table (populated by the `resolve_packages` pipeline stage) rather than from columns on `pull_requests`. The synthesize stage joins `pull_requests` with `packages` on `(owner, repo, merge_commit_sha = sha)` to obtain resolution data. A PR without a corresponding `packages` row is skipped; **a PR whose seed the probe disliked is not.** `can_install` no longer gates — stage 4 gates nothing — and `probe_status` orders the queue instead, best-first. See [the pipeline guide](../../guide/pipeline.md#stage-4-resolve-packages).
 
 ## What was removed (2026-03-21)