From 6ed0dc55710518e34b5a4d13ca353bb5e975f53a Mon Sep 17 00:00:00 2001 From: Jonas Ries Date: Tue, 17 Nov 2026 17:34:07 +0100 Subject: [PATCH] tests: a bundled package's slow tier runs only when the package changed; --release runs them all Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 21 ++++++++- tests/conftest.py | 118 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a5883a34..346736d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,7 +81,9 @@ in one line of the summary, saying what has gathered since the last tag (`git log --oneline $(git describe --tags --abbrev=0)..origin/main`). Do not release yourself: a tag publishes to PyPI, which cannot be taken back. -A release is: `version` in `pyproject.toml` raised (and pushed to `main`), +A release is: `python -m pytest tests -q -n auto --release` passing on +`origin/main` (`--slow` with every package's slow tier, changed or not; see +below), `version` in `pyproject.toml` raised (and pushed to `main`), then an annotated tag of that commit, `git tag -a vX.Y.Z -m "smappy-smlm X.Y.Z"` and `git push origin vX.Y.Z`; `.github/workflows/wheels.yml` builds the wheels on every platform, tests them and publishes them to PyPI. @@ -93,11 +95,26 @@ A fresh container has **no numpy**. Before anything else: pip install -e . # builds the C++ extensions too pip install -e .[test] # pytest, pytest-timeout (a 300 s limit per test), pytest-xdist -Running the tests -- three tiers: +Running the tests -- three tiers, and the run before a release: python -m pytest tests -q -n auto # after each change: ~1 min, the slow tier skipped python -m pytest tests -q -n auto --slow # before a PR or a push of major work: ~4.5 min on four cores python -m pytest tests -q -n auto --long # by hand, when uipsf's learning changes: --slow and the studies + python -m pytest tests -q -n auto --release # before a release: --slow with every package's slow tier + +* **The packages beside smappy (uipsf, decode_smlm, and those to come) run + their slow tier only when they changed.** Their fast tests always run; + under `--slow` a package's slow tests run only if one of its paths differs + from the merge base with `origin/main` (committed, staged, unstaged or + untracked) -- the paths are `PACKAGES` in `tests/conftest.py`: the package, + its tests and the parts of smappy its tests lean on most. A change to + `tests/conftest.py` or `pyproject.toml` counts for all. `--package NAME` + runs one anyway, naming its tests folder on the command line does too, + `--since REF` moves the comparison, and `--release` and `--long` run them + all. The run's summary ends with a line per package saying whether its + slow tier ran. A new package bundled here is an entry in `PACKAGES` and a + folder `tests/_tests/`, with its tests marked `slow` / `long` as + smappy's are. * **Run the slow tier before you open a PR, and before you call major work done** -- a new plugin, a change to the GUI, a tutorial, a page or what an diff --git a/tests/conftest.py b/tests/conftest.py index a40ea88b..89fbd1f9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,6 +53,93 @@ LONG_TIMEOUT = 1200 +#: The packages kept in this repository beside smappy, each with its own tests +#: folder (the first path). Their fast tests always run; their slow tests run +#: under `--slow` only when one of the package's paths changed +#: (`package_changes`), and always under `--release`, `--long`, `--package +#: NAME` or with the package's tests named on the command line. The paths are +#: the package, its tests and the parts of smappy its tests lean on most -- +#: not all of them: uipsf's tests import a dozen smappy modules, and a change +#: to `smappy.simulate` reaches them through the fast tier and `--release`, +#: not here. A new package is an entry here and a folder of its tests. +PACKAGES = { + "uipsf": ("tests/uipsf_tests", "src/uipsf", "benchmarks/uipsf", "scripts/uipsf", + "src/smappy/calibrate", "src/smappy/psf.py", "src/smappy/io/calibration.py", + "src/smappy/plugins/fit.py", "csrc"), + "decode": ("tests/decode_tests", "src/decode_smlm", "src/smappy/training", + "src/smappy/plugins/fit_decode.py", "src/smappy/decode_remote.py", + "src/smappy/window_engine.py", "src/smappy/io/calibration.py"), +} + +#: a change here is a change to every package: how the tests run, what is installed +SHARED_PATHS = ("tests/conftest.py", "pyproject.toml", "setup.py") + + +def _git(root, *args): + import subprocess + return subprocess.run(["git", *args], cwd=root, capture_output=True, text=True, + check=True, timeout=30).stdout + + +def package_changes(root, since): + """Which packages' paths differ from `since`: a set of names, and a line + saying what they were compared with. The comparison is with the merge + base of HEAD and `since`, so a branch is judged by its own commits and not + by what landed upstream meanwhile, plus what is staged, unstaged and + untracked. Every package counts as changed when git cannot answer -- no + git, no such ref, an sdist -- since a skip nobody asked for is the failure + to avoid.""" + try: + base = _git(root, "merge-base", "HEAD", since).strip() + files = _git(root, "diff", "--name-only", base).splitlines() + files += _git(root, "ls-files", "--others", "--exclude-standard").splitlines() + except Exception as exc: # noqa: BLE001 -- any failure means "assume changed" + return set(PACKAGES), f"git could not compare with {since} ({type(exc).__name__}): all run" + + def touches(prefixes): + return any(f == p or f.startswith(p + "/") for f in files for p in prefixes) + + if touches(SHARED_PATHS): + return set(PACKAGES), f"shared files changed since {since} ({base[:8]})" + return ({name for name, paths in PACKAGES.items() if touches(paths)}, + f"compared with {since} ({base[:8]})") + + +def _packages_to_run(config): + """The packages whose slow tier runs, and why; computed once per process + (each xdist worker asks git itself, a few milliseconds).""" + cached = getattr(config, "_smappy_packages", None) + if cached is not None: + return cached + asked = set(config.getoption("--package") or ()) + unknown = asked - set(PACKAGES) + if unknown: + raise pytest.UsageError(f"--package {', '.join(sorted(unknown))}: " + f"the packages are {', '.join(PACKAGES)}") + root = str(config.rootpath) + # a package's tests named on the command line are wanted whatever changed + for arg in config.args: + path = os.path.relpath(os.path.abspath(arg.split("::")[0]), root).replace(os.sep, "/") + for name, paths in PACKAGES.items(): + if path == paths[0] or path.startswith(paths[0] + "/"): + asked.add(name) + if config.getoption("--release") or config.getoption("--long"): + result = set(PACKAGES), "--release" if config.getoption("--release") else "--long" + else: + changed, why = package_changes(root, config.getoption("--since")) + result = changed | asked, why + config._smappy_packages = result + return result + + +def _package_of(item, root): + path = os.path.relpath(str(item.path), root).replace(os.sep, "/") + for name, paths in PACKAGES.items(): + if path.startswith(paths[0] + "/"): + return name + return None + + def pytest_addoption(parser): """`timeout` in pyproject.toml belongs to pytest-timeout; without that plugin it is declared here, so that it is not an unknown-option warning. @@ -65,6 +152,15 @@ def pytest_addoption(parser): parser.addoption("--long", action="store_true", help="also run the tests marked long, uipsf's learning studies " "of 20 s to minutes each (implies --slow)") + parser.addoption("--release", action="store_true", + help="before a release: --slow with every package's slow tier, " + "changed or not") + parser.addoption("--package", action="append", metavar="NAME", + help=f"run this package's slow tier under --slow even if it did not " + f"change (repeatable): {', '.join(PACKAGES)}") + parser.addoption("--since", default="origin/main", metavar="REF", + help="what a package's change is measured against under --slow " + "(default origin/main, through the merge base)") try: import pytest_timeout # noqa: F401 except ImportError: @@ -116,7 +212,15 @@ def pytest_collection_modifyitems(config, items): for item in items: if "long" in item.keywords: item.add_marker(skip) - if long_ or config.getoption("--slow"): + if long_ or config.getoption("--slow") or config.getoption("--release"): + # a package's slow tier only where the package changed (`PACKAGES`) + run, why = _packages_to_run(config) + root = str(config.rootpath) + for item in items: + name = _package_of(item, root) + if name is not None and name not in run and "slow" in item.keywords: + item.add_marker(pytest.mark.skip( + reason=f"{name} slow: unchanged ({why}); run with --package {name} or --release")) # longest first: collected last (`test_tutorial`), the tutorials were # a tail of 20-50 s tests that four workers sat through after # everything else had finished @@ -129,6 +233,18 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip) +def pytest_terminal_summary(terminalreporter, config): + """Say which packages' slow tiers were left out: a skip count alone hid + that the run had not checked them.""" + if not (config.getoption("--slow") or config.getoption("--long") + or config.getoption("--release")): + return + run, why = _packages_to_run(config) + for name in PACKAGES: + state = "run" if name in run else f"SKIPPED (unchanged) -- --package {name} to run it" + terminalreporter.write_line(f"{name} slow tier: {state}; {why}") + + _exitstatus = 0