diff --git a/asv/contrib/lightspeed/__init__.py b/asv/contrib/lightspeed/__init__.py index 97aa480..3b45270 100644 --- a/asv/contrib/lightspeed/__init__.py +++ b/asv/contrib/lightspeed/__init__.py @@ -8,4 +8,6 @@ ConfigError, BenchmarkError, NoBenchmarksError, + ProjectShadowed, + check_project_imports, ) diff --git a/asv/contrib/lightspeed/project_imports.py b/asv/contrib/lightspeed/project_imports.py new file mode 100644 index 0000000..966b395 --- /dev/null +++ b/asv/contrib/lightspeed/project_imports.py @@ -0,0 +1,88 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +"""Find where the benchmark processes import the project packages from.""" + +import json +import os +import shutil +import tempfile +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +from ...runner import get_spawner + +_PROBE = '''import importlib, json, os +from asv_runner._aux import update_sys_path + +def track_import_probe(): + try: # the suite's own __init__ can change sys.path, as in a real run + update_sys_path({bench!r}) + importlib.import_module(os.path.basename({bench!r})) + except Exception: + pass + out = {{}} + for name in {names!r}: + try: + mod = importlib.import_module(name) + out[name] = getattr(mod, "__file__", None) or next(iter(getattr(mod, "__path__", [])), None) + except Exception: + out[name] = None + with open({out!r}, "w") as f: + json.dump(out, f) + return 0 +''' + + +def import_packages(paths: Iterable[str], repo_root: str) -> List[str]: + """Top-level import names of the files: numpy/core/x.c -> numpy, src/skimage/a.py -> skimage.""" + root = Path(repo_root).resolve() + names = set() + for p in paths: + try: + parts = Path(p).resolve().relative_to(root).parts + except ValueError: + continue + for i in range(1, len(parts)): + if (root.joinpath(*parts[:i]) / "__init__.py").is_file(): + names.add(parts[i - 1]) + break + return sorted(names) + + +def probe_imports(env, benchmark_dir, launch_method, packages) -> Dict[str, Optional[str]]: + """Import ``packages`` in a process started by the benchmark spawner; map each to its file, None if not importable.""" + tmp = tempfile.mkdtemp(prefix="lsv-import-probe-") + try: + bench = os.path.join(tmp, "lsv_import_probe") + os.mkdir(bench) + Path(bench, "__init__.py").write_text("") + out = os.path.join(tmp, "out.json") + Path(bench, "probe.py").write_text( + _PROBE.format(bench=os.path.abspath(str(benchmark_dir)), names=list(packages), out=out) + ) + spawner = get_spawner(env, bench, launch_method) + try: + log, code = spawner.run( + name="probe.track_import_probe", + params_str="{}", + profile_path="None", + result_file_name=os.path.join(tmp, "res.json"), + timeout=300, + cwd=tmp, + ) + finally: + spawner.close() + if code != 0 or not os.path.exists(out): + raise RuntimeError(f"import probe failed (exit {code}): {log[-2000:]}") + return json.loads(Path(out).read_text()) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def outside_root(found: Dict[str, Optional[str]], repo_root) -> Dict[str, str]: + """The packages that resolve to a file outside ``repo_root``.""" + root = os.path.realpath(str(repo_root)) + return { + name: path + for name, path in found.items() + if path and os.path.commonpath([root, os.path.realpath(path)]) != root + } diff --git a/asv/contrib/lightspeed/session.py b/asv/contrib/lightspeed/session.py index fcbb420..1e55fba 100644 --- a/asv/contrib/lightspeed/session.py +++ b/asv/contrib/lightspeed/session.py @@ -34,6 +34,7 @@ from ... import util from .deps_db import BenchmarkId, LightspeedDB, _parse_bid from .fingerprint import changed_files_with_fingerprints +from .project_imports import import_packages, outside_root, probe_imports from .survey import run_survey _DEPS_DB_FILENAME = ".lightspeed_deps.db" @@ -63,6 +64,23 @@ class NoBenchmarksError(ASVError): """No benchmarks were selected or found.""" +class ProjectShadowed(ASVError): + """The benchmark processes import a project package from outside the repository.""" + def __init__(self, paths): + super().__init__("benchmarks import the project from outside the repository: " + + ", ".join(f"{name} -> {path}" for name, path in paths.items())) + self.paths = paths + + +def check_project_imports(env, benchmark_dir, launch_method, packages, repo_root): + """Raise ProjectShadowed when a package resolves outside ``repo_root`` in a benchmark process.""" + found = probe_imports(env, benchmark_dir, launch_method, packages) if packages else {} + bad = outside_root(found, repo_root) + if bad: + raise ProjectShadowed(bad) + return found + + # --------------------------------------------------------------------------- # Result dataclasses # --------------------------------------------------------------------------- @@ -557,6 +575,7 @@ def measure_impacted( rounds: Optional[int] = None, repeat: Optional[int] = None, warmup_time: Optional[float] = None, + packages: Optional[List[str]] = None, ) -> MeasureResult: """ Selectively re-run benchmarks affected by code changes. @@ -578,6 +597,9 @@ def measure_impacted( Samples per round. ``None`` means auto. warmup_time : float, optional Warmup seconds. ``None`` means auto. + packages : list of str, optional + Import names of the project. ``None`` derives them from the changed files. + Raises ProjectShadowed before timing when one imports from outside the repository. """ if not from_git_diff and changed_files is None: raise ValueError("Provide either from_git_diff=True or changed_files=[...]") @@ -638,6 +660,8 @@ def measure_impacted( extra_params = _timing_params(rounds, repeat, warmup_time) env = self._get_env() lm = getattr(self._conf, "launch_method", None) or "auto" + root = _git_toplevel(os.path.dirname(paths[0])) or self.repo + check_project_imports(env, self.benchmark_dir, lm, packages or import_packages(paths, root), root) asv_results = run_benchmarks(filtered, env, extra_params=extra_params, launch_method=lm) deltas = _extract_deltas(asv_results, filtered, baseline) diff --git a/test/test_lightspeed_project_imports.py b/test/test_lightspeed_project_imports.py new file mode 100644 index 0000000..7c45e3e --- /dev/null +++ b/test/test_lightspeed_project_imports.py @@ -0,0 +1,60 @@ +"""The import probe reports where a benchmark process imports the project from.""" + +import sys + +import pytest + +from asv import config, environment +from asv.contrib.lightspeed import ProjectShadowed, check_project_imports +from asv.contrib.lightspeed.project_imports import import_packages + + +@pytest.fixture +def layout(tmp_path): + for d in ("repo/pkg", "shadow/pkg", "repo/benchmarks"): + (tmp_path / d).mkdir(parents=True) + (tmp_path / d / "__init__.py").write_text("") + conf = config.Config.from_json( + { + "repo": str(tmp_path / "repo"), + "benchmark_dir": "benchmarks", + "env_dir": "env", + "results_dir": "results", + "html_dir": "html", + } + ) + return tmp_path, environment.ExistingEnvironment(conf, sys.executable, {}, {}) + + +def _check(tmp_path, env, launch): + return check_project_imports( + env, tmp_path / "repo/benchmarks", launch, ["pkg"], tmp_path / "repo" + ) + + +@pytest.mark.parametrize("launch", ["spawn", "forkserver"]) +def test_package_in_repo_passes(layout, monkeypatch, launch): + tmp_path, env = layout + monkeypatch.setenv("ASV_PYTHONPATH", str(tmp_path / "repo")) + assert _check(tmp_path, env, launch) == {"pkg": str(tmp_path / "repo/pkg/__init__.py")} + + +@pytest.mark.parametrize("launch", ["spawn", "forkserver"]) +def test_shadow_copy_earlier_on_path_is_reported(layout, monkeypatch, launch): + tmp_path, env = layout + monkeypatch.setenv("ASV_PYTHONPATH", f"{tmp_path / 'shadow'}:{tmp_path / 'repo'}") + with pytest.raises(ProjectShadowed) as exc: + _check(tmp_path, env, launch) + assert exc.value.paths == {"pkg": str(tmp_path / "shadow/pkg/__init__.py")} + + +def test_import_packages_from_changed_files(tmp_path): + for d in ("numpy/core", "src/skimage/filters", "benchmarks"): + (tmp_path / d).mkdir(parents=True) + for f in ("numpy/__init__.py", "numpy/core/__init__.py", "src/skimage/__init__.py"): + (tmp_path / f).write_text("") + files = ["numpy/core/x.c", "src/skimage/filters/a.py", "benchmarks/b.py", "README.rst"] + assert import_packages([str(tmp_path / f) for f in files], str(tmp_path)) == [ + "numpy", + "skimage", + ]