lightspeed: stop measure_impacted when the benchmarks import the project from outside the repository

asv/contrib/lightspeed/__init__.py

@@ -8,4 +8,6 @@
     ConfigError,
     BenchmarkError,
     NoBenchmarksError,
+    ProjectShadowed,
+    check_project_imports,
 )

asv/contrib/lightspeed/project_imports.py

@@ -0,0 +1,88 @@
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+"""Find where the benchmark processes import the project packages from."""
+
+import json
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional
+
+from ...runner import get_spawner
+
+_PROBE = '''import importlib, json, os
+from asv_runner._aux import update_sys_path
+
+def track_import_probe():
+    try:  # the suite's own __init__ can change sys.path, as in a real run
+        update_sys_path({bench!r})
+        importlib.import_module(os.path.basename({bench!r}))
+    except Exception:
+        pass
+    out = {{}}
+    for name in {names!r}:
+        try:
+            mod = importlib.import_module(name)
+            out[name] = getattr(mod, "__file__", None) or next(iter(getattr(mod, "__path__", [])), None)
+        except Exception:
+            out[name] = None
+    with open({out!r}, "w") as f:
+        json.dump(out, f)
+    return 0
+'''
+
+
+def import_packages(paths: Iterable[str], repo_root: str) -> List[str]:
+    """Top-level import names of the files: numpy/core/x.c -> numpy, src/skimage/a.py -> skimage."""
+    root = Path(repo_root).resolve()
+    names = set()
+    for p in paths:
+        try:
+            parts = Path(p).resolve().relative_to(root).parts
+        except ValueError:
+            continue
+        for i in range(1, len(parts)):
+            if (root.joinpath(*parts[:i]) / "__init__.py").is_file():
+                names.add(parts[i - 1])
+                break
+    return sorted(names)
+
+
+def probe_imports(env, benchmark_dir, launch_method, packages) -> Dict[str, Optional[str]]:
+    """Import ``packages`` in a process started by the benchmark spawner; map each to its file, None if not importable."""
+    tmp = tempfile.mkdtemp(prefix="lsv-import-probe-")
+    try:
+        bench = os.path.join(tmp, "lsv_import_probe")
+        os.mkdir(bench)
+        Path(bench, "__init__.py").write_text("")
+        out = os.path.join(tmp, "out.json")
+        Path(bench, "probe.py").write_text(
+            _PROBE.format(bench=os.path.abspath(str(benchmark_dir)), names=list(packages), out=out)
+        )
+        spawner = get_spawner(env, bench, launch_method)
+        try:
+            log, code = spawner.run(
+                name="probe.track_import_probe",
+                params_str="{}",
+                profile_path="None",
+                result_file_name=os.path.join(tmp, "res.json"),
+                timeout=300,
+                cwd=tmp,
+            )
+        finally:
+            spawner.close()
+        if code != 0 or not os.path.exists(out):
+            raise RuntimeError(f"import probe failed (exit {code}): {log[-2000:]}")
+        return json.loads(Path(out).read_text())
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def outside_root(found: Dict[str, Optional[str]], repo_root) -> Dict[str, str]:
+    """The packages that resolve to a file outside ``repo_root``."""
+    root = os.path.realpath(str(repo_root))
+    return {
+        name: path
+        for name, path in found.items()
+        if path and os.path.commonpath([root, os.path.realpath(path)]) != root
+    }

asv/contrib/lightspeed/session.py

@@ -34,6 +34,7 @@
 from ... import util
 from .deps_db import BenchmarkId, LightspeedDB, _parse_bid
 from .fingerprint import changed_files_with_fingerprints
+from .project_imports import import_packages, outside_root, probe_imports
 from .survey import run_survey
 
 _DEPS_DB_FILENAME = ".lightspeed_deps.db"
@@ -63,6 +64,23 @@ class NoBenchmarksError(ASVError):
     """No benchmarks were selected or found."""
 
 
+class ProjectShadowed(ASVError):
+    """The benchmark processes import a project package from outside the repository."""
+    def __init__(self, paths):
+        super().__init__("benchmarks import the project from outside the repository: "
+                         + ", ".join(f"{name} -> {path}" for name, path in paths.items()))
+        self.paths = paths
+
+
+def check_project_imports(env, benchmark_dir, launch_method, packages, repo_root):
+    """Raise ProjectShadowed when a package resolves outside ``repo_root`` in a benchmark process."""
+    found = probe_imports(env, benchmark_dir, launch_method, packages) if packages else {}
+    bad = outside_root(found, repo_root)
+    if bad:
+        raise ProjectShadowed(bad)
+    return found
+
+
 # ---------------------------------------------------------------------------
 # Result dataclasses
 # ---------------------------------------------------------------------------
@@ -557,6 +575,7 @@ def measure_impacted(
         rounds: Optional[int] = None,
         repeat: Optional[int] = None,
         warmup_time: Optional[float] = None,
+        packages: Optional[List[str]] = None,
     ) -> MeasureResult:
         """
         Selectively re-run benchmarks affected by code changes.
@@ -578,6 +597,9 @@ def measure_impacted(
             Samples per round.  ``None`` means auto.
         warmup_time : float, optional
             Warmup seconds.  ``None`` means auto.
+        packages : list of str, optional
+            Import names of the project.  ``None`` derives them from the changed files.
+            Raises ProjectShadowed before timing when one imports from outside the repository.
         """
         if not from_git_diff and changed_files is None:
             raise ValueError("Provide either from_git_diff=True or changed_files=[...]")
@@ -638,6 +660,8 @@ def measure_impacted(
         extra_params = _timing_params(rounds, repeat, warmup_time)
         env = self._get_env()
         lm = getattr(self._conf, "launch_method", None) or "auto"
+        root = _git_toplevel(os.path.dirname(paths[0])) or self.repo
+        check_project_imports(env, self.benchmark_dir, lm, packages or import_packages(paths, root), root)
         asv_results = run_benchmarks(filtered, env, extra_params=extra_params, launch_method=lm)
 
         deltas = _extract_deltas(asv_results, filtered, baseline)

test/test_lightspeed_project_imports.py

@@ -0,0 +1,60 @@
+"""The import probe reports where a benchmark process imports the project from."""
+
+import sys
+
+import pytest
+
+from asv import config, environment
+from asv.contrib.lightspeed import ProjectShadowed, check_project_imports
+from asv.contrib.lightspeed.project_imports import import_packages
+
+
+@pytest.fixture
+def layout(tmp_path):
+    for d in ("repo/pkg", "shadow/pkg", "repo/benchmarks"):
+        (tmp_path / d).mkdir(parents=True)
+        (tmp_path / d / "__init__.py").write_text("")
+    conf = config.Config.from_json(
+        {
+            "repo": str(tmp_path / "repo"),
+            "benchmark_dir": "benchmarks",
+            "env_dir": "env",
+            "results_dir": "results",
+            "html_dir": "html",
+        }
+    )
+    return tmp_path, environment.ExistingEnvironment(conf, sys.executable, {}, {})
+
+
+def _check(tmp_path, env, launch):
+    return check_project_imports(
+        env, tmp_path / "repo/benchmarks", launch, ["pkg"], tmp_path / "repo"
+    )
+
+
+@pytest.mark.parametrize("launch", ["spawn", "forkserver"])
+def test_package_in_repo_passes(layout, monkeypatch, launch):
+    tmp_path, env = layout
+    monkeypatch.setenv("ASV_PYTHONPATH", str(tmp_path / "repo"))
+    assert _check(tmp_path, env, launch) == {"pkg": str(tmp_path / "repo/pkg/__init__.py")}
+
+
+@pytest.mark.parametrize("launch", ["spawn", "forkserver"])
+def test_shadow_copy_earlier_on_path_is_reported(layout, monkeypatch, launch):
+    tmp_path, env = layout
+    monkeypatch.setenv("ASV_PYTHONPATH", f"{tmp_path / 'shadow'}:{tmp_path / 'repo'}")
+    with pytest.raises(ProjectShadowed) as exc:
+        _check(tmp_path, env, launch)
+    assert exc.value.paths == {"pkg": str(tmp_path / "shadow/pkg/__init__.py")}
+
+
+def test_import_packages_from_changed_files(tmp_path):
+    for d in ("numpy/core", "src/skimage/filters", "benchmarks"):
+        (tmp_path / d).mkdir(parents=True)
+    for f in ("numpy/__init__.py", "numpy/core/__init__.py", "src/skimage/__init__.py"):
+        (tmp_path / f).write_text("")
+    files = ["numpy/core/x.c", "src/skimage/filters/a.py", "benchmarks/b.py", "README.rst"]
+    assert import_packages([str(tmp_path / f) for f in files], str(tmp_path)) == [
+        "numpy",
+        "skimage",
+    ]