SEP 46: calibrating the native backend to its machine

README.md

@@ -159,7 +159,9 @@ on x86-64 (win-64) and Windows on Arm (win-arm64). On any other platform the ins
 reports that it found no matching distribution:
 [Installation](https://solprop.readthedocs.io/en/stable/introduction/installation.html) explains that error and how to
 report the platform, and covers environment checks and a first run. For GPUs, build the
-native backend from source (below).
+native backend from source (below). Then run `solprop calibrate` once, to fit the native
+backend's threads, waiting and transform plans to the machine
+([Calibrating solprop to your machine](https://solprop.readthedocs.io/en/stable/guide/performance/calibration.html)).
 
 ## Building from source
 

benchmarks/ab.py

@@ -26,20 +26,23 @@
 )
 from benchmarks.registry import select
 from benchmarks.runner import (
+    calibrate_builds,
     freeze_harness,
     launch_worker,
     plan_rows,
     prepared_python,
     receipt,
+    worker_calibration,
 )
 from benchmarks.system import eligible_cpus, guard, physical_cores, thread_environment
-from benchmarks.track import BACKENDS, selection_options, steppers
+from benchmarks.track import BACKENDS, calibration_option, selection_options, steppers
 
 __all__ = [
     "ARMS",
     "ab",
     "ab_options",
     "arm_configuration",
+    "arm_environment",
     "arm_order",
     "field_difference",
     "show_summary",
@@ -77,6 +80,28 @@ def ab_options(parser: argparse.ArgumentParser) -> None:
         choices=BACKENDS,
         help="the candidate arm's backend; default: --backend",
     )
+    calibration_option(parser)
+    for arm in ARMS:
+        parser.add_argument(
+            f"--{arm}-env",
+            action="append",
+            default=[],
+            metavar="NAME=VALUE",
+            help=f"an environment variable for the {arm} arm alone, over the "
+            "threading preset (repeatable); SOLPROP_CALIBRATION=off against =use "
+            "compares the built-in table with a machine calibration (SEP 46)",
+        )
+
+
+def arm_environment(settings: list[str]) -> dict[str, str]:
+    """Return an arm's own variables from ``NAME=VALUE`` settings."""
+    environment: dict[str, str] = {}
+    for setting in settings:
+        name, separator, value = setting.partition("=")
+        if not separator or not name:
+            raise SuiteError(f"an arm's environment takes NAME=VALUE, got {setting!r}")
+        environment[name] = value
+    return environment
 
 
 def arm_configuration(configuration: Record, backend: str) -> Record:
@@ -231,6 +256,10 @@ def ab(args: argparse.Namespace) -> int:
         "baseline": prepared_python(args.baseline_python),
         "candidate": prepared_python(args.candidate_python),
     }
+    environments = {
+        "baseline": arm_environment(args.baseline_env),
+        "candidate": arm_environment(args.candidate_env),
+    }
     harness = output / "harness"
     atomic_record(
         output / "plan.json",
@@ -239,6 +268,7 @@ def ab(args: argparse.Namespace) -> int:
             "purpose": "exploratory A/B",
             "pythons": pythons,
             "backends": backends,
+            "environments": environments,
             "rounds": args.rounds,
             "repeats": args.repeats,
             "length": args.length,
@@ -250,6 +280,7 @@ def ab(args: argparse.Namespace) -> int:
         },
         immutable=True,
     )
+    calibrate_builds(pythons, args.calibrate, output, environments)
     samples: list[Record] = []
     provenance: dict[str, list[Record]] = {arm: [] for arm in ARMS}
     allocation = eligible_cpus()
@@ -274,7 +305,9 @@ def ab(args: argparse.Namespace) -> int:
                         if round_index == 0
                         else None,
                     },
-                    thread_environment(width, dict(os.environ), args.threading),
+                    thread_environment(width, dict(os.environ), args.threading)
+                    | worker_calibration(dict(os.environ) | environments[arm])
+                    | environments[arm],
                     output / "raw" / f"r{round_index}-{arm}-{row['id']}",
                     args.timeout,
                     harness,
@@ -315,6 +348,7 @@ def ab(args: argparse.Namespace) -> int:
         same = (
             pythons["baseline"] == pythons["candidate"]
             and backends["baseline"] == backends["candidate"]
+            and environments["baseline"] == environments["candidate"]
         )
         print(
             ("A/A control: the same interpreter for both arms. " if same else "")

benchmarks/builders.py

@@ -13,6 +13,7 @@
 from __future__ import annotations
 
 import dataclasses
+import os
 from typing import TYPE_CHECKING, Any
 
 from benchmarks.records import Record, SuiteError
@@ -34,6 +35,37 @@
     from solprop.stepper import Multistep, RungeKutta
 
 
+NODE_BUDGET = "SOLPROP_BENCH_NODE_BUDGET"
+"""A node budget in bytes for every case's sources, to measure the node chunk (#361).
+
+Set in one arm of ``benchmarks ab`` (``--candidate-env``), it compares two chunks with
+one build. Unset, each case keeps its translation's default.
+"""
+
+
+def _node_budget() -> int | None:
+    """Return the node budget :data:`NODE_BUDGET` asks for, if any.
+
+    Returns
+    -------
+    int or None
+        Bytes, or ``None`` when the variable is unset.
+
+    Raises
+    ------
+    SuiteError
+        When the variable is not a positive integer.
+    """
+    value = os.environ.get(NODE_BUDGET, "").strip()
+    if not value:
+        return None
+    if not value.isdigit() or int(value) <= 0:
+        raise SuiteError(
+            f"{NODE_BUDGET} must be a positive number of bytes, got {value!r}"
+        )
+    return int(value)
+
+
 def integrator(stepper: str) -> Multistep | RungeKutta | None:
     """Return the integrator a stepper override names; ``None`` keeps the default."""
     from solprop.stepper import Multistep, RungeKutta  # noqa: PLC0415
@@ -539,6 +571,7 @@ def build(configuration: Record) -> Simulation:
             backend=configuration["backend"],
             fields=fields,
             field_planes=observed["field_planes"],
+            node_budget=_node_budget(),
         )
         if case.step is not None:
             settings = dataclasses.replace(
@@ -583,5 +616,6 @@ def build(configuration: Record) -> Simulation:
         raise SuiteError(f"unknown builder {case.builder}")
     if chosen is not None:
         observed["integrator"] = chosen
+    observed["node_budget"] = _node_budget()  # None: the translation's default
     simulation.settings = dataclasses.replace(simulation.settings, **observed)
     return simulation

benchmarks/history.py

@@ -40,6 +40,7 @@
     "numa_domains",
     "threads",
     "threading",
+    "calibration",
     "case",
     "revision",
     "case_hash",

benchmarks/runner.py

@@ -161,6 +161,71 @@ def show_plan(rows: list[Record], skipped: list[str], runs: int) -> None:
     )
 
 
+CALIBRATION_LEVELS = ("thorough", "standard", "quick", "none")
+"""``--calibrate``'s levels: how each build is calibrated before a series (SEP 46)."""
+
+
+def worker_calibration(environ: dict[str, str]) -> dict[str, str]:
+    """Return a worker's ``SOLPROP_CALIBRATION``: ``use``, or ``off`` if asked.
+
+    A worker applies the profile the series was calibrated with and never measures
+    inside a timed window (SEP 46 section 6.13); ``off`` holds the built-in table for
+    an experiment that must keep the settings fixed while the code changes.
+    """
+    chosen = environ.get("SOLPROP_CALIBRATION", "").strip().lower()
+    return {"SOLPROP_CALIBRATION": "off" if chosen == "off" else "use"}
+
+
+def calibrate_builds(
+    pythons: dict[str, str],
+    level: str,
+    output: Path,
+    environments: dict[str, dict[str, str]] | None = None,
+) -> Record:
+    """Calibrate each distinct build at `level` before a series (SEP 46 section 6.13).
+
+    Each interpreter runs ``python -m solprop calibrate --LEVEL`` once, with its arm's
+    own variables, so that an A/B's two builds each get their own profile, as two
+    releases would for a user; its report is kept under ``calibration/``. ``none``
+    calibrates nothing and the series runs on whatever profiles exist; an arm under
+    ``SOLPROP_CALIBRATION=off`` runs the built-in table and calibrates nothing either.
+    """
+    if level == "none":
+        return {}
+    import os  # noqa: PLC0415
+
+    done: dict[str, Record] = {}
+    found: Record = {}
+    for arm, python in pythons.items():
+        if python in done:
+            found[arm] = done[python]
+            continue
+        environment = dict(os.environ) | (environments or {}).get(arm, {})
+        if worker_calibration(environment)["SOLPROP_CALIBRATION"] == "off":
+            found[arm] = {"level": "off"}
+            continue
+        completed = subprocess.run(
+            [python, "-m", "solprop", "calibrate", f"--{level}"],
+            capture_output=True,
+            text=True,
+            check=False,
+            env=environment,
+        )
+        report = output / "calibration" / f"{arm}.txt"
+        report.parent.mkdir(parents=True, exist_ok=True)
+        report.write_text(completed.stdout + completed.stderr, encoding="utf-8")
+        done[python] = {
+            "level": level,
+            "status": completed.returncode,
+            "summary": completed.stdout.splitlines()[:1],
+        }
+        found[arm] = done[python]
+        print(
+            f"calibrated {arm} ({level}): {' '.join(completed.stdout.splitlines()[:1])}"
+        )
+    return found
+
+
 def launch_worker(
     python: str,
     request: Record,

benchmarks/track.py

@@ -19,11 +19,14 @@
 from benchmarks.records import LOCAL_ROOT, Record, SuiteError, atomic_record, now
 from benchmarks.registry import CASES, SETS, STEPPERS, select
 from benchmarks.runner import (
+    CALIBRATION_LEVELS,
     ROOT,
+    calibrate_builds,
     launch_worker,
     plan_rows,
     receipt,
     show_plan,
+    worker_calibration,
 )
 from benchmarks.system import (
     THREADING,
@@ -107,11 +110,23 @@ def selection_options(parser: argparse.ArgumentParser) -> None:
     )
 
 
+def calibration_option(parser: argparse.ArgumentParser) -> None:
+    """Declare ``--calibrate``: how each build is calibrated before the series."""
+    parser.add_argument(
+        "--calibrate",
+        choices=CALIBRATION_LEVELS,
+        default="thorough",
+        help="calibrate each build at this level first, as a user would (SEP 46); "
+        "none keeps the profiles there are (default: thorough)",
+    )
+
+
 def track_options(parser: argparse.ArgumentParser) -> None:
     """Declare the tracking run's output and history options."""
     selection_options(parser)
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--out", type=Path)
+    calibration_option(parser)
     parser.add_argument("--instance-type", help="cloud instance type, if any")
     parser.add_argument(
         "--append-history",
@@ -157,6 +172,7 @@ def summarise(
         "threading": threading,
         "status": result["status"],
         "busy": busy,
+        "calibration": result.get("calibration"),
     }
     if result["status"] != "ok":
         return {**base, "error": result.get("error")}
@@ -250,6 +266,7 @@ def history_rows(
             "numa_domains": machine["numa_domains"],
             "threads": row["threads"],
             "threading": row["threading"],
+            "calibration": row.get("calibration"),
             "case": row["case"],
             "revision": row["revision"],
             "case_hash": CASES[row["case"]].digest[:12],
@@ -378,6 +395,7 @@ def track(args: argparse.Namespace) -> int:
         },
         immutable=True,
     )
+    calibrate_builds({"build": sys.executable}, args.calibrate, output)
     started = time.perf_counter()
     results: list[Record] = []
     provenance: Record = {}
@@ -388,7 +406,8 @@ def track(args: argparse.Namespace) -> int:
         events, elapsed = launch_worker(
             sys.executable,
             {"configuration": row["configuration"], "repeats": args.repeats},
-            thread_environment(width, dict(os.environ), args.threading),
+            thread_environment(width, dict(os.environ), args.threading)
+            | worker_calibration(dict(os.environ)),
             output / "raw" / row["id"],
             args.timeout,
         )

benchmarks/worker.py

@@ -310,6 +310,7 @@ def measure(configuration: Record, repeats: int, fields: Path | None) -> Record:
             fields.parent.mkdir(parents=True, exist_ok=True)
             np.save(fields, final, allow_pickle=False)
         return {
+            "calibration": calibration_label(engine.metadata()),
             "construction_seconds": construction,
             "backend_seconds": backend_seconds,
             "warmup_seconds": warmup,
@@ -341,6 +342,22 @@ def measure(configuration: Record, repeats: int, fields: Path | None) -> Record:
         engine.close()
 
 
+def calibration_label(metadata: Record) -> str | None:
+    """Name the machine profile a run applied: ``built-in``, or its level and
+    generation, with ``+refinement`` where a stage-2 refinement applied (SEP 46)."""
+    found = metadata.get("calibration")
+    if not isinstance(found, dict):
+        return None
+    profile = found.get("profile")
+    label = (
+        f"{profile['level']}:{profile['generation']}"
+        if isinstance(profile, dict)
+        else "built-in"
+    )
+    source = str(found.get("workspace_source", ""))
+    return label + ("+refinement" if source.startswith("refinement") else "")
+
+
 def main() -> int:
     """Execute the JSON request passed by the controller."""
     diagnostics = Diagnostics()

changelog.d/sep-0046-machine-calibration.added.md

@@ -0,0 +1 @@
+`solprop calibrate` fits the native backend to the machine it runs on ([SEP 46](docs/dev/seps/archive/sep-0046-machine-calibration.md); [Calibrating solprop to your machine](docs/guide/performance/calibration.md)): in about 20 s (`--quick`), a minute and a half, or five minutes (`--thorough`), child processes pinned to the job's CPUs measure how idle threads should wait, how wide the team should be, where each kind of operation should start to thread and which FFTW plan effort runs fastest inside the step, and a setting replaces the built-in value only if it wins every round by more than the measured noise, slows none of the problems (production-sized ones among them) and validates with the other winners. The profile is kept per user in the platform's cache directory (`SOLPROP_CACHE_DIR` moves it; `SOLPROP_CALIBRATION_PATH` adds shared read-only directories, such as a cluster's per node type), keyed by machine, build and allocation, and every later run applies it and records the settings it applied in its metadata (`calibration`). The first native run with no profile for its machine, build and allocation calibrates quickly by itself after a `CalibrationNotice`, and a scan calibrates its workers' allocation first (unless they run one thread each, which has nothing to choose); `SOLPROP_CALIBRATION=use` never measures and `off` ignores profiles. `Simulation.calibrate()` and `solprop calibrate --recipe` refine the settings for one simulation, reused by every run and scan point of the same execution. `--show`, `--export`, `--import` and `--clear` manage profiles. Benchmark series calibrate each build first and record the profile per row. Without a profile, every run uses exactly the former settings, now a built-in table keyed by operating system; `BuildInfo`'s threshold fields report that table for the process.

changelog.d/sep-0046-scan-worker-wait.changed.md

@@ -0,0 +1 @@
+A scan's workers ask for passive waiting instead of setting `OMP_WAIT_POLICY=passive` themselves, so that a user's own `OMP_WAIT_POLICY`, `GOMP_SPINCOUNT` or `KMP_BLOCKTIME` now reaches them, and so does a machine calibration's row for the workers' allocation ([SEP 46](docs/dev/seps/archive/sep-0046-machine-calibration.md)); without either they wait as before. Their recorded `openmp_wait_policy` reads `passive`.

docs/conf.py

@@ -31,6 +31,11 @@
 if _watchdog := os.environ.get("SOLPROP_DOCS_WATCHDOG"):
     faulthandler.dump_traceback_later(float(_watchdog), exit=False)
 
+# A machine calibration (SEP 46) neither runs nor applies while the galleries run:
+# their committed output must not show a first-use notice, nor depend on the machine
+# that refreshed it.
+os.environ.setdefault("SOLPROP_CALIBRATION", "off")
+
 # Gallery examples may select the native backend automatically. Load it and its OpenMP
 # runtime before autodoc or a tutorial imports the rest of the stack, as the tests do.
 if importlib.util.find_spec("solprop_native") is not None:

docs/dev/architecture.md

@@ -222,6 +222,7 @@ against `src/solprop` by `tests/test_structure_figures.py`.
 | `backends/numpy_ref/` | Reference execution | B |
 | `stepper/`, `log.py`, `stats.py` | Tableaux, controllers, schedules, progress reporters and statistics specifications | D |
 | `driver/`, `driver/executors/`, `scan.py` | Simulation and convenience interfaces, scan/batch orchestration, Local/Pool/Queue/Slurm execution and scan CLI | D |
+| `calibration/` | The native backend's machine calibration ([SEP 46](seps/archive/sep-0046-machine-calibration.md)): reading, showing, exporting, importing and clearing profiles in the per-user store, through solprop-native's standard-library modules loaded by path so that the native runtime never starts (`calibration/_native.py`), the measuring controller, its child processes, the synthetic problems they time, the operation-grain candidates and the selection rule (`calibration/controller.py`, `child.py`, `problems.py`, `grain.py`, `selection.py`), the refinement of one simulation (`calibration/refine.py`, stage 2, which `Simulation.calibrate` runs), and the `solprop calibrate` command (`calibration/command.py`) | D |
 | `output/` | Memory and HDF5 outputs, provenance/resume, scan storage and artifacts | D |
 | `processing/`, `plotting/`, `sol.py` | Field, spectrum, time, coherence, compression, mixture and scan analysis; a saved plane read back as a field plane or a fluence map (`processing/planes.py`), and a field plane's file form (`processing/plane_file.py`); propagation/statistics plots and spectrograms; the interactive session namespace, the one module whose import has side effects | X |
 | `service/` | Front-end support, above every other layer: the messages between a front end and solprop, the kernel process that answers a front end's checks, previews and readings of a results file as plain data, each run as a job that keeps its record in its folder, and the helper and connection that reach a machine's runs over SSH ([the front end's design](gui/design.md), sections 13.3 and 13.4). Not part of the user API and outside the stability promise; nothing else in solprop imports it | X |