diff --git a/doc/api.rst b/doc/api.rst index cb08ebed..0b3e85d8 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -512,6 +512,7 @@ HookBase class Called when the hook is completed for a process. May add any information collected to the passed-in ``metadata`` dictionary. + The ``phase`` metadata identifies the run phase and sample type. Runner class @@ -780,6 +781,9 @@ Other: * ``perf_version``: Version of the ``pyperf`` module * ``unit``: Unit of values: ``byte``, ``integer`` or ``second`` +* ``phase`` (tuple or list of two non-empty strings): run phase and sample + type, set to one of ``("calibration", "loops")``, ``("calibration", + "warmup")``, ``("measurement", "warmup")`` or ``("measurement", "value")``. * ``calibrate_loops`` (``int >= 1``): number of loops computed in a loops calibration run * ``recalibrate_loops`` (``int >= 1``): number of loops computed in a loops diff --git a/doc/changelog.rst b/doc/changelog.rst index 066beb8f..e486842a 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,12 @@ Changelog ========= +* Feature: pyperf now stores ``phase`` run metadata in JSON files to identify + calibration and measurement samples. Hooks can also inspect this metadata in + :meth:`HookBase.teardown` to distinguish calibration loops, calibration + warmups, measurement warmups, and measurement values. + Patch by Maurycy Pawłowski-Wieroński. + Version 2.10.0 (2026-02-07) --------------------------- diff --git a/pyperf/_metadata.py b/pyperf/_metadata.py index e2ff96c5..fd8bc2b1 100644 --- a/pyperf/_metadata.py +++ b/pyperf/_metadata.py @@ -48,6 +48,12 @@ def is_tags(value): return all(isinstance(x, str) and x not in ('all', '') for x in value) +def is_phase(value): + if not isinstance(value, (tuple, list)): + return False + return len(value) == 2 and all(isinstance(x, str) and x for x in value) + + def parse_load_avg(value): if isinstance(value, NUMBER_TYPES): return value @@ -69,6 +75,7 @@ def format_noop(value): WARMUPS = _MetadataInfo(format_number, (int,), is_positive, 'integer') SECONDS = _MetadataInfo(format_seconds, NUMBER_TYPES, is_positive, 'second') TAGS = _MetadataInfo(format_generic, (list,), is_tags, 'tag') +PHASE = _MetadataInfo(format_generic, (tuple, list), is_phase, None) # Registry of metadata keys METADATA = { @@ -92,6 +99,7 @@ def format_noop(value): 'calibrate_warmups': WARMUPS, 'recalibrate_warmups': WARMUPS, 'tags': TAGS, + 'phase': PHASE, } DEFAULT_METADATA_INFO = _MetadataInfo(format_generic, METADATA_VALUE_TYPES, None, None) diff --git a/pyperf/_worker.py b/pyperf/_worker.py index 3547faeb..c9fe965b 100644 --- a/pyperf/_worker.py +++ b/pyperf/_worker.py @@ -114,6 +114,15 @@ def _compute_values(self, values, nvalue, index += 1 + if calibrate_loops: + self.metadata["phase"] = ("calibration", "loops") + elif is_warmup and (args.calibrate_warmups or args.recalibrate_warmups): + self.metadata["phase"] = ("calibration", "warmup") + elif is_warmup: + self.metadata["phase"] = ("measurement", "warmup") + else: + self.metadata["phase"] = ("measurement", "value") + for hook in hook_managers.values(): hook.teardown(self.metadata) diff --git a/pyperf/tests/test_bench.py b/pyperf/tests/test_bench.py index 8fd4e2a1..48aa4439 100644 --- a/pyperf/tests/test_bench.py +++ b/pyperf/tests/test_bench.py @@ -1,6 +1,7 @@ import datetime import errno import gzip +import json import unittest import pyperf @@ -69,6 +70,25 @@ def test_constructor(self): collect_metadata=False) self.assertEqual(run.get_metadata()['load_avg_1min'], 0.0) + def test_phase_metadata(self): + for value in (('measurement', 'value'), ['calibration', 'loops']): + with self.subTest(value=value): + run = pyperf.Run([1.0], metadata={'phase': value}, + collect_metadata=False) + self.assertEqual(run.get_metadata()['phase'], value) + + invalid_values = ( + 'measurement:value', + ('measurement',), + ('measurement', ''), + ('measurement', 1), + ) + for value in invalid_values: + with self.subTest(value=value): + with self.assertRaises(ValueError): + pyperf.Run([1.0], metadata={'phase': value}, + collect_metadata=False) + def test_name(self): # name must be non-empty with self.assertRaises(ValueError): @@ -198,6 +218,40 @@ def test_dump_load(self): self.check_benchmarks_equal(bench, bench2) + def test_dump_phase_metadata(self): + runs = [ + pyperf.Run( + (), + warmups=[(1, 2.92e-7), (64, 2.65e-9)], + metadata={'name': 'bench', + 'loops': 64, + 'calibrate_loops': 64, + 'phase': ('calibration', 'loops')}, + collect_metadata=False), + pyperf.Run( + [2.5949900301497353e-9, 2.633662268304582e-9], + warmups=[(64, 2.6324198845045776e-9)], + metadata={'name': 'bench', + 'loops': 64, + 'phase': ('measurement', 'value')}, + collect_metadata=False), + ] + bench = pyperf.Benchmark(runs) + + bench_json = tests.benchmark_as_json(bench) + data = json.loads(bench_json) + json_runs = data['benchmarks'][0]['runs'] + self.assertEqual(json_runs[0]['metadata']['phase'], + ['calibration', 'loops']) + self.assertEqual(json_runs[1]['metadata']['phase'], + ['measurement', 'value']) + + bench2 = pyperf.Benchmark.loads(bench_json) + phases = [run.get_metadata()['phase'] for run in bench2.get_runs()] + self.assertEqual(phases, + [['calibration', 'loops'], + ['measurement', 'value']]) + def test_dump_replace(self): bench = self.create_dummy_benchmark() diff --git a/pyperf/tests/test_runner.py b/pyperf/tests/test_runner.py index c77e0251..181cb8de 100644 --- a/pyperf/tests/test_runner.py +++ b/pyperf/tests/test_runner.py @@ -519,6 +519,48 @@ def test_hook_command(self): self.assertEqual(bench.get_metadata()["hooks"], "_test_hook") + def test_hook_teardown_phase_metadata(self): + def time_func(loops): + return 1.0 + + def hook_phases(args): + phases = [] + + class PhaseHook(HookBase): + name = "phase_hook" + + @staticmethod + def load(): + return PhaseHook + + def teardown(self, metadata): + phases.append(metadata["phase"]) + + args = args + ["--hook", PhaseHook.name] + runner = self.create_runner(args, hooks=[PhaseHook]) + with tests.capture_stdout(): + runner.bench_time_func('bench', time_func) + return phases + + cases = ( + ('measurement', + '-l1 -w1 -n1 --worker', + [('measurement', 'warmup'), ('measurement', 'value')]), + ('calibrate loops', + '--calibrate-loops -w0 -n1 --min-time=1.0 --worker', + [('calibration', 'loops')]), + ) + for name, args, expected in cases: + with self.subTest(name=name): + self.assertEqual(hook_phases(args.split()), expected) + + with self.subTest(name='calibrate warmups'): + with mock.patch('pyperf._worker.WorkerTask.test_calibrate_warmups', + return_value=True): + self.assertEqual( + hook_phases('--calibrate-warmups -l1 -n1 --worker'.split()), + [('calibration', 'warmup')]) + def test_custom_hook(self): class State: inited = 0