From f875198fcabcb970a16b20ab2fb32e8115be8333 Mon Sep 17 00:00:00 2001 From: yixi ning Date: Sun, 18 Oct 2026 08:43:40 +0000 Subject: [PATCH 1/2] fix: honor remaining bootstrap resamples --- src/lighteval/metrics/utils/stderr.py | 5 ++-- tests/unit/metrics/test_stderr.py | 36 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 tests/unit/metrics/test_stderr.py diff --git a/src/lighteval/metrics/utils/stderr.py b/src/lighteval/metrics/utils/stderr.py index e8d47e444..5b05b950d 100644 --- a/src/lighteval/metrics/utils/stderr.py +++ b/src/lighteval/metrics/utils/stderr.py @@ -82,12 +82,13 @@ def bootstrap_stderr(metric: Callable, population: list, number_experiments: int """ res = [] number_draws = min(1000, number_experiments) - number_seeds = number_experiments // number_draws + number_seeds = (number_experiments + number_draws - 1) // number_draws logger.info(f"Bootstrapping {metric.__name__}'s stderr with {number_seeds} seeds.") for seed in range(number_seeds): # sample w replacement - res.extend(_bootstrap_internal(metric=metric, number_draws=number_draws)((population, seed))) + draws_for_seed = min(number_draws, number_experiments - seed * number_draws) + res.extend(_bootstrap_internal(metric=metric, number_draws=draws_for_seed)((population, seed))) return mean_stderr(res) diff --git a/tests/unit/metrics/test_stderr.py b/tests/unit/metrics/test_stderr.py new file mode 100644 index 000000000..48e77012f --- /dev/null +++ b/tests/unit/metrics/test_stderr.py @@ -0,0 +1,36 @@ +# MIT License + +# Copyright (c) 2026 ningyixi + +import math +import multiprocessing as mp +import statistics + +import pytest + +from lighteval.metrics.utils.stderr import get_stderr_function + + +class _RecordingMetric: + __name__ = "recording_metric" + + def __init__(self, draws): + self.draws = draws + + def __call__(self, sample): + value = statistics.mean(sample) + self.draws.append(value) + return value + + +@pytest.mark.parametrize("number_experiments", [25, 1000, 1001, 1500]) +def test_bootstrap_uses_every_requested_experiment(monkeypatch, number_experiments): + monkeypatch.setattr(mp, "cpu_count", lambda: 2) + with mp.Manager() as manager: + draws = manager.list() + stderr = get_stderr_function(_RecordingMetric(draws), number_experiments=number_experiments) + + result = stderr([0, 1, 2, 4]) + + assert len(draws) == number_experiments + assert result == pytest.approx(statistics.stdev(draws) / math.sqrt(number_experiments)) From f3fe6d654779364f6f0f14d19bd9f16be2ac783b Mon Sep 17 00:00:00 2001 From: yixi ning Date: Sun, 18 Oct 2026 14:48:42 +0000 Subject: [PATCH 2/2] fix: use bootstrap sample deviation as the standard error --- src/lighteval/metrics/utils/stderr.py | 5 +++-- tests/unit/metrics/test_stderr.py | 24 +++++++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/lighteval/metrics/utils/stderr.py b/src/lighteval/metrics/utils/stderr.py index 5b05b950d..393ed658d 100644 --- a/src/lighteval/metrics/utils/stderr.py +++ b/src/lighteval/metrics/utils/stderr.py @@ -78,7 +78,8 @@ def __call__(self, cur_experiment): def bootstrap_stderr(metric: Callable, population: list, number_experiments: int): """Bootstraps the stderr of the given metric for the given population of samples, by sampling said population for number_experiments and recomputing the metric on the - different samplings. + different samplings. The standard error is the sample standard deviation of + these bootstrap metric values, without dividing by the number of replicates. """ res = [] number_draws = min(1000, number_experiments) @@ -90,7 +91,7 @@ def bootstrap_stderr(metric: Callable, population: list, number_experiments: int draws_for_seed = min(number_draws, number_experiments - seed * number_draws) res.extend(_bootstrap_internal(metric=metric, number_draws=draws_for_seed)((population, seed))) - return mean_stderr(res) + return _stddev(res) def get_stderr_function(aggregation: Callable, number_experiments: int = 1000): diff --git a/tests/unit/metrics/test_stderr.py b/tests/unit/metrics/test_stderr.py index 48e77012f..74d06d8d0 100644 --- a/tests/unit/metrics/test_stderr.py +++ b/tests/unit/metrics/test_stderr.py @@ -33,4 +33,26 @@ def test_bootstrap_uses_every_requested_experiment(monkeypatch, number_experimen result = stderr([0, 1, 2, 4]) assert len(draws) == number_experiments - assert result == pytest.approx(statistics.stdev(draws) / math.sqrt(number_experiments)) + assert result == pytest.approx(statistics.stdev(draws)) + + +def _accuracy(sample): + return sum(sample) / len(sample) + + +@pytest.mark.parametrize("number_experiments", [1000, 4000]) +def test_bootstrap_standard_error_matches_bernoulli_sampling_variance(monkeypatch, number_experiments): + monkeypatch.setattr(mp, "cpu_count", lambda: 2) + population = [0, 1] * 250 + stderr = get_stderr_function(_accuracy, number_experiments=number_experiments) + + # Resampling this empirical Bernoulli population gives variance p(1-p)/n + # for its accuracy, independent of the number of bootstrap replicates. + assert stderr(population) == pytest.approx(math.sqrt(0.5 * 0.5 / len(population)), rel=0.08) + + +def test_mean_aggregation_keeps_analytic_standard_error(): + population = [0, 1, 2, 4] + stderr = get_stderr_function(statistics.mean) + + assert stderr(population) == pytest.approx(statistics.stdev(population) / math.sqrt(len(population)))