fix: correct bootstrap standard errors and retain all resamples

src/lighteval/metrics/utils/stderr.py

@@ -78,18 +78,20 @@ def __call__(self, cur_experiment):
 def bootstrap_stderr(metric: Callable, population: list, number_experiments: int):
     """Bootstraps the stderr of the given metric for the given population of samples,
     by sampling said population for number_experiments and recomputing the metric on the
-    different samplings.
+    different samplings. The standard error is the sample standard deviation of
+    these bootstrap metric values, without dividing by the number of replicates.
     """
     res = []
     number_draws = min(1000, number_experiments)
-    number_seeds = number_experiments // number_draws
+    number_seeds = (number_experiments + number_draws - 1) // number_draws
 
     logger.info(f"Bootstrapping {metric.__name__}'s stderr with {number_seeds} seeds.")
     for seed in range(number_seeds):
         # sample w replacement
-        res.extend(_bootstrap_internal(metric=metric, number_draws=number_draws)((population, seed)))
+        draws_for_seed = min(number_draws, number_experiments - seed * number_draws)
+        res.extend(_bootstrap_internal(metric=metric, number_draws=draws_for_seed)((population, seed)))
 
-    return mean_stderr(res)
+    return _stddev(res)
 
 
 def get_stderr_function(aggregation: Callable, number_experiments: int = 1000):

tests/unit/metrics/test_stderr.py

@@ -0,0 +1,58 @@
+# MIT License
+
+# Copyright (c) 2026 ningyixi
+
+import math
+import multiprocessing as mp
+import statistics
+
+import pytest
+
+from lighteval.metrics.utils.stderr import get_stderr_function
+
+
+class _RecordingMetric:
+    __name__ = "recording_metric"
+
+    def __init__(self, draws):
+        self.draws = draws
+
+    def __call__(self, sample):
+        value = statistics.mean(sample)
+        self.draws.append(value)
+        return value
+
+
+@pytest.mark.parametrize("number_experiments", [25, 1000, 1001, 1500])
+def test_bootstrap_uses_every_requested_experiment(monkeypatch, number_experiments):
+    monkeypatch.setattr(mp, "cpu_count", lambda: 2)
+    with mp.Manager() as manager:
+        draws = manager.list()
+        stderr = get_stderr_function(_RecordingMetric(draws), number_experiments=number_experiments)
+
+        result = stderr([0, 1, 2, 4])
+
+        assert len(draws) == number_experiments
+        assert result == pytest.approx(statistics.stdev(draws))
+
+
+def _accuracy(sample):
+    return sum(sample) / len(sample)
+
+
+@pytest.mark.parametrize("number_experiments", [1000, 4000])
+def test_bootstrap_standard_error_matches_bernoulli_sampling_variance(monkeypatch, number_experiments):
+    monkeypatch.setattr(mp, "cpu_count", lambda: 2)
+    population = [0, 1] * 250
+    stderr = get_stderr_function(_accuracy, number_experiments=number_experiments)
+
+    # Resampling this empirical Bernoulli population gives variance p(1-p)/n
+    # for its accuracy, independent of the number of bootstrap replicates.
+    assert stderr(population) == pytest.approx(math.sqrt(0.5 * 0.5 / len(population)), rel=0.08)
+
+
+def test_mean_aggregation_keeps_analytic_standard_error():
+    population = [0, 1, 2, 4]
+    stderr = get_stderr_function(statistics.mean)
+
+    assert stderr(population) == pytest.approx(statistics.stdev(population) / math.sqrt(len(population)))