From 4b1f8a0682f8903d21484d3f1f6a88427aba105b Mon Sep 17 00:00:00 2001 From: Sarthak Gupta Date: Mon, 9 Nov 2026 23:38:55 -0800 Subject: [PATCH 1/2] ENH: Add max_undetectable_understatement to backtest (xref #904) Adds the sixth function from #904 that #965 left out pending a design decision. It finds the largest fraction by which a VaR series could have been scaled down, with the same returns, and still have passed kupiec_pof, christoffersen_independence, christoffersen_conditional_coverage and traffic_light. The search is over proportional scalings of var and is exact: the exception set only changes where a loss crosses the scaled VaR, so one pass over the breakpoints finds the limit with no optimizer. The result, MaxUndetectableUnderstatementResult, reports the limit, the test that binds just beyond it, the exceptions the limit would add, and whether the reported VaR passes to begin with. kupiec_pof and christoffersen_conditional_coverage are two-sided, so by default (ignore_too_few=True) a rejection caused by too few exceptions is not counted, since a conservative VaR is not evidence of understatement. This is a lower bound: a series that understates VaR on some days and overstates it on others can produce the same, or fewer, exceptions than a proportional scaling, and so can be harder to detect. The docstring states this caveat. Reuses _aligned, _exceptions, _lr_pof and _lr_independence for the search and the four public test functions for the baseline check, so the baseline validates p, length and the traffic_light window with the same errors as calling them directly. Tests add a hand-computed case, a brute-force grid check that the limit is the smallest passing proportional scaling, the no-exceptions and all-exceptions edges, both settings of ignore_too_few, and agreement with the public functions at the computed boundary. Co-Authored-By: Claude Opus 5.5 --- arch/tests/univariate/test_backtest.py | 224 +++++++++++++++++ arch/univariate/__init__.py | 4 + arch/univariate/backtest.py | 326 ++++++++++++++++++++++++- doc/source/changes/9.0.rst | 7 + doc/source/univariate/utility.rst | 29 +++ 5 files changed, 589 insertions(+), 1 deletion(-) diff --git a/arch/tests/univariate/test_backtest.py b/arch/tests/univariate/test_backtest.py index b042dc4cf7..b6b84aa34f 100644 --- a/arch/tests/univariate/test_backtest.py +++ b/arch/tests/univariate/test_backtest.py @@ -29,10 +29,12 @@ from scipy import stats from arch.univariate import ( + MaxUndetectableUnderstatementResult, TrafficLightResult, christoffersen_conditional_coverage, christoffersen_independence, kupiec_pof, + max_undetectable_understatement, traffic_light, var_exceptions, ) @@ -1051,3 +1053,225 @@ def test_p_boundaries(): # Values that are close to the lower boundary and 0.5 are valid assert kupiec_pof(returns, var, 1e-12).stat > 0.0 assert kupiec_pof(returns, var, 0.5).stat > 0.0 + + +# Maximum undetectable understatement (xref #904) + + +def _mul_passes(ret, var, scale, p=0.05, size=0.05, ignore_too_few=True, window=250): + """Independent pass/fail check using the public tests directly.""" + scaled = scale * var + hits = var_exceptions(ret, scaled) + too_few = bool(hits.mean() < p) + ok = kupiec_pof(ret, scaled, p).pval > size or (ignore_too_few and too_few) + ok &= christoffersen_independence(ret, scaled).pval > size + cc = christoffersen_conditional_coverage(ret, scaled, p) + ok &= cc.pval > size or (ignore_too_few and too_few) + ok &= traffic_light(ret, scaled, p, window).zone == "green" + return bool(ok) + + +@pytest.fixture(scope="module") +def mul_sim(): + rng = np.random.default_rng(20261109) + nobs = 600 + sigma = np.exp(0.25 * rng.standard_normal(nobs)) + ret = sigma * rng.standard_normal(nobs) * 0.01 + var = -stats.norm.ppf(0.05) * sigma * 0.01 # a generous, correctly sized 95% VaR + return ret, var + + +def test_hand_computed_kupiec_only(): + # 4 observations, ratios -returns/var are 2, 0.5, 0.5, 0.5. Only one exception + # (2 > 1) at the reported VaR, and the three ties at 0.5 do not become + # exceptions until the VaR is scaled below 0.5, so the limit is exactly 0.5. + returns = np.array([-2.0, -0.5, -0.5, -0.5]) + var = np.ones(4) + res = max_undetectable_understatement(returns, var, p=0.5, tests=["kupiec_pof"]) + assert isinstance(res, MaxUndetectableUnderstatementResult) + assert res.passes + assert res.failed == () + assert_allclose(res.understatement, 0.5) + assert res.exceptions == 1 + assert res.extra_exceptions == 0 + assert res.binding == ("kupiec_pof",) + assert res.nobs == 4 + assert res.p == 0.5 + + +def test_passes_and_reports_counts(mul_sim): + ret, var = mul_sim + res = max_undetectable_understatement(ret, var) + assert res.passes + assert res.failed == () + assert 0.0 <= res.understatement <= 1.0 + assert res.extra_exceptions >= 0 + assert res.exceptions == int(var_exceptions(ret, var).sum()) + assert res.nobs == ret.shape[0] + + +def test_brute_force_grid_matches_public_tests(mul_sim): + # The limit is the SMALLEST proportional scaling that still passes: scale=1 (no + # understatement) always passes, since the baseline passes, so a grid search for + # the computed limit has to look for the minimum passing point, not the maximum. + ret, var = mul_sim + res = max_undetectable_understatement(ret, var) + scale = 1.0 - res.understatement + grid = np.linspace(0.0, 1.0, 4001) + passing = [s for s in grid if _mul_passes(ret, var, s)] + best = min(passing) + # the exact limit is at or below the best grid point, and within one cell of it + assert scale <= best + 1e-9 + assert best - scale <= grid[1] - grid[0] + 1e-9 + assert _mul_passes(ret, var, scale) + + +def test_boundary_agrees_with_public_functions(mul_sim): + ret, var = mul_sim + res = max_undetectable_understatement(ret, var) + scale = 1.0 - res.understatement + assert _mul_passes(ret, var, scale) + hits = var_exceptions(ret, scale * var) + assert int(hits.sum()) - res.exceptions == res.extra_exceptions + + +def test_more_tests_never_increase_understatement(mul_sim): + ret, var = mul_sim + only_kupiec = max_undetectable_understatement(ret, var, tests=["kupiec_pof"]) + all_tests = max_undetectable_understatement(ret, var) + assert all_tests.understatement <= only_kupiec.understatement + 1e-12 + + +def test_stricter_size_never_increases_understatement(mul_sim): + # Restricted to kupiec_pof, whose baseline passes at every size tried here; + # Christoffersen's independence test can reject the baseline itself at a loose + # size, which makes the comparison undefined rather than merely smaller. + ret, var = mul_sim + loose = max_undetectable_understatement(ret, var, size=0.01, tests=["kupiec_pof"]) + strict = max_undetectable_understatement(ret, var, size=0.20, tests=["kupiec_pof"]) + assert loose.passes + assert strict.passes + assert strict.understatement <= loose.understatement + 1e-12 + + +def test_no_exceptions_can_be_fully_understated(): + # Every return is a gain: var can be scaled all the way to 0 undetected + ret = np.full(300, 0.01) + var = np.ones(300) + res = max_undetectable_understatement(ret, var) + assert res.passes + assert_allclose(res.understatement, 1.0) + assert res.exceptions == 0 + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_all_exceptions_fails_baseline(): + # Every day is an exception at the reported VaR: far too many for any p < 1 + ret = np.full(20, -5.0) + var = np.ones(20) + res = max_undetectable_understatement(ret, var, p=0.3, window=20) + assert not res.passes + assert "kupiec_pof" in res.failed + assert math.isnan(res.understatement) + assert res.exceptions == 20 + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_ignore_too_few_true_passes_a_conservative_var(): + # No exceptions in 250 observations at p=0.01 is conservative, not understated + ret = np.full(250, 0.001) + var = np.ones(250) + res = max_undetectable_understatement(ret, var, p=0.01, ignore_too_few=True) + assert res.passes + assert res.failed == () + + +def test_ignore_too_few_false_fails_the_same_var(): + ret = np.full(250, 0.001) + var = np.ones(250) + res = max_undetectable_understatement(ret, var, p=0.01, ignore_too_few=False) + assert not res.passes + assert "kupiec_pof" in res.failed + assert math.isnan(res.understatement) + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_overstated_var_fails_baseline(mul_sim): + ret, var = mul_sim + res = max_undetectable_understatement(ret, 0.3 * var) + assert not res.passes + assert math.isnan(res.understatement) + assert res.failed + + +def test_traffic_light_only_sets_the_limit(): + nobs = 250 + var = np.ones(nobs) + # ratios -returns / var are strictly increasing and all in (0.5, 1) + returns = -0.5 * (1.0 + np.arange(nobs) / nobs) + res = max_undetectable_understatement( + returns, var, p=0.01, tests=["traffic_light"], window=nobs + ) + ratios = np.sort(-returns / var)[::-1] + # green allows 4 exceptions at p=0.01: the 5th largest ratio sets the limit + assert_allclose(1.0 - res.understatement, ratios[4]) + assert res.exceptions == 0 + assert res.extra_exceptions == 4 + assert res.binding == ("traffic_light",) + + +def test_result_is_immutable(mul_sim): + ret, var = mul_sim + res = max_undetectable_understatement(ret, var) + assert dataclasses.is_dataclass(res) + assert [f.name for f in dataclasses.fields(res)] == [ + "understatement", + "exceptions", + "extra_exceptions", + "binding", + "passes", + "failed", + "nobs", + "p", + ] + with pytest.raises(dataclasses.FrozenInstanceError): + res.passes = False + + +def test_mul_pandas_alignment(): + returns, var = _series_pair() # 120 observations + scaled_var = var.abs() + 1.0 # strictly positive and generous + result = max_undetectable_understatement(returns, scaled_var, window=100) + expected = max_undetectable_understatement( + returns.to_numpy(), scaled_var.to_numpy(), window=100 + ) + assert result == expected + + +def test_mul_input_validation(mul_sim): + ret, var = mul_sim + with pytest.raises(ValueError, match="strictly positive"): + max_undetectable_understatement(ret, np.zeros_like(var)) + with pytest.raises(ValueError, match="one of"): + max_undetectable_understatement(ret, var, tests=["not_a_test"]) + with pytest.raises(ValueError, match="non-empty"): + max_undetectable_understatement(ret, var, tests=[]) + with pytest.raises(TypeError, match="single str"): + max_undetectable_understatement(ret, var, tests="kupiec_pof") + with pytest.raises(ValueError, match="size must be"): + max_undetectable_understatement(ret, var, size=1.5) + with pytest.raises(ValueError, match=r"p must be a finite float in \(0, 0.5\]"): + max_undetectable_understatement(ret, var, p=0.9) + with pytest.raises(ValueError, match="window must be"): + max_undetectable_understatement(ret, var, window=0) + with pytest.raises(TypeError, match="ignore_too_few"): + max_undetectable_understatement(ret, var, ignore_too_few="yes") + with pytest.raises(ValueError, match="same length as returns"): + max_undetectable_understatement(ret, var[:-1]) + with pytest.raises(ValueError, match="at least 2 observations"): + max_undetectable_understatement( + ret[:1], var[:1], tests=["christoffersen_independence"] + ) diff --git a/arch/univariate/__init__.py b/arch/univariate/__init__.py index 3a6a4d9070..54b3306c6e 100644 --- a/arch/univariate/__init__.py +++ b/arch/univariate/__init__.py @@ -2,10 +2,12 @@ from arch.univariate import recursions_python from arch.univariate.backtest import ( + MaxUndetectableUnderstatementResult, TrafficLightResult, christoffersen_conditional_coverage, christoffersen_independence, kupiec_pof, + max_undetectable_understatement, traffic_light, var_exceptions, ) @@ -70,6 +72,7 @@ "GeneralizedError", "MAMean", "MIDASHyperbolic", + "MaxUndetectableUnderstatementResult", "Normal", "RiskMetrics2006", "SkewStudent", @@ -82,6 +85,7 @@ "excess_kurtosis", "hill_estimator", "kupiec_pof", + "max_undetectable_understatement", "recursions", "recursions_python", "traffic_light", diff --git a/arch/univariate/backtest.py b/arch/univariate/backtest.py index fc8dd172b4..5b7ce5be94 100644 --- a/arch/univariate/backtest.py +++ b/arch/univariate/backtest.py @@ -10,6 +10,7 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass import numpy as np @@ -18,18 +19,35 @@ from scipy.special import xlogy from arch._typing import ArrayLike1D, BoolArray, Float64Array1D -from arch.utility._validation import check_array, check_float, check_int +from arch.utility._validation import ( + check_array, + check_bool, + check_float, + check_int, + check_option, +) from arch.utility.testing import WaldTestStatistic __all__ = [ + "MaxUndetectableUnderstatementResult", "TrafficLightResult", "christoffersen_conditional_coverage", "christoffersen_independence", "kupiec_pof", + "max_undetectable_understatement", "traffic_light", "var_exceptions", ] +# The tests that max_undetectable_understatement can select among, and the order in +# which failed and binding tests are reported +_UNDERSTATEMENT_TESTS = ( + "kupiec_pof", + "christoffersen_independence", + "christoffersen_conditional_coverage", + "traffic_light", +) + # The cumulative probabilities that begin the yellow and the red zone (BCBS, 1996) _YELLOW_PROBABILITY = 0.95 _RED_PROBABILITY = 0.9999 @@ -582,3 +600,309 @@ def traffic_light( nobs=window, p=p, ) + + +@dataclass(frozen=True) +class MaxUndetectableUnderstatementResult: # numpydoc ignore=PR02 + """ + Result of the maximum undetectable understatement backtest of VaR. + + Parameters + ---------- + understatement : float + The largest fraction by which ``var`` could have been scaled down and + still pass every selected test. NaN if the reported VaR does not pass + to begin with. + exceptions : int + The number of exceptions of the reported VaR. + extra_exceptions : int + The number of additional exceptions produced by scaling ``var`` down by + ``understatement``, beyond ``exceptions``. 0 if the reported VaR does + not pass. + binding : tuple of str + The selected tests that reject a VaR scaled down by just beyond + ``understatement``. Empty if ``var`` can be scaled to 0 without being + rejected, and empty if the reported VaR does not pass, since no limit + is computed. + passes : bool + Whether the reported VaR passes every selected test. + failed : tuple of str + The selected tests that reject the reported VaR. Empty if ``passes``. + nobs : int + The number of observations used. + p : float + The probability of an exception that the VaR is designed for. + + Attributes + ---------- + understatement : float + The largest fraction by which ``var`` could have been scaled down and + still pass every selected test. NaN if the reported VaR does not pass + to begin with. + exceptions : int + The number of exceptions of the reported VaR. + extra_exceptions : int + The number of additional exceptions produced by scaling ``var`` down by + ``understatement``, beyond ``exceptions``. 0 if the reported VaR does + not pass. + binding : tuple of str + The selected tests that reject a VaR scaled down by just beyond + ``understatement``. Empty if ``var`` can be scaled to 0 without being + rejected, and empty if the reported VaR does not pass, since no limit + is computed. + passes : bool + Whether the reported VaR passes every selected test. + failed : tuple of str + The selected tests that reject the reported VaR. Empty if ``passes``. + nobs : int + The number of observations used. + p : float + The probability of an exception that the VaR is designed for. + """ + + understatement: float + exceptions: int + extra_exceptions: int + binding: tuple[str, ...] + passes: bool + failed: tuple[str, ...] + nobs: int + p: float + + +def _lr_pof_critical(size: float, df: int) -> float: + return float(stats.chi2.ppf(1.0 - size, df)) + + +def max_undetectable_understatement( + returns: ArrayLike1D | list[float], + var: ArrayLike1D | list[float], + p: float = 0.05, + size: float = 0.05, + tests: Sequence[str] = _UNDERSTATEMENT_TESTS, + ignore_too_few: bool = True, + window: int = 250, +) -> MaxUndetectableUnderstatementResult: + r""" + Maximum undetectable understatement of a VaR series. + + A passing backtest only says that a VaR was not rejected. It does not say + how close it was. This function finds the largest fraction by which the + whole VaR series could have been scaled down, with the same returns, and + still not have been rejected by the selected tests. + + Parameters + ---------- + returns : array_like + Realized returns, a 1-d array of finite numbers. + var : array_like + VaR forecasts as positive losses, a 1-d array of finite, strictly + positive numbers. See `var_exceptions` for the sign convention and for + how the inputs are aligned. + p : float, optional + The probability of an exception that the VaR is designed for, which is + the size of the tail, in the interval (0, 0.5]. It is 0.05 for a 95% + VaR and 0.01 for a 99% VaR. The default is 0.05, a 95% VaR, and is used + by every selected test, including `traffic_light`. + size : float, optional + The size of the Kupiec and the Christoffersen tests, in (0, 1). The + default is 0.05. `traffic_light` does not use a size, since its zones + are already defined at fixed probabilities. + tests : sequence of str, optional + The tests to pass, a subset of ``"kupiec_pof"``, + ``"christoffersen_independence"``, ``"christoffersen_conditional_coverage"`` + and ``"traffic_light"``. All four are selected by default. + ignore_too_few : bool, optional + If True, the default, a rejection of `kupiec_pof` or + `christoffersen_conditional_coverage` caused by too few exceptions, + rather than too many, is not treated as a failure. `kupiec_pof` is + two-sided, so a VaR with too few exceptions is otherwise rejected for + being too conservative, which is not evidence of an understated VaR. + window : int, optional + The window of `traffic_light`, used only if it is selected. The + default is 250. + + Returns + ------- + MaxUndetectableUnderstatementResult + The largest fraction by which `var` could have been understated, and + supporting information. + + Raises + ------ + TypeError + If `returns` or `var` cannot be converted to numbers, `p` or `size` is + not a real number, `window` is not an integer, `ignore_too_few` is not + a bool, or `tests` is not a sequence of str. + ValueError + If `returns` or `var` is not valid, as in `var_exceptions`, `var` is + not strictly positive, `p` is not in (0, 0.5], `size` is not in (0, 1), + `window` is not positive, `tests` is empty or has a value that is not + one of the tests listed above, or there are too few observations for a + selected test, as in `christoffersen_independence` or `traffic_light`. + + See Also + -------- + kupiec_pof : Test of the proportion of exceptions. + christoffersen_independence : Test of the independence of the exceptions. + christoffersen_conditional_coverage : Joint test of coverage and independence. + traffic_light : The Basel zone of the number of exceptions. + + Notes + ----- + The search is over the proportional scalings :math:`(1-s)\mathrm{VaR}_t` for + :math:`s` in :math:`[0, 1]`. An observation becomes an exception at + :math:`s = 1 - (-r_t/\mathrm{VaR}_t)`, so the exception set, and so every + selected test, is constant between these breakpoints. The search evaluates + the tests once at each distinct breakpoint, which is exact for this family + of series and needs no optimizer. + + This is a lower bound on what a series that is not a proportional scaling + of `var` could hide. A series that understates the VaR on some days and + overstates it on others can produce the same, or a smaller, number of + exceptions than a proportional scaling, and so can be harder to detect. + + References + ---------- + .. [1] Basel Committee on Banking Supervision (1996). Supervisory framework + for the use of "backtesting" in conjunction with the internal models + approach to market risk capital requirements. + .. [2] Christoffersen, P. F. (1998). Evaluating interval forecasts. + International Economic Review, 39(4), 841-862. + .. [3] Kupiec, P. H. (1995). Techniques for verifying the accuracy of risk + measurement models. The Journal of Derivatives, 3(2), 73-84. + """ + p = check_float("p", p, lower=0.0, upper=0.5, inclusive="right") + size = check_float("size", size, lower=0.0, upper=1.0, inclusive="neither") + window = check_int("window", window, lower=1) + ignore_too_few = check_bool("ignore_too_few", ignore_too_few) + if isinstance(tests, str): + raise TypeError( + f"tests must be a sequence of str, not a single str; got {tests!r} (str)" + ) + tests_seq = list(tests) + if not tests_seq: + raise ValueError( + "tests must be a non-empty sequence of 'kupiec_pof', " + "'christoffersen_independence', 'christoffersen_conditional_coverage' " + f"or 'traffic_light'; got {tests!r} ({type(tests).__name__})" + ) + selected = frozenset( + check_option("tests", test, _UNDERSTATEMENT_TESTS) for test in tests_seq + ) + + min_obs = ( + 2 + if ( + "christoffersen_independence" in selected + or "christoffersen_conditional_coverage" in selected + ) + else 1 + ) + exceptions0, _ = _exceptions(returns, var, min_obs) + ret, var_arr, _ = _aligned(returns, var) + if not np.all(var_arr > 0.0): + raise ValueError( + "var must be strictly positive to compute a proportional " + f"understatement; got a minimum of {float(var_arr.min()):g} (ndarray)" + ) + nobs = ret.shape[0] + + # The baseline (s = 0) is checked with the public tests, which validates p, the + # length of the series and, for traffic_light, the window, with the same error + # messages as calling them directly. + too_few = float(exceptions0.mean()) < p + failed = [] + if "kupiec_pof" in selected: + if kupiec_pof(ret, var_arr, p).pval <= size and not ( + ignore_too_few and too_few + ): + failed.append("kupiec_pof") + if "christoffersen_independence" in selected: + if christoffersen_independence(ret, var_arr).pval <= size: + failed.append("christoffersen_independence") + if "christoffersen_conditional_coverage" in selected: + cc = christoffersen_conditional_coverage(ret, var_arr, p) + if cc.pval <= size and not (ignore_too_few and too_few): + failed.append("christoffersen_conditional_coverage") + if "traffic_light" in selected: + if traffic_light(ret, var_arr, p, window).zone != "green": + failed.append("traffic_light") + + n_reported = int(exceptions0.sum()) + if failed: + return MaxUndetectableUnderstatementResult( + understatement=float("nan"), + exceptions=n_reported, + extra_exceptions=0, + binding=(), + passes=False, + failed=tuple(failed), + nobs=nobs, + p=p, + ) + + # Beyond the baseline, the tests are evaluated with the private likelihoods and + # the traffic light boundaries directly, which is exact and avoids building a + # WaldTestStatistic or a TrafficLightResult at every breakpoint. + crit1 = _lr_pof_critical(size, 1) + crit2 = _lr_pof_critical(size, 2) + + def _gate(hits: BoolArray) -> tuple[str, ...]: + failed_now = [] + count = hits.sum() + hits_too_few = (float(count) / nobs) < p + lr_pof = None + if ( + "kupiec_pof" in selected + or "christoffersen_conditional_coverage" in selected + ): + lr_pof = _lr_pof(int(count), nobs, p) + if "kupiec_pof" in selected: + if lr_pof > crit1 and not (ignore_too_few and hits_too_few): + failed_now.append("kupiec_pof") + lr_ind = None + if ( + "christoffersen_independence" in selected + or "christoffersen_conditional_coverage" in selected + ): + lr_ind = _lr_independence(hits) + if "christoffersen_independence" in selected: + if lr_ind > crit1: + failed_now.append("christoffersen_independence") + if "christoffersen_conditional_coverage" in selected: + if lr_pof is not None and lr_ind is not None: + if (lr_pof + lr_ind) > crit2 and not (ignore_too_few and hits_too_few): + failed_now.append("christoffersen_conditional_coverage") + if "traffic_light" in selected: + window_count = int(hits[-window:].sum()) + if window_count > 0: + prob = float(stats.binom.cdf(window_count, window, p)) + if prob >= _YELLOW_PROBABILITY: + failed_now.append("traffic_light") + return tuple(t for t in _UNDERSTATEMENT_TESTS if t in failed_now) + + ratio = -ret / var_arr + breakpoints = np.unique(ratio[(ratio > 0.0) & (ratio < 1.0)]) + candidates = np.concatenate(([0.0], breakpoints, [1.0])) + limit_idx = candidates.shape[0] - 1 # thr = 1 is the baseline, which passes + for i in range(candidates.shape[0] - 1): + if not _gate(ratio > candidates[i]): + limit_idx = i + break + limit = float(candidates[limit_idx]) + binding: tuple[str, ...] = () + if limit_idx > 0: + binding = _gate(ratio > candidates[limit_idx - 1]) + hits_at_limit = ratio > limit + + return MaxUndetectableUnderstatementResult( + understatement=1.0 - limit, + exceptions=n_reported, + extra_exceptions=int(hits_at_limit.sum()) - n_reported, + binding=binding, + passes=True, + failed=(), + nobs=nobs, + p=p, + ) diff --git a/doc/source/changes/9.0.rst b/doc/source/changes/9.0.rst index 69f2fd2670..d9d100a8f9 100644 --- a/doc/source/changes/9.0.rst +++ b/doc/source/changes/9.0.rst @@ -120,6 +120,13 @@ New features probability of an exception, such as 0.01 for a 99% VaR, and must be in (0, 0.5]. The unconditional and conditional coverage statistics equal those of ``VaRTest`` in the R package rugarch (:issue:`904`, :issue:`965`). +- Added :func:`~arch.univariate.max_undetectable_understatement`, which finds the + largest fraction by which a VaR series could have been scaled down, with the same + returns, and still have passed the selected backtests. The search over proportional + scalings is exact, since the exceptions only change at the points where a loss + crosses the scaled VaR, and is a lower bound on what a non-proportional series could + hide. The result is a :class:`~arch.univariate.MaxUndetectableUnderstatementResult` + with the limit, the binding test, and the exceptions it would add (:issue:`904`). - Added :class:`~arch.bootstrap.LocalBlockBootstrap`, the local block bootstrap of Paparoditis and Politis (2002), for series whose properties change slowly over time. Each block of the data is replaced by a block that starts at most ``locality`` diff --git a/doc/source/univariate/utility.rst b/doc/source/univariate/utility.rst index 8d6ed1c20a..132d842cf0 100644 --- a/doc/source/univariate/utility.rst +++ b/doc/source/univariate/utility.rst @@ -170,3 +170,32 @@ exception, or with only exceptions, has a statistic. traffic_light .. autoclass:: TrafficLightResult + +A passing backtest only says that the reported VaR was not rejected. It does not say +how close it was. :func:`~arch.univariate.max_undetectable_understatement` answers a +more specific question: by what fraction could the whole VaR series have been scaled +down, with the same returns, and still have passed the selected tests. The search is +over proportional scalings of ``var``, is exact, and needs no optimizer, since the set +of exceptions only changes at the breakpoints where an observation crosses the scaled +VaR. It is a lower bound on what a series that understates VaR on some days and +overstates it on others could hide. + +``kupiec_pof`` and ``christoffersen_conditional_coverage`` are two-sided, so a VaR with +too few exceptions is otherwise rejected for being too conservative. By default, +``ignore_too_few=True`` does not count that as a failure, since it is not evidence of +an understated VaR. + +.. ipython:: python + + from arch.univariate import max_undetectable_understatement + + result = max_undetectable_understatement(returns, var, p=0.01) + result.understatement + result.binding + +.. autosummary:: + :toctree: generated/ + + max_undetectable_understatement + +.. autoclass:: MaxUndetectableUnderstatementResult From c99ac1ee6325b5848e938bd53b8353d151aa2bfa Mon Sep 17 00:00:00 2001 From: Sarthak Gupta Date: Wed, 11 Nov 2026 21:22:37 -0800 Subject: [PATCH 2/2] BUG: Report the first rejection in max_undetectable_understatement Address review on #1015. - Scan the breakpoints downward from the reported VaR and stop at the first rejected scaling. The independence test, and so the conditional coverage test, is not monotone in the scaling, so the old upward scan could report a limit beyond a rejected scaling. The result is now the largest s for which every scaling from 0 to s passes. - With ignore_too_few, set aside only the coverage part of a test. christoffersen_conditional_coverage now uses LR_pof = 0 with the chi2(2) critical value, so a rejection caused by clustering still counts. - Use one rejection rule, pval <= size, for the reported VaR and for every scaling, through the same gate. - Tests for the non-monotone case, clustered exceptions with too few exceptions, the shared rejection rule and the remaining branches. - Notes relate the result to Table 1 of the 1996 Basel framework. Co-Authored-By: Claude Opus 5.5 --- arch/tests/univariate/test_backtest.py | 146 ++++++++++++++++++++-- arch/univariate/backtest.py | 166 +++++++++++++------------ doc/source/changes/9.0.rst | 8 +- 3 files changed, 224 insertions(+), 96 deletions(-) diff --git a/arch/tests/univariate/test_backtest.py b/arch/tests/univariate/test_backtest.py index b6b84aa34f..828d09e3a7 100644 --- a/arch/tests/univariate/test_backtest.py +++ b/arch/tests/univariate/test_backtest.py @@ -1064,9 +1064,14 @@ def _mul_passes(ret, var, scale, p=0.05, size=0.05, ignore_too_few=True, window= hits = var_exceptions(ret, scaled) too_few = bool(hits.mean() < p) ok = kupiec_pof(ret, scaled, p).pval > size or (ignore_too_few and too_few) - ok &= christoffersen_independence(ret, scaled).pval > size + ind = christoffersen_independence(ret, scaled) + ok &= ind.pval > size cc = christoffersen_conditional_coverage(ret, scaled, p) - ok &= cc.pval > size or (ignore_too_few and too_few) + if ignore_too_few and too_few: + # only the coverage part is set aside: the independence part keeps chi2(2) + ok &= stats.chi2.sf(ind.stat, 2) > size + else: + ok &= cc.pval > size ok &= traffic_light(ret, scaled, p, window).zone == "green" return bool(ok) @@ -1111,19 +1116,15 @@ def test_passes_and_reports_counts(mul_sim): def test_brute_force_grid_matches_public_tests(mul_sim): - # The limit is the SMALLEST proportional scaling that still passes: scale=1 (no - # understatement) always passes, since the baseline passes, so a grid search for - # the computed limit has to look for the minimum passing point, not the maximum. + # The limit is reached scanning down from the reported VaR (scale=1): every + # scaling between the limit and 1 passes, and a scaling just below it fails. ret, var = mul_sim res = max_undetectable_understatement(ret, var) scale = 1.0 - res.understatement - grid = np.linspace(0.0, 1.0, 4001) - passing = [s for s in grid if _mul_passes(ret, var, s)] - best = min(passing) - # the exact limit is at or below the best grid point, and within one cell of it - assert scale <= best + 1e-9 - assert best - scale <= grid[1] - grid[0] + 1e-9 - assert _mul_passes(ret, var, scale) + assert res.understatement < 1.0 + grid = np.linspace(scale, 1.0, 2001) + assert all(_mul_passes(ret, var, s) for s in grid) + assert not _mul_passes(ret, var, scale - 1e-9) def test_boundary_agrees_with_public_functions(mul_sim): @@ -1207,6 +1208,127 @@ def test_overstated_var_fails_baseline(mul_sim): assert res.failed +def test_clustered_exceptions_fail_baseline_independence(): + nobs = 100 + var = np.ones(nobs) + returns = np.full(nobs, 0.1) + returns[40:50] = -2.0 # one block of 10 consecutive exceptions, 10% of the sample + res = max_undetectable_understatement( + returns, var, p=0.1, tests=["christoffersen_independence"] + ) + assert not res.passes + assert res.failed == ("christoffersen_independence",) + assert math.isnan(res.understatement) + assert res.exceptions == 10 + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_no_losses_has_no_breakpoints(): + nobs = 100 + var = np.ones(nobs) + returns = np.full(nobs, 0.1) # no losses: no breakpoints in (0, 1) + res = max_undetectable_understatement(returns, var, tests=["kupiec_pof"]) + assert res.passes + assert res.failed == () + assert res.understatement == 1.0 # the smallest scaling, 0, already passes + assert res.exceptions == 0 + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_non_monotone_independence_stops_at_first_rejection(): + # The independence test is not monotone in the scaling: for this seed the VaR + # scaled down by 0.187 is rejected while a scaling down by 0.226 passes again. + # The limit is the first rejection met scanning down from the reported VaR. + rng = np.random.default_rng(1234) + for _ in range(32): + returns = rng.standard_t(5, 250) * 0.01 + var = np.full(250, 1.645 * 0.01 * rng.uniform(1.0, 1.4)) + assert christoffersen_independence(returns, var * (1 - 0.187)).pval <= 0.05 + assert christoffersen_independence(returns, var * (1 - 0.226)).pval > 0.05 + res = max_undetectable_understatement(returns, var, p=0.05) + assert res.passes + assert res.understatement < 0.187 + assert res.binding == ("christoffersen_independence",) + scale = 1.0 - res.understatement + ratio = -returns / var + # every distinct exception set between the limit and the reported VaR passes + points = np.concatenate(([scale, 1.0], ratio[(ratio >= scale) & (ratio < 1.0)])) + assert all(_mul_passes(returns, var, s) for s in points) + assert not _mul_passes(returns, var, scale - 1e-9) + + +def test_ignore_too_few_keeps_clustering_rejection(): + # 10 consecutive exceptions in 500 is a rate of 0.02, below p, but the + # conditional coverage test rejects because of the clustering + returns = np.full(500, 0.001) + returns[200:210] = -0.05 + var = np.full(500, 0.02) + assert christoffersen_conditional_coverage(returns, var, 0.05).pval <= 0.05 + res = max_undetectable_understatement( + returns, var, p=0.05, tests=("christoffersen_conditional_coverage",) + ) + assert not res.passes + assert res.failed == ("christoffersen_conditional_coverage",) + assert math.isnan(res.understatement) + + +def test_ignore_too_few_sets_aside_only_coverage(): + # 3 scattered exceptions in 500 at p=0.05: the joint test rejects for too few, + # but the independence part alone, against chi2(2), does not + returns = np.full(500, 0.001) + returns[[100, 250, 400]] = -0.05 + var = np.full(500, 0.02) + tests = ("christoffersen_conditional_coverage",) + assert christoffersen_conditional_coverage(returns, var, 0.05).pval <= 0.05 + strict = max_undetectable_understatement( + returns, var, p=0.05, tests=tests, ignore_too_few=False + ) + assert not strict.passes + res = max_undetectable_understatement(returns, var, p=0.05, tests=tests) + assert res.passes + assert _mul_passes(returns, var, 1.0 - res.understatement, p=0.05) + + +def test_first_breakpoint_rejected_gives_zero_understatement(): + # 9 exceptions in 100 pass Kupiec at p=0.05. A loss exactly equal to the VaR is + # not an exception, but it becomes the 10th, which is rejected, at any scaling + # down, so no understatement at all is undetectable + returns = np.full(100, 0.1) + returns[:9] = -2.0 + returns[9] = -1.0 + var = np.ones(100) + res = max_undetectable_understatement(returns, var, p=0.05, tests=["kupiec_pof"]) + assert res.passes + assert res.understatement == 0.0 + assert res.exceptions == 9 + assert res.extra_exceptions == 0 + assert res.binding == ("kupiec_pof",) + + +def test_baseline_and_search_share_the_rejection_rule(mul_sim): + # A test rejects when its p-value is at most size. With size set to the + # p-value of the reported VaR, the baseline is rejected, as it is by the + # public test. + ret, var = mul_sim + pval = kupiec_pof(ret, var, 0.05).pval + res = max_undetectable_understatement( + ret, var, size=pval, tests=["kupiec_pof"], ignore_too_few=False + ) + assert not res.passes + assert res.failed == ("kupiec_pof",) + + +def test_mul_traffic_light_short_series_raises(): + returns = np.full(100, 0.01) + var = np.ones(100) + with pytest.raises(ValueError, match=r"window must be .*\(100\); got 250 \(int\)"): + max_undetectable_understatement(returns, var) + res = max_undetectable_understatement(returns, var, tests=["kupiec_pof"]) + assert res.passes + + def test_traffic_light_only_sets_the_limit(): nobs = 250 var = np.ones(nobs) diff --git a/arch/univariate/backtest.py b/arch/univariate/backtest.py index 5b7ce5be94..8a07fa00fb 100644 --- a/arch/univariate/backtest.py +++ b/arch/univariate/backtest.py @@ -610,9 +610,9 @@ class MaxUndetectableUnderstatementResult: # numpydoc ignore=PR02 Parameters ---------- understatement : float - The largest fraction by which ``var`` could have been scaled down and - still pass every selected test. NaN if the reported VaR does not pass - to begin with. + The largest fraction by which ``var`` could have been scaled down such + that it, and every smaller scaling down, passes every selected test. + NaN if the reported VaR does not pass to begin with. exceptions : int The number of exceptions of the reported VaR. extra_exceptions : int @@ -636,9 +636,9 @@ class MaxUndetectableUnderstatementResult: # numpydoc ignore=PR02 Attributes ---------- understatement : float - The largest fraction by which ``var`` could have been scaled down and - still pass every selected test. NaN if the reported VaR does not pass - to begin with. + The largest fraction by which ``var`` could have been scaled down such + that it, and every smaller scaling down, passes every selected test. + NaN if the reported VaR does not pass to begin with. exceptions : int The number of exceptions of the reported VaR. extra_exceptions : int @@ -670,10 +670,6 @@ class MaxUndetectableUnderstatementResult: # numpydoc ignore=PR02 p: float -def _lr_pof_critical(size: float, df: int) -> float: - return float(stats.chi2.ppf(1.0 - size, df)) - - def max_undetectable_understatement( returns: ArrayLike1D | list[float], var: ArrayLike1D | list[float], @@ -688,8 +684,9 @@ def max_undetectable_understatement( A passing backtest only says that a VaR was not rejected. It does not say how close it was. This function finds the largest fraction by which the - whole VaR series could have been scaled down, with the same returns, and - still not have been rejected by the selected tests. + whole VaR series could have been scaled down, with the same returns, such + that it and every smaller scaling down are not rejected by the selected + tests. Parameters ---------- @@ -713,11 +710,16 @@ def max_undetectable_understatement( ``"christoffersen_independence"``, ``"christoffersen_conditional_coverage"`` and ``"traffic_light"``. All four are selected by default. ignore_too_few : bool, optional - If True, the default, a rejection of `kupiec_pof` or - `christoffersen_conditional_coverage` caused by too few exceptions, - rather than too many, is not treated as a failure. `kupiec_pof` is - two-sided, so a VaR with too few exceptions is otherwise rejected for - being too conservative, which is not evidence of an understated VaR. + If True, the default, the coverage part of a test is set aside when the + rate of exceptions is below `p`, since too few exceptions are not + evidence of an understated VaR. `kupiec_pof` is two-sided, and a VaR + with too few exceptions is otherwise rejected for being too + conservative, so it is not treated as a failure. For + `christoffersen_conditional_coverage`, the coverage statistic is set to + 0 and the independence statistic is compared with the same + :math:`\chi^2_2` distribution, so a rejection caused by clustered + exceptions is still a failure. `christoffersen_independence` and + `traffic_light` are not affected. window : int, optional The window of `traffic_light`, used only if it is selected. The default is 250. @@ -757,11 +759,27 @@ def max_undetectable_understatement( the tests once at each distinct breakpoint, which is exact for this family of series and needs no optimizer. + The scan starts at the reported VaR, :math:`s = 0`, and moves toward larger + understatement, stopping at the first scaling that is rejected. The result is + the largest :math:`s` such that every scaling from 0 to :math:`s` passes. + The independence test, and so the conditional coverage test, is not monotone + in :math:`s`, so a larger scaling beyond the first rejection can pass again, + and is not reported. `binding` lists the tests that reject at that first + rejected scaling. + + A test rejects when its p-value is at most `size`, as in the public tests, + and the same rule is used for the reported VaR and for every scaling. + This is a lower bound on what a series that is not a proportional scaling of `var` could hide. A series that understates the VaR on some days and overstates it on others can produce the same, or a smaller, number of exceptions than a proportional scaling, and so can be harder to detect. + Table 1 of [1] reports how often the traffic light accepts a VaR whose true + coverage is lower than stated. This function asks a related question of one + realised series: how much lower the VaR could have been before any selected + test rejected it. + References ---------- .. [1] Basel Committee on Banking Supervision (1996). Supervisory framework @@ -808,71 +826,39 @@ def max_undetectable_understatement( ) nobs = ret.shape[0] - # The baseline (s = 0) is checked with the public tests, which validates p, the - # length of the series and, for traffic_light, the window, with the same error - # messages as calling them directly. - too_few = float(exceptions0.mean()) < p - failed = [] - if "kupiec_pof" in selected: - if kupiec_pof(ret, var_arr, p).pval <= size and not ( - ignore_too_few and too_few - ): - failed.append("kupiec_pof") - if "christoffersen_independence" in selected: - if christoffersen_independence(ret, var_arr).pval <= size: - failed.append("christoffersen_independence") - if "christoffersen_conditional_coverage" in selected: - cc = christoffersen_conditional_coverage(ret, var_arr, p) - if cc.pval <= size and not (ignore_too_few and too_few): - failed.append("christoffersen_conditional_coverage") - if "traffic_light" in selected: - if traffic_light(ret, var_arr, p, window).zone != "green": - failed.append("traffic_light") - - n_reported = int(exceptions0.sum()) - if failed: - return MaxUndetectableUnderstatementResult( - understatement=float("nan"), - exceptions=n_reported, - extra_exceptions=0, - binding=(), - passes=False, - failed=tuple(failed), - nobs=nobs, - p=p, + if "traffic_light" in selected and window > nobs: + raise ValueError( + f"window must be an integer no larger than the number of aligned " + f"observations ({nobs}); got {window} (int)" ) - # Beyond the baseline, the tests are evaluated with the private likelihoods and - # the traffic light boundaries directly, which is exact and avoids building a - # WaldTestStatistic or a TrafficLightResult at every breakpoint. - crit1 = _lr_pof_critical(size, 1) - crit2 = _lr_pof_critical(size, 2) + def _rejects(stat: float, df: int) -> bool: + # The rule of the public tests: reject when the p-value is at most size + return bool(stats.chi2.sf(stat, df) <= size) + # The tests are evaluated with the private likelihoods and the traffic light + # boundaries directly, which avoids building a WaldTestStatistic or a + # TrafficLightResult at every breakpoint. The p-values are the same as those of + # the public tests, and the baseline and the search use the same gate. def _gate(hits: BoolArray) -> tuple[str, ...]: failed_now = [] - count = hits.sum() - hits_too_few = (float(count) / nobs) < p - lr_pof = None - if ( - "kupiec_pof" in selected - or "christoffersen_conditional_coverage" in selected - ): - lr_pof = _lr_pof(int(count), nobs, p) - if "kupiec_pof" in selected: - if lr_pof > crit1 and not (ignore_too_few and hits_too_few): + count = int(hits.sum()) + # Too few exceptions only sets aside the coverage part of a test, which is + # the part that rejects a conservative VaR + skip_coverage = ignore_too_few and (count / nobs) < p + lr_pof = 0.0 if skip_coverage else _lr_pof(count, nobs, p) + if "kupiec_pof" in selected and not skip_coverage: + if _rejects(lr_pof, 1): failed_now.append("kupiec_pof") - lr_ind = None if ( "christoffersen_independence" in selected or "christoffersen_conditional_coverage" in selected ): lr_ind = _lr_independence(hits) - if "christoffersen_independence" in selected: - if lr_ind > crit1: + if "christoffersen_independence" in selected and _rejects(lr_ind, 1): failed_now.append("christoffersen_independence") - if "christoffersen_conditional_coverage" in selected: - if lr_pof is not None and lr_ind is not None: - if (lr_pof + lr_ind) > crit2 and not (ignore_too_few and hits_too_few): + if "christoffersen_conditional_coverage" in selected: + if _rejects(lr_pof + lr_ind, 2): failed_now.append("christoffersen_conditional_coverage") if "traffic_light" in selected: window_count = int(hits[-window:].sum()) @@ -882,19 +868,39 @@ def _gate(hits: BoolArray) -> tuple[str, ...]: failed_now.append("traffic_light") return tuple(t for t in _UNDERSTATEMENT_TESTS if t in failed_now) + n_reported = int(exceptions0.sum()) + failed = _gate(exceptions0) + if failed: + return MaxUndetectableUnderstatementResult( + understatement=float("nan"), + exceptions=n_reported, + extra_exceptions=0, + binding=(), + passes=False, + failed=failed, + nobs=nobs, + p=p, + ) + + # A scaling 1 - s turns observation t into an exception when the ratio + # -r_t / VaR_t exceeds 1 - s, so the exceptions at a threshold thr = 1 - s are + # ratio > thr, constant for thr in [b_k, b_k+1) between consecutive candidates. + # The scan starts at the reported VaR (thr = 1) and moves down, toward larger + # understatement, and stops at the first candidate that fails. The tests are not + # monotone in thr, so a passing candidate below a failing one is not a limit. ratio = -ret / var_arr breakpoints = np.unique(ratio[(ratio > 0.0) & (ratio < 1.0)]) - candidates = np.concatenate(([0.0], breakpoints, [1.0])) - limit_idx = candidates.shape[0] - 1 # thr = 1 is the baseline, which passes - for i in range(candidates.shape[0] - 1): - if not _gate(ratio > candidates[i]): - limit_idx = i - break - limit = float(candidates[limit_idx]) + candidates = np.concatenate(([0.0], breakpoints))[::-1] + first_fail = next( + (i for i, thr in enumerate(candidates) if _gate(ratio > thr)), None + ) binding: tuple[str, ...] = () - if limit_idx > 0: - binding = _gate(ratio > candidates[limit_idx - 1]) - hits_at_limit = ratio > limit + if first_fail is None: + limit = 0.0 + else: + binding = _gate(ratio > candidates[first_fail]) + limit = 1.0 if first_fail == 0 else float(candidates[first_fail - 1]) + hits_at_limit = ratio > limit if limit < 1.0 else exceptions0 return MaxUndetectableUnderstatementResult( understatement=1.0 - limit, diff --git a/doc/source/changes/9.0.rst b/doc/source/changes/9.0.rst index d9d100a8f9..a720a2f375 100644 --- a/doc/source/changes/9.0.rst +++ b/doc/source/changes/9.0.rst @@ -122,10 +122,10 @@ New features the R package rugarch (:issue:`904`, :issue:`965`). - Added :func:`~arch.univariate.max_undetectable_understatement`, which finds the largest fraction by which a VaR series could have been scaled down, with the same - returns, and still have passed the selected backtests. The search over proportional - scalings is exact, since the exceptions only change at the points where a loss - crosses the scaled VaR, and is a lower bound on what a non-proportional series could - hide. The result is a :class:`~arch.univariate.MaxUndetectableUnderstatementResult` + returns, such that it and every smaller scaling down pass the selected backtests. + The search over proportional scalings is exact, since the exceptions only change at + the points where a loss crosses the scaled VaR, and is a lower bound on what a + non-proportional series could hide. The result is a :class:`~arch.univariate.MaxUndetectableUnderstatementResult` with the limit, the binding test, and the exceptions it would add (:issue:`904`). - Added :class:`~arch.bootstrap.LocalBlockBootstrap`, the local block bootstrap of Paparoditis and Politis (2002), for series whose properties change slowly over time.