diff --git a/arch/tests/univariate/test_backtest.py b/arch/tests/univariate/test_backtest.py index b042dc4cf7..828d09e3a7 100644 --- a/arch/tests/univariate/test_backtest.py +++ b/arch/tests/univariate/test_backtest.py @@ -29,10 +29,12 @@ from scipy import stats from arch.univariate import ( + MaxUndetectableUnderstatementResult, TrafficLightResult, christoffersen_conditional_coverage, christoffersen_independence, kupiec_pof, + max_undetectable_understatement, traffic_light, var_exceptions, ) @@ -1051,3 +1053,347 @@ def test_p_boundaries(): # Values that are close to the lower boundary and 0.5 are valid assert kupiec_pof(returns, var, 1e-12).stat > 0.0 assert kupiec_pof(returns, var, 0.5).stat > 0.0 + + +# Maximum undetectable understatement (xref #904) + + +def _mul_passes(ret, var, scale, p=0.05, size=0.05, ignore_too_few=True, window=250): + """Independent pass/fail check using the public tests directly.""" + scaled = scale * var + hits = var_exceptions(ret, scaled) + too_few = bool(hits.mean() < p) + ok = kupiec_pof(ret, scaled, p).pval > size or (ignore_too_few and too_few) + ind = christoffersen_independence(ret, scaled) + ok &= ind.pval > size + cc = christoffersen_conditional_coverage(ret, scaled, p) + if ignore_too_few and too_few: + # only the coverage part is set aside: the independence part keeps chi2(2) + ok &= stats.chi2.sf(ind.stat, 2) > size + else: + ok &= cc.pval > size + ok &= traffic_light(ret, scaled, p, window).zone == "green" + return bool(ok) + + +@pytest.fixture(scope="module") +def mul_sim(): + rng = np.random.default_rng(20261109) + nobs = 600 + sigma = np.exp(0.25 * rng.standard_normal(nobs)) + ret = sigma * rng.standard_normal(nobs) * 0.01 + var = -stats.norm.ppf(0.05) * sigma * 0.01 # a generous, correctly sized 95% VaR + return ret, var + + +def test_hand_computed_kupiec_only(): + # 4 observations, ratios -returns/var are 2, 0.5, 0.5, 0.5. Only one exception + # (2 > 1) at the reported VaR, and the three ties at 0.5 do not become + # exceptions until the VaR is scaled below 0.5, so the limit is exactly 0.5. + returns = np.array([-2.0, -0.5, -0.5, -0.5]) + var = np.ones(4) + res = max_undetectable_understatement(returns, var, p=0.5, tests=["kupiec_pof"]) + assert isinstance(res, MaxUndetectableUnderstatementResult) + assert res.passes + assert res.failed == () + assert_allclose(res.understatement, 0.5) + assert res.exceptions == 1 + assert res.extra_exceptions == 0 + assert res.binding == ("kupiec_pof",) + assert res.nobs == 4 + assert res.p == 0.5 + + +def test_passes_and_reports_counts(mul_sim): + ret, var = mul_sim + res = max_undetectable_understatement(ret, var) + assert res.passes + assert res.failed == () + assert 0.0 <= res.understatement <= 1.0 + assert res.extra_exceptions >= 0 + assert res.exceptions == int(var_exceptions(ret, var).sum()) + assert res.nobs == ret.shape[0] + + +def test_brute_force_grid_matches_public_tests(mul_sim): + # The limit is reached scanning down from the reported VaR (scale=1): every + # scaling between the limit and 1 passes, and a scaling just below it fails. + ret, var = mul_sim + res = max_undetectable_understatement(ret, var) + scale = 1.0 - res.understatement + assert res.understatement < 1.0 + grid = np.linspace(scale, 1.0, 2001) + assert all(_mul_passes(ret, var, s) for s in grid) + assert not _mul_passes(ret, var, scale - 1e-9) + + +def test_boundary_agrees_with_public_functions(mul_sim): + ret, var = mul_sim + res = max_undetectable_understatement(ret, var) + scale = 1.0 - res.understatement + assert _mul_passes(ret, var, scale) + hits = var_exceptions(ret, scale * var) + assert int(hits.sum()) - res.exceptions == res.extra_exceptions + + +def test_more_tests_never_increase_understatement(mul_sim): + ret, var = mul_sim + only_kupiec = max_undetectable_understatement(ret, var, tests=["kupiec_pof"]) + all_tests = max_undetectable_understatement(ret, var) + assert all_tests.understatement <= only_kupiec.understatement + 1e-12 + + +def test_stricter_size_never_increases_understatement(mul_sim): + # Restricted to kupiec_pof, whose baseline passes at every size tried here; + # Christoffersen's independence test can reject the baseline itself at a loose + # size, which makes the comparison undefined rather than merely smaller. + ret, var = mul_sim + loose = max_undetectable_understatement(ret, var, size=0.01, tests=["kupiec_pof"]) + strict = max_undetectable_understatement(ret, var, size=0.20, tests=["kupiec_pof"]) + assert loose.passes + assert strict.passes + assert strict.understatement <= loose.understatement + 1e-12 + + +def test_no_exceptions_can_be_fully_understated(): + # Every return is a gain: var can be scaled all the way to 0 undetected + ret = np.full(300, 0.01) + var = np.ones(300) + res = max_undetectable_understatement(ret, var) + assert res.passes + assert_allclose(res.understatement, 1.0) + assert res.exceptions == 0 + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_all_exceptions_fails_baseline(): + # Every day is an exception at the reported VaR: far too many for any p < 1 + ret = np.full(20, -5.0) + var = np.ones(20) + res = max_undetectable_understatement(ret, var, p=0.3, window=20) + assert not res.passes + assert "kupiec_pof" in res.failed + assert math.isnan(res.understatement) + assert res.exceptions == 20 + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_ignore_too_few_true_passes_a_conservative_var(): + # No exceptions in 250 observations at p=0.01 is conservative, not understated + ret = np.full(250, 0.001) + var = np.ones(250) + res = max_undetectable_understatement(ret, var, p=0.01, ignore_too_few=True) + assert res.passes + assert res.failed == () + + +def test_ignore_too_few_false_fails_the_same_var(): + ret = np.full(250, 0.001) + var = np.ones(250) + res = max_undetectable_understatement(ret, var, p=0.01, ignore_too_few=False) + assert not res.passes + assert "kupiec_pof" in res.failed + assert math.isnan(res.understatement) + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_overstated_var_fails_baseline(mul_sim): + ret, var = mul_sim + res = max_undetectable_understatement(ret, 0.3 * var) + assert not res.passes + assert math.isnan(res.understatement) + assert res.failed + + +def test_clustered_exceptions_fail_baseline_independence(): + nobs = 100 + var = np.ones(nobs) + returns = np.full(nobs, 0.1) + returns[40:50] = -2.0 # one block of 10 consecutive exceptions, 10% of the sample + res = max_undetectable_understatement( + returns, var, p=0.1, tests=["christoffersen_independence"] + ) + assert not res.passes + assert res.failed == ("christoffersen_independence",) + assert math.isnan(res.understatement) + assert res.exceptions == 10 + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_no_losses_has_no_breakpoints(): + nobs = 100 + var = np.ones(nobs) + returns = np.full(nobs, 0.1) # no losses: no breakpoints in (0, 1) + res = max_undetectable_understatement(returns, var, tests=["kupiec_pof"]) + assert res.passes + assert res.failed == () + assert res.understatement == 1.0 # the smallest scaling, 0, already passes + assert res.exceptions == 0 + assert res.extra_exceptions == 0 + assert res.binding == () + + +def test_non_monotone_independence_stops_at_first_rejection(): + # The independence test is not monotone in the scaling: for this seed the VaR + # scaled down by 0.187 is rejected while a scaling down by 0.226 passes again. + # The limit is the first rejection met scanning down from the reported VaR. + rng = np.random.default_rng(1234) + for _ in range(32): + returns = rng.standard_t(5, 250) * 0.01 + var = np.full(250, 1.645 * 0.01 * rng.uniform(1.0, 1.4)) + assert christoffersen_independence(returns, var * (1 - 0.187)).pval <= 0.05 + assert christoffersen_independence(returns, var * (1 - 0.226)).pval > 0.05 + res = max_undetectable_understatement(returns, var, p=0.05) + assert res.passes + assert res.understatement < 0.187 + assert res.binding == ("christoffersen_independence",) + scale = 1.0 - res.understatement + ratio = -returns / var + # every distinct exception set between the limit and the reported VaR passes + points = np.concatenate(([scale, 1.0], ratio[(ratio >= scale) & (ratio < 1.0)])) + assert all(_mul_passes(returns, var, s) for s in points) + assert not _mul_passes(returns, var, scale - 1e-9) + + +def test_ignore_too_few_keeps_clustering_rejection(): + # 10 consecutive exceptions in 500 is a rate of 0.02, below p, but the + # conditional coverage test rejects because of the clustering + returns = np.full(500, 0.001) + returns[200:210] = -0.05 + var = np.full(500, 0.02) + assert christoffersen_conditional_coverage(returns, var, 0.05).pval <= 0.05 + res = max_undetectable_understatement( + returns, var, p=0.05, tests=("christoffersen_conditional_coverage",) + ) + assert not res.passes + assert res.failed == ("christoffersen_conditional_coverage",) + assert math.isnan(res.understatement) + + +def test_ignore_too_few_sets_aside_only_coverage(): + # 3 scattered exceptions in 500 at p=0.05: the joint test rejects for too few, + # but the independence part alone, against chi2(2), does not + returns = np.full(500, 0.001) + returns[[100, 250, 400]] = -0.05 + var = np.full(500, 0.02) + tests = ("christoffersen_conditional_coverage",) + assert christoffersen_conditional_coverage(returns, var, 0.05).pval <= 0.05 + strict = max_undetectable_understatement( + returns, var, p=0.05, tests=tests, ignore_too_few=False + ) + assert not strict.passes + res = max_undetectable_understatement(returns, var, p=0.05, tests=tests) + assert res.passes + assert _mul_passes(returns, var, 1.0 - res.understatement, p=0.05) + + +def test_first_breakpoint_rejected_gives_zero_understatement(): + # 9 exceptions in 100 pass Kupiec at p=0.05. A loss exactly equal to the VaR is + # not an exception, but it becomes the 10th, which is rejected, at any scaling + # down, so no understatement at all is undetectable + returns = np.full(100, 0.1) + returns[:9] = -2.0 + returns[9] = -1.0 + var = np.ones(100) + res = max_undetectable_understatement(returns, var, p=0.05, tests=["kupiec_pof"]) + assert res.passes + assert res.understatement == 0.0 + assert res.exceptions == 9 + assert res.extra_exceptions == 0 + assert res.binding == ("kupiec_pof",) + + +def test_baseline_and_search_share_the_rejection_rule(mul_sim): + # A test rejects when its p-value is at most size. With size set to the + # p-value of the reported VaR, the baseline is rejected, as it is by the + # public test. + ret, var = mul_sim + pval = kupiec_pof(ret, var, 0.05).pval + res = max_undetectable_understatement( + ret, var, size=pval, tests=["kupiec_pof"], ignore_too_few=False + ) + assert not res.passes + assert res.failed == ("kupiec_pof",) + + +def test_mul_traffic_light_short_series_raises(): + returns = np.full(100, 0.01) + var = np.ones(100) + with pytest.raises(ValueError, match=r"window must be .*\(100\); got 250 \(int\)"): + max_undetectable_understatement(returns, var) + res = max_undetectable_understatement(returns, var, tests=["kupiec_pof"]) + assert res.passes + + +def test_traffic_light_only_sets_the_limit(): + nobs = 250 + var = np.ones(nobs) + # ratios -returns / var are strictly increasing and all in (0.5, 1) + returns = -0.5 * (1.0 + np.arange(nobs) / nobs) + res = max_undetectable_understatement( + returns, var, p=0.01, tests=["traffic_light"], window=nobs + ) + ratios = np.sort(-returns / var)[::-1] + # green allows 4 exceptions at p=0.01: the 5th largest ratio sets the limit + assert_allclose(1.0 - res.understatement, ratios[4]) + assert res.exceptions == 0 + assert res.extra_exceptions == 4 + assert res.binding == ("traffic_light",) + + +def test_result_is_immutable(mul_sim): + ret, var = mul_sim + res = max_undetectable_understatement(ret, var) + assert dataclasses.is_dataclass(res) + assert [f.name for f in dataclasses.fields(res)] == [ + "understatement", + "exceptions", + "extra_exceptions", + "binding", + "passes", + "failed", + "nobs", + "p", + ] + with pytest.raises(dataclasses.FrozenInstanceError): + res.passes = False + + +def test_mul_pandas_alignment(): + returns, var = _series_pair() # 120 observations + scaled_var = var.abs() + 1.0 # strictly positive and generous + result = max_undetectable_understatement(returns, scaled_var, window=100) + expected = max_undetectable_understatement( + returns.to_numpy(), scaled_var.to_numpy(), window=100 + ) + assert result == expected + + +def test_mul_input_validation(mul_sim): + ret, var = mul_sim + with pytest.raises(ValueError, match="strictly positive"): + max_undetectable_understatement(ret, np.zeros_like(var)) + with pytest.raises(ValueError, match="one of"): + max_undetectable_understatement(ret, var, tests=["not_a_test"]) + with pytest.raises(ValueError, match="non-empty"): + max_undetectable_understatement(ret, var, tests=[]) + with pytest.raises(TypeError, match="single str"): + max_undetectable_understatement(ret, var, tests="kupiec_pof") + with pytest.raises(ValueError, match="size must be"): + max_undetectable_understatement(ret, var, size=1.5) + with pytest.raises(ValueError, match=r"p must be a finite float in \(0, 0.5\]"): + max_undetectable_understatement(ret, var, p=0.9) + with pytest.raises(ValueError, match="window must be"): + max_undetectable_understatement(ret, var, window=0) + with pytest.raises(TypeError, match="ignore_too_few"): + max_undetectable_understatement(ret, var, ignore_too_few="yes") + with pytest.raises(ValueError, match="same length as returns"): + max_undetectable_understatement(ret, var[:-1]) + with pytest.raises(ValueError, match="at least 2 observations"): + max_undetectable_understatement( + ret[:1], var[:1], tests=["christoffersen_independence"] + ) diff --git a/arch/univariate/__init__.py b/arch/univariate/__init__.py index 3a6a4d9070..54b3306c6e 100644 --- a/arch/univariate/__init__.py +++ b/arch/univariate/__init__.py @@ -2,10 +2,12 @@ from arch.univariate import recursions_python from arch.univariate.backtest import ( + MaxUndetectableUnderstatementResult, TrafficLightResult, christoffersen_conditional_coverage, christoffersen_independence, kupiec_pof, + max_undetectable_understatement, traffic_light, var_exceptions, ) @@ -70,6 +72,7 @@ "GeneralizedError", "MAMean", "MIDASHyperbolic", + "MaxUndetectableUnderstatementResult", "Normal", "RiskMetrics2006", "SkewStudent", @@ -82,6 +85,7 @@ "excess_kurtosis", "hill_estimator", "kupiec_pof", + "max_undetectable_understatement", "recursions", "recursions_python", "traffic_light", diff --git a/arch/univariate/backtest.py b/arch/univariate/backtest.py index fc8dd172b4..8a07fa00fb 100644 --- a/arch/univariate/backtest.py +++ b/arch/univariate/backtest.py @@ -10,6 +10,7 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass import numpy as np @@ -18,18 +19,35 @@ from scipy.special import xlogy from arch._typing import ArrayLike1D, BoolArray, Float64Array1D -from arch.utility._validation import check_array, check_float, check_int +from arch.utility._validation import ( + check_array, + check_bool, + check_float, + check_int, + check_option, +) from arch.utility.testing import WaldTestStatistic __all__ = [ + "MaxUndetectableUnderstatementResult", "TrafficLightResult", "christoffersen_conditional_coverage", "christoffersen_independence", "kupiec_pof", + "max_undetectable_understatement", "traffic_light", "var_exceptions", ] +# The tests that max_undetectable_understatement can select among, and the order in +# which failed and binding tests are reported +_UNDERSTATEMENT_TESTS = ( + "kupiec_pof", + "christoffersen_independence", + "christoffersen_conditional_coverage", + "traffic_light", +) + # The cumulative probabilities that begin the yellow and the red zone (BCBS, 1996) _YELLOW_PROBABILITY = 0.95 _RED_PROBABILITY = 0.9999 @@ -582,3 +600,315 @@ def traffic_light( nobs=window, p=p, ) + + +@dataclass(frozen=True) +class MaxUndetectableUnderstatementResult: # numpydoc ignore=PR02 + """ + Result of the maximum undetectable understatement backtest of VaR. + + Parameters + ---------- + understatement : float + The largest fraction by which ``var`` could have been scaled down such + that it, and every smaller scaling down, passes every selected test. + NaN if the reported VaR does not pass to begin with. + exceptions : int + The number of exceptions of the reported VaR. + extra_exceptions : int + The number of additional exceptions produced by scaling ``var`` down by + ``understatement``, beyond ``exceptions``. 0 if the reported VaR does + not pass. + binding : tuple of str + The selected tests that reject a VaR scaled down by just beyond + ``understatement``. Empty if ``var`` can be scaled to 0 without being + rejected, and empty if the reported VaR does not pass, since no limit + is computed. + passes : bool + Whether the reported VaR passes every selected test. + failed : tuple of str + The selected tests that reject the reported VaR. Empty if ``passes``. + nobs : int + The number of observations used. + p : float + The probability of an exception that the VaR is designed for. + + Attributes + ---------- + understatement : float + The largest fraction by which ``var`` could have been scaled down such + that it, and every smaller scaling down, passes every selected test. + NaN if the reported VaR does not pass to begin with. + exceptions : int + The number of exceptions of the reported VaR. + extra_exceptions : int + The number of additional exceptions produced by scaling ``var`` down by + ``understatement``, beyond ``exceptions``. 0 if the reported VaR does + not pass. + binding : tuple of str + The selected tests that reject a VaR scaled down by just beyond + ``understatement``. Empty if ``var`` can be scaled to 0 without being + rejected, and empty if the reported VaR does not pass, since no limit + is computed. + passes : bool + Whether the reported VaR passes every selected test. + failed : tuple of str + The selected tests that reject the reported VaR. Empty if ``passes``. + nobs : int + The number of observations used. + p : float + The probability of an exception that the VaR is designed for. + """ + + understatement: float + exceptions: int + extra_exceptions: int + binding: tuple[str, ...] + passes: bool + failed: tuple[str, ...] + nobs: int + p: float + + +def max_undetectable_understatement( + returns: ArrayLike1D | list[float], + var: ArrayLike1D | list[float], + p: float = 0.05, + size: float = 0.05, + tests: Sequence[str] = _UNDERSTATEMENT_TESTS, + ignore_too_few: bool = True, + window: int = 250, +) -> MaxUndetectableUnderstatementResult: + r""" + Maximum undetectable understatement of a VaR series. + + A passing backtest only says that a VaR was not rejected. It does not say + how close it was. This function finds the largest fraction by which the + whole VaR series could have been scaled down, with the same returns, such + that it and every smaller scaling down are not rejected by the selected + tests. + + Parameters + ---------- + returns : array_like + Realized returns, a 1-d array of finite numbers. + var : array_like + VaR forecasts as positive losses, a 1-d array of finite, strictly + positive numbers. See `var_exceptions` for the sign convention and for + how the inputs are aligned. + p : float, optional + The probability of an exception that the VaR is designed for, which is + the size of the tail, in the interval (0, 0.5]. It is 0.05 for a 95% + VaR and 0.01 for a 99% VaR. The default is 0.05, a 95% VaR, and is used + by every selected test, including `traffic_light`. + size : float, optional + The size of the Kupiec and the Christoffersen tests, in (0, 1). The + default is 0.05. `traffic_light` does not use a size, since its zones + are already defined at fixed probabilities. + tests : sequence of str, optional + The tests to pass, a subset of ``"kupiec_pof"``, + ``"christoffersen_independence"``, ``"christoffersen_conditional_coverage"`` + and ``"traffic_light"``. All four are selected by default. + ignore_too_few : bool, optional + If True, the default, the coverage part of a test is set aside when the + rate of exceptions is below `p`, since too few exceptions are not + evidence of an understated VaR. `kupiec_pof` is two-sided, and a VaR + with too few exceptions is otherwise rejected for being too + conservative, so it is not treated as a failure. For + `christoffersen_conditional_coverage`, the coverage statistic is set to + 0 and the independence statistic is compared with the same + :math:`\chi^2_2` distribution, so a rejection caused by clustered + exceptions is still a failure. `christoffersen_independence` and + `traffic_light` are not affected. + window : int, optional + The window of `traffic_light`, used only if it is selected. The + default is 250. + + Returns + ------- + MaxUndetectableUnderstatementResult + The largest fraction by which `var` could have been understated, and + supporting information. + + Raises + ------ + TypeError + If `returns` or `var` cannot be converted to numbers, `p` or `size` is + not a real number, `window` is not an integer, `ignore_too_few` is not + a bool, or `tests` is not a sequence of str. + ValueError + If `returns` or `var` is not valid, as in `var_exceptions`, `var` is + not strictly positive, `p` is not in (0, 0.5], `size` is not in (0, 1), + `window` is not positive, `tests` is empty or has a value that is not + one of the tests listed above, or there are too few observations for a + selected test, as in `christoffersen_independence` or `traffic_light`. + + See Also + -------- + kupiec_pof : Test of the proportion of exceptions. + christoffersen_independence : Test of the independence of the exceptions. + christoffersen_conditional_coverage : Joint test of coverage and independence. + traffic_light : The Basel zone of the number of exceptions. + + Notes + ----- + The search is over the proportional scalings :math:`(1-s)\mathrm{VaR}_t` for + :math:`s` in :math:`[0, 1]`. An observation becomes an exception at + :math:`s = 1 - (-r_t/\mathrm{VaR}_t)`, so the exception set, and so every + selected test, is constant between these breakpoints. The search evaluates + the tests once at each distinct breakpoint, which is exact for this family + of series and needs no optimizer. + + The scan starts at the reported VaR, :math:`s = 0`, and moves toward larger + understatement, stopping at the first scaling that is rejected. The result is + the largest :math:`s` such that every scaling from 0 to :math:`s` passes. + The independence test, and so the conditional coverage test, is not monotone + in :math:`s`, so a larger scaling beyond the first rejection can pass again, + and is not reported. `binding` lists the tests that reject at that first + rejected scaling. + + A test rejects when its p-value is at most `size`, as in the public tests, + and the same rule is used for the reported VaR and for every scaling. + + This is a lower bound on what a series that is not a proportional scaling + of `var` could hide. A series that understates the VaR on some days and + overstates it on others can produce the same, or a smaller, number of + exceptions than a proportional scaling, and so can be harder to detect. + + Table 1 of [1] reports how often the traffic light accepts a VaR whose true + coverage is lower than stated. This function asks a related question of one + realised series: how much lower the VaR could have been before any selected + test rejected it. + + References + ---------- + .. [1] Basel Committee on Banking Supervision (1996). Supervisory framework + for the use of "backtesting" in conjunction with the internal models + approach to market risk capital requirements. + .. [2] Christoffersen, P. F. (1998). Evaluating interval forecasts. + International Economic Review, 39(4), 841-862. + .. [3] Kupiec, P. H. (1995). Techniques for verifying the accuracy of risk + measurement models. The Journal of Derivatives, 3(2), 73-84. + """ + p = check_float("p", p, lower=0.0, upper=0.5, inclusive="right") + size = check_float("size", size, lower=0.0, upper=1.0, inclusive="neither") + window = check_int("window", window, lower=1) + ignore_too_few = check_bool("ignore_too_few", ignore_too_few) + if isinstance(tests, str): + raise TypeError( + f"tests must be a sequence of str, not a single str; got {tests!r} (str)" + ) + tests_seq = list(tests) + if not tests_seq: + raise ValueError( + "tests must be a non-empty sequence of 'kupiec_pof', " + "'christoffersen_independence', 'christoffersen_conditional_coverage' " + f"or 'traffic_light'; got {tests!r} ({type(tests).__name__})" + ) + selected = frozenset( + check_option("tests", test, _UNDERSTATEMENT_TESTS) for test in tests_seq + ) + + min_obs = ( + 2 + if ( + "christoffersen_independence" in selected + or "christoffersen_conditional_coverage" in selected + ) + else 1 + ) + exceptions0, _ = _exceptions(returns, var, min_obs) + ret, var_arr, _ = _aligned(returns, var) + if not np.all(var_arr > 0.0): + raise ValueError( + "var must be strictly positive to compute a proportional " + f"understatement; got a minimum of {float(var_arr.min()):g} (ndarray)" + ) + nobs = ret.shape[0] + + if "traffic_light" in selected and window > nobs: + raise ValueError( + f"window must be an integer no larger than the number of aligned " + f"observations ({nobs}); got {window} (int)" + ) + + def _rejects(stat: float, df: int) -> bool: + # The rule of the public tests: reject when the p-value is at most size + return bool(stats.chi2.sf(stat, df) <= size) + + # The tests are evaluated with the private likelihoods and the traffic light + # boundaries directly, which avoids building a WaldTestStatistic or a + # TrafficLightResult at every breakpoint. The p-values are the same as those of + # the public tests, and the baseline and the search use the same gate. + def _gate(hits: BoolArray) -> tuple[str, ...]: + failed_now = [] + count = int(hits.sum()) + # Too few exceptions only sets aside the coverage part of a test, which is + # the part that rejects a conservative VaR + skip_coverage = ignore_too_few and (count / nobs) < p + lr_pof = 0.0 if skip_coverage else _lr_pof(count, nobs, p) + if "kupiec_pof" in selected and not skip_coverage: + if _rejects(lr_pof, 1): + failed_now.append("kupiec_pof") + if ( + "christoffersen_independence" in selected + or "christoffersen_conditional_coverage" in selected + ): + lr_ind = _lr_independence(hits) + if "christoffersen_independence" in selected and _rejects(lr_ind, 1): + failed_now.append("christoffersen_independence") + if "christoffersen_conditional_coverage" in selected: + if _rejects(lr_pof + lr_ind, 2): + failed_now.append("christoffersen_conditional_coverage") + if "traffic_light" in selected: + window_count = int(hits[-window:].sum()) + if window_count > 0: + prob = float(stats.binom.cdf(window_count, window, p)) + if prob >= _YELLOW_PROBABILITY: + failed_now.append("traffic_light") + return tuple(t for t in _UNDERSTATEMENT_TESTS if t in failed_now) + + n_reported = int(exceptions0.sum()) + failed = _gate(exceptions0) + if failed: + return MaxUndetectableUnderstatementResult( + understatement=float("nan"), + exceptions=n_reported, + extra_exceptions=0, + binding=(), + passes=False, + failed=failed, + nobs=nobs, + p=p, + ) + + # A scaling 1 - s turns observation t into an exception when the ratio + # -r_t / VaR_t exceeds 1 - s, so the exceptions at a threshold thr = 1 - s are + # ratio > thr, constant for thr in [b_k, b_k+1) between consecutive candidates. + # The scan starts at the reported VaR (thr = 1) and moves down, toward larger + # understatement, and stops at the first candidate that fails. The tests are not + # monotone in thr, so a passing candidate below a failing one is not a limit. + ratio = -ret / var_arr + breakpoints = np.unique(ratio[(ratio > 0.0) & (ratio < 1.0)]) + candidates = np.concatenate(([0.0], breakpoints))[::-1] + first_fail = next( + (i for i, thr in enumerate(candidates) if _gate(ratio > thr)), None + ) + binding: tuple[str, ...] = () + if first_fail is None: + limit = 0.0 + else: + binding = _gate(ratio > candidates[first_fail]) + limit = 1.0 if first_fail == 0 else float(candidates[first_fail - 1]) + hits_at_limit = ratio > limit if limit < 1.0 else exceptions0 + + return MaxUndetectableUnderstatementResult( + understatement=1.0 - limit, + exceptions=n_reported, + extra_exceptions=int(hits_at_limit.sum()) - n_reported, + binding=binding, + passes=True, + failed=(), + nobs=nobs, + p=p, + ) diff --git a/doc/source/changes/9.0.rst b/doc/source/changes/9.0.rst index 69f2fd2670..a720a2f375 100644 --- a/doc/source/changes/9.0.rst +++ b/doc/source/changes/9.0.rst @@ -120,6 +120,13 @@ New features probability of an exception, such as 0.01 for a 99% VaR, and must be in (0, 0.5]. The unconditional and conditional coverage statistics equal those of ``VaRTest`` in the R package rugarch (:issue:`904`, :issue:`965`). +- Added :func:`~arch.univariate.max_undetectable_understatement`, which finds the + largest fraction by which a VaR series could have been scaled down, with the same + returns, such that it and every smaller scaling down pass the selected backtests. + The search over proportional scalings is exact, since the exceptions only change at + the points where a loss crosses the scaled VaR, and is a lower bound on what a + non-proportional series could hide. The result is a :class:`~arch.univariate.MaxUndetectableUnderstatementResult` + with the limit, the binding test, and the exceptions it would add (:issue:`904`). - Added :class:`~arch.bootstrap.LocalBlockBootstrap`, the local block bootstrap of Paparoditis and Politis (2002), for series whose properties change slowly over time. Each block of the data is replaced by a block that starts at most ``locality`` diff --git a/doc/source/univariate/utility.rst b/doc/source/univariate/utility.rst index 8d6ed1c20a..132d842cf0 100644 --- a/doc/source/univariate/utility.rst +++ b/doc/source/univariate/utility.rst @@ -170,3 +170,32 @@ exception, or with only exceptions, has a statistic. traffic_light .. autoclass:: TrafficLightResult + +A passing backtest only says that the reported VaR was not rejected. It does not say +how close it was. :func:`~arch.univariate.max_undetectable_understatement` answers a +more specific question: by what fraction could the whole VaR series have been scaled +down, with the same returns, and still have passed the selected tests. The search is +over proportional scalings of ``var``, is exact, and needs no optimizer, since the set +of exceptions only changes at the breakpoints where an observation crosses the scaled +VaR. It is a lower bound on what a series that understates VaR on some days and +overstates it on others could hide. + +``kupiec_pof`` and ``christoffersen_conditional_coverage`` are two-sided, so a VaR with +too few exceptions is otherwise rejected for being too conservative. By default, +``ignore_too_few=True`` does not count that as a failure, since it is not evidence of +an understated VaR. + +.. ipython:: python + + from arch.univariate import max_undetectable_understatement + + result = max_undetectable_understatement(returns, var, p=0.01) + result.understatement + result.binding + +.. autosummary:: + :toctree: generated/ + + max_undetectable_understatement + +.. autoclass:: MaxUndetectableUnderstatementResult