Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
346 changes: 346 additions & 0 deletions arch/tests/univariate/test_backtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@
from scipy import stats

from arch.univariate import (
MaxUndetectableUnderstatementResult,
TrafficLightResult,
christoffersen_conditional_coverage,
christoffersen_independence,
kupiec_pof,
max_undetectable_understatement,
traffic_light,
var_exceptions,
)
Expand Down Expand Up @@ -1051,3 +1053,347 @@ def test_p_boundaries():
# Values that are close to the lower boundary and 0.5 are valid
assert kupiec_pof(returns, var, 1e-12).stat > 0.0
assert kupiec_pof(returns, var, 0.5).stat > 0.0


# Maximum undetectable understatement (xref #904)


def _mul_passes(ret, var, scale, p=0.05, size=0.05, ignore_too_few=True, window=250):
"""Independent pass/fail check using the public tests directly."""
scaled = scale * var
hits = var_exceptions(ret, scaled)
too_few = bool(hits.mean() < p)
ok = kupiec_pof(ret, scaled, p).pval > size or (ignore_too_few and too_few)
ind = christoffersen_independence(ret, scaled)
ok &= ind.pval > size
cc = christoffersen_conditional_coverage(ret, scaled, p)
if ignore_too_few and too_few:
# only the coverage part is set aside: the independence part keeps chi2(2)
ok &= stats.chi2.sf(ind.stat, 2) > size
else:
ok &= cc.pval > size
ok &= traffic_light(ret, scaled, p, window).zone == "green"
return bool(ok)


@pytest.fixture(scope="module")
def mul_sim():
rng = np.random.default_rng(20261109)
nobs = 600
sigma = np.exp(0.25 * rng.standard_normal(nobs))
ret = sigma * rng.standard_normal(nobs) * 0.01
var = -stats.norm.ppf(0.05) * sigma * 0.01 # a generous, correctly sized 95% VaR
return ret, var


def test_hand_computed_kupiec_only():
# 4 observations, ratios -returns/var are 2, 0.5, 0.5, 0.5. Only one exception
# (2 > 1) at the reported VaR, and the three ties at 0.5 do not become
# exceptions until the VaR is scaled below 0.5, so the limit is exactly 0.5.
returns = np.array([-2.0, -0.5, -0.5, -0.5])
var = np.ones(4)
res = max_undetectable_understatement(returns, var, p=0.5, tests=["kupiec_pof"])
assert isinstance(res, MaxUndetectableUnderstatementResult)
assert res.passes
assert res.failed == ()
assert_allclose(res.understatement, 0.5)
assert res.exceptions == 1
assert res.extra_exceptions == 0
assert res.binding == ("kupiec_pof",)
assert res.nobs == 4
assert res.p == 0.5


def test_passes_and_reports_counts(mul_sim):
ret, var = mul_sim
res = max_undetectable_understatement(ret, var)
assert res.passes
assert res.failed == ()
assert 0.0 <= res.understatement <= 1.0
assert res.extra_exceptions >= 0
assert res.exceptions == int(var_exceptions(ret, var).sum())
assert res.nobs == ret.shape[0]


def test_brute_force_grid_matches_public_tests(mul_sim):
# The limit is reached scanning down from the reported VaR (scale=1): every
# scaling between the limit and 1 passes, and a scaling just below it fails.
ret, var = mul_sim
res = max_undetectable_understatement(ret, var)
scale = 1.0 - res.understatement
assert res.understatement < 1.0
grid = np.linspace(scale, 1.0, 2001)
assert all(_mul_passes(ret, var, s) for s in grid)
assert not _mul_passes(ret, var, scale - 1e-9)


def test_boundary_agrees_with_public_functions(mul_sim):
ret, var = mul_sim
res = max_undetectable_understatement(ret, var)
scale = 1.0 - res.understatement
assert _mul_passes(ret, var, scale)
hits = var_exceptions(ret, scale * var)
assert int(hits.sum()) - res.exceptions == res.extra_exceptions


def test_more_tests_never_increase_understatement(mul_sim):
ret, var = mul_sim
only_kupiec = max_undetectable_understatement(ret, var, tests=["kupiec_pof"])
all_tests = max_undetectable_understatement(ret, var)
assert all_tests.understatement <= only_kupiec.understatement + 1e-12


def test_stricter_size_never_increases_understatement(mul_sim):
# Restricted to kupiec_pof, whose baseline passes at every size tried here;
# Christoffersen's independence test can reject the baseline itself at a loose
# size, which makes the comparison undefined rather than merely smaller.
ret, var = mul_sim
loose = max_undetectable_understatement(ret, var, size=0.01, tests=["kupiec_pof"])
strict = max_undetectable_understatement(ret, var, size=0.20, tests=["kupiec_pof"])
assert loose.passes
assert strict.passes
assert strict.understatement <= loose.understatement + 1e-12


def test_no_exceptions_can_be_fully_understated():
# Every return is a gain: var can be scaled all the way to 0 undetected
ret = np.full(300, 0.01)
var = np.ones(300)
res = max_undetectable_understatement(ret, var)
assert res.passes
assert_allclose(res.understatement, 1.0)
assert res.exceptions == 0
assert res.extra_exceptions == 0
assert res.binding == ()


def test_all_exceptions_fails_baseline():
# Every day is an exception at the reported VaR: far too many for any p < 1
ret = np.full(20, -5.0)
var = np.ones(20)
res = max_undetectable_understatement(ret, var, p=0.3, window=20)
assert not res.passes
assert "kupiec_pof" in res.failed
assert math.isnan(res.understatement)
assert res.exceptions == 20
assert res.extra_exceptions == 0
assert res.binding == ()


def test_ignore_too_few_true_passes_a_conservative_var():
# No exceptions in 250 observations at p=0.01 is conservative, not understated
ret = np.full(250, 0.001)
var = np.ones(250)
res = max_undetectable_understatement(ret, var, p=0.01, ignore_too_few=True)
assert res.passes
assert res.failed == ()


def test_ignore_too_few_false_fails_the_same_var():
ret = np.full(250, 0.001)
var = np.ones(250)
res = max_undetectable_understatement(ret, var, p=0.01, ignore_too_few=False)
assert not res.passes
assert "kupiec_pof" in res.failed
assert math.isnan(res.understatement)
assert res.extra_exceptions == 0
assert res.binding == ()


def test_overstated_var_fails_baseline(mul_sim):
ret, var = mul_sim
res = max_undetectable_understatement(ret, 0.3 * var)
assert not res.passes
assert math.isnan(res.understatement)
assert res.failed


def test_clustered_exceptions_fail_baseline_independence():
nobs = 100
var = np.ones(nobs)
returns = np.full(nobs, 0.1)
returns[40:50] = -2.0 # one block of 10 consecutive exceptions, 10% of the sample
res = max_undetectable_understatement(
returns, var, p=0.1, tests=["christoffersen_independence"]
)
assert not res.passes
assert res.failed == ("christoffersen_independence",)
assert math.isnan(res.understatement)
assert res.exceptions == 10
assert res.extra_exceptions == 0
assert res.binding == ()


def test_no_losses_has_no_breakpoints():
nobs = 100
var = np.ones(nobs)
returns = np.full(nobs, 0.1) # no losses: no breakpoints in (0, 1)
res = max_undetectable_understatement(returns, var, tests=["kupiec_pof"])
assert res.passes
assert res.failed == ()
assert res.understatement == 1.0 # the smallest scaling, 0, already passes
assert res.exceptions == 0
assert res.extra_exceptions == 0
assert res.binding == ()


def test_non_monotone_independence_stops_at_first_rejection():
# The independence test is not monotone in the scaling: for this seed the VaR
# scaled down by 0.187 is rejected while a scaling down by 0.226 passes again.
# The limit is the first rejection met scanning down from the reported VaR.
rng = np.random.default_rng(1234)
for _ in range(32):
returns = rng.standard_t(5, 250) * 0.01
var = np.full(250, 1.645 * 0.01 * rng.uniform(1.0, 1.4))
assert christoffersen_independence(returns, var * (1 - 0.187)).pval <= 0.05
assert christoffersen_independence(returns, var * (1 - 0.226)).pval > 0.05
res = max_undetectable_understatement(returns, var, p=0.05)
assert res.passes
assert res.understatement < 0.187
assert res.binding == ("christoffersen_independence",)
scale = 1.0 - res.understatement
ratio = -returns / var
# every distinct exception set between the limit and the reported VaR passes
points = np.concatenate(([scale, 1.0], ratio[(ratio >= scale) & (ratio < 1.0)]))
assert all(_mul_passes(returns, var, s) for s in points)
assert not _mul_passes(returns, var, scale - 1e-9)


def test_ignore_too_few_keeps_clustering_rejection():
# 10 consecutive exceptions in 500 is a rate of 0.02, below p, but the
# conditional coverage test rejects because of the clustering
returns = np.full(500, 0.001)
returns[200:210] = -0.05
var = np.full(500, 0.02)
assert christoffersen_conditional_coverage(returns, var, 0.05).pval <= 0.05
res = max_undetectable_understatement(
returns, var, p=0.05, tests=("christoffersen_conditional_coverage",)
)
assert not res.passes
assert res.failed == ("christoffersen_conditional_coverage",)
assert math.isnan(res.understatement)


def test_ignore_too_few_sets_aside_only_coverage():
# 3 scattered exceptions in 500 at p=0.05: the joint test rejects for too few,
# but the independence part alone, against chi2(2), does not
returns = np.full(500, 0.001)
returns[[100, 250, 400]] = -0.05
var = np.full(500, 0.02)
tests = ("christoffersen_conditional_coverage",)
assert christoffersen_conditional_coverage(returns, var, 0.05).pval <= 0.05
strict = max_undetectable_understatement(
returns, var, p=0.05, tests=tests, ignore_too_few=False
)
assert not strict.passes
res = max_undetectable_understatement(returns, var, p=0.05, tests=tests)
assert res.passes
assert _mul_passes(returns, var, 1.0 - res.understatement, p=0.05)


def test_first_breakpoint_rejected_gives_zero_understatement():
# 9 exceptions in 100 pass Kupiec at p=0.05. A loss exactly equal to the VaR is
# not an exception, but it becomes the 10th, which is rejected, at any scaling
# down, so no understatement at all is undetectable
returns = np.full(100, 0.1)
returns[:9] = -2.0
returns[9] = -1.0
var = np.ones(100)
res = max_undetectable_understatement(returns, var, p=0.05, tests=["kupiec_pof"])
assert res.passes
assert res.understatement == 0.0
assert res.exceptions == 9
assert res.extra_exceptions == 0
assert res.binding == ("kupiec_pof",)


def test_baseline_and_search_share_the_rejection_rule(mul_sim):
# A test rejects when its p-value is at most size. With size set to the
# p-value of the reported VaR, the baseline is rejected, as it is by the
# public test.
ret, var = mul_sim
pval = kupiec_pof(ret, var, 0.05).pval
res = max_undetectable_understatement(
ret, var, size=pval, tests=["kupiec_pof"], ignore_too_few=False
)
assert not res.passes
assert res.failed == ("kupiec_pof",)


def test_mul_traffic_light_short_series_raises():
returns = np.full(100, 0.01)
var = np.ones(100)
with pytest.raises(ValueError, match=r"window must be .*\(100\); got 250 \(int\)"):
max_undetectable_understatement(returns, var)
res = max_undetectable_understatement(returns, var, tests=["kupiec_pof"])
assert res.passes


def test_traffic_light_only_sets_the_limit():
nobs = 250
var = np.ones(nobs)
# ratios -returns / var are strictly increasing and all in (0.5, 1)
returns = -0.5 * (1.0 + np.arange(nobs) / nobs)
res = max_undetectable_understatement(
returns, var, p=0.01, tests=["traffic_light"], window=nobs
)
ratios = np.sort(-returns / var)[::-1]
# green allows 4 exceptions at p=0.01: the 5th largest ratio sets the limit
assert_allclose(1.0 - res.understatement, ratios[4])
assert res.exceptions == 0
assert res.extra_exceptions == 4
assert res.binding == ("traffic_light",)


def test_result_is_immutable(mul_sim):
ret, var = mul_sim
res = max_undetectable_understatement(ret, var)
assert dataclasses.is_dataclass(res)
assert [f.name for f in dataclasses.fields(res)] == [
"understatement",
"exceptions",
"extra_exceptions",
"binding",
"passes",
"failed",
"nobs",
"p",
]
with pytest.raises(dataclasses.FrozenInstanceError):
res.passes = False


def test_mul_pandas_alignment():
returns, var = _series_pair() # 120 observations
scaled_var = var.abs() + 1.0 # strictly positive and generous
result = max_undetectable_understatement(returns, scaled_var, window=100)
expected = max_undetectable_understatement(
returns.to_numpy(), scaled_var.to_numpy(), window=100
)
assert result == expected


def test_mul_input_validation(mul_sim):
ret, var = mul_sim
with pytest.raises(ValueError, match="strictly positive"):
max_undetectable_understatement(ret, np.zeros_like(var))
with pytest.raises(ValueError, match="one of"):
max_undetectable_understatement(ret, var, tests=["not_a_test"])
with pytest.raises(ValueError, match="non-empty"):
max_undetectable_understatement(ret, var, tests=[])
with pytest.raises(TypeError, match="single str"):
max_undetectable_understatement(ret, var, tests="kupiec_pof")
with pytest.raises(ValueError, match="size must be"):
max_undetectable_understatement(ret, var, size=1.5)
with pytest.raises(ValueError, match=r"p must be a finite float in \(0, 0.5\]"):
max_undetectable_understatement(ret, var, p=0.9)
with pytest.raises(ValueError, match="window must be"):
max_undetectable_understatement(ret, var, window=0)
with pytest.raises(TypeError, match="ignore_too_few"):
max_undetectable_understatement(ret, var, ignore_too_few="yes")
with pytest.raises(ValueError, match="same length as returns"):
max_undetectable_understatement(ret, var[:-1])
with pytest.raises(ValueError, match="at least 2 observations"):
max_undetectable_understatement(
ret[:1], var[:1], tests=["christoffersen_independence"]
)
4 changes: 4 additions & 0 deletions arch/univariate/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

from arch.univariate import recursions_python
from arch.univariate.backtest import (
MaxUndetectableUnderstatementResult,
TrafficLightResult,
christoffersen_conditional_coverage,
christoffersen_independence,
kupiec_pof,
max_undetectable_understatement,
traffic_light,
var_exceptions,
)
Expand Down Expand Up @@ -70,6 +72,7 @@
"GeneralizedError",
"MAMean",
"MIDASHyperbolic",
"MaxUndetectableUnderstatementResult",
"Normal",
"RiskMetrics2006",
"SkewStudent",
Expand All @@ -82,6 +85,7 @@
"excess_kurtosis",
"hill_estimator",
"kupiec_pof",
"max_undetectable_understatement",
"recursions",
"recursions_python",
"traffic_light",
Expand Down
Loading