diff --git a/arch/bootstrap/base.py b/arch/bootstrap/base.py index 49685e13d4..bf3c21b66f 100644 --- a/arch/bootstrap/base.py +++ b/arch/bootstrap/base.py @@ -389,6 +389,40 @@ def _check_real_statistic(base: np.ndarray) -> None: ) +def _check_null_type(name: str, null: float | ArrayLike) -> np.ndarray: + """Validate null type and finiteness without knowing the parameter count.""" + if null is None: + _fail(TypeError, name, "a real scalar or a 1-dimensional array", "None") + try: + null_arr: np.ndarray = np.asarray(null, dtype=np.float64) + except (TypeError, ValueError): + _fail( + TypeError, name, "a real scalar or a 1-dimensional array", _describe(null) + ) + if np.isnan(null_arr).any(): + _fail( + ValueError, + name, + "a finite scalar or a 1-dimensional array without NaN", + _describe(null), + ) + return null_arr + + +def _broadcast_null(name: str, null_arr: np.ndarray, num_params: int) -> Float64Array: + """Broadcast a validated null array to one entry per parameter.""" + if null_arr.ndim == 0: + return np.full(num_params, null_arr.item(), dtype=np.float64) + if null_arr.shape == (num_params,): + return null_arr + _fail( + ValueError, + name, + "a scalar or a 1-dimensional array with one value per parameter", + _describe(null_arr), + ) + + def _check_extra_kwargs( kwargs: dict[str, Any], extra_kwargs: Mapping[str, Any] | None = None ) -> Mapping[str, Any] | None: @@ -966,84 +1000,348 @@ def conf_int( real_only=True, ) + return self._conf_int_from_estimates( + size, tail_name, method, func, extra_kwargs + ) + + def _conf_int_from_estimates( + self, + size: float, + tail_name: Literal["two", "upper", "lower"], + method: str, + func: Callable[..., Float64Array], + extra_kwargs: dict[str, Any] | None, + ) -> Float64Array: + """Build a confidence interval from already-computed bootstrap estimates. + + This is the interval-building half of :meth:`conf_int`. It reads the + bootstrap replications cached on the instance and is kept separate so + that :meth:`p_value` can reuse the same replications while searching over + the confidence level. + """ base, results = self._base, self._results assert results is not None assert base is not None _check_real_statistic(base) - studentized_results = self._studentized_results + num_params = 1 if base.ndim == 0 else base.shape[0] - std_err = np.empty((0,)) - if method in ("norm", studentized): - errors = results - results.mean(axis=0) - std_err = np.asarray(np.sqrt(np.diag(errors.T.dot(errors) / reps))) + bias = None + acceleration = None + if method in ("bc", "bca"): + bias = np.asarray(self._bca_bias()).ravel() + if method == "bca": + lens = [len(arg) for arg in self._args] + [ + len(kwarg) for kwarg in self._kwargs.values() + ] + if min(lens) != max(lens): + raise ValueError( + "BCa cannot be applied to statistics " + "computed from datasets with " + "different lengths" + ) + acceleration = np.asarray( + self._bca_acceleration(func, extra_kwargs) + ).ravel() + else: + acceleration = np.zeros(num_params) + + lower = np.zeros(num_params) + upper = np.zeros(num_params) + for param in range(num_params): + lower[param], upper[param] = self._conf_int_parameter_from_estimates( + size, tail_name, method, param, bias, acceleration + ) + return np.vstack((lower, upper)) + + def _conf_int_parameter_from_estimates( + self, + size: float, + tail_name: Literal["two", "upper", "lower"], + method: str, + param: int, + bias: Float64Array | None, + acceleration: Float64Array | None, + ) -> tuple[float, float]: + """Build the interval for a single parameter from cached estimates.""" + base_all = self._base + results = self._results + assert base_all is not None + assert results is not None + base = base_all[param] if base_all.ndim else base_all + std_err = 0.0 + if method in ("norm", "studentized"): + raw_values = results[:, param] + errors = raw_values - raw_values.mean() + std_err = float(np.sqrt(np.dot(errors, errors) / errors.shape[0])) + + values = results[:, param] + if method == "studentized": + studentized_results = self._studentized_results + assert studentized_results is not None + values = studentized_results[:, param] if tail_name == "two": alpha = (1.0 - size) / 2 else: alpha = 1.0 - size - nreps = 1 if not base.shape else base.shape[0] - percentiles = np.array([[alpha, 1.0 - alpha]] * nreps) - norm_quantiles = stats.norm.ppf(percentiles) + norm_quantiles = stats.norm.ppf(np.array([alpha, 1.0 - alpha])) if method == "norm": - lower = base + norm_quantiles[:, 0] * std_err - upper = base + norm_quantiles[:, 1] * std_err - + lower = float(base + norm_quantiles[0] * std_err) + upper = float(base + norm_quantiles[1] * std_err) else: - values = results - if method == studentized: - # studentized uses studentized parameter estimates - values = cast("Float64Array", studentized_results) - + percentiles = np.array([alpha, 1.0 - alpha]) if method in ("bc", "bca"): - # bias corrected uses modified percentiles, but is - # otherwise identical to the percentile method - b = self._bca_bias() - if method == "bca": - lens = [len(arg) for arg in self._args] + [ - len(kwarg) for kwarg in self._kwargs.values() - ] - if min(lens) != max(lens): - raise ValueError( - "BCa cannot be applied to statistics " - "computed from datasets with " - "different lengths" - ) - a = self._bca_acceleration(func, extra_kwargs) + b = bias[param] if bias is not None else 0.0 + a = acceleration[param] if acceleration is not None else 0.0 + z = np.clip(norm_quantiles, -30.0, 30.0) + if a == 0.0: + adjusted = b + b + z else: - a = np.zeros((1, 1)) - percentiles = stats.norm.cdf( - b + (b + norm_quantiles) / (1.0 - a * (b + norm_quantiles)) - ) - percentiles *= 100 + adjusted = b + (b + z) / (1.0 - a * (b + z)) + percentiles = stats.norm.cdf(adjusted) * 100.0 + percentiles = np.clip(percentiles, 0.0, 100.0) else: - percentiles = np.array([100 * p for p in percentiles]) # Rescale - - k = values.shape[1] - lower = np.zeros(k) - upper = np.zeros(k) - for i in range(k): - lower[i], upper[i] = np.percentile(values[:, i], list(percentiles[i])) + percentiles = 100.0 * percentiles + lower_q, upper_q = np.percentile(values, list(percentiles)) # Basic and studentized use the lower empirical quantile to # compute upper and vice versa. Bias corrected and percentile use # upper to estimate the upper, and lower to estimate the lower if method == "basic": - lower_copy = lower + 0.0 - lower = 2.0 * base - upper - upper = 2.0 * base - lower_copy - elif method == studentized: - lower_copy = lower + 0.0 - lower = base - upper * std_err - upper = base - lower_copy * std_err + lower = float(2.0 * base - upper_q) + upper = float(2.0 * base - lower_q) + elif method == "studentized": + lower = float(base - upper_q * std_err) + upper = float(base - lower_q * std_err) + else: + lower = float(lower_q) + upper = float(upper_q) if tail_name == "lower": - upper = np.zeros_like(base) - upper.fill(np.inf) + upper = np.inf elif tail_name == "upper": - lower = np.zeros_like(base) - lower.fill(-1 * np.inf) + lower = -np.inf + return lower, upper - return np.vstack((lower, upper)) + def p_value( + self, + func: Callable[..., Float64Array], + null: float | ArrayLike = 0.0, + alternative: Literal["two-sided", "less", "greater"] = "two-sided", + reps: int = 1000, + method: Literal[ + "basic", "percentile", "studentized", "norm", "bc", "bca" + ] = "basic", + reuse: bool = False, + extra_kwargs: dict[str, Any] | None = None, + sampling: Literal[ + "nonparametric", "semi-parametric", "semi", "parametric", "semiparametric" + ] = "nonparametric", + std_err_func: Callable[..., ArrayLike] | None = None, + studentize_reps: int = 1000, + ) -> Float64Array: + """ + Compute a bootstrap p-value by inverting a confidence interval. + + Parameters + ---------- + func : callable + Function that computes parameter values. See :meth:`conf_int` for the + required signature. + null : float or array-like, optional + Value(s) of the parameter under the null hypothesis. Must be a scalar + or a 1-dimensional array with one value per parameter returned by + ``func``. + alternative : {"two-sided", "less", "greater"}, optional + The alternative hypothesis. ``"less"`` tests that the parameter is + less than ``null``, ``"greater"`` that it is greater than ``null``, + and ``"two-sided"`` that it is different from ``null``. + reps : int, optional + Number of bootstrap replications. Must be at least 1. + method : {"basic", "percentile", "studentized", "norm", "bc", "bca"}, optional + The confidence interval method to invert. Not case sensitive. + reuse : bool, optional + Flag indicating whether to reuse previously computed bootstrap + results. The conditions are the same as in :meth:`conf_int`. + extra_kwargs : dict, optional + Extra keyword arguments to use when calling ``func`` and + ``std_err_func``, when appropriate. + sampling : {"nonparametric", "semi-parametric", "parametric"}, optional + Type of sampling to use. Not case sensitive. + std_err_func : callable, optional + Function to use when standardizing estimated parameters when using + the studentized bootstrap. + studentize_reps : int, optional + Number of bootstraps to use in the inner bootstrap when using the + studentized bootstrap without ``std_err_func``. + + Returns + ------- + ndarray + A 1-dimensional array of p-values, one per parameter returned by + ``func``. + + Raises + ------ + TypeError + If ``func`` or ``std_err_func`` is not callable, ``reps`` or + ``studentize_reps`` is not an int, ``method``, ``alternative`` or + ``sampling`` is not a str, or ``extra_kwargs`` is not a dict. + ValueError + If ``reps`` is less than 1, ``studentize_reps`` is less than 1 when it + is used, ``null`` contains NaN or is not one value per parameter, + ``method``, ``alternative`` or ``sampling`` is not one of the options, + or ``extra_kwargs`` has a key that is the name of bootstrapped data. + + Notes + ----- + The p-value is the smallest significance level at which the + :meth:`conf_int` interval excludes ``null``, computed by inverting the + interval. When the same bootstrap replications are used, the p-value + matches the corresponding ``conf_int`` call by construction, for every + interval method, including bias-corrected and accelerated intervals. + + ``alternative="less"`` inverts the upper confidence bound, + ``alternative="greater"`` inverts the lower bound, and + ``alternative="two-sided"`` inverts the two-sided interval. + + The p-value is resolved by 60 bisection steps, so its effective + resolution is about ``1 / reps``: a p-value is exactly 0 when every + replication lies on one side of ``null``. + + The studentized method uses the same nested bootstrap as + :meth:`conf_int`. The bias-corrected accelerated method computes the + acceleration constant once, before the search. + + Examples + -------- + >>> import numpy as np + >>> from arch.bootstrap import IIDBootstrap + >>> def func(x): + ... return x.mean(0) + >>> y = np.random.default_rng(1234).standard_normal((1000, 2)) + >>> bs = IIDBootstrap(y, seed=1234) + >>> p = bs.p_value(func, null=0.0, reps=1000, method="basic") + >>> p.shape + (2,) + """ + check_callable("func", func) + reps = check_int("reps", reps, lower=1) + method = check_option("method", method, _CONF_INT_METHODS) + alternative = check_option( + "alternative", alternative, ("two-sided", "less", "greater") + ) + reuse = check_bool("reuse", reuse) + _check_extra_kwargs(self._kwargs, extra_kwargs) + std_err_func = check_callable("std_err_func", std_err_func, allow_none=True) + used = method == "studentized" and std_err_func is None + studentize_reps = check_int( + "studentize_reps", studentize_reps, lower=1 if used else None + ) + studentize_reps = studentize_reps if method == "studentized" else 0 + sampling = check_option( + "sampling", + sampling, + { + "nonparametric": "nonparametric", + "parametric": "parametric", + "semi": "semiparametric", + "semi-parametric": "semiparametric", + "semiparametric": "semiparametric", + }, + ) + + # Validate null type and finiteness before any computation. The length + # check needs the number of parameters, which is obtained by evaluating + # func once on the original data, below. + null_arr = _check_null_type("null", null) + + # Evaluate func once on the original data so the null length can be + # validated before the bootstrap runs. + kwargs = _add_extra_kwargs(self._kwargs, extra_kwargs) + base = np.asarray(func(*self._args, **kwargs)) + num_params = 1 if base.ndim == 0 else base.shape[0] + null_values = _broadcast_null("null", null_arr, num_params) + + _reuse = False + if reuse: + _reuse = ( + self._results is not None + and len(self._results) == reps + and method != "studentized" + and self._last_func is func + ) + if not _reuse: + if reuse: + warn = ( + "The conditions to reuse the previous bootstrap has " + "not been satisfied. A new bootstrap will be used." + ) + warnings.warn(warn, RuntimeWarning, stacklevel=2) + self._construct_bootstrap_estimates( + func, + reps, + extra_kwargs, + std_err_func=std_err_func, + studentize_reps=studentize_reps, + sampling=sampling, + real_only=True, + ) + + base = self._base + assert base is not None + num_params = 1 if base.ndim == 0 else base.shape[0] + tail_map: dict[str, Literal["two", "upper", "lower"]] = { + "two-sided": "two", + "less": "upper", + "greater": "lower", + } + tail_name = tail_map[alternative] + + bias = None + acceleration = None + if method in ("bc", "bca"): + bias = np.asarray(self._bca_bias()).ravel() + if method == "bca": + lens = [len(arg) for arg in self._args] + [ + len(kwarg) for kwarg in self._kwargs.values() + ] + if min(lens) != max(lens): + raise ValueError( + "BCa cannot be applied to statistics " + "computed from datasets with " + "different lengths" + ) + acceleration = np.asarray( + self._bca_acceleration(func, extra_kwargs) + ).ravel() + else: + acceleration = np.zeros(num_params) + + pvals = np.zeros(num_params) + for param in range(num_params): + low = 0.0 + high = 1.0 + for _ in range(60): + size = (low + high) / 2.0 + lower, upper = self._conf_int_parameter_from_estimates( + size, tail_name, method, param, bias, acceleration + ) + if tail_name == "two": + inside = lower <= null_values[param] <= upper + elif tail_name == "upper": + inside = null_values[param] <= upper + else: + inside = null_values[param] >= lower + # ``inside`` is monotone increasing in ``size``: wider + # intervals include the null more often. Track the smallest + # size that includes the null for this parameter. + if inside: + high = size + else: + low = size + pvals[param] = 1.0 - high + return cast("Float64Array", pvals) def _check_data(self) -> None: supported = (np.ndarray, pd.DataFrame, pd.Series) diff --git a/arch/tests/bootstrap/test_bootstrap.py b/arch/tests/bootstrap/test_bootstrap.py index 8ed1bf8ff1..c44c13d7cd 100644 --- a/arch/tests/bootstrap/test_bootstrap.py +++ b/arch/tests/bootstrap/test_bootstrap.py @@ -322,6 +322,135 @@ def test_conf_int_norm(bs_setup): assert_equal(-1 * inf, ci_u[0, :]) +@pytest.mark.parametrize("method", ["basic", "percentile", "norm", "bc", "bca"]) +@pytest.mark.parametrize("alternative", ["two-sided", "less", "greater"]) +def test_p_value_inverts_conf_int(bs_setup, method, alternative): + num_bootstrap = 500 + seed = 1234 + tail = {"two-sided": "two", "less": "upper", "greater": "lower"}[alternative] + bs = IIDBootstrap(bs_setup.x, seed=seed) + null = bs_setup.func(bs_setup.x) + 1.0 + + p = bs.p_value( + bs_setup.func, + null=null, + alternative=alternative, + reps=num_bootstrap, + method=method, + ) + assert p.shape == (2,) + + sizes = np.linspace(0.01, 0.9999, 30) + included = np.zeros((sizes.shape[0], 2), dtype=bool) + for i, size in enumerate(sizes): + ci = bs.conf_int( + bs_setup.func, + reps=num_bootstrap, + size=size, + method=method, + tail=tail, + reuse=True, + ) + if alternative == "less": + included[i] = null <= ci[1] + elif alternative == "greater": + included[i] = null >= ci[0] + else: + included[i] = (ci[0] <= null) & (null <= ci[1]) + + boundary = np.array( + [sizes[included[:, j]].min() if included[:, j].any() else 1.0 for j in range(2)] + ) + assert_allclose(p, 1.0 - boundary, atol=0.05) + + +def test_p_value_studentized_inverts_conf_int(bs_setup): + num_bootstrap = 500 + seed = 1234 + + def std_err_func(params, x): + return x.std(0) / np.sqrt(x.shape[0]) + + bs = IIDBootstrap(bs_setup.x, seed=seed) + null = bs_setup.func(bs_setup.x) + 0.05 + p = bs.p_value( + bs_setup.func, + null=null, + reps=num_bootstrap, + method="studentized", + std_err_func=std_err_func, + ) + + for j in range(2): + assert 0.0 < p[j] < 1.0 + fresh = IIDBootstrap(bs_setup.x, seed=seed) + ci = fresh.conf_int( + bs_setup.func, + reps=num_bootstrap, + size=1.0 - p[j], + method="studentized", + std_err_func=std_err_func, + ) + assert min(abs(ci[0, j] - null[j]), abs(ci[1, j] - null[j])) < 1e-6 + + +@pytest.mark.parametrize("bootstrap_cls", [IIDBootstrap, StationaryBootstrap]) +def test_p_value_block_and_iid_bootstrap(bs_setup, bootstrap_cls): + if bootstrap_cls is StationaryBootstrap: + bs = bootstrap_cls(12, bs_setup.x, seed=1234) + else: + bs = bootstrap_cls(bs_setup.x, seed=1234) + + p = bs.p_value(bs_setup.func, null=[0.0, 0.0], reps=200, method="basic") + assert p.shape == (2,) + assert np.all((p >= 0.0) & (p <= 1.0)) + + +def test_p_value_independent_samples_bootstrap(bs_setup): + def two_means(a, b): + return np.array([a.mean(), b.mean()]) + + bs = IndependentSamplesBootstrap(bs_setup.x[:, 0], bs_setup.x[:, 1], seed=1234) + p = bs.p_value(two_means, null=[0.0, 0.0], reps=200, method="basic") + assert p.shape == (2,) + assert np.all((p >= 0.0) & (p <= 1.0)) + + +def test_p_value_bca_scalar(bs_setup): + bs = IIDBootstrap(bs_setup.x, seed=1234) + p = bs.p_value(np.mean, null=0.0, reps=200, method="bca") + assert p.shape == (1,) + assert np.all((p >= 0.0) & (p <= 1.0)) + + +def test_p_value_rejects_bad_null_before_bootstrap(bs_setup): + bs = IIDBootstrap(bs_setup.x, seed=0) + calls: list[int] = [] + + def counting_func(x): + calls.append(1) + return x.mean(axis=0) + + with pytest.raises(TypeError, match="real scalar or a 1-dimensional array"): + bs.p_value(counting_func, null=None, reps=500) + assert calls == [] + + with pytest.raises(ValueError, match="without NaN"): + bs.p_value(counting_func, null=np.nan, reps=500) + assert calls == [] + + with pytest.raises(TypeError, match="real scalar or a 1-dimensional array"): + bs.p_value(counting_func, null="zero", reps=500) + assert calls == [] + + with pytest.raises(ValueError, match="one value per parameter"): + bs.p_value(counting_func, null=np.zeros(1), reps=500) + assert len(calls) == 1 + + with pytest.raises(ValueError, match="alternative"): + bs.p_value(counting_func, alternative="dragon") + + def test_reuse(bs_setup, seed): num_bootstrap = 100 bs = IIDBootstrap(bs_setup.x, seed=seed) diff --git a/doc/source/bootstrap/confidence-intervals.rst b/doc/source/bootstrap/confidence-intervals.rst index 447faa41f8..bcd0a98cd1 100644 --- a/doc/source/bootstrap/confidence-intervals.rst +++ b/doc/source/bootstrap/confidence-intervals.rst @@ -101,6 +101,29 @@ Parametric Confidence Intervals See :doc:`Parametric Bootstraps ` +P-values +======== + +``p_value`` inverts a bootstrap confidence interval to produce a p-value. It +uses the same interval methods as ``conf_int``, so the p-value matches the +interval by construction whenever both calls share the same bootstrap +replications. Pass ``reuse=True`` to reuse replications from a preceding call, +in the same way as ``conf_int``. + +.. code-block:: python + + from arch.bootstrap import IIDBootstrap + + bs = IIDBootstrap(returns) + p = bs.p_value(sharpe_ratio, null=0.0, reps=1000, method='basic') + +By default ``p_value`` tests the two-sided alternative that a parameter differs +from ``null``. Pass ``alternative='less'`` or ``alternative='greater'`` for a +one-sided test, and pass an array with one value per parameter when the statistic +is vector-valued. The effective resolution of a p-value is about ``1 / reps``; +it is exactly 0 when every replication lies on one side of ``null``. + + Confidence Interval Methods ===========================