From 8e0c108b6040eab98c51c08a58d74fd42d0338ac Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:43:53 +0000 Subject: [PATCH 01/14] ENH: Regenerate the ADF t-test tables from the complete simulation run Run adf_t_simulation_process.py on all 8,000 simulation files, 2,000 per trend with 250,000 draws each. P-values below 0.15 change by at most 0.00001 and the critical values by at most 0.00003. With "c" the cut-off moves from -1.72243 to -1.70276, which changes p by up to 0.0006. --- .../unitroot/critical_values/dickey_fuller.py | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/arch/unitroot/critical_values/dickey_fuller.py b/arch/unitroot/critical_values/dickey_fuller.py index d17ca46206..55ff984e33 100644 --- a/arch/unitroot/critical_values/dickey_fuller.py +++ b/arch/unitroot/critical_values/dickey_fuller.py @@ -369,50 +369,49 @@ } -# t values from new simulations, 250,000 per ex., exercises -# n 1,966, c 1,954, ct 1,947, ctt 1,949. +# t values from new simulations, 250,000 per ex., 2,000 exercises. # One unit root. The critical values and the p-values are fit to the same # simulations, and the p-value at an asymptotic critical value is its size. adf_t_min = {"n": -inf, "c": -inf, "ct": -inf, "ctt": -inf} -adf_t_star = {"n": -1.03884, "c": -1.72243, "ct": -1.92756, "ctt": -2.13529} +adf_t_star = {"n": -1.03884, "c": -1.70276, "ct": -1.92756, "ctt": -2.13529} adf_t_max = {"n": inf, "c": inf, "ct": inf, "ctt": inf} adf_t_small_p = { - "n": [0.6833411665, 1.315029094, 0.07244639622, 0.006663824941], - "c": [2.313783659, 1.593247201, 0.0894645463, 0.005636419168], - "ct": [3.507025793, 1.833226361, 0.1179704595, 0.006853449833], - "ctt": [4.350765654, 1.937712641, 0.1226570161, 0.006592764095], + "n": [0.6836852219, 1.315480901, 0.07263159277, 0.006687123265], + "c": [2.308678903, 1.588493406, 0.08800898388, 0.005489394431], + "ct": [3.505829042, 1.832153392, 0.1176540505, 0.006822842442], + "ctt": [4.350249461, 1.937373616, 0.1225858139, 0.006588008193], } adf_t_large_p = { - "n": [0.4778283114, 0.9445076761, -0.06959215169, 0.02995625396], - "c": [1.724023251, 0.9854879487, -0.01986811093, 0.03160449619], - "ct": [2.65925795, 1.009846397, -0.05183902784, 0.02199224487], - "ctt": [3.282981007, 0.9677512622, -0.08327592936, 0.01320964366], + "n": [0.4778795534, 0.9444210777, -0.06962952399, 0.03000302019], + "c": [1.72315283, 0.9857849826, -0.01406669498, 0.03481596722], + "ct": [2.659007943, 1.008840017, -0.05285077557, 0.02171517328], + "ctt": [3.281745927, 0.9651128956, -0.08502778083, 0.01284826647], } adf_t_cv_approx = { "n": [ - [-2.56583, -2.2208, -3.93978, 0.0], - [-1.94092, -0.29436, -1.50446, 0.0], - [-1.61673, 0.24128, -1.09348, 0.0], + [-2.56583, -2.21905, -3.98064, 0.0], + [-1.94091, -0.29351, -1.53148, 0.0], + [-1.61674, 0.24192, -1.11074, 0.0], ], "c": [ - [-3.43028, -6.5442, -16.78148, -78.88107], - [-2.86153, -2.88378, -4.85339, -27.21776], - [-2.56678, -1.54286, -2.01148, -13.71388], + [-3.4303, -6.54325, -16.79014, -78.74877], + [-2.86153, -2.88397, -4.85708, -26.98494], + [-2.56677, -1.54355, -1.97303, -14.18317], ], "ct": [ - [-3.95852, -9.14587, -23.49649, -192.96108], - [-3.41043, -4.41263, -8.00584, -56.60505], - [-3.12704, -2.59751, -3.33152, -29.29239], + [-3.95851, -9.14992, -23.30656, -194.98745], + [-3.41043, -4.41328, -7.99116, -56.543], + [-3.12704, -2.59659, -3.40084, -28.09664], ], "ctt": [ - [-4.37119, -11.58943, -34.74746, -359.09432], - [-3.83233, -5.91188, -12.44579, -117.08312], - [-3.55323, -3.66531, -5.18029, -62.89369], + [-4.3712, -11.59065, -34.60391, -361.28589], + [-3.83233, -5.90996, -12.50339, -116.72904], + [-3.55323, -3.66321, -5.26853, -61.94961], ], } From f6d1162884b9d4954fc4082b4b026be546225cd8 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:43:54 +0000 Subject: [PATCH 02/14] TST: Update the ADF t-test reference values for the complete run --- arch/tests/unitroot/test_adf_t_pvalue.py | 6 +++--- arch/tests/unitroot/test_unitroot.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/tests/unitroot/test_adf_t_pvalue.py b/arch/tests/unitroot/test_adf_t_pvalue.py index 98f9d39ceb..dcc18141c0 100644 --- a/arch/tests/unitroot/test_adf_t_pvalue.py +++ b/arch/tests/unitroot/test_adf_t_pvalue.py @@ -45,17 +45,17 @@ "ct": [-3.9585, -3.4104, -3.1270, -2.1806, -1.2464], "ctt": [-4.3712, -3.8323, -3.5532, -2.6201, -1.7365], } -# The largest error seen is 3.3e-5 for the 1, 5 and 10 percentiles and 6.3e-3 for the +# The largest error seen is 3.3e-5 for the 1, 5 and 10 percentiles and 6.2e-3 for the # 50 and 90 percentiles (cubic polynomials cannot track the whole distribution) ATOL = np.array([1e-4, 1e-4, 1e-4, 7.5e-3, 7.5e-3]) # Mean over the simulation files of the 1, 5 and 10 percentiles with nobs = 24 and # nobs = 99 observations in the regression (series of 25 and 100 observations), # rounded to 4 decimals. Their standard errors are at most 0.00022. SIMULATED_CRITICAL_VALUES = { - "n": {24: [-2.6649, -1.9557, -1.6086], 99: [-2.5886, -1.9440, -1.6144]}, + "n": {24: [-2.6649, -1.9558, -1.6086], 99: [-2.5887, -1.9440, -1.6144]}, "c": {24: [-3.7379, -2.9922, -2.6356], 99: [-3.4983, -2.8913, -2.5826]}, "ct": {24: [-4.3940, -3.6122, -3.2431], 99: [-4.0535, -3.4558, -3.1536]}, - "ctt": {24: [-4.9400, -4.1087, -3.7194], 99: [-4.4923, -3.8935, -3.5909]}, + "ctt": {24: [-4.9399, -4.1087, -3.7194], 99: [-4.4923, -3.8935, -3.5909]}, } diff --git a/arch/tests/unitroot/test_unitroot.py b/arch/tests/unitroot/test_unitroot.py index 4dd458a731..69ad7e5dc0 100644 --- a/arch/tests/unitroot/test_unitroot.py +++ b/arch/tests/unitroot/test_unitroot.py @@ -73,7 +73,7 @@ def test_adf_no_options(self): adf = ADF(self.inflation) assert_almost_equal(adf.stat, -3.09310, DECIMAL_4) assert_equal(adf.lags, 2) - assert_almost_equal(adf.pvalue, 0.027104, DECIMAL_4) + assert_almost_equal(adf.pvalue, 0.027106, DECIMAL_4) adf.regression.summary() adf2 = ADF(self.inflation, low_memory=True) assert_equal(adf2.lags, 2) From c2fda06cf4f00a9b659b45771943158fed5e6763 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:43:54 +0000 Subject: [PATCH 03/14] DOC: Update the ADF t-test release note for the complete run --- doc/source/changes/9.0.rst | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/source/changes/9.0.rst b/doc/source/changes/9.0.rst index b0603b3f72..817132a866 100644 --- a/doc/source/changes/9.0.rst +++ b/doc/source/changes/9.0.rst @@ -444,26 +444,26 @@ to restore the previous results. :class:`~arch.unitroot.PhillipsPerron` uses with ``test_type="tau"`` and ``mackinnonp`` returns with ``dist_type="adf-t"`` and one unit root, is now a smooth and increasing function of the statistic, and the critical values with one unit root - are fit to the same simulations. The simulations have 486,750,000 to 491,500,000 - draws for each trend. The p-value is approximated by one polynomial below a cut-off - and another above it. These were estimated separately (MacKinnon, 1994), so the - p-value jumped up at the cut-off, by 0.004 with ``trend="n"`` and by up to 0.0006 - otherwise, and jumped up to 1 at the largest statistic covered, by up to 0.003. The - two are now cubics, estimated jointly so that they have the same value and slope at - the cut-off. They are fit to the asymptotic quantiles, which are the constants of the - regressions of the simulated quantiles on powers of 1/nobs, and the critical values - are the regressions of the 1%, 5% and 10% quantiles. The p-values of the asymptotic - critical values are then 1%, 5% and 10%, to 0.00002, where the values of MacKinnon - (1994, 2010) differed by up to 0.0001. The approximation is also more accurate in - the left tail: the largest error of the p-values of the 1%, 5% and 10% quantiles - falls from 0.00008 to 0.00001, and the largest relative error of p-values below 0.15 - from 5.3% to 0.05%. In the middle of the distribution, the largest difference from - the simulated distribution is 0.006 with ``"n"`` and 0.004 with ``"c"`` (0.004 and - 0.003 before), and at most 0.002 otherwise, as before. P-values below 0.15 change by - at most 0.0002, and the largest change is 0.004, at a p-value near 0.6 with - ``"c"``. The critical values change by at most 0.0008 in the smallest samples and by - at most 0.0003 with 100 or more observations. The p-values and critical values with - more than one unit root are unchanged (:issue:`1008`, :issue:`1017`). + are fit to the same simulations, which have 500,000,000 draws for each trend. The + p-value is approximated by one polynomial below a cut-off and another above it. These + were estimated separately (MacKinnon, 1994), so the p-value jumped up at the cut-off, + by 0.004 with ``trend="n"`` and by up to 0.0006 otherwise, and jumped up to 1 at the + largest statistic covered, by up to 0.003. The two are now cubics, estimated jointly + so that they have the same value and slope at the cut-off. They are fit to the + asymptotic quantiles, which are the constants of the regressions of the simulated + quantiles on powers of 1/nobs, and the critical values are the regressions of the 1%, + 5% and 10% quantiles. The p-values of the asymptotic critical values are then 1%, 5% + and 10%, to 0.00002, where the values of MacKinnon (1994, 2010) differed by up to + 0.0001. The approximation is also more accurate in the left tail: the largest error of + the p-values of the 1%, 5% and 10% quantiles falls from 0.00008 to 0.00001, and the + largest relative error of p-values below 0.15 from 5.3% to 0.06%. In the middle of the + distribution, the largest difference from the simulated distribution is 0.006 with + ``"n"`` and 0.004 with ``"c"`` (0.004 and 0.003 before), and at most 0.002 otherwise, + as before. P-values below 0.15 change by at most 0.0002, and the largest change is + 0.005, at a p-value near 0.6 with ``"c"``. The critical values change by at most + 0.0008 in the smallest samples and by at most 0.0003 with 100 or more observations. + The p-values and critical values with more than one unit root are unchanged + (:issue:`1008`, :issue:`1017`). - Integer, unsigned integer and Boolean data are now converted to ``float64`` when they are passed to an estimator, so that no calculation is done in integer arithmetic. Previously a calculation that used the products or differences of integer data could From 8f5ce391a2f3af37e024eb8bfb1e378a826a3d9b Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:40:28 +0000 Subject: [PATCH 04/14] ENH: Fit the ADF z-test p-value to the asymptotic quantiles Regress every simulated quantile of the ADF z-statistic on a constant and powers of 1/nobs, where nobs = T - 1 is the number of observations in the regression, which ADF and PhillipsPerron pass to mackinnoncrit. The simulations were labeled with the series length T, so the critical values were those of a sample one observation smaller. The regressions of the 1%, 5% and 10% quantiles are the critical values, adf_z_cv_approx. The p-value is fit to the constants of all 124 regressions, the asymptotic quantiles, in place of the quantiles with T=2000, so the p-values of the asymptotic critical values are their sizes. Use estimate_response_surface, which drops the highest power of 1/nobs while it is insignificant and fits the regression again, in place of estimate_cv_regression, which set the 1/T**3 terms of the 5% and 10% critical values with "n" and "c" to 0 without refitting the others. The files are sorted so that the averages do not depend on the order of the directory, and the script takes --data-path, --output and --ex-size, and prints the errors of the fits, as adf_t_simulation_process.py does. --- .../simulation/adf_z_simlation_process.py | 158 ++++++++++++++---- 1 file changed, 130 insertions(+), 28 deletions(-) diff --git a/arch/unitroot/critical_values/simulation/adf_z_simlation_process.py b/arch/unitroot/critical_values/simulation/adf_z_simlation_process.py index ceef3f4c67..c5e4ceaee4 100644 --- a/arch/unitroot/critical_values/simulation/adf_z_simlation_process.py +++ b/arch/unitroot/critical_values/simulation/adf_z_simlation_process.py @@ -1,10 +1,84 @@ +""" +Process the ADF z-test simulations into the adf_z values of dickey_fuller.py. + +adf_z_critical_values_simulation_joblib.py writes the quantiles of the z-test +statistic to adf-z/adf_z_-.npz. Run from this directory: + + python adf_z_simlation_process.py + +The statistic of a series of length T is nobs * (gamma - 1), computed from the +nobs = T - 1 observations of the regression of ADF, which is the nobs that ADF and +PhillipsPerron give mackinnoncrit. Each simulated quantile is regressed on a +constant and powers of 1/nobs. The regressions of the 1%, 5% and 10% quantiles +are the critical values. The constants of the regressions are the asymptotic +quantiles, and the p-value is fit to them, so that the p-value at an asymptotic +critical value is its size, up to the error of the fit. The values are written to +the last section of dickey_fuller.py, which is replaced, and the sections before it +are kept. +""" + +import argparse +import os +import pathlib + from adf_simulation import OUTPUT_PATH, PERCENTILES, TIME_SERIES_LENGTHS, TRENDS from black import FileMode, TargetVersion, format_file_contents import numpy as np import pandas as pd -from shared import estimate_cv_regression, fit_smooth_pval_model, format_dict +from scipy import stats +from shared import ( + PvalueResult, + estimate_response_surface, + fit_smooth_pval_model, + format_dict, +) + +Z_MARKER = "# Z values from" + +parser = argparse.ArgumentParser(description="Process the ADF z-test simulations") +parser.add_argument( + "--data-path", + default=str(OUTPUT_PATH), + help="Directory containing the adf_z_-.npz files", +) +parser.add_argument( + "--output", + default=os.path.join("..", "dickey_fuller.py"), + help="The dickey_fuller.py to update", +) +# The files hold quantiles only, so the number of simulations per file, EX_SIZE in +# adf_z_critical_values_simulation_joblib.py, is only known from the run +parser.add_argument( + "--ex-size", + type=int, + default=250_000, + help="Simulations per file, for the comment in the output", +) +args = parser.parse_args() +output = pathlib.Path(args.output) + +def pvalue(stat: np.ndarray, pvals: PvalueResult) -> np.ndarray: + # The small p polynomial is in log|stat|, and tau_star is negative + small, large = np.array(pvals.small_p), np.array(pvals.large_p) + below = stat <= pvals.tau_star + with np.errstate(divide="ignore", over="ignore"): + return stats.norm.cdf( + np.where( + below, + np.polyval(small[::-1], np.log(np.abs(np.where(below, stat, -1.0)))), + np.polyval(large[::-1], stat), + ) + ) + + +data_path = pathlib.Path(args.data_path) critical_values = (1.0, 5.0, 10.0) +cv_rows = [int(np.argmin(np.abs(PERCENTILES - cv))) for cv in critical_values] +nobs = TIME_SERIES_LENGTHS - 1 +rhs = (1.0 / nobs[:, None]) ** np.arange(4) +probs = PERCENTILES / 100.0 +counts = {} adf_z_cv_approx = {} adf_z_max = {} adf_z_min = {} @@ -13,51 +87,79 @@ adf_z_large_p = {} for trend in TRENDS: - data_files = list(OUTPUT_PATH.glob(f"adf_z_{trend}-*.npz")) - percentiles = PERCENTILES + # Sorted so that the averages do not depend on the order of the directory + data_files = sorted(data_path.glob(f"adf_z_{trend}-*.npz")) + if not data_files: + raise FileNotFoundError(f"No adf_z_{trend}-*.npz files in {data_path}") all_results = [] for df in data_files: with np.load(df) as data: all_results.append(data["results"]) results = np.hstack(all_results) - cols = TIME_SERIES_LENGTHS.tolist() * len(data_files) - results_df = pd.DataFrame( - results, index=pd.Index(PERCENTILES / 100.0), columns=cols - ) + cols = nobs.tolist() * len(data_files) + results_df = pd.DataFrame(results, index=pd.Index(probs), columns=cols) + counts[trend] = len(data_files) - cv_approx = estimate_cv_regression(results_df, critical_values) + surface = estimate_response_surface(results_df) adf_z_cv_approx[trend] = [ - [float(val) for val in cv_approx[cv]] for cv in critical_values + [float(val) for val in surface.iloc[row]] for row in cv_rows ] + asymptotic = surface.iloc[:, :1] # The two p-value polynomials are fit jointly so that the p-value is continuous, # has a continuous first derivative and is monotone in the statistic - pvals = fit_smooth_pval_model(results_df[2000]) + pvals = fit_smooth_pval_model(asymptotic) adf_z_max[trend] = pvals.tau_max adf_z_min[trend] = pvals.tau_min adf_z_star[trend] = float(pvals.tau_star) adf_z_small_p[trend] = [float(val) for val in pvals.small_p] adf_z_large_p[trend] = [float(val) for val in pvals.large_p] -formatted_code = "adf_z_min = " + format_dict(adf_z_min) -formatted_code += "\n\nadf_z_star = " + format_dict(adf_z_star) -formatted_code += "\n\nadf_z_max = " + format_dict(adf_z_max) -formatted_code += "\n\n# The small p parameters are for np.log(np.abs(stat))\n" -formatted_code += "adf_z_small_p = " + format_dict(adf_z_small_p) -formatted_code += "\n\nadf_z_large_p = " + format_dict(adf_z_large_p) -formatted_code += "\n\nadf_z_cv_approx = " + format_dict(adf_z_cv_approx) - -with open("../dickey_fuller.py") as cvs: - lines = cvs.readlines() + # The fit at the asymptotic quantiles that it is estimated from + err = np.abs(pvalue(asymptotic.iloc[:, 0].to_numpy(), pvals) - probs) + tail = probs <= 0.15 + print( + f"{trend}: {len(data_files)} files, tau_star {pvals.tau_star}, largest " + f"error in p {err[tail].max():.1e} if p <= 0.15 and {err[~tail].max():.4f} " + f"otherwise, relative error at p={probs[0]:g} {err[0] / probs[0]:.2%}" + ) + asymptotic_cv = np.array([cv[0] for cv in adf_z_cv_approx[trend]]) + print( + " p-value at the asymptotic critical values: " + + ", ".join(f"{val:.6f}" for val in pvalue(asymptotic_cv, pvals)) + ) + # The critical values compared with the average simulated quantiles + for j, cv in enumerate(critical_values): + grouped = results_df.iloc[cv_rows[j]].groupby(level=0) + mean = grouped.mean().to_numpy() + se = (grouped.std() / np.sqrt(grouped.count())).to_numpy() + fit_err = rhs @ np.array(adf_z_cv_approx[trend][j]) - mean + worst_fit = int(np.argmax(np.abs(fit_err / se))) + print( + f" {cv:g}%: {adf_z_cv_approx[trend][j]}, largest error " + f"{fit_err[worst_fit]:+.4f} at nobs={nobs[worst_fit]} " + f"({fit_err[worst_fit] / se[worst_fit]:+.1f} s.e.)" + ) -retain = [] -for line in lines: - if "# Z values from" in line: - break - retain.append(line) -retain.append("\n\n# Z values from new simulations, 2,000 exercises, 250,000 per ex.\n") +z_section = f"{Z_MARKER} new simulations, {args.ex_size:,} per ex., " +if len(set(counts.values())) == 1: + z_section += f"{counts[TRENDS[0]]:,} exercises.\n" +else: + z_section += "exercises\n# " + ", ".join(f"{t} {c:,}" for t, c in counts.items()) + z_section += ".\n" +z_section += "# The critical values and the p-values are fit to the same simulations,\n" +z_section += "# and the p-value at an asymptotic critical value is its size.\n" +z_section += "adf_z_min = " + format_dict(adf_z_min) +z_section += "\n\nadf_z_star = " + format_dict(adf_z_star) +z_section += "\n\nadf_z_max = " + format_dict(adf_z_max) +z_section += "\n\n# The small p parameters are for np.log(np.abs(stat))\n" +z_section += "adf_z_small_p = " + format_dict(adf_z_small_p) +z_section += "\n\nadf_z_large_p = " + format_dict(adf_z_large_p) +z_section += "\n\nadf_z_cv_approx = " + format_dict(adf_z_cv_approx) -formatted_code = "".join(retain) + formatted_code +lines = output.read_text().splitlines(keepends=True) +z_start = next(i for i, line in enumerate(lines) if line.startswith(Z_MARKER)) +formatted_code = "".join(lines[:z_start]) + "\n\n" + z_section targets = { TargetVersion.PY310, @@ -70,5 +172,5 @@ formatted_code = format_file_contents(formatted_code, fast=False, mode=fm) # LF line endings on every platform, as in the repository -with open("../dickey_fuller.py", "w", newline="\n") as cvs: +with open(output, "w", newline="\n") as cvs: cvs.write(formatted_code) From 964e08b783f9dc9cd6ae296dca8c2f9a88fb8b70 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:40:46 +0000 Subject: [PATCH 05/14] ENH: Regenerate the ADF z-test tables Run adf_z_simlation_process.py on the 2,000 simulation files of each trend (250,000 draws each). The critical values are fit in 1/nobs, and the p-value to the asymptotic quantiles, so the p-values of the asymptotic critical values are within 0.00008 of 1%, 5% and 10%, where they were as low as 0.0097, 0.0491 and 0.0987. --- .../unitroot/critical_values/dickey_fuller.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/arch/unitroot/critical_values/dickey_fuller.py b/arch/unitroot/critical_values/dickey_fuller.py index 55ff984e33..f32f1895f2 100644 --- a/arch/unitroot/critical_values/dickey_fuller.py +++ b/arch/unitroot/critical_values/dickey_fuller.py @@ -416,47 +416,49 @@ } -# Z values from new simulations, 2,000 exercises, 250,000 per ex. +# Z values from new simulations, 250,000 per ex., 2,000 exercises. +# The critical values and the p-values are fit to the same simulations, +# and the p-value at an asymptotic critical value is its size. adf_z_min = {"n": -inf, "c": -inf, "ct": -inf, "ctt": -inf} -adf_z_star = {"n": -4.19211, "c": -8.14944, "ct": -13.099, "ctt": -15.93082} +adf_z_star = {"n": -4.0064, "c": -8.16013, "ct": -13.12342, "ctt": -16.38077} adf_z_max = {"n": inf, "c": inf, "ct": inf, "ctt": inf} # The small p parameters are for np.log(np.abs(stat)) adf_z_small_p = { - "n": [0.1128585132, -0.7519533397, 0.05363711911, -0.04697405824], - "c": [2.088134483, -1.636716716, 0.2693618616, -0.06974896038], - "ct": [4.111050235, -2.389235384, 0.427668324, -0.08437837715], - "ctt": [5.673700637, -2.973275585, 0.5518855813, -0.09562857546], + "n": [0.1062408749, -0.7407032906, 0.04794614031, -0.04588555161], + "c": [2.044724996, -1.584478157, 0.2484627075, -0.06678450574], + "ct": [4.13107837, -2.402397513, 0.4292874893, -0.08397972402], + "ctt": [5.80363306, -3.072930766, 0.5755748454, -0.09702934255], } adf_z_large_p = { - "n": [0.5094184144, 0.6839706602, 0.120355783, 0.01018310207], - "c": [1.716168045, 0.5500431791, 0.04263836138, 0.001606489684], - "ct": [2.704353828, 0.4512169076, 0.0212562381, 0.0004817109902], - "ctt": [3.429989758, 0.4149740654, 0.01567632693, 0.000285455431], + "n": [0.5085359126, 0.6825608646, 0.1221162317, 0.01066952761], + "c": [1.717115673, 0.5501294487, 0.0426198626, 0.001601767266], + "ct": [2.705575065, 0.4514206297, 0.02130532041, 0.0004831813997], + "ctt": [3.428402762, 0.4139435455, 0.01556637139, 0.0002808289518], } adf_z_cv_approx = { "n": [ - [-13.69464, 63.39558, -188.65735, 286.67928], - [-8.03941, 26.16037, -55.17125, 0.0], - [-5.71384, 15.38915, -23.26257, 0.0], + [-13.69457, 63.35494, -247.92292, 591.36974], + [-8.03939, 26.15035, -80.25024, 175.36092], + [-5.71369, 15.32536, -34.85607, 0.0], ], "c": [ - [-20.6251, 119.00752, -422.57873, 743.47694], - [-14.0929, 57.98623, -137.0056, 0.0], - [-11.25026, 38.2072, -70.72371, 0.0], + [-20.62498, 118.92923, -532.7576, 1394.08562], + [-14.09288, 57.97266, -193.30391, 362.3315], + [-11.25026, 38.20468, -108.45849, 164.38619], ], "ct": [ - [-29.35612, 232.23486, -1170.13934, 3209.13496], - [-21.71065, 130.0962, -500.98106, 1113.20613], - [-18.24497, 93.26993, -305.55437, 585.03699], + [-29.3556, 231.91837, -1368.25043, 4604.98636], + [-21.71045, 129.97569, -618.20581, 1804.67391], + [-18.24487, 93.2048, -391.76978, 1045.62368], ], "ctt": [ - [-36.59763, 354.7435, -2242.38386, 7798.9413], - [-28.11149, 212.35851, -1062.25936, 3096.6951], - [-24.1856, 158.61472, -696.85452, 1863.55439], + [-36.59641, 354.00743, -2518.58072, 9948.58706], + [-28.11099, 212.05398, -1241.9985, 4313.92147], + [-24.18529, 158.42745, -835.35719, 2724.38175], ], } From e3e40a6160e857dd6593c25e403b248a6214f684 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:45:36 +0000 Subject: [PATCH 06/14] TST: Test the ADF z-test critical values and their p-values Check that the p-values of the asymptotic critical values are 1%, 5% and 10%, and that the critical values match the simulated quantiles with 24, 99 and infinitely many observations in the regression. The p-value reference quantiles are now the asymptotic quantiles. --- arch/tests/unitroot/test_adf_z_pvalue.py | 67 +++++++++++++++++++----- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/arch/tests/unitroot/test_adf_z_pvalue.py b/arch/tests/unitroot/test_adf_z_pvalue.py index 70ffb724e7..4cebcdd73b 100644 --- a/arch/tests/unitroot/test_adf_z_pvalue.py +++ b/arch/tests/unitroot/test_adf_z_pvalue.py @@ -1,9 +1,12 @@ """ -Tests that the ADF-Z p-value is a smooth function of the test statistic. +Tests of the ADF-Z p-value and critical values. The p-value is a cubic in log|stat| below adf_z_star and a cubic in stat above it. The two pieces are estimated jointly so that they have the same value and the same -slope at adf_z_star, and the p-value is monotone in the statistic. +slope at adf_z_star, and the p-value is monotone in the statistic. The critical +values are fit to the same simulations, and the p-value is fit to the asymptotic +quantiles that are the constants of the critical value regressions, so the p-values +of the asymptotic critical values are 1%, 5% and 10%. """ import importlib.util @@ -15,26 +18,40 @@ import pytest from scipy import stats -from arch.unitroot.critical_values.dickey_fuller import adf_z_star -from arch.unitroot.unitroot import mackinnonp +from arch.unitroot.critical_values.dickey_fuller import adf_z_cv_approx, adf_z_star +from arch.unitroot.unitroot import mackinnoncrit, mackinnonp TRENDS = ["n", "c", "ct", "ctt"] -# Mean over the 2,000 simulation files adf_z_-NNNN.npz (written by the -# adf_z_critical_values_simulation scripts in critical_values/simulation) of the 1, 5, -# 10, 50 and 90 percentiles of the ADF-Z statistic with T=2000 observations, rounded -# to 4 decimals. These are the data that the p-value models are fit to, and are the +# The asymptotic 1, 5, 10, 50 and 90 percentiles of the ADF-Z statistic, rounded to 4 +# decimals. These are the constants of the regressions of the percentiles in the 2,000 +# simulation files adf_z_-NNNN.npz (written by +# adf_z_critical_values_simulation_joblib.py in critical_values/simulation) on powers +# of 1/nobs, which adf_z_simlation_process.py fits the p-value models to. They are the # reference for the accuracy of the fitted p-values. PERCENTILES = np.array([0.01, 0.05, 0.10, 0.50, 0.90]) SIMULATED_QUANTILES = { - "n": [-13.6630, -8.0260, -5.7053, -0.8518, 0.9285], - "c": [-20.5667, -14.0644, -11.2311, -4.3534, -0.8439], - "ct": [-29.2396, -21.6459, -18.1980, -9.0901, -3.7642], - "ctt": [-36.4242, -28.0074, -24.1075, -13.3801, -6.6372], + "n": [-13.6946, -8.0394, -5.7137, -0.8528, 0.9280], + "c": [-20.6250, -14.0929, -11.2503, -4.3570, -0.8455], + "ct": [-29.3556, -21.7105, -18.2449, -9.1027, -3.7673], + "ctt": [-36.5964, -28.1110, -24.1853, -13.4049, -6.6436], } -# The largest error seen is 7.8e-5 for the 1, 5 and 10 percentiles and 3.1e-3 for the +# The largest error seen is 7.7e-5 for the 1, 5 and 10 percentiles and 4.6e-3 for the # 50 and 90 percentiles (cubic polynomials cannot track the whole distribution) -ATOL = np.array([1.5e-4, 1.5e-4, 1.5e-4, 5e-3, 5e-3]) +ATOL = np.array([1e-4, 1e-4, 1e-4, 7.5e-3, 7.5e-3]) +# Mean over the simulation files of the 1, 5 and 10 percentiles with nobs = 24 and +# nobs = 99 observations in the regression (series of 25 and 100 observations), +# rounded to 4 decimals. Their standard errors are at most 0.0019. +SIMULATED_CRITICAL_VALUES = { + "n": {24: [-11.4410, -7.0767, -5.1360], 99: [-13.0796, -7.7830, -5.5619]}, + "c": {24: [-16.4941, -11.9877, -9.8359], 99: [-19.4769, -13.5272, -10.8754]}, + "ct": {24: [-21.7334, -17.2364, -14.9653], 99: [-27.1485, -20.4596, -17.3423]}, + "ctt": {24: [-25.4915, -21.1170, -18.8359], 99: [-33.2677, -26.0924, -22.6684]}, +} +# The largest error of the critical value regressions is 0.0075 with "ctt" at 1% and +# nobs = 24 (7 standard errors of the simulated percentile), and 0.0014 (2.5 standard +# errors) otherwise. The asymptotic critical values are the reference quantiles. +CV_ATOL = {24: 1e-2, 99: 2.5e-3, np.inf: 1e-4} # Wide enough that the p-value is 0 or 1 to machine precision at the ends GRID_LIMITS = {"n": -200.0, "c": -200.0, "ct": -400.0, "ctt": -600.0} @@ -112,6 +129,28 @@ def test_pvalue_matches_simulated_distribution(trend): assert np.all(np.abs(pvals - PERCENTILES) <= ATOL) +@pytest.mark.parametrize("trend", TRENDS) +def test_pvalue_at_asymptotic_critical_values(trend): + # The p-value is fit to the constants of the critical value regressions + critical_values = mackinnoncrit(1, trend, np.inf, "adf-z") + pvals = [adf_z_pvalue(float(cv), trend) for cv in critical_values] + assert_allclose(pvals, [0.01, 0.05, 0.10], rtol=1e-3) + + +@pytest.mark.parametrize("trend", TRENDS) +@pytest.mark.parametrize("nobs", [24, 99, np.inf]) +def test_critical_values_match_simulated_distribution(trend, nobs): + coef = np.array(adf_z_cv_approx[trend]) + critical_values = mackinnoncrit(1, trend, nobs, "adf-z") + if nobs == np.inf: + assert_allclose(critical_values, coef[:, 0]) + expected = SIMULATED_QUANTILES[trend][:3] + else: + assert_allclose(critical_values, np.polyval(coef[:, ::-1].T, 1.0 / nobs)) + expected = SIMULATED_CRITICAL_VALUES[trend][nobs] + assert_allclose(critical_values, expected, rtol=0, atol=CV_ATOL[nobs]) + + @pytest.fixture(scope="module") def simulation_shared(): path = ( From bc8f370d64f9c6f97a7fe7c8013454dfc620c20a Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:45:37 +0000 Subject: [PATCH 07/14] DOC: Describe the ADF z-test critical values in mackinnoncrit The ADF z-test critical values are now fit to the same simulations as the p-values, and the p-values of the asymptotic critical values are 1%, 5% and 10%, as for the ADF t-test with one unit root. --- arch/unitroot/unitroot.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/unitroot/unitroot.py b/arch/unitroot/unitroot.py index cacc49cc16..92b6341f6d 100644 --- a/arch/unitroot/unitroot.py +++ b/arch/unitroot/unitroot.py @@ -2396,11 +2396,11 @@ def mackinnoncrit( Notes ----- - The values for the ADF t-test with one unit root are fit to the same - simulations as the p-values of mackinnonp, and the p-values of the asymptotic - critical values are 1%, 5% and 10%. The values for the ADF z-test and the - DFGLS test are also fit to new simulations. The values for the ADF t-test with - more than one unit root are from MacKinnon (2010). + The values for the ADF t-test with one unit root and for the ADF z-test are fit + to the same simulations as the p-values of mackinnonp, and the p-values of the + asymptotic critical values are 1%, 5% and 10%. The values for the DFGLS test + are also fit to new simulations. The values for the ADF t-test with more than + one unit root are from MacKinnon (2010). References ---------- From dabb6272924adbe82f8d36b27472d83ed07cf84b Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:47:02 +0000 Subject: [PATCH 08/14] DOC: Revise the ADF z-test release notes for the new critical values The critical values are now fit to the same simulations as the p-values, in powers of 1/nobs, and the p-value to the asymptotic quantiles. Update the numbers of the ADF-Z entries, which compare with 8.0. --- doc/source/changes/9.0.rst | 44 +++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/doc/source/changes/9.0.rst b/doc/source/changes/9.0.rst index 817132a866..5d11c5fd42 100644 --- a/doc/source/changes/9.0.rst +++ b/doc/source/changes/9.0.rst @@ -396,26 +396,40 @@ to restore the previous results. ``test_type="rho"`` and ``trend="n"``, and which ``mackinnoncrit`` and ``mackinnonp`` return with ``dist_type="adf-z"``. The values are from a new simulation that starts the random walk at 0. The 1%, 5% and 10% critical values with 100 observations change - from -10.78, -6.13 and -4.27 to -13.08, -7.78 and -5.56, and the asymptotic ones from + from -10.78, -6.13 and -4.27 to -13.09, -7.79 and -5.56, and the asymptotic ones from -13.30, -7.83 and -5.57 to -13.69, -8.04 and -5.71. In small samples the previous - critical values were wrong: with 10 observations they were 20.32, 13.84 and 10.54, - and are now -8.95, -5.98 and -4.41. For example the p-value of a statistic of -8 - changes from 0.0477 to 0.0504. The tables with a constant or a trend are also from the - new simulation. Their critical values change by less than 0.005 with 25 or more - observations and their p-values below 0.15 by at most 0.0001. Contributed by mkzung + critical values were wrong: with 10 observations they were 20.32, 13.84 and 10.54, and + are now -9.25, -6.05 and -4.53. For example the p-value of a statistic of -8 changes + from 0.0477 to 0.0506. The tables with a constant or a trend are also from the new + simulation, and their changes are described in the next item. Contributed by mkzung (:issue:`971`, :issue:`972`, :issue:`973`, :issue:`980`). - The p-value of the ADF-Z statistic, which :class:`~arch.unitroot.PhillipsPerron` uses with ``test_type="rho"`` and ``mackinnonp`` returns with ``dist_type="adf-z"``, is now - a smooth and increasing function of the statistic. It is approximated by one - polynomial below a cut-off and another above it. These were estimated separately, so - the p-value jumped down at the cut-off, by 0.033 with ``"n"``, 0.014 with ``"c"``, - 0.010 with ``"ct"`` and 0.008 with ``"ctt"``. The two are now estimated jointly, so - that they have the same value and slope at the cut-off. The approximation of the - middle of the distribution is also more accurate: the largest difference from the - simulated distribution is now 0.012 with ``"n"`` and at most 0.0015 otherwise. + a smooth and increasing function of the statistic, and the critical values are fit to + the same simulations. The p-value is approximated by one polynomial below a cut-off + and another above it. These were estimated separately, so the p-value jumped down at + the cut-off, by 0.033 with ``"n"``, 0.014 with ``"c"``, 0.010 with ``"ct"`` and 0.008 + with ``"ctt"``. The two are now estimated jointly, so that they have the same value + and slope at the cut-off. They are fit to the asymptotic quantiles, which are the + constants of the regressions of the simulated quantiles on powers of 1/nobs, and the + critical values are the regressions of the 1%, 5% and 10% quantiles. The p-values of + the asymptotic critical values are then 1%, 5% and 10%, to 0.00008, where they were as + low as 0.0097, 0.0491 and 0.0987 with ``"ctt"``. The approximation is also more + accurate in the left tail: the largest relative error of p-values below 0.15 is now + 0.13% with ``"n"`` and at most 0.07% otherwise, where it was up to 6.0% with a + constant or a trend. In the middle of the distribution, the largest difference from + the simulated distribution is 0.012 with ``"n"`` and at most 0.0017 otherwise. Compared with 8.0, with a constant or a trend, p-values below 0.15 change by at most - 0.0001, and the largest change is 0.013, at p-values between 0.7 and 0.8. The - critical values are unchanged (:issue:`989`). + 0.0016, and the largest change is 0.012, at p-values between 0.7 and 0.8. The critical + values were regressions on powers of one over the length of the series, which is one + more than the number of observations in the regression of the test that + :class:`~arch.unitroot.PhillipsPerron` passes to ``mackinnoncrit``, so they were those + of a sample with one fewer observation. They are now regressions on powers of 1/nobs. + With 25 observations in the regression, the 1% critical value with ``"ctt"`` changes + from -25.49 to -25.83, and with 100 observations from -33.27 to -33.30. With a + constant or a trend, the critical values change by up to 0.49 with 19 observations, by + at most 0.032 with 100 or more, and the asymptotic ones by at most 0.002 + (:issue:`989`, :issue:`1017`). - The p-value of :func:`~arch.unitroot.cointegration.engle_granger` is now a smooth and increasing function of the statistic. It is approximated by one polynomial below a cut-off and another above it. These were estimated separately, so the p-value jumped From 53aa58c613d94c8c4d8c82b6f1f48f3fcbe75181 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:59:44 +0000 Subject: [PATCH 09/14] DOC: Say which nobs engle_granger_cv expects The Engle-Granger critical values are fit to simulations in the number of observations of the ADF regression of the residuals, which engle_granger passes. The docstring said the number of observations in the time series. --- arch/unitroot/_engle_granger.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/unitroot/_engle_granger.py b/arch/unitroot/_engle_granger.py index c06b2de4ac..d8a4f3832e 100644 --- a/arch/unitroot/_engle_granger.py +++ b/arch/unitroot/_engle_granger.py @@ -322,7 +322,9 @@ def engle_granger_cv(trend: str, num_x: int, nobs: int) -> pd.Series: The number of cross-sectional regressors in the model. Must be between 1 and 11. nobs : int - The number of observations in the time series. + The number of observations in the ADF regression of the residuals, which + is the number of observations in the time series less the number of lags + less 1. The critical values are fit to simulations in this nobs. Returns ------- From c80b83cc7e31c85036bce49f1297fa50e21c3450 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 13:59:45 +0000 Subject: [PATCH 10/14] DOC: Cite #1018 in the ADF t-test and z-test release notes --- doc/source/changes/9.0.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/changes/9.0.rst b/doc/source/changes/9.0.rst index 5d11c5fd42..7c58e87620 100644 --- a/doc/source/changes/9.0.rst +++ b/doc/source/changes/9.0.rst @@ -429,7 +429,7 @@ to restore the previous results. from -25.49 to -25.83, and with 100 observations from -33.27 to -33.30. With a constant or a trend, the critical values change by up to 0.49 with 19 observations, by at most 0.032 with 100 or more, and the asymptotic ones by at most 0.002 - (:issue:`989`, :issue:`1017`). + (:issue:`989`, :issue:`1018`). - The p-value of :func:`~arch.unitroot.cointegration.engle_granger` is now a smooth and increasing function of the statistic. It is approximated by one polynomial below a cut-off and another above it. These were estimated separately, so the p-value jumped @@ -477,7 +477,7 @@ to restore the previous results. 0.005, at a p-value near 0.6 with ``"c"``. The critical values change by at most 0.0008 in the smallest samples and by at most 0.0003 with 100 or more observations. The p-values and critical values with more than one unit root are unchanged - (:issue:`1008`, :issue:`1017`). + (:issue:`1008`, :issue:`1017`, :issue:`1018`). - Integer, unsigned integer and Boolean data are now converted to ``float64`` when they are passed to an estimator, so that no calculation is done in integer arithmetic. Previously a calculation that used the products or differences of integer data could From 9bb4bb21bea7a195db0422a22d897e9e2cbde622 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 14:12:08 +0000 Subject: [PATCH 11/14] ENH: Allow a quartic small p polynomial in fit_smooth_pval_model small_order may be 5. The junction constraints and the monotonicity checks already handle any order. Test a quartic fit and its constraints, and that orders 2 and 6 are rejected. --- arch/tests/unitroot/test_smooth_pvalue_fit.py | 12 +++++++----- arch/unitroot/critical_values/simulation/shared.py | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/tests/unitroot/test_smooth_pvalue_fit.py b/arch/tests/unitroot/test_smooth_pvalue_fit.py index 39552ceeb5..4b79d4e5ff 100644 --- a/arch/tests/unitroot/test_smooth_pvalue_fit.py +++ b/arch/tests/unitroot/test_smooth_pvalue_fit.py @@ -48,6 +48,7 @@ ), # Similar to ADF-Z "exponential-log": (-4.0 * np.exp(-0.7 * NORMAL_QUANTILES), 4, True), + "exponential-log-quartic": (-4.0 * np.exp(-0.7 * NORMAL_QUANTILES), 5, True), } MAX_TAIL_STEP = 1e-12 @@ -203,16 +204,17 @@ def test_fit_significant_digits(simulation_shared, digits): assert float(f"{value:.{digits - 1}e}") == value -def test_fit_small_order_invalid(simulation_shared): +@pytest.mark.parametrize("small_order", [2, 6]) +def test_fit_small_order_invalid(simulation_shared, small_order): quantiles = simulated(CASES["exponential-log"][0]) - with pytest.raises(ValueError, match="small_order must be 3 or 4"): - simulation_shared.fit_smooth_pval_model(quantiles, small_order=5) + with pytest.raises(ValueError, match="small_order must be 3, 4 or 5"): + simulation_shared.fit_smooth_pval_model(quantiles, small_order=small_order) @pytest.mark.parametrize( ("small_order", "use_log"), - [(3, False), (4, False), (3, True), (4, True)], - ids=["quadratic", "cubic", "log-quadratic", "log-cubic"], + [(3, False), (4, False), (5, False), (3, True), (4, True), (5, True)], + ids=["quadratic", "cubic", "quartic", "log-quadratic", "log-cubic", "log-quartic"], ) def test_junction_constraints(simulation_shared, small_order, use_log): rng = np.random.default_rng(1234) diff --git a/arch/unitroot/critical_values/simulation/shared.py b/arch/unitroot/critical_values/simulation/shared.py index 3c4ef57935..5cbbb3afd6 100644 --- a/arch/unitroot/critical_values/simulation/shared.py +++ b/arch/unitroot/critical_values/simulation/shared.py @@ -355,8 +355,8 @@ def fit_smooth_pval_model( digits must be large enough that rounding does not make the p-value discontinuous. small_order : int, default 4 - The number of coefficients of the small p polynomial, 3 for a quadratic - and 4 for a cubic. + The number of coefficients of the small p polynomial, 3 for a quadratic, + 4 for a cubic and 5 for a quartic. use_log : bool, default True Whether the small p polynomial is in log|stat|. If False, it is in stat. max_tail_step : float, optional @@ -374,8 +374,8 @@ def fit_smooth_pval_model( below and above which the p-value is 0 and 1. They are -inf and inf when both tails are monotone and tend to 0 and 1. """ - if small_order not in (3, 4): - raise ValueError("small_order must be 3 or 4") + if small_order not in (3, 4, 5): + raise ValueError("small_order must be 3, 4 or 5") quantiles = quantiles.sort_index(ascending=False) probs = quantiles.index.to_numpy(dtype=float) data = np.asarray(quantiles, dtype=float) From 4a4d8fb680a6db954708cb8e9404bc091b03e68e Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 14:12:09 +0000 Subject: [PATCH 12/14] ENH: Fit the ADF z-test small p-value with a quartic in log|stat| With a cubic, the p-values of the asymptotic 10% critical values were off by up to 7.6e-5. With a quartic they are within 4.2e-5 of 1%, 5% and 10% with "n" and within 5e-6 otherwise. The largest error of p-values below 0.15 falls from 2.0e-4 to 6.9e-5 with "n" and from 8.4e-5 to 1.4e-5 or less otherwise. In the body the largest error is 0.013 with "n" (0.012 with the cubic) and at most 0.0015 otherwise (0.0017). The critical values are unchanged. --- arch/unitroot/critical_values/dickey_fuller.py | 18 +++++++++--------- .../simulation/adf_z_simlation_process.py | 5 ++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/arch/unitroot/critical_values/dickey_fuller.py b/arch/unitroot/critical_values/dickey_fuller.py index f32f1895f2..77735cd5ab 100644 --- a/arch/unitroot/critical_values/dickey_fuller.py +++ b/arch/unitroot/critical_values/dickey_fuller.py @@ -421,23 +421,23 @@ # and the p-value at an asymptotic critical value is its size. adf_z_min = {"n": -inf, "c": -inf, "ct": -inf, "ctt": -inf} -adf_z_star = {"n": -4.0064, "c": -8.16013, "ct": -13.12342, "ctt": -16.38077} +adf_z_star = {"n": -1.83702, "c": -3.14129, "ct": -5.93276, "ctt": -8.24519} adf_z_max = {"n": inf, "c": inf, "ct": inf, "ctt": inf} # The small p parameters are for np.log(np.abs(stat)) adf_z_small_p = { - "n": [0.1062408749, -0.7407032906, 0.04794614031, -0.04588555161], - "c": [2.044724996, -1.584478157, 0.2484627075, -0.06678450574], - "ct": [4.13107837, -2.402397513, 0.4292874893, -0.08397972402], - "ctt": [5.80363306, -3.072930766, 0.5755748454, -0.09702934255], + "n": [-0.1780983743, -0.2224742066, -0.2985627781, 0.05499834644, -0.01080878312], + "c": [1.186687586, -0.3554158055, -0.4073498794, 0.0877299448, -0.01356550117], + "ct": [2.370196501, -0.208289258, -0.5926765005, 0.1269131906, -0.0162693314], + "ctt": [3.238426923, -0.1006542359, -0.7131286635, 0.1507726862, -0.01783062046], } adf_z_large_p = { - "n": [0.5085359126, 0.6825608646, 0.1221162317, 0.01066952761], - "c": [1.717115673, 0.5501294487, 0.0426198626, 0.001601767266], - "ct": [2.705575065, 0.4514206297, 0.02130532041, 0.0004831813997], - "ctt": [3.428402762, 0.4139435455, 0.01556637139, 0.0002808289518], + "n": [0.5060346197, 0.7114864934, 0.116003454, 0.0005506503572], + "c": [1.719416611, 0.5435502671, 0.0349093361, 6.449528063e-05], + "ct": [2.683617312, 0.4310766477, 0.01600015499, 7.659418533e-05], + "ctt": [3.396767671, 0.3966952367, 0.01267343771, 0.0001299056083], } adf_z_cv_approx = { diff --git a/arch/unitroot/critical_values/simulation/adf_z_simlation_process.py b/arch/unitroot/critical_values/simulation/adf_z_simlation_process.py index c5e4ceaee4..43a898aa40 100644 --- a/arch/unitroot/critical_values/simulation/adf_z_simlation_process.py +++ b/arch/unitroot/critical_values/simulation/adf_z_simlation_process.py @@ -33,6 +33,9 @@ format_dict, ) +# The small p polynomial is a quartic in log|stat|. A cubic misses the asymptotic +# 10% critical value by up to 7.6e-5 in p with "n". +SMALL_ORDER = 5 Z_MARKER = "# Z values from" parser = argparse.ArgumentParser(description="Process the ADF z-test simulations") @@ -108,7 +111,7 @@ def pvalue(stat: np.ndarray, pvals: PvalueResult) -> np.ndarray: # The two p-value polynomials are fit jointly so that the p-value is continuous, # has a continuous first derivative and is monotone in the statistic - pvals = fit_smooth_pval_model(asymptotic) + pvals = fit_smooth_pval_model(asymptotic, small_order=SMALL_ORDER) adf_z_max[trend] = pvals.tau_max adf_z_min[trend] = pvals.tau_min adf_z_star[trend] = float(pvals.tau_star) From 0841a1cfdce5822d19ae1db438635ea455726a9a Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 14:12:09 +0000 Subject: [PATCH 13/14] TST: Describe the quartic ADF z-test p-value in its tests --- arch/tests/unitroot/test_adf_z_pvalue.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/tests/unitroot/test_adf_z_pvalue.py b/arch/tests/unitroot/test_adf_z_pvalue.py index 4cebcdd73b..050cef8232 100644 --- a/arch/tests/unitroot/test_adf_z_pvalue.py +++ b/arch/tests/unitroot/test_adf_z_pvalue.py @@ -1,7 +1,7 @@ """ Tests of the ADF-Z p-value and critical values. -The p-value is a cubic in log|stat| below adf_z_star and a cubic in stat above it. +The p-value is a quartic in log|stat| below adf_z_star and a cubic in stat above it. The two pieces are estimated jointly so that they have the same value and the same slope at adf_z_star, and the p-value is monotone in the statistic. The critical values are fit to the same simulations, and the p-value is fit to the asymptotic @@ -36,8 +36,8 @@ "ct": [-29.3556, -21.7105, -18.2449, -9.1027, -3.7673], "ctt": [-36.5964, -28.1110, -24.1853, -13.4049, -6.6436], } -# The largest error seen is 7.7e-5 for the 1, 5 and 10 percentiles and 4.6e-3 for the -# 50 and 90 percentiles (cubic polynomials cannot track the whole distribution) +# The largest error seen is 4.1e-5 for the 1, 5 and 10 percentiles and 6.7e-3 for the +# 50 and 90 percentiles (the polynomials cannot track the whole distribution) ATOL = np.array([1e-4, 1e-4, 1e-4, 7.5e-3, 7.5e-3]) # Mean over the simulation files of the 1, 5 and 10 percentiles with nobs = 24 and # nobs = 99 observations in the regression (series of 25 and 100 observations), From 176e2436e90d0cb1045ac09f1952c47fb8cc5f62 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 11 Nov 2026 14:12:09 +0000 Subject: [PATCH 14/14] DOC: Update the ADF z-test release note for the quartic --- doc/source/changes/9.0.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/source/changes/9.0.rst b/doc/source/changes/9.0.rst index 7c58e87620..a43e00660a 100644 --- a/doc/source/changes/9.0.rst +++ b/doc/source/changes/9.0.rst @@ -409,20 +409,20 @@ to restore the previous results. the same simulations. The p-value is approximated by one polynomial below a cut-off and another above it. These were estimated separately, so the p-value jumped down at the cut-off, by 0.033 with ``"n"``, 0.014 with ``"c"``, 0.010 with ``"ct"`` and 0.008 - with ``"ctt"``. The two are now estimated jointly, so that they have the same value - and slope at the cut-off. They are fit to the asymptotic quantiles, which are the - constants of the regressions of the simulated quantiles on powers of 1/nobs, and the - critical values are the regressions of the 1%, 5% and 10% quantiles. The p-values of - the asymptotic critical values are then 1%, 5% and 10%, to 0.00008, where they were as - low as 0.0097, 0.0491 and 0.0987 with ``"ctt"``. The approximation is also more - accurate in the left tail: the largest relative error of p-values below 0.15 is now - 0.13% with ``"n"`` and at most 0.07% otherwise, where it was up to 6.0% with a - constant or a trend. In the middle of the distribution, the largest difference from - the simulated distribution is 0.012 with ``"n"`` and at most 0.0017 otherwise. + with ``"ctt"``. The two are now a quartic in the log of the statistic and a cubic, + estimated jointly so that they have the same value and slope at the cut-off. They are + fit to the asymptotic quantiles, which are the constants of the regressions of the + simulated quantiles on powers of 1/nobs, and the critical values are the regressions + of the 1%, 5% and 10% quantiles. The p-values of the asymptotic critical values are + then 1%, 5% and 10%, to 0.00005, where they were as low as 0.0097, 0.0491 and 0.0987 + with ``"ctt"``. The approximation is also more accurate in the left tail: the largest + relative error of p-values below 0.15 is now at most 0.05%, where it was up to 6.0% + with a constant or a trend. In the middle of the distribution, the largest difference + from the simulated distribution is 0.013 with ``"n"`` and at most 0.0015 otherwise. Compared with 8.0, with a constant or a trend, p-values below 0.15 change by at most - 0.0016, and the largest change is 0.012, at p-values between 0.7 and 0.8. The critical - values were regressions on powers of one over the length of the series, which is one - more than the number of observations in the regression of the test that + 0.0015, and the largest change is 0.014, at a p-value near 0.7 with ``"c"``. The + critical values were regressions on powers of one over the length of the series, which + is one more than the number of observations in the regression of the test that :class:`~arch.unitroot.PhillipsPerron` passes to ``mackinnoncrit``, so they were those of a sample with one fewer observation. They are now regressions on powers of 1/nobs. With 25 observations in the regression, the 1% critical value with ``"ctt"`` changes