diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ceec5c6..6938cdfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -416,15 +416,28 @@ jobs: # values are what identify the bug - then the end of the log. # `set +e` above is deliberate: diagnostic helpers may find no # matches, and only the captured test status controls this step. - grep -E "Test Failed at |Error During Test at |^ Expression: |^ Evaluated: |^ Test threw exception" \ - /tmp/validated-statistics-tests.log > /tmp/validated-statistics-failures.log + # One annotation cannot hold a failing run, and a tail hides every + # failure but the last one. Pair each location with its assertion, + # cut long lines (an evaluated HTML page is useless in a digest), + # then publish the first few chunks. + # Print each failure's location and the lines that explain it: the + # expression and evaluated value for an assertion, the exception for + # an error. Long lines are cut because an evaluated HTML page would + # otherwise fill the whole digest. + awk ' + /Test Failed at / { print; remaining = 2; next } + /Error During Test at / { print; remaining = 4; next } + /statistics UI request failed/ { print; remaining = 6; next } + remaining > 0 { print; remaining-- } + ' /tmp/validated-statistics-tests.log | + cut -c1-260 > /tmp/validated-statistics-failures.log rm -f /tmp/validated-statistics-digest-* - split -b 2500 /tmp/validated-statistics-failures.log /tmp/validated-statistics-digest- + split -b 3800 /tmp/validated-statistics-failures.log /tmp/validated-statistics-digest- index=0 for chunk in /tmp/validated-statistics-digest-*; do - [ -e "$chunk" ] || continue + [[ -e "$chunk" ]] || continue index=$((index + 1)) - [ "$index" -le 8 ] || break + [[ "$index" -le 8 ]] || break echo "::error title=Validated-statistics failure digest $index::$(enc < "$chunk")" done echo "::error title=Validated-statistics log tail::$(tail -c 3000 /tmp/validated-statistics-tests.log | enc)" diff --git a/docs/statistics/numeric-contracts.md b/docs/statistics/numeric-contracts.md index 52c10293..6b66d003 100644 --- a/docs/statistics/numeric-contracts.md +++ b/docs/statistics/numeric-contracts.md @@ -43,7 +43,11 @@ bytes, per `hyperpolymath/standards` JSON-POLICY (D306) and the - JSON numbers carry exactly the safe I-JSON integer range `-(2^53-1) .. 2^53-1`. - **A count above `2^53-1` is a decimal string, never a JSON number.** `1` is a - number; `"9007199254740993"` is a string. + number; `"9007199254740993"` is a string. A JSON number outside + `-(2^53-1) .. 2^53-1` is refused by the canonical reader as a `400`: the + document itself is not I-JSON, so the statistics layer never sees a count it + would have to round. The same magnitude sent as a whole-number string is + accepted and analysed exactly. - **Decimal and exponent tokens are refused even when their value is integral.** `1.0` and `1e0` are float tokens for a field that is contractually integral; accepting them would silently claim a precision the diff --git a/scripts/statistics/simulation_study.jl b/scripts/statistics/simulation_study.jl index 43f383e8..a0862fe6 100755 --- a/scripts/statistics/simulation_study.jl +++ b/scripts/statistics/simulation_study.jl @@ -101,7 +101,7 @@ function main(args::Vector{String}) rng = MersenneTwister(seed) exact_failures = String[] metric_failures = String[] - maxima = Dict(Int, Dict{String,BigFloat})( + maxima = Dict{Int,Dict{String,BigFloat}}( bits => Dict("shannon_abs" => big(0.0), "shannon_rel" => big(0.0), "simpson_abs" => big(0.0), "simpson_rel" => big(0.0)) for bits in PRECISIONS) diff --git a/src/analysis/numeric_policy.jl b/src/analysis/numeric_policy.jl index a2cf02d2..1b97cf78 100644 --- a/src/analysis/numeric_policy.jl +++ b/src/analysis/numeric_policy.jl @@ -424,7 +424,10 @@ function to_display(v::NumericValue; digits::Integer=6) kept = v.kind === ROUNDED && !isnothing(v.digits) ? v.digits : Int(digits) rendered = _rendered_decimal(Rational{BigInt}(v.value), kept) bits = isnothing(v.precision_bits) ? "unknown precision" : "$(v.precision_bits)-bit" - "$rendered ($(v.kind), $bits, $(kept)dp)" + # The display is always a rounding, whatever the value's kind: an + # approximate MPFR value shown at 6dp has still been rounded, and a reader + # must not mistake the short string for the stored value. + "$rendered ($(v.kind), $bits, rounded to $(kept)dp)" end function encode_json_number(x) diff --git a/src/server/routes/statistics_ui.jl b/src/server/routes/statistics_ui.jl index d365eaca..b91df8d8 100644 --- a/src/server/routes/statistics_ui.jl +++ b/src/server/routes/statistics_ui.jl @@ -18,8 +18,11 @@ using ..DerivedAnalyses const _STATISTICS_UI_TABLE_CELLS = 20_000 const _STATISTICS_UI_TITLE = "Validated statistics" +# `string`, not `String`: the storage dictionaries carry integers as JSON +# numbers, and `String(::Int)` is a MethodError. Every value that reaches the +# page is rendered, so accepting any printable value here is the contract. function _html_escape(value) - replace(String(value), + replace(string(value), '&' => "&", '<' => "<", '>' => ">", '"' => """, '\'' => "'") end @@ -348,7 +351,7 @@ function _ui_high_precision_value(value) display = get(value, "display", "") bits = get(value, "precision_bits", "") kind = get(value, "kind", "approximate") - String("", _html(print_value), "
", + string("", _html(print_value), "
", _html(kind), " · ", _html(string(bits)), "-bit · display: ", _html(display), "") end diff --git a/test/unit/test_method_catalogue.jl b/test/unit/test_method_catalogue.jl index d608e4dc..e9c7f656 100644 --- a/test/unit/test_method_catalogue.jl +++ b/test/unit/test_method_catalogue.jl @@ -97,9 +97,12 @@ end @test rejected(doc -> doc["methods"] = Any[]) @test rejected(doc -> doc["methods"][1]["status"] = "maybe") @test rejected(doc -> doc["methods"][1]["evidence"] = Any[]) - # Flipping an unsupported entry to supported without evidence is refused. + # Flipping an unsupported entry to supported is refused: it has no + # implementation and no evidence. Clearing the evidence of a *supported* + # entry is refused too, which is why the second mutation targets entry 2: + # an unsupported entry has no evidence to clear, and removing it is legal. @test rejected(doc -> doc["methods"][3]["status"] = "supported") - @test rejected(doc -> doc["methods"][3]["evidence"] = Any[]) + @test rejected(doc -> doc["methods"][2]["evidence"] = Any[]) @test rejected(doc -> doc["methods"][1]["id"] = doc["methods"][2]["id"]) @test rejected(doc -> doc["limits"]["max_cells"] = MAX_CELLS + 1) @test rejected(doc -> doc["limits"]["max_precision_bits"] = MAX_PRECISION_BITS + 1) diff --git a/test/unit/test_numeric_policy.jl b/test/unit/test_numeric_policy.jl index 29d700e3..ffd95e46 100644 --- a/test/unit/test_numeric_policy.jl +++ b/test/unit/test_numeric_policy.jl @@ -113,7 +113,7 @@ end @test occursin("approximate", to_display(approx)) rounded = rounded_value(1.0/3; digits=6) @test rounded.kind == ROUNDED - @test to_display(rounded) == "0.333333 (rounded, 53-bit, 6dp)" + @test to_display(rounded) == "0.333333 (rounded, 53-bit, rounded to 6dp)" @test_throws UnsupportedRepresentationError to_storage(approximate_value(NaN)) @test_throws UnsupportedRepresentationError encode_json_number(Inf) diff --git a/test/unit/test_statistics_adversarial.jl b/test/unit/test_statistics_adversarial.jl index b77a28cb..0745758a 100644 --- a/test/unit/test_statistics_adversarial.jl +++ b/test/unit/test_statistics_adversarial.jl @@ -49,6 +49,10 @@ const _ADV_BASE = Dict{String,Any}( ) _adv_body(overrides::Dict) = merge(_ADV_BASE, overrides) +# The call sites pass pairs, not keyword arguments: `_adv_body("k" => v)` is +# one positional Pair, and without this method it is a MethodError rather than +# a request body. +_adv_body(pairs::Pair...) = merge(_ADV_BASE, Dict{String,Any}(pairs...)) _adv_body(key::String, value) = merge(_ADV_BASE, Dict(key => value)) """Assert a refused request: documented status, documented code, canonical body.""" @@ -207,15 +211,37 @@ end @test response.status != 500 end - # Booleans are not counts, and neither is a JSON float token. - for value in (true, false, 1.5, -1) + # Booleans are not counts. `Any[Any[true, 2], Any[3, 4]]` keeps the + # Boolean in the document: `[[true, 2], [3, 4]]` is an Int array in + # Julia, so it would serialise as 1 and test nothing. A JSON float + # token and a negative integer are refused for the same reason. + for value in (true, false) + body = _adv_body("counts" => Any[Any[value, 2], Any[3, 4]]) + status = _adv_post("/api/v1/statistics/summaries", body).status + @test (repr(value), status) == (repr(value), 422) + end + for value in (1.5, -1) body = _adv_body("counts" => [[value, 2], [3, 4]]) - @test _adv_post("/api/v1/statistics/summaries", body).status == 422 + status = _adv_post("/api/v1/statistics/summaries", body).status + @test (repr(value), status) == (repr(value), 422) end - # Counts at 2^53 and above must arrive as whole-number strings. - for bad in (9007199254740992, "-1", "1.5", "1e3") + + # A JSON number beyond 2^53-1 is not I-JSON-exact, so the canonical + # reader refuses the document itself (400) before the statistics layer + # sees a count. The same magnitude is accepted when it arrives as a + # whole-number string, which is the documented route for large counts. + huge_number = _adv_post("/api/v1/statistics/summaries", + _adv_body("counts" => [[9007199254740992, 2], [3, 4]])).status + @test huge_number == 400 + huge_string = _adv_post("/api/v1/statistics/summaries", + _adv_body("counts" => [["9007199254740993", 2], [3, 4]])).status + @test huge_string == 200 + # Non-integral or negative strings are refused by the statistics layer + # (422), not by the JSON reader: they are well-formed JSON. + for bad in ("-1", "1.5", "1e3") body = _adv_body("counts" => [[bad, 2], [3, 4]]) - @test _adv_post("/api/v1/statistics/summaries", body).status == 422 + status = _adv_post("/api/v1/statistics/summaries", body).status + @test (repr(bad), status) == (repr(bad), 422) end end end diff --git a/test/unit/test_statistics_properties.jl b/test/unit/test_statistics_properties.jl index 6daa1e23..4a3cd201 100644 --- a/test/unit/test_statistics_properties.jl +++ b/test/unit/test_statistics_properties.jl @@ -166,8 +166,13 @@ end @test permuted.samples[j].total == summary.samples[j].total @test permuted.samples[j].richness == summary.samples[j].richness if !isnothing(summary.samples[j].shannon) - @test permuted.samples[j].shannon == summary.samples[j].shannon - @test permuted.samples[j].gini_simpson == summary.samples[j].gini_simpson + # Summation order changes with the permutation, so the last bit + # of a 256-bit sum may differ. The metric must agree to within + # the declared tolerance, not bit-for-bit: asserting equality + # here would test the summation order, not the statistic. + @test _close(summary.samples[j].shannon, permuted.samples[j].shannon) + @test _close(summary.samples[j].gini_simpson, + permuted.samples[j].gini_simpson) end end end diff --git a/test/unit/test_statistics_ui.jl b/test/unit/test_statistics_ui.jl index 4593d52d..fc4ad442 100644 --- a/test/unit/test_statistics_ui.jl +++ b/test/unit/test_statistics_ui.jl @@ -144,6 +144,22 @@ end end end +@testset "The run path is callable without the HTTP handler" begin + # The handler turns any exception into a 500 page, which hides the cause + # behind "No result is claimed". Calling the run path directly means a + # failure reaches the test as the exception it is. + _ui_with_root() do _ + params = Dict{String,String}( + "mode" => "high_precision", "precision_bits" => "128", + "sample_labels" => "A, B", "feature_labels" => "t1, t2", + "counts" => "1, 0\n1, 0", "round_digits" => "6") + outcome = _UI_SV._ui_run(params) + @test outcome.result["status"] == "partial" + @test outcome.result["samples"][1]["shannon"]["precision_bits"] == 128 + @test isnothing(outcome.result["samples"][2]["shannon"]) + end +end + @testset "High-precision form labels approximations as approximate" begin _ui_with_root() do _ fields = merge(_ui_form(), Dict{String,String}(