From f091eb47deb615ee78edf42fe3fe255100d62d5d Mon Sep 17 00:00:00 2001 From: Shuxin Lin Date: Mon, 12 Oct 2026 23:20:26 -0400 Subject: [PATCH 1/2] fix(evaluation): restore term-based CAR scoring Reverts the scorer change from f8805be. A CAR answer again passes only when it is a single-key object whose key matches the gold mode and whose value contains every required term extracted from the gold value; the score is the F1 over the mode key plus those terms. This is the scorer every aa_v1 job before 9584e28 was graded with. Re-scoring their saved CAR answers with it reproduces the recorded f1 exactly, so restoring it puts the 9584e28+ runs back on the same scale. The pre-f8805be CAR tests and docs describe the groundtruth_eval.json metadata scoring added in 79d6356 (car_score = 0.6 mode + 0.4 coverage, optional terms, pass on any one required term). 8f1e538 took that out of the scorer two days later but left the tests and docs, so six tests fail against this code. They are replaced with tests of the behaviour the code has, and the docs are corrected to match. Signed-off-by: Shuxin Lin --- docs/static-json-evaluation.md | 46 +++++++----- src/evaluation/scorers/static_json.py | 71 ++++++++++++------- .../tests/test_static_json_scorer.py | 53 ++++++++++---- 3 files changed, 114 insertions(+), 56 deletions(-) diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index c2085560..61a5c5bf 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -101,8 +101,9 @@ The evaluator then joins trajectories and ground truth by scenario id. Scenario folders may also include `groundtruth_eval.json`, which carries scorer-specific metadata alongside the canonical answer in `groundtruth.txt`. Clarification-abstain-response (CAR) scenarios ship one, but the `static_json` -scorer does not read it: a CAR answer is scored on its top-level mode key, and -the expected mode is the single key of `groundtruth.txt`. +scorer does not read it: the expected mode is the single key of +`groundtruth.txt`, and the required terms are extracted from the value under +that key. Example `groundtruth.txt`: @@ -224,31 +225,44 @@ The scorer marks a scenario as passed only when the structured answer is a stric ### CAR metrics and pass criteria -For CAR scenarios, the scorer evaluates only the mode the model chose. The -model answer should be a single JSON object using exactly one of these -top-level keys: +For CAR scenarios, the scorer evaluates the answer as a mode-selection task plus +required-term coverage. The model answer should be a single JSON object using +exactly one of these top-level keys: ```text response | clarification | abstain ``` +The required terms are extracted from the gold value: quoted phrases, asset or +fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` stance, and a small +allowlist of domain terms. Matching is case-insensitive on normalised text. + A CAR scenario passes when: -1. the model returns a JSON object with exactly one top-level key, and -2. that key matches the expected mode (case-insensitive). +1. the model returns a JSON object with exactly one top-level key, +2. that key matches the expected mode (case-insensitive), and +3. every required term appears in the value under that key. + +The score is the F1 over the mode key plus each required term. Gold has +`1 + len(required_terms)` entries; the model side has the same plus any extra +top-level keys. For gold `{"clarification": "Which asset do you mean by 'the +main unit'?"}` (one required term, `main unit`): -The value under the key is not scored. `score`, `f1` and -`strict_exact_match_accuracy` are all 1.0 on a pass and 0.0 otherwise. +| Model answer | Passed | Score | +| ---------------------------------------------------- | ------ | ----- | +| `{"clarification": "Which asset is the main unit?"}` | yes | 1.0 | +| `{"clarification": "Which pump?"}` | no | 0.5 | +| `{"response": "The main unit is PMP42144."}` | no | 0.4 | Additional CAR report fields: -| Metric | Meaning | -| ---------------------- | -------------------------------------------------------------------- | -| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | -| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | -| `mode_required_terms` | Diagnostic terms extracted from the gold answer value | -| `mode_matched_terms` | Diagnostic terms found in the model answer value | -| `mode_term_coverage` | Fraction of diagnostic terms found; reported only, does not affect pass or score | +| Metric | Meaning | +| ---------------------- | ---------------------------------------------------------- | +| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | +| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | +| `mode_required_terms` | Required terms extracted from the gold answer value | +| `mode_matched_terms` | Required terms found in the model answer value | +| `mode_term_coverage` | Fraction of required terms found | --- diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index 160d3f98..a70b5d65 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -505,11 +505,13 @@ def _dedupe_terms(terms: list[str]) -> list[str]: def _extract_required_mode_terms(value: Any) -> list[str]: - """Extract lightweight diagnostic terms for mode-selection scenarios. + """Extract lightweight required terms for mode-selection scenarios. - The terms are reported alongside the CAR score but do not affect it. We - extract only high-signal terms: quoted phrases, asset/fault identifiers, - yes/no stance, and a small domain-term allowlist. + The mode scenarios are not exact-string tasks: answers can be phrased + differently as long as they choose the correct mode and mention the + important ambiguity/evidence. We therefore extract only high-signal terms: + quoted phrases, asset/fault identifiers, yes/no stance, and a small + domain-term allowlist. """ text = str(value) terms: list[str] = [] @@ -547,13 +549,6 @@ def _is_mode_gold_answer(value: Any) -> bool: def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: - """Score a CAR answer on its top-level mode key alone. - - The answer passes when it is a single-key object whose key is the gold - mode (``response``, ``clarification`` or ``abstain``). The value under - the key does not affect pass or score; required-term coverage is still - reported in the ``mode_*`` fields for analysis. - """ gold = parse_structured_answer(gold_answer) model = parse_structured_answer(model_answer) @@ -567,8 +562,6 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: model_value = next(iter(model.values())) if model_is_dict and model_exactly_one_key else "" key_match = model_exactly_one_key and model_key == gold_key - - # Diagnostics only: which gold terms the answer value mentions. required_terms = _extract_required_mode_terms(gold_value) model_text = _normalize_text_for_terms(model_value) matched_terms = [ @@ -590,6 +583,20 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: ) ] + for term in required_terms: + matched = term in matched_terms + details.append( + KeyComparison( + key=f"answer.required_term.{term}", + gold_value=term, + model_value=term if matched else "MISSING", + exact=matched, + match_type="term_present" if matched else "term_missing", + similarity=1.0 if matched else 0.0, + accepted=matched, + ) + ) + missing_keys = [] if key_match else [f"answer.{gold_key}"] extra_keys = [] if model_is_dict: @@ -601,24 +608,36 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: else: extra_keys = ["answer"] if model is not None else [] - key_score = 1.0 if key_match else 0.0 + total_gold_keys = 1 + len(required_terms) + total_model_keys = 1 + len(required_terms) + len(extra_keys) + exact_matches = (1 if key_match else 0) + len(matched_terms) + + precision = exact_matches / total_model_keys if total_model_keys else 0.0 + recall = exact_matches / total_gold_keys if total_gold_keys else 0.0 + f1 = ( + 2 * precision * recall / (precision + recall) + if precision + recall > 0 + else 0.0 + ) + strict_exact = 1.0 if key_match and term_coverage == 1.0 and not extra_keys else 0.0 return StaticJsonScore( - partial_match_accuracy=key_score, - partial_exact_match_accuracy=key_score, - strict_exact_match_accuracy=key_score, - partial_similarity_score=details[0].similarity, + partial_match_accuracy=recall, + partial_exact_match_accuracy=recall, + strict_exact_match_accuracy=strict_exact, + partial_similarity_score=sum(item.similarity for item in details) + / total_gold_keys, partial_numeric_match_accuracy=0.0, range_match_accuracy=0.0, delta_1_match_accuracy=0.0, - precision=key_score, - recall=key_score, - f1=key_score, - total_gold_keys=1, - total_model_keys=len(model_keys) if model_is_dict else len(extra_keys), + precision=precision, + recall=recall, + f1=f1, + total_gold_keys=total_gold_keys, + total_model_keys=total_model_keys, matched_keys=1 if key_match else 0, - accepted_value_matches=1 if key_match else 0, - exact_value_matches=1 if key_match else 0, + accepted_value_matches=exact_matches, + exact_value_matches=exact_matches, numeric_gold_keys=0, numeric_value_matches=0, range_eligible_keys=0, @@ -628,7 +647,7 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: missing_keys=missing_keys, extra_keys=extra_keys, details=details, - mode_key_match=key_score, + mode_key_match=1.0 if key_match else 0.0, mode_exactly_one_key=1.0 if model_exactly_one_key else 0.0, mode_required_terms=required_terms, mode_matched_terms=matched_terms, diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 34003588..2ab9db7e 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -1,3 +1,5 @@ +import pytest + from evaluation.scorers.static_json import ( evaluate_static_json, evaluate_static_json_batch, @@ -6,6 +8,7 @@ parse_structured_answer, ) + def test_parse_json_object_from_noisy_markdown_answer(): raw = 'Answer:\n```json\n{"energy": 3, "material": 12}\n```' @@ -328,7 +331,7 @@ def test_mode_clarification_fails_when_mode_key_is_wrong(): assert score.extra_keys == ["answer.response"] -def test_mode_abstain_reports_lately_dataset_date_time_terms(): +def test_mode_abstain_requires_lately_dataset_date_time_terms(): score = evaluate_static_json( { "abstain": ( @@ -396,34 +399,52 @@ def test_mode_requires_exactly_one_top_level_key(): assert score.extra_keys == ["answer.clarification", "answer.response"] -def test_mode_passes_on_key_alone_when_terms_are_missing(): +def test_mode_fails_when_a_required_term_is_missing(): score = evaluate_static_json( {"clarification": "Which asset do you mean by 'the main unit'?"}, {"clarification": "Which asset should I analyze?"}, ) - assert score.strict_exact_match_accuracy == 1.0 - assert score.f1 == 1.0 + assert score.strict_exact_match_accuracy == 0.0 assert score.mode_key_match == 1.0 - assert score.mode_required_terms == ["main unit"] assert score.mode_matched_terms == [] assert score.mode_term_coverage == 0.0 + assert score.f1 == 0.5 + + +def test_mode_needs_every_required_term_to_pass(): + score = evaluate_static_json( + { + "abstain": ( + "Cannot determine what has been giving trouble 'lately' because " + "the dataset does not contain date or time information." + ) + }, + {"abstain": "Cannot determine this because the dataset has no date information."}, + ) + + assert score.strict_exact_match_accuracy == 0.0 + assert score.mode_key_match == 1.0 + assert score.mode_matched_terms == ["cannot determine", "date", "dataset"] + assert score.mode_term_coverage == 0.6 + assert score.f1 == pytest.approx(2 / 3) -def test_mode_fails_on_wrong_key_even_when_terms_match(): +def test_mode_wrong_key_still_earns_partial_f1_for_terms(): score = evaluate_static_json( {"clarification": "Which asset do you mean by 'the main unit'?"}, {"response": "The main unit appears to be PMP42144."}, ) assert score.strict_exact_match_accuracy == 0.0 - assert score.f1 == 0.0 assert score.mode_key_match == 0.0 + assert score.mode_term_coverage == 1.0 + assert score.f1 == pytest.approx(0.4) def test_mode_key_is_case_insensitive(): score = evaluate_static_json( - {"abstain": "Cannot determine because date or time is missing."}, + {"abstain": "Not enough information."}, '{"Abstain": "Not enough information."}', ) @@ -438,8 +459,9 @@ def test_mode_fails_when_answer_is_not_an_object(): ) assert score.strict_exact_match_accuracy == 0.0 - assert score.mode_key_match == 0.0 + assert score.f1 == 0.0 assert score.mode_exactly_one_key == 0.0 + assert score.extra_keys == ["answer"] def test_count_only_exact_match(): @@ -489,7 +511,7 @@ def test_static_json_scorer_wrapper_exact_match(): assert result.details["strict_exact_match_accuracy"] == 1.0 -def test_static_json_scorer_scores_car_on_mode_key_only(): +def test_static_json_scorer_scores_car_on_mode_key_and_required_terms(): scenario = Scenario.from_raw( { "id": "151", @@ -500,13 +522,16 @@ def test_static_json_scorer_scores_car_on_mode_key_only(): ) scorer = StaticJsonScorer() - right_key = scorer(scenario, '{"clarification": "Which pump?"}', "") + full = scorer(scenario, '{"clarification": "Which asset is the main unit?"}', "") + term_missing = scorer(scenario, '{"clarification": "Which pump?"}', "") wrong_key = scorer(scenario, '{"response": "The main unit is PMP42144."}', "") - assert right_key.passed is True - assert right_key.score == 1.0 + assert full.passed is True + assert full.score == 1.0 + assert term_missing.passed is False + assert term_missing.score == 0.5 assert wrong_key.passed is False - assert wrong_key.score == 0.0 + assert wrong_key.score == 0.4 def test_identifier_with_a_separator_is_not_read_as_a_count(): From 1015e9e51b77d55a8006fe8209a6f52a80dbf8c6 Mon Sep 17 00:00:00 2001 From: Shuxin Lin Date: Mon, 12 Oct 2026 23:38:48 -0400 Subject: [PATCH 2/2] docs(evaluation): stop claiming CAR scenarios ship groundtruth_eval.json The suite's CAR groundtruth_eval.json files were removed on 2026-10-07, and static_json never read them. Signed-off-by: Shuxin Lin --- docs/static-json-evaluation.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index 61a5c5bf..daf2bd23 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -100,10 +100,9 @@ The evaluator then joins trajectories and ground truth by scenario id. Scenario folders may also include `groundtruth_eval.json`, which carries scorer-specific metadata alongside the canonical answer in `groundtruth.txt`. -Clarification-abstain-response (CAR) scenarios ship one, but the `static_json` -scorer does not read it: the expected mode is the single key of -`groundtruth.txt`, and the required terms are extracted from the value under -that key. +The `static_json` scorer does not read it. For clarification-abstain-response +(CAR) scenarios the expected mode is the single key of `groundtruth.txt`, and +the required terms are extracted from the value under that key. Example `groundtruth.txt`: