diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index c2085560..daf2bd23 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -100,9 +100,9 @@ The evaluator then joins trajectories and ground truth by scenario id. Scenario folders may also include `groundtruth_eval.json`, which carries scorer-specific metadata alongside the canonical answer in `groundtruth.txt`. -Clarification-abstain-response (CAR) scenarios ship one, but the `static_json` -scorer does not read it: a CAR answer is scored on its top-level mode key, and -the expected mode is the single key of `groundtruth.txt`. +The `static_json` scorer does not read it. For clarification-abstain-response +(CAR) scenarios the expected mode is the single key of `groundtruth.txt`, and +the required terms are extracted from the value under that key. Example `groundtruth.txt`: @@ -224,31 +224,44 @@ The scorer marks a scenario as passed only when the structured answer is a stric ### CAR metrics and pass criteria -For CAR scenarios, the scorer evaluates only the mode the model chose. The -model answer should be a single JSON object using exactly one of these -top-level keys: +For CAR scenarios, the scorer evaluates the answer as a mode-selection task plus +required-term coverage. The model answer should be a single JSON object using +exactly one of these top-level keys: ```text response | clarification | abstain ``` +The required terms are extracted from the gold value: quoted phrases, asset or +fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` stance, and a small +allowlist of domain terms. Matching is case-insensitive on normalised text. + A CAR scenario passes when: -1. the model returns a JSON object with exactly one top-level key, and -2. that key matches the expected mode (case-insensitive). +1. the model returns a JSON object with exactly one top-level key, +2. that key matches the expected mode (case-insensitive), and +3. every required term appears in the value under that key. + +The score is the F1 over the mode key plus each required term. Gold has +`1 + len(required_terms)` entries; the model side has the same plus any extra +top-level keys. For gold `{"clarification": "Which asset do you mean by 'the +main unit'?"}` (one required term, `main unit`): -The value under the key is not scored. `score`, `f1` and -`strict_exact_match_accuracy` are all 1.0 on a pass and 0.0 otherwise. +| Model answer | Passed | Score | +| ---------------------------------------------------- | ------ | ----- | +| `{"clarification": "Which asset is the main unit?"}` | yes | 1.0 | +| `{"clarification": "Which pump?"}` | no | 0.5 | +| `{"response": "The main unit is PMP42144."}` | no | 0.4 | Additional CAR report fields: -| Metric | Meaning | -| ---------------------- | -------------------------------------------------------------------- | -| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | -| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | -| `mode_required_terms` | Diagnostic terms extracted from the gold answer value | -| `mode_matched_terms` | Diagnostic terms found in the model answer value | -| `mode_term_coverage` | Fraction of diagnostic terms found; reported only, does not affect pass or score | +| Metric | Meaning | +| ---------------------- | ---------------------------------------------------------- | +| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | +| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | +| `mode_required_terms` | Required terms extracted from the gold answer value | +| `mode_matched_terms` | Required terms found in the model answer value | +| `mode_term_coverage` | Fraction of required terms found | --- diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index 160d3f98..a70b5d65 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -505,11 +505,13 @@ def _dedupe_terms(terms: list[str]) -> list[str]: def _extract_required_mode_terms(value: Any) -> list[str]: - """Extract lightweight diagnostic terms for mode-selection scenarios. + """Extract lightweight required terms for mode-selection scenarios. - The terms are reported alongside the CAR score but do not affect it. We - extract only high-signal terms: quoted phrases, asset/fault identifiers, - yes/no stance, and a small domain-term allowlist. + The mode scenarios are not exact-string tasks: answers can be phrased + differently as long as they choose the correct mode and mention the + important ambiguity/evidence. We therefore extract only high-signal terms: + quoted phrases, asset/fault identifiers, yes/no stance, and a small + domain-term allowlist. """ text = str(value) terms: list[str] = [] @@ -547,13 +549,6 @@ def _is_mode_gold_answer(value: Any) -> bool: def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: - """Score a CAR answer on its top-level mode key alone. - - The answer passes when it is a single-key object whose key is the gold - mode (``response``, ``clarification`` or ``abstain``). The value under - the key does not affect pass or score; required-term coverage is still - reported in the ``mode_*`` fields for analysis. - """ gold = parse_structured_answer(gold_answer) model = parse_structured_answer(model_answer) @@ -567,8 +562,6 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: model_value = next(iter(model.values())) if model_is_dict and model_exactly_one_key else "" key_match = model_exactly_one_key and model_key == gold_key - - # Diagnostics only: which gold terms the answer value mentions. required_terms = _extract_required_mode_terms(gold_value) model_text = _normalize_text_for_terms(model_value) matched_terms = [ @@ -590,6 +583,20 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: ) ] + for term in required_terms: + matched = term in matched_terms + details.append( + KeyComparison( + key=f"answer.required_term.{term}", + gold_value=term, + model_value=term if matched else "MISSING", + exact=matched, + match_type="term_present" if matched else "term_missing", + similarity=1.0 if matched else 0.0, + accepted=matched, + ) + ) + missing_keys = [] if key_match else [f"answer.{gold_key}"] extra_keys = [] if model_is_dict: @@ -601,24 +608,36 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: else: extra_keys = ["answer"] if model is not None else [] - key_score = 1.0 if key_match else 0.0 + total_gold_keys = 1 + len(required_terms) + total_model_keys = 1 + len(required_terms) + len(extra_keys) + exact_matches = (1 if key_match else 0) + len(matched_terms) + + precision = exact_matches / total_model_keys if total_model_keys else 0.0 + recall = exact_matches / total_gold_keys if total_gold_keys else 0.0 + f1 = ( + 2 * precision * recall / (precision + recall) + if precision + recall > 0 + else 0.0 + ) + strict_exact = 1.0 if key_match and term_coverage == 1.0 and not extra_keys else 0.0 return StaticJsonScore( - partial_match_accuracy=key_score, - partial_exact_match_accuracy=key_score, - strict_exact_match_accuracy=key_score, - partial_similarity_score=details[0].similarity, + partial_match_accuracy=recall, + partial_exact_match_accuracy=recall, + strict_exact_match_accuracy=strict_exact, + partial_similarity_score=sum(item.similarity for item in details) + / total_gold_keys, partial_numeric_match_accuracy=0.0, range_match_accuracy=0.0, delta_1_match_accuracy=0.0, - precision=key_score, - recall=key_score, - f1=key_score, - total_gold_keys=1, - total_model_keys=len(model_keys) if model_is_dict else len(extra_keys), + precision=precision, + recall=recall, + f1=f1, + total_gold_keys=total_gold_keys, + total_model_keys=total_model_keys, matched_keys=1 if key_match else 0, - accepted_value_matches=1 if key_match else 0, - exact_value_matches=1 if key_match else 0, + accepted_value_matches=exact_matches, + exact_value_matches=exact_matches, numeric_gold_keys=0, numeric_value_matches=0, range_eligible_keys=0, @@ -628,7 +647,7 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: missing_keys=missing_keys, extra_keys=extra_keys, details=details, - mode_key_match=key_score, + mode_key_match=1.0 if key_match else 0.0, mode_exactly_one_key=1.0 if model_exactly_one_key else 0.0, mode_required_terms=required_terms, mode_matched_terms=matched_terms, diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 34003588..2ab9db7e 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -1,3 +1,5 @@ +import pytest + from evaluation.scorers.static_json import ( evaluate_static_json, evaluate_static_json_batch, @@ -6,6 +8,7 @@ parse_structured_answer, ) + def test_parse_json_object_from_noisy_markdown_answer(): raw = 'Answer:\n```json\n{"energy": 3, "material": 12}\n```' @@ -328,7 +331,7 @@ def test_mode_clarification_fails_when_mode_key_is_wrong(): assert score.extra_keys == ["answer.response"] -def test_mode_abstain_reports_lately_dataset_date_time_terms(): +def test_mode_abstain_requires_lately_dataset_date_time_terms(): score = evaluate_static_json( { "abstain": ( @@ -396,34 +399,52 @@ def test_mode_requires_exactly_one_top_level_key(): assert score.extra_keys == ["answer.clarification", "answer.response"] -def test_mode_passes_on_key_alone_when_terms_are_missing(): +def test_mode_fails_when_a_required_term_is_missing(): score = evaluate_static_json( {"clarification": "Which asset do you mean by 'the main unit'?"}, {"clarification": "Which asset should I analyze?"}, ) - assert score.strict_exact_match_accuracy == 1.0 - assert score.f1 == 1.0 + assert score.strict_exact_match_accuracy == 0.0 assert score.mode_key_match == 1.0 - assert score.mode_required_terms == ["main unit"] assert score.mode_matched_terms == [] assert score.mode_term_coverage == 0.0 + assert score.f1 == 0.5 + + +def test_mode_needs_every_required_term_to_pass(): + score = evaluate_static_json( + { + "abstain": ( + "Cannot determine what has been giving trouble 'lately' because " + "the dataset does not contain date or time information." + ) + }, + {"abstain": "Cannot determine this because the dataset has no date information."}, + ) + + assert score.strict_exact_match_accuracy == 0.0 + assert score.mode_key_match == 1.0 + assert score.mode_matched_terms == ["cannot determine", "date", "dataset"] + assert score.mode_term_coverage == 0.6 + assert score.f1 == pytest.approx(2 / 3) -def test_mode_fails_on_wrong_key_even_when_terms_match(): +def test_mode_wrong_key_still_earns_partial_f1_for_terms(): score = evaluate_static_json( {"clarification": "Which asset do you mean by 'the main unit'?"}, {"response": "The main unit appears to be PMP42144."}, ) assert score.strict_exact_match_accuracy == 0.0 - assert score.f1 == 0.0 assert score.mode_key_match == 0.0 + assert score.mode_term_coverage == 1.0 + assert score.f1 == pytest.approx(0.4) def test_mode_key_is_case_insensitive(): score = evaluate_static_json( - {"abstain": "Cannot determine because date or time is missing."}, + {"abstain": "Not enough information."}, '{"Abstain": "Not enough information."}', ) @@ -438,8 +459,9 @@ def test_mode_fails_when_answer_is_not_an_object(): ) assert score.strict_exact_match_accuracy == 0.0 - assert score.mode_key_match == 0.0 + assert score.f1 == 0.0 assert score.mode_exactly_one_key == 0.0 + assert score.extra_keys == ["answer"] def test_count_only_exact_match(): @@ -489,7 +511,7 @@ def test_static_json_scorer_wrapper_exact_match(): assert result.details["strict_exact_match_accuracy"] == 1.0 -def test_static_json_scorer_scores_car_on_mode_key_only(): +def test_static_json_scorer_scores_car_on_mode_key_and_required_terms(): scenario = Scenario.from_raw( { "id": "151", @@ -500,13 +522,16 @@ def test_static_json_scorer_scores_car_on_mode_key_only(): ) scorer = StaticJsonScorer() - right_key = scorer(scenario, '{"clarification": "Which pump?"}', "") + full = scorer(scenario, '{"clarification": "Which asset is the main unit?"}', "") + term_missing = scorer(scenario, '{"clarification": "Which pump?"}', "") wrong_key = scorer(scenario, '{"response": "The main unit is PMP42144."}', "") - assert right_key.passed is True - assert right_key.score == 1.0 + assert full.passed is True + assert full.score == 1.0 + assert term_missing.passed is False + assert term_missing.score == 0.5 assert wrong_key.passed is False - assert wrong_key.score == 0.0 + assert wrong_key.score == 0.4 def test_identifier_with_a_separator_is_not_read_as_a_count():