fix(evaluation): restore term-based CAR scoring

docs/static-json-evaluation.md

@@ -100,9 +100,9 @@ The evaluator then joins trajectories and ground truth by scenario id.
 
 Scenario folders may also include `groundtruth_eval.json`, which carries
 scorer-specific metadata alongside the canonical answer in `groundtruth.txt`.
-Clarification-abstain-response (CAR) scenarios ship one, but the `static_json`
-scorer does not read it: a CAR answer is scored on its top-level mode key, and
-the expected mode is the single key of `groundtruth.txt`.
+The `static_json` scorer does not read it. For clarification-abstain-response
+(CAR) scenarios the expected mode is the single key of `groundtruth.txt`, and
+the required terms are extracted from the value under that key.
 
 Example `groundtruth.txt`:
 
@@ -224,31 +224,44 @@ The scorer marks a scenario as passed only when the structured answer is a stric
 
 ### CAR metrics and pass criteria
 
-For CAR scenarios, the scorer evaluates only the mode the model chose. The
-model answer should be a single JSON object using exactly one of these
-top-level keys:
+For CAR scenarios, the scorer evaluates the answer as a mode-selection task plus
+required-term coverage. The model answer should be a single JSON object using
+exactly one of these top-level keys:
 
 ```text
 response | clarification | abstain
 ```
 
+The required terms are extracted from the gold value: quoted phrases, asset or
+fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` stance, and a small
+allowlist of domain terms. Matching is case-insensitive on normalised text.
+
 A CAR scenario passes when:
 
-1. the model returns a JSON object with exactly one top-level key, and
-2. that key matches the expected mode (case-insensitive).
+1. the model returns a JSON object with exactly one top-level key,
+2. that key matches the expected mode (case-insensitive), and
+3. every required term appears in the value under that key.
+
+The score is the F1 over the mode key plus each required term. Gold has
+`1 + len(required_terms)` entries; the model side has the same plus any extra
+top-level keys. For gold `{"clarification": "Which asset do you mean by 'the
+main unit'?"}` (one required term, `main unit`):
 
-The value under the key is not scored. `score`, `f1` and
-`strict_exact_match_accuracy` are all 1.0 on a pass and 0.0 otherwise.
+| Model answer                                         | Passed | Score |
+| ---------------------------------------------------- | ------ | ----- |
+| `{"clarification": "Which asset is the main unit?"}` | yes    | 1.0   |
+| `{"clarification": "Which pump?"}`                   | no     | 0.5   |
+| `{"response": "The main unit is PMP42144."}`         | no     | 0.4   |
 
 Additional CAR report fields:
 
-| Metric                 | Meaning                                                              |
-| ---------------------- | -------------------------------------------------------------------- |
-| `mode_key_match`       | 1.0 when the returned mode key matches the expected mode             |
-| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key                    |
-| `mode_required_terms`  | Diagnostic terms extracted from the gold answer value                |
-| `mode_matched_terms`   | Diagnostic terms found in the model answer value                     |
-| `mode_term_coverage`   | Fraction of diagnostic terms found; reported only, does not affect pass or score |
+| Metric                 | Meaning                                                    |
+| ---------------------- | ---------------------------------------------------------- |
+| `mode_key_match`       | 1.0 when the returned mode key matches the expected mode   |
+| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key          |
+| `mode_required_terms`  | Required terms extracted from the gold answer value        |
+| `mode_matched_terms`   | Required terms found in the model answer value             |
+| `mode_term_coverage`   | Fraction of required terms found                           |
 
 ---
 

src/evaluation/scorers/static_json.py

@@ -505,11 +505,13 @@ def _dedupe_terms(terms: list[str]) -> list[str]:
 
 
 def _extract_required_mode_terms(value: Any) -> list[str]:
-    """Extract lightweight diagnostic terms for mode-selection scenarios.
+    """Extract lightweight required terms for mode-selection scenarios.
 
-    The terms are reported alongside the CAR score but do not affect it. We
-    extract only high-signal terms: quoted phrases, asset/fault identifiers,
-    yes/no stance, and a small domain-term allowlist.
+    The mode scenarios are not exact-string tasks: answers can be phrased
+    differently as long as they choose the correct mode and mention the
+    important ambiguity/evidence. We therefore extract only high-signal terms:
+    quoted phrases, asset/fault identifiers, yes/no stance, and a small
+    domain-term allowlist.
     """
     text = str(value)
     terms: list[str] = []
@@ -547,13 +549,6 @@ def _is_mode_gold_answer(value: Any) -> bool:
 
 
 def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore:
-    """Score a CAR answer on its top-level mode key alone.
-
-    The answer passes when it is a single-key object whose key is the gold
-    mode (``response``, ``clarification`` or ``abstain``). The value under
-    the key does not affect pass or score; required-term coverage is still
-    reported in the ``mode_*`` fields for analysis.
-    """
     gold = parse_structured_answer(gold_answer)
     model = parse_structured_answer(model_answer)
 
@@ -567,8 +562,6 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore:
     model_value = next(iter(model.values())) if model_is_dict and model_exactly_one_key else ""
 
     key_match = model_exactly_one_key and model_key == gold_key
-
-    # Diagnostics only: which gold terms the answer value mentions.
     required_terms = _extract_required_mode_terms(gold_value)
     model_text = _normalize_text_for_terms(model_value)
     matched_terms = [
@@ -590,6 +583,20 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore:
         )
     ]
 
+    for term in required_terms:
+        matched = term in matched_terms
+        details.append(
+            KeyComparison(
+                key=f"answer.required_term.{term}",
+                gold_value=term,
+                model_value=term if matched else "MISSING",
+                exact=matched,
+                match_type="term_present" if matched else "term_missing",
+                similarity=1.0 if matched else 0.0,
+                accepted=matched,
+            )
+        )
+
     missing_keys = [] if key_match else [f"answer.{gold_key}"]
     extra_keys = []
     if model_is_dict:
@@ -601,24 +608,36 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore:
     else:
         extra_keys = ["answer"] if model is not None else []
 
-    key_score = 1.0 if key_match else 0.0
+    total_gold_keys = 1 + len(required_terms)
+    total_model_keys = 1 + len(required_terms) + len(extra_keys)
+    exact_matches = (1 if key_match else 0) + len(matched_terms)
+
+    precision = exact_matches / total_model_keys if total_model_keys else 0.0
+    recall = exact_matches / total_gold_keys if total_gold_keys else 0.0
+    f1 = (
+        2 * precision * recall / (precision + recall)
+        if precision + recall > 0
+        else 0.0
+    )
+    strict_exact = 1.0 if key_match and term_coverage == 1.0 and not extra_keys else 0.0
 
     return StaticJsonScore(
-        partial_match_accuracy=key_score,
-        partial_exact_match_accuracy=key_score,
-        strict_exact_match_accuracy=key_score,
-        partial_similarity_score=details[0].similarity,
+        partial_match_accuracy=recall,
+        partial_exact_match_accuracy=recall,
+        strict_exact_match_accuracy=strict_exact,
+        partial_similarity_score=sum(item.similarity for item in details)
+        / total_gold_keys,
         partial_numeric_match_accuracy=0.0,
         range_match_accuracy=0.0,
         delta_1_match_accuracy=0.0,
-        precision=key_score,
-        recall=key_score,
-        f1=key_score,
-        total_gold_keys=1,
-        total_model_keys=len(model_keys) if model_is_dict else len(extra_keys),
+        precision=precision,
+        recall=recall,
+        f1=f1,
+        total_gold_keys=total_gold_keys,
+        total_model_keys=total_model_keys,
         matched_keys=1 if key_match else 0,
-        accepted_value_matches=1 if key_match else 0,
-        exact_value_matches=1 if key_match else 0,
+        accepted_value_matches=exact_matches,
+        exact_value_matches=exact_matches,
         numeric_gold_keys=0,
         numeric_value_matches=0,
         range_eligible_keys=0,
@@ -628,7 +647,7 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore:
         missing_keys=missing_keys,
         extra_keys=extra_keys,
         details=details,
-        mode_key_match=key_score,
+        mode_key_match=1.0 if key_match else 0.0,
         mode_exactly_one_key=1.0 if model_exactly_one_key else 0.0,
         mode_required_terms=required_terms,
         mode_matched_terms=matched_terms,

src/evaluation/tests/test_static_json_scorer.py

@@ -1,3 +1,5 @@
+import pytest
+
 from evaluation.scorers.static_json import (
     evaluate_static_json,
     evaluate_static_json_batch,
@@ -6,6 +8,7 @@
     parse_structured_answer,
 )
 
+
 def test_parse_json_object_from_noisy_markdown_answer():
     raw = 'Answer:\n```json\n{"energy": 3, "material": 12}\n```'
 
@@ -328,7 +331,7 @@ def test_mode_clarification_fails_when_mode_key_is_wrong():
     assert score.extra_keys == ["answer.response"]
 
 
-def test_mode_abstain_reports_lately_dataset_date_time_terms():
+def test_mode_abstain_requires_lately_dataset_date_time_terms():
     score = evaluate_static_json(
         {
             "abstain": (
@@ -396,34 +399,52 @@ def test_mode_requires_exactly_one_top_level_key():
     assert score.extra_keys == ["answer.clarification", "answer.response"]
 
 
-def test_mode_passes_on_key_alone_when_terms_are_missing():
+def test_mode_fails_when_a_required_term_is_missing():
     score = evaluate_static_json(
         {"clarification": "Which asset do you mean by 'the main unit'?"},
         {"clarification": "Which asset should I analyze?"},
     )
 
-    assert score.strict_exact_match_accuracy == 1.0
-    assert score.f1 == 1.0
+    assert score.strict_exact_match_accuracy == 0.0
     assert score.mode_key_match == 1.0
-    assert score.mode_required_terms == ["main unit"]
     assert score.mode_matched_terms == []
     assert score.mode_term_coverage == 0.0
+    assert score.f1 == 0.5
+
+
+def test_mode_needs_every_required_term_to_pass():
+    score = evaluate_static_json(
+        {
+            "abstain": (
+                "Cannot determine what has been giving trouble 'lately' because "
+                "the dataset does not contain date or time information."
+            )
+        },
+        {"abstain": "Cannot determine this because the dataset has no date information."},
+    )
+
+    assert score.strict_exact_match_accuracy == 0.0
+    assert score.mode_key_match == 1.0
+    assert score.mode_matched_terms == ["cannot determine", "date", "dataset"]
+    assert score.mode_term_coverage == 0.6
+    assert score.f1 == pytest.approx(2 / 3)
 
 
-def test_mode_fails_on_wrong_key_even_when_terms_match():
+def test_mode_wrong_key_still_earns_partial_f1_for_terms():
     score = evaluate_static_json(
         {"clarification": "Which asset do you mean by 'the main unit'?"},
         {"response": "The main unit appears to be PMP42144."},
     )
 
     assert score.strict_exact_match_accuracy == 0.0
-    assert score.f1 == 0.0
     assert score.mode_key_match == 0.0
+    assert score.mode_term_coverage == 1.0
+    assert score.f1 == pytest.approx(0.4)
 
 
 def test_mode_key_is_case_insensitive():
     score = evaluate_static_json(
-        {"abstain": "Cannot determine because date or time is missing."},
+        {"abstain": "Not enough information."},
         '{"Abstain": "Not enough information."}',
     )
 
@@ -438,8 +459,9 @@ def test_mode_fails_when_answer_is_not_an_object():
     )
 
     assert score.strict_exact_match_accuracy == 0.0
-    assert score.mode_key_match == 0.0
+    assert score.f1 == 0.0
     assert score.mode_exactly_one_key == 0.0
+    assert score.extra_keys == ["answer"]
 
 
 def test_count_only_exact_match():
@@ -489,7 +511,7 @@ def test_static_json_scorer_wrapper_exact_match():
     assert result.details["strict_exact_match_accuracy"] == 1.0
 
 
-def test_static_json_scorer_scores_car_on_mode_key_only():
+def test_static_json_scorer_scores_car_on_mode_key_and_required_terms():
     scenario = Scenario.from_raw(
         {
             "id": "151",
@@ -500,13 +522,16 @@ def test_static_json_scorer_scores_car_on_mode_key_only():
     )
 
     scorer = StaticJsonScorer()
-    right_key = scorer(scenario, '{"clarification": "Which pump?"}', "")
+    full = scorer(scenario, '{"clarification": "Which asset is the main unit?"}', "")
+    term_missing = scorer(scenario, '{"clarification": "Which pump?"}', "")
     wrong_key = scorer(scenario, '{"response": "The main unit is PMP42144."}', "")
 
-    assert right_key.passed is True
-    assert right_key.score == 1.0
+    assert full.passed is True
+    assert full.score == 1.0
+    assert term_missing.passed is False
+    assert term_missing.score == 0.5
     assert wrong_key.passed is False
-    assert wrong_key.score == 0.0
+    assert wrong_key.score == 0.4
 
 
 def test_identifier_with_a_separator_is_not_read_as_a_count():