From 7fad33d43ac07ca4e51a4da9e4a6a81860abb03a Mon Sep 17 00:00:00 2001 From: Shuxin Lin Date: Mon, 12 Oct 2026 23:58:10 -0400 Subject: [PATCH] feat(evaluation): score CAR concepts from groundtruth_eval.json A CAR scenario's groundtruth_eval.json now supplies its required and optional concepts. Each concept is {"concept", "any_of": [...]} or a plain string, and is covered when any surface form appears as whole words in the answer. Without the file, concepts are extracted from the gold value as before. Ported from the CAR part of d1a668c (#617, closed unmerged): - concepts are read only when the mode key is right, so a wrong-mode answer scores 0 instead of earning credit for echoed terms; - pass needs the mode plus every required concept; - the file's "mode" is checked against the gold key, never used in its place, and reported as mode_spec_matches_gold; - optional concepts and car_score are reported, never scored. Unlike d1a668c, f1 stays the F1 over the mode key plus required concepts, (1 + k) / (1 + n), the same counting as every other static_json scenario. StaticJsonScorer now passes scenario.evaluation_metadata through; the loader and the Harbor task generator already carry groundtruth_eval.json. Signed-off-by: Shuxin Lin --- docs/static-json-evaluation.md | 80 ++++++++--- src/evaluation/report.py | 4 + src/evaluation/scorers/static_json.py | 119 +++++++++++++-- .../tests/test_static_json_scorer.py | 135 +++++++++++++++++- 4 files changed, 300 insertions(+), 38 deletions(-) diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index daf2bd23a..a12bbfe47 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -100,9 +100,8 @@ The evaluator then joins trajectories and ground truth by scenario id. Scenario folders may also include `groundtruth_eval.json`, which carries scorer-specific metadata alongside the canonical answer in `groundtruth.txt`. -The `static_json` scorer does not read it. For clarification-abstain-response -(CAR) scenarios the expected mode is the single key of `groundtruth.txt`, and -the required terms are extracted from the value under that key. +The `static_json` scorer reads it for clarification-abstain-response (CAR) +scenarios, where it lists the concepts a correct answer must cover. Example `groundtruth.txt`: @@ -112,6 +111,30 @@ Example `groundtruth.txt`: } ``` +Example `groundtruth_eval.json`: + +```json +{ + "mode": "clarification", + "required_terms": [ + {"concept": "ambiguous asset", "any_of": ["main unit", "which asset", "which pump"]} + ], + "optional_terms": [ + {"concept": "asks for tag", "any_of": ["asset tag", "tag number"]} + ] +} +``` + +| Field | Meaning | +| ---------------- | ---------------------------------------------------------------------------------------- | +| `mode` | Expected top-level key. Checked against the key of `groundtruth.txt`, never used instead of it | +| `required_terms` | Concepts that gate pass and count toward the score | +| `optional_terms` | Concepts reported for analysis only; they never affect pass or score | + +Each concept is `{"concept": , "any_of": []}` or a plain +string (a concept whose only form is itself). Other fields, such as `_comment`, +are ignored. + --- ## How Matching Works @@ -225,43 +248,54 @@ The scorer marks a scenario as passed only when the structured answer is a stric ### CAR metrics and pass criteria For CAR scenarios, the scorer evaluates the answer as a mode-selection task plus -required-term coverage. The model answer should be a single JSON object using -exactly one of these top-level keys: +concept coverage. The model answer should be a single JSON object using exactly +one of these top-level keys: ```text response | clarification | abstain ``` -The required terms are extracted from the gold value: quoted phrases, asset or -fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` stance, and a small -allowlist of domain terms. Matching is case-insensitive on normalised text. +The expected mode is the single key of `groundtruth.txt`. The required concepts +come from `groundtruth_eval.json` when it lists `required_terms` or +`optional_terms`. Otherwise they are extracted from the gold value: quoted +phrases, asset or fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` +stance, and a small allowlist of domain terms, each a concept with one form. + +A concept is covered when any of its surface forms appears as whole words in +the value under the model's key, after lowercasing and replacing punctuation +with spaces. Concepts are read only when the mode key is right, so a wrong-mode +answer covers none. A CAR scenario passes when: 1. the model returns a JSON object with exactly one top-level key, 2. that key matches the expected mode (case-insensitive), and -3. every required term appears in the value under that key. +3. every required concept is covered. -The score is the F1 over the mode key plus each required term. Gold has -`1 + len(required_terms)` entries; the model side has the same plus any extra -top-level keys. For gold `{"clarification": "Which asset do you mean by 'the -main unit'?"}` (one required term, `main unit`): +The score is the F1 over the mode key plus each required concept: with the right +mode and `k` of `n` concepts covered it is `(1 + k) / (1 + n)`, and a wrong mode +scores 0. With the example `groundtruth_eval.json` above (one required concept): | Model answer | Passed | Score | | ---------------------------------------------------- | ------ | ----- | -| `{"clarification": "Which asset is the main unit?"}` | yes | 1.0 | -| `{"clarification": "Which pump?"}` | no | 0.5 | -| `{"response": "The main unit is PMP42144."}` | no | 0.4 | +| `{"clarification": "Which pump do you mean?"}` | yes | 1.0 | +| `{"clarification": "Could you clarify?"}` | no | 0.5 | +| `{"response": "The main unit is PMP42144."}` | no | 0.0 | Additional CAR report fields: -| Metric | Meaning | -| ---------------------- | ---------------------------------------------------------- | -| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | -| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | -| `mode_required_terms` | Required terms extracted from the gold answer value | -| `mode_matched_terms` | Required terms found in the model answer value | -| `mode_term_coverage` | Fraction of required terms found | +| Metric | Meaning | +| ----------------------------- | -------------------------------------------------------------------- | +| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | +| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | +| `mode_required_terms` | Required concept names | +| `mode_matched_terms` | Required concepts covered by the answer | +| `mode_term_coverage` | Fraction of required concepts covered | +| `mode_optional_terms` | Optional concept names | +| `mode_matched_optional_terms` | Optional concepts covered by the answer | +| `mode_optional_term_coverage` | Fraction of optional concepts covered, or null if none are declared | +| `car_score` | `0.6 + 0.4 * mode_term_coverage` with the right mode, else 0; reported only | +| `mode_spec_matches_gold` | 0.0 when `groundtruth_eval.json` declares a mode the gold does not use | --- diff --git a/src/evaluation/report.py b/src/evaluation/report.py index 0be6769d9..7351e37a0 100644 --- a/src/evaluation/report.py +++ b/src/evaluation/report.py @@ -59,6 +59,7 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: "mode_term_coverage", "mode_optional_term_coverage", "car_score", + "mode_spec_matches_gold", ] score_values: dict[str, list[float]] = {name: [] for name in metric_names} @@ -152,6 +153,9 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: score_values["mode_optional_term_coverage"] ), "car_score_avg": _avg(score_values["car_score"]), + # Below 1.0 means a groundtruth_eval.json declares a mode its gold + # answer does not use: a scenario defect, not a model result. + "mode_spec_matches_gold_avg": _avg(score_values["mode_spec_matches_gold"]), "missing_keys_total": missing_keys_total, "extra_keys_total": extra_keys_total, "detail_entries_total": detail_entries_total, diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index a70b5d65f..7fed21eec 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -78,6 +78,11 @@ class StaticJsonScore: mode_required_terms: list[str] = field(default_factory=list) mode_matched_terms: list[str] = field(default_factory=list) mode_term_coverage: float | None = None + mode_optional_terms: list[str] = field(default_factory=list) + mode_matched_optional_terms: list[str] = field(default_factory=list) + mode_optional_term_coverage: float | None = None + car_score: float | None = None + mode_spec_matches_gold: float | None = None def to_dict(self) -> dict[str, Any]: """Return a JSON-serializable dictionary.""" @@ -540,6 +545,58 @@ def _extract_required_mode_terms(value: Any) -> list[str]: return _dedupe_terms(terms) +_CAR_KEY_WEIGHT = 0.6 +_CAR_TERM_WEIGHT = 0.4 + + +def _mode_concepts(raw: Any) -> list[tuple[str, list[str]]]: + """Normalize a CAR term list into (concept, accepted surface forms). + + Each entry is either a plain string or ``{"concept": str, "any_of": [str]}``. + A concept is covered when ANY of its surface forms occurs in the answer, so + a correct answer is not penalized for paraphrasing. Forms are normalized + like the answer text, so case and hyphenation cannot cause a miss. + """ + concepts: list[tuple[str, list[str]]] = [] + if not isinstance(raw, (list, tuple)): + return concepts + for entry in raw: + if isinstance(entry, str): + name, forms = entry, [entry] + elif isinstance(entry, dict): + name = str(entry.get("concept") or "").strip() + forms = entry.get("any_of") or ([name] if name else []) + if not isinstance(forms, (list, tuple)): + forms = [forms] + else: + continue + normalized: list[str] = [] + for form in forms: + if not isinstance(form, str): + continue + text = _normalize_text_for_terms(form) + if text and text not in normalized: + normalized.append(text) + if name and normalized: + concepts.append((name, normalized)) + return concepts + + +def _cover_concepts( + concepts: list[tuple[str, list[str]]], text: str +) -> tuple[list[str], float]: + """Return the covered concept names and the coverage fraction.""" + if not concepts: + return [], 1.0 + haystack = f" {text} " + covered = [ + name + for name, forms in concepts + if any(f" {form} " in haystack for form in forms) + ] + return covered, len(covered) / len(concepts) + + def _is_mode_gold_answer(value: Any) -> bool: parsed = parse_structured_answer(value) if not isinstance(parsed, dict) or len(parsed) != 1: @@ -548,12 +605,29 @@ def _is_mode_gold_answer(value: Any) -> bool: return key in _MODE_KEYS -def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: +def _evaluate_mode_json( + gold_answer: Any, + model_answer: Any, + *, + evaluation_metadata: dict[str, Any] | None = None, +) -> StaticJsonScore: + """Score a CAR answer on its mode key plus required concepts. + + Concepts come from the scenario's ``groundtruth_eval.json`` when it lists + ``required_terms`` or ``optional_terms``; otherwise they are extracted from + the gold value. They are read only when the mode key is right, so a + wrong-mode answer scores 0. Pass needs the mode and every required concept; + f1 counts the mode key and each required concept as one item. + """ gold = parse_structured_answer(gold_answer) model = parse_structured_answer(model_answer) + metadata = evaluation_metadata or {} + # The gold answer's key is the expected mode. A sidecar "mode" is only + # checked against it, so a typo there cannot invert the scenario. gold_key = str(next(iter(gold))).strip().lower() gold_value = next(iter(gold.values())) + declared_mode = str(metadata.get("mode") or "").strip().lower() model_is_dict = isinstance(model, dict) model_keys = [str(key).strip().lower() for key in model.keys()] if model_is_dict else [] @@ -562,14 +636,25 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: model_value = next(iter(model.values())) if model_is_dict and model_exactly_one_key else "" key_match = model_exactly_one_key and model_key == gold_key - required_terms = _extract_required_mode_terms(gold_value) - model_text = _normalize_text_for_terms(model_value) - matched_terms = [ - term for term in required_terms if f" {term} " in f" {model_text} " - ] - term_coverage = ( - len(matched_terms) / len(required_terms) if required_terms else 1.0 + + if "required_terms" in metadata or "optional_terms" in metadata: + required_concepts = _mode_concepts(metadata.get("required_terms")) + optional_concepts = _mode_concepts(metadata.get("optional_terms")) + else: + required_concepts = _mode_concepts(_extract_required_mode_terms(gold_value)) + optional_concepts = [] + + required_terms = [name for name, _ in required_concepts] + optional_terms = [name for name, _ in optional_concepts] + + model_text = _normalize_text_for_terms(model_value) if key_match else "" + matched_terms, term_coverage = _cover_concepts(required_concepts, model_text) + matched_optional_terms, optional_coverage = _cover_concepts( + optional_concepts, model_text ) + if not required_concepts: + term_coverage = 1.0 if key_match else 0.0 + car_score = _CAR_KEY_WEIGHT + _CAR_TERM_WEIGHT * term_coverage if key_match else 0.0 details = [ KeyComparison( @@ -619,7 +704,7 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: if precision + recall > 0 else 0.0 ) - strict_exact = 1.0 if key_match and term_coverage == 1.0 and not extra_keys else 0.0 + strict_exact = 1.0 if key_match and term_coverage == 1.0 else 0.0 return StaticJsonScore( partial_match_accuracy=recall, @@ -652,6 +737,13 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: mode_required_terms=required_terms, mode_matched_terms=matched_terms, mode_term_coverage=term_coverage, + mode_optional_terms=optional_terms, + mode_matched_optional_terms=matched_optional_terms, + # None when the scenario declares no optional terms, so it does not + # drag down mode_optional_term_coverage_avg in the report. + mode_optional_term_coverage=optional_coverage if optional_concepts else None, + car_score=car_score, + mode_spec_matches_gold=1.0 if declared_mode in ("", gold_key) else 0.0, ) @@ -838,10 +930,13 @@ def evaluate_static_json( model_answer: Any, *, similarity_threshold: float = 0.0, + evaluation_metadata: dict[str, Any] | None = None, ) -> StaticJsonScore: """Evaluate one structured gold answer against one model answer.""" if _is_mode_gold_answer(gold_answer): - return _evaluate_mode_json(gold_answer, model_answer) + return _evaluate_mode_json( + gold_answer, model_answer, evaluation_metadata=evaluation_metadata + ) if _is_choice_scalar(gold_answer): model_answer = _normalize_choice_answer(model_answer) @@ -1067,7 +1162,9 @@ def __call__( ), ) - static_score = evaluate_static_json(gold_answer, answer) + static_score = evaluate_static_json( + gold_answer, answer, evaluation_metadata=scenario.evaluation_metadata + ) passed = static_score.strict_exact_match_accuracy == 1.0 return ScorerResult( diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 2ab9db7e0..5b481eb8e 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -430,7 +430,7 @@ def test_mode_needs_every_required_term_to_pass(): assert score.f1 == pytest.approx(2 / 3) -def test_mode_wrong_key_still_earns_partial_f1_for_terms(): +def test_mode_wrong_key_reads_no_terms(): score = evaluate_static_json( {"clarification": "Which asset do you mean by 'the main unit'?"}, {"response": "The main unit appears to be PMP42144."}, @@ -438,8 +438,10 @@ def test_mode_wrong_key_still_earns_partial_f1_for_terms(): assert score.strict_exact_match_accuracy == 0.0 assert score.mode_key_match == 0.0 - assert score.mode_term_coverage == 1.0 - assert score.f1 == pytest.approx(0.4) + assert score.mode_matched_terms == [] + assert score.mode_term_coverage == 0.0 + assert score.f1 == 0.0 + assert score.car_score == 0.0 def test_mode_key_is_case_insensitive(): @@ -464,6 +466,108 @@ def test_mode_fails_when_answer_is_not_an_object(): assert score.extra_keys == ["answer"] +_MAIN_UNIT_GOLD = {"clarification": "Which asset do you mean by 'the main unit'?"} + + +def test_car_sidecar_concepts_replace_terms_from_the_gold(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which pump are you referring to?"}, + evaluation_metadata={ + "mode": "clarification", + "required_terms": [ + {"concept": "ambiguous asset", "any_of": ["which pump", "which asset"]} + ], + }, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.f1 == 1.0 + assert score.mode_required_terms == ["ambiguous asset"] + assert score.mode_matched_terms == ["ambiguous asset"] + + +def test_car_sidecar_needs_every_required_concept(): + metadata = { + "required_terms": [ + {"concept": "ambiguous asset", "any_of": ["main unit"]}, + {"concept": "asks for tag", "any_of": ["asset tag", "tag number"]}, + ] + } + + partial = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which asset is the main unit?"}, + evaluation_metadata=metadata, + ) + full = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "What is the tag number of the main unit?"}, + evaluation_metadata=metadata, + ) + + assert partial.strict_exact_match_accuracy == 0.0 + assert partial.mode_term_coverage == 0.5 + assert partial.f1 == pytest.approx(2 / 3) + assert partial.car_score == pytest.approx(0.8) + assert full.strict_exact_match_accuracy == 1.0 + assert full.f1 == 1.0 + + +def test_car_sidecar_optional_terms_never_gate_or_grade(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which asset is the main unit?"}, + evaluation_metadata={ + "required_terms": [{"concept": "ambiguous asset", "any_of": ["main unit"]}], + "optional_terms": [{"concept": "asks for tag", "any_of": ["asset tag"]}], + }, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.f1 == 1.0 + assert score.mode_optional_terms == ["asks for tag"] + assert score.mode_matched_optional_terms == [] + assert score.mode_optional_term_coverage == 0.0 + + +def test_car_sidecar_accepts_plain_strings_and_normalizes_forms(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which asset is the MAIN-UNIT?"}, + evaluation_metadata={"required_terms": ["Main Unit"]}, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.mode_matched_terms == ["Main Unit"] + + +def test_car_sidecar_mode_never_overrides_the_gold_key(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"abstain": "Cannot tell which asset the main unit is."}, + evaluation_metadata={ + "mode": "abstain", + "required_terms": [{"concept": "ambiguous asset", "any_of": ["main unit"]}], + }, + ) + + assert score.mode_key_match == 0.0 + assert score.strict_exact_match_accuracy == 0.0 + assert score.mode_spec_matches_gold == 0.0 + + +def test_car_without_sidecar_reports_no_optional_coverage(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which asset is the main unit?"}, + ) + + assert score.mode_optional_terms == [] + assert score.mode_optional_term_coverage is None + assert score.mode_spec_matches_gold == 1.0 + + def test_count_only_exact_match(): score = evaluate_static_json("34", "The answer is 34.") @@ -531,7 +635,30 @@ def test_static_json_scorer_scores_car_on_mode_key_and_required_terms(): assert term_missing.passed is False assert term_missing.score == 0.5 assert wrong_key.passed is False - assert wrong_key.score == 0.4 + assert wrong_key.score == 0.0 + + +def test_static_json_scorer_reads_car_sidecar_from_the_scenario(): + scenario = Scenario.from_raw( + { + "id": "151", + "text": "Clarify main unit.", + "expected_answer": '{"clarification": "Which asset do you mean by the main unit?"}', + "evaluation_metadata": { + "mode": "clarification", + "required_terms": [ + {"concept": "ambiguous asset", "any_of": ["which pump", "main unit"]} + ], + }, + "scoring_method": "static_json", + } + ) + + result = StaticJsonScorer()(scenario, '{"clarification": "Which pump?"}', "") + + assert result.passed is True + assert result.score == 1.0 + assert result.details["mode_matched_terms"] == ["ambiguous asset"] def test_identifier_with_a_separator_is_not_read_as_a_count():