diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index daf2bd23..a12bbfe4 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -100,9 +100,8 @@ The evaluator then joins trajectories and ground truth by scenario id. Scenario folders may also include `groundtruth_eval.json`, which carries scorer-specific metadata alongside the canonical answer in `groundtruth.txt`. -The `static_json` scorer does not read it. For clarification-abstain-response -(CAR) scenarios the expected mode is the single key of `groundtruth.txt`, and -the required terms are extracted from the value under that key. +The `static_json` scorer reads it for clarification-abstain-response (CAR) +scenarios, where it lists the concepts a correct answer must cover. Example `groundtruth.txt`: @@ -112,6 +111,30 @@ Example `groundtruth.txt`: } ``` +Example `groundtruth_eval.json`: + +```json +{ + "mode": "clarification", + "required_terms": [ + {"concept": "ambiguous asset", "any_of": ["main unit", "which asset", "which pump"]} + ], + "optional_terms": [ + {"concept": "asks for tag", "any_of": ["asset tag", "tag number"]} + ] +} +``` + +| Field | Meaning | +| ---------------- | ---------------------------------------------------------------------------------------- | +| `mode` | Expected top-level key. Checked against the key of `groundtruth.txt`, never used instead of it | +| `required_terms` | Concepts that gate pass and count toward the score | +| `optional_terms` | Concepts reported for analysis only; they never affect pass or score | + +Each concept is `{"concept": , "any_of": []}` or a plain +string (a concept whose only form is itself). Other fields, such as `_comment`, +are ignored. + --- ## How Matching Works @@ -225,43 +248,54 @@ The scorer marks a scenario as passed only when the structured answer is a stric ### CAR metrics and pass criteria For CAR scenarios, the scorer evaluates the answer as a mode-selection task plus -required-term coverage. The model answer should be a single JSON object using -exactly one of these top-level keys: +concept coverage. The model answer should be a single JSON object using exactly +one of these top-level keys: ```text response | clarification | abstain ``` -The required terms are extracted from the gold value: quoted phrases, asset or -fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` stance, and a small -allowlist of domain terms. Matching is case-insensitive on normalised text. +The expected mode is the single key of `groundtruth.txt`. The required concepts +come from `groundtruth_eval.json` when it lists `required_terms` or +`optional_terms`. Otherwise they are extracted from the gold value: quoted +phrases, asset or fault identifiers (e.g. `PMP42144`), a leading `yes` / `no` +stance, and a small allowlist of domain terms, each a concept with one form. + +A concept is covered when any of its surface forms appears as whole words in +the value under the model's key, after lowercasing and replacing punctuation +with spaces. Concepts are read only when the mode key is right, so a wrong-mode +answer covers none. A CAR scenario passes when: 1. the model returns a JSON object with exactly one top-level key, 2. that key matches the expected mode (case-insensitive), and -3. every required term appears in the value under that key. +3. every required concept is covered. -The score is the F1 over the mode key plus each required term. Gold has -`1 + len(required_terms)` entries; the model side has the same plus any extra -top-level keys. For gold `{"clarification": "Which asset do you mean by 'the -main unit'?"}` (one required term, `main unit`): +The score is the F1 over the mode key plus each required concept: with the right +mode and `k` of `n` concepts covered it is `(1 + k) / (1 + n)`, and a wrong mode +scores 0. With the example `groundtruth_eval.json` above (one required concept): | Model answer | Passed | Score | | ---------------------------------------------------- | ------ | ----- | -| `{"clarification": "Which asset is the main unit?"}` | yes | 1.0 | -| `{"clarification": "Which pump?"}` | no | 0.5 | -| `{"response": "The main unit is PMP42144."}` | no | 0.4 | +| `{"clarification": "Which pump do you mean?"}` | yes | 1.0 | +| `{"clarification": "Could you clarify?"}` | no | 0.5 | +| `{"response": "The main unit is PMP42144."}` | no | 0.0 | Additional CAR report fields: -| Metric | Meaning | -| ---------------------- | ---------------------------------------------------------- | -| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | -| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | -| `mode_required_terms` | Required terms extracted from the gold answer value | -| `mode_matched_terms` | Required terms found in the model answer value | -| `mode_term_coverage` | Fraction of required terms found | +| Metric | Meaning | +| ----------------------------- | -------------------------------------------------------------------- | +| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | +| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | +| `mode_required_terms` | Required concept names | +| `mode_matched_terms` | Required concepts covered by the answer | +| `mode_term_coverage` | Fraction of required concepts covered | +| `mode_optional_terms` | Optional concept names | +| `mode_matched_optional_terms` | Optional concepts covered by the answer | +| `mode_optional_term_coverage` | Fraction of optional concepts covered, or null if none are declared | +| `car_score` | `0.6 + 0.4 * mode_term_coverage` with the right mode, else 0; reported only | +| `mode_spec_matches_gold` | 0.0 when `groundtruth_eval.json` declares a mode the gold does not use | --- diff --git a/src/evaluation/report.py b/src/evaluation/report.py index 0be6769d..7351e37a 100644 --- a/src/evaluation/report.py +++ b/src/evaluation/report.py @@ -59,6 +59,7 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: "mode_term_coverage", "mode_optional_term_coverage", "car_score", + "mode_spec_matches_gold", ] score_values: dict[str, list[float]] = {name: [] for name in metric_names} @@ -152,6 +153,9 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: score_values["mode_optional_term_coverage"] ), "car_score_avg": _avg(score_values["car_score"]), + # Below 1.0 means a groundtruth_eval.json declares a mode its gold + # answer does not use: a scenario defect, not a model result. + "mode_spec_matches_gold_avg": _avg(score_values["mode_spec_matches_gold"]), "missing_keys_total": missing_keys_total, "extra_keys_total": extra_keys_total, "detail_entries_total": detail_entries_total, diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index a70b5d65..7fed21ee 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -78,6 +78,11 @@ class StaticJsonScore: mode_required_terms: list[str] = field(default_factory=list) mode_matched_terms: list[str] = field(default_factory=list) mode_term_coverage: float | None = None + mode_optional_terms: list[str] = field(default_factory=list) + mode_matched_optional_terms: list[str] = field(default_factory=list) + mode_optional_term_coverage: float | None = None + car_score: float | None = None + mode_spec_matches_gold: float | None = None def to_dict(self) -> dict[str, Any]: """Return a JSON-serializable dictionary.""" @@ -540,6 +545,58 @@ def _extract_required_mode_terms(value: Any) -> list[str]: return _dedupe_terms(terms) +_CAR_KEY_WEIGHT = 0.6 +_CAR_TERM_WEIGHT = 0.4 + + +def _mode_concepts(raw: Any) -> list[tuple[str, list[str]]]: + """Normalize a CAR term list into (concept, accepted surface forms). + + Each entry is either a plain string or ``{"concept": str, "any_of": [str]}``. + A concept is covered when ANY of its surface forms occurs in the answer, so + a correct answer is not penalized for paraphrasing. Forms are normalized + like the answer text, so case and hyphenation cannot cause a miss. + """ + concepts: list[tuple[str, list[str]]] = [] + if not isinstance(raw, (list, tuple)): + return concepts + for entry in raw: + if isinstance(entry, str): + name, forms = entry, [entry] + elif isinstance(entry, dict): + name = str(entry.get("concept") or "").strip() + forms = entry.get("any_of") or ([name] if name else []) + if not isinstance(forms, (list, tuple)): + forms = [forms] + else: + continue + normalized: list[str] = [] + for form in forms: + if not isinstance(form, str): + continue + text = _normalize_text_for_terms(form) + if text and text not in normalized: + normalized.append(text) + if name and normalized: + concepts.append((name, normalized)) + return concepts + + +def _cover_concepts( + concepts: list[tuple[str, list[str]]], text: str +) -> tuple[list[str], float]: + """Return the covered concept names and the coverage fraction.""" + if not concepts: + return [], 1.0 + haystack = f" {text} " + covered = [ + name + for name, forms in concepts + if any(f" {form} " in haystack for form in forms) + ] + return covered, len(covered) / len(concepts) + + def _is_mode_gold_answer(value: Any) -> bool: parsed = parse_structured_answer(value) if not isinstance(parsed, dict) or len(parsed) != 1: @@ -548,12 +605,29 @@ def _is_mode_gold_answer(value: Any) -> bool: return key in _MODE_KEYS -def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: +def _evaluate_mode_json( + gold_answer: Any, + model_answer: Any, + *, + evaluation_metadata: dict[str, Any] | None = None, +) -> StaticJsonScore: + """Score a CAR answer on its mode key plus required concepts. + + Concepts come from the scenario's ``groundtruth_eval.json`` when it lists + ``required_terms`` or ``optional_terms``; otherwise they are extracted from + the gold value. They are read only when the mode key is right, so a + wrong-mode answer scores 0. Pass needs the mode and every required concept; + f1 counts the mode key and each required concept as one item. + """ gold = parse_structured_answer(gold_answer) model = parse_structured_answer(model_answer) + metadata = evaluation_metadata or {} + # The gold answer's key is the expected mode. A sidecar "mode" is only + # checked against it, so a typo there cannot invert the scenario. gold_key = str(next(iter(gold))).strip().lower() gold_value = next(iter(gold.values())) + declared_mode = str(metadata.get("mode") or "").strip().lower() model_is_dict = isinstance(model, dict) model_keys = [str(key).strip().lower() for key in model.keys()] if model_is_dict else [] @@ -562,14 +636,25 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: model_value = next(iter(model.values())) if model_is_dict and model_exactly_one_key else "" key_match = model_exactly_one_key and model_key == gold_key - required_terms = _extract_required_mode_terms(gold_value) - model_text = _normalize_text_for_terms(model_value) - matched_terms = [ - term for term in required_terms if f" {term} " in f" {model_text} " - ] - term_coverage = ( - len(matched_terms) / len(required_terms) if required_terms else 1.0 + + if "required_terms" in metadata or "optional_terms" in metadata: + required_concepts = _mode_concepts(metadata.get("required_terms")) + optional_concepts = _mode_concepts(metadata.get("optional_terms")) + else: + required_concepts = _mode_concepts(_extract_required_mode_terms(gold_value)) + optional_concepts = [] + + required_terms = [name for name, _ in required_concepts] + optional_terms = [name for name, _ in optional_concepts] + + model_text = _normalize_text_for_terms(model_value) if key_match else "" + matched_terms, term_coverage = _cover_concepts(required_concepts, model_text) + matched_optional_terms, optional_coverage = _cover_concepts( + optional_concepts, model_text ) + if not required_concepts: + term_coverage = 1.0 if key_match else 0.0 + car_score = _CAR_KEY_WEIGHT + _CAR_TERM_WEIGHT * term_coverage if key_match else 0.0 details = [ KeyComparison( @@ -619,7 +704,7 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: if precision + recall > 0 else 0.0 ) - strict_exact = 1.0 if key_match and term_coverage == 1.0 and not extra_keys else 0.0 + strict_exact = 1.0 if key_match and term_coverage == 1.0 else 0.0 return StaticJsonScore( partial_match_accuracy=recall, @@ -652,6 +737,13 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: mode_required_terms=required_terms, mode_matched_terms=matched_terms, mode_term_coverage=term_coverage, + mode_optional_terms=optional_terms, + mode_matched_optional_terms=matched_optional_terms, + # None when the scenario declares no optional terms, so it does not + # drag down mode_optional_term_coverage_avg in the report. + mode_optional_term_coverage=optional_coverage if optional_concepts else None, + car_score=car_score, + mode_spec_matches_gold=1.0 if declared_mode in ("", gold_key) else 0.0, ) @@ -838,10 +930,13 @@ def evaluate_static_json( model_answer: Any, *, similarity_threshold: float = 0.0, + evaluation_metadata: dict[str, Any] | None = None, ) -> StaticJsonScore: """Evaluate one structured gold answer against one model answer.""" if _is_mode_gold_answer(gold_answer): - return _evaluate_mode_json(gold_answer, model_answer) + return _evaluate_mode_json( + gold_answer, model_answer, evaluation_metadata=evaluation_metadata + ) if _is_choice_scalar(gold_answer): model_answer = _normalize_choice_answer(model_answer) @@ -1067,7 +1162,9 @@ def __call__( ), ) - static_score = evaluate_static_json(gold_answer, answer) + static_score = evaluate_static_json( + gold_answer, answer, evaluation_metadata=scenario.evaluation_metadata + ) passed = static_score.strict_exact_match_accuracy == 1.0 return ScorerResult( diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 2ab9db7e..5b481eb8 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -430,7 +430,7 @@ def test_mode_needs_every_required_term_to_pass(): assert score.f1 == pytest.approx(2 / 3) -def test_mode_wrong_key_still_earns_partial_f1_for_terms(): +def test_mode_wrong_key_reads_no_terms(): score = evaluate_static_json( {"clarification": "Which asset do you mean by 'the main unit'?"}, {"response": "The main unit appears to be PMP42144."}, @@ -438,8 +438,10 @@ def test_mode_wrong_key_still_earns_partial_f1_for_terms(): assert score.strict_exact_match_accuracy == 0.0 assert score.mode_key_match == 0.0 - assert score.mode_term_coverage == 1.0 - assert score.f1 == pytest.approx(0.4) + assert score.mode_matched_terms == [] + assert score.mode_term_coverage == 0.0 + assert score.f1 == 0.0 + assert score.car_score == 0.0 def test_mode_key_is_case_insensitive(): @@ -464,6 +466,108 @@ def test_mode_fails_when_answer_is_not_an_object(): assert score.extra_keys == ["answer"] +_MAIN_UNIT_GOLD = {"clarification": "Which asset do you mean by 'the main unit'?"} + + +def test_car_sidecar_concepts_replace_terms_from_the_gold(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which pump are you referring to?"}, + evaluation_metadata={ + "mode": "clarification", + "required_terms": [ + {"concept": "ambiguous asset", "any_of": ["which pump", "which asset"]} + ], + }, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.f1 == 1.0 + assert score.mode_required_terms == ["ambiguous asset"] + assert score.mode_matched_terms == ["ambiguous asset"] + + +def test_car_sidecar_needs_every_required_concept(): + metadata = { + "required_terms": [ + {"concept": "ambiguous asset", "any_of": ["main unit"]}, + {"concept": "asks for tag", "any_of": ["asset tag", "tag number"]}, + ] + } + + partial = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which asset is the main unit?"}, + evaluation_metadata=metadata, + ) + full = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "What is the tag number of the main unit?"}, + evaluation_metadata=metadata, + ) + + assert partial.strict_exact_match_accuracy == 0.0 + assert partial.mode_term_coverage == 0.5 + assert partial.f1 == pytest.approx(2 / 3) + assert partial.car_score == pytest.approx(0.8) + assert full.strict_exact_match_accuracy == 1.0 + assert full.f1 == 1.0 + + +def test_car_sidecar_optional_terms_never_gate_or_grade(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which asset is the main unit?"}, + evaluation_metadata={ + "required_terms": [{"concept": "ambiguous asset", "any_of": ["main unit"]}], + "optional_terms": [{"concept": "asks for tag", "any_of": ["asset tag"]}], + }, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.f1 == 1.0 + assert score.mode_optional_terms == ["asks for tag"] + assert score.mode_matched_optional_terms == [] + assert score.mode_optional_term_coverage == 0.0 + + +def test_car_sidecar_accepts_plain_strings_and_normalizes_forms(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which asset is the MAIN-UNIT?"}, + evaluation_metadata={"required_terms": ["Main Unit"]}, + ) + + assert score.strict_exact_match_accuracy == 1.0 + assert score.mode_matched_terms == ["Main Unit"] + + +def test_car_sidecar_mode_never_overrides_the_gold_key(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"abstain": "Cannot tell which asset the main unit is."}, + evaluation_metadata={ + "mode": "abstain", + "required_terms": [{"concept": "ambiguous asset", "any_of": ["main unit"]}], + }, + ) + + assert score.mode_key_match == 0.0 + assert score.strict_exact_match_accuracy == 0.0 + assert score.mode_spec_matches_gold == 0.0 + + +def test_car_without_sidecar_reports_no_optional_coverage(): + score = evaluate_static_json( + _MAIN_UNIT_GOLD, + {"clarification": "Which asset is the main unit?"}, + ) + + assert score.mode_optional_terms == [] + assert score.mode_optional_term_coverage is None + assert score.mode_spec_matches_gold == 1.0 + + def test_count_only_exact_match(): score = evaluate_static_json("34", "The answer is 34.") @@ -531,7 +635,30 @@ def test_static_json_scorer_scores_car_on_mode_key_and_required_terms(): assert term_missing.passed is False assert term_missing.score == 0.5 assert wrong_key.passed is False - assert wrong_key.score == 0.4 + assert wrong_key.score == 0.0 + + +def test_static_json_scorer_reads_car_sidecar_from_the_scenario(): + scenario = Scenario.from_raw( + { + "id": "151", + "text": "Clarify main unit.", + "expected_answer": '{"clarification": "Which asset do you mean by the main unit?"}', + "evaluation_metadata": { + "mode": "clarification", + "required_terms": [ + {"concept": "ambiguous asset", "any_of": ["which pump", "main unit"]} + ], + }, + "scoring_method": "static_json", + } + ) + + result = StaticJsonScorer()(scenario, '{"clarification": "Which pump?"}', "") + + assert result.passed is True + assert result.score == 1.0 + assert result.details["mode_matched_terms"] == ["ambiguous asset"] def test_identifier_with_a_separator_is_not_read_as_a_count():