From 4c389c49adc222dd9a23006388878911c105b588 Mon Sep 17 00:00:00 2001
From: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
Date: Thu, 22 Oct 2026 09:16:27 -0700
Subject: [PATCH] Restore TriviaQA prediction normalization

I noticed that the TriviaQA task lowercases and strips punctuation from the gold aliases but no longer normalizes the model's answer, so a correct answer such as "John Nash" scores 0. I restored the harness_triviaqa_normalizer prediction normalizer that the task used before the tasks were moved to individual files in #1016, and added a regression test.

Assisted-by: Claude
Assisted-by: GPT-6 Astra
Signed-off-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
---
 src/lighteval/tasks/tasks/triviaqa.py |  5 ++-
 tests/unit/tasks/test_triviaqa.py     | 62 +++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100644 tests/unit/tasks/test_triviaqa.py

diff --git a/src/lighteval/tasks/tasks/triviaqa.py b/src/lighteval/tasks/tasks/triviaqa.py
index f158eb67a..2bcb3c386 100644
--- a/src/lighteval/tasks/tasks/triviaqa.py
+++ b/src/lighteval/tasks/tasks/triviaqa.py
@@ -25,6 +25,7 @@
 import string
 
 from lighteval.metrics.metrics import Metrics
+from lighteval.metrics.normalizations import harness_triviaqa_normalizer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
@@ -61,7 +62,9 @@ def _remove_prefixes(aliases):
     few_shots_split=None,
     few_shots_select=None,
     generation_size=20,
-    metrics=[Metrics.exact_match],
+    metrics=[
+        Metrics.exact_match(sample_params={"strip_strings": True, "normalize_pred": harness_triviaqa_normalizer})
+    ],
     stop_sequence=["\n", ".", ","],
     version=0,
 )
diff --git a/tests/unit/tasks/test_triviaqa.py b/tests/unit/tasks/test_triviaqa.py
new file mode 100644
index 000000000..126c78c56
--- /dev/null
+++ b/tests/unit/tasks/test_triviaqa.py
@@ -0,0 +1,62 @@
+# MIT License
+
+# Copyright (c) 2026 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import pytest
+
+from lighteval.models.model_output import ModelResponse
+from lighteval.tasks.tasks.triviaqa import triviaqa, triviaqa_prompt
+
+
+# Questions bb_5713 and wh_4325 from the TriviaQA rc.nocontext validation split
+JOHN_NASH = {
+    "question": "Name the pioneering Game Theory US mathematician featured in the film A Beautiful Mind?",
+    "answer": {
+        "aliases": [
+            "John Nash (disambiguation)",
+            "Johnny Nash (disambiguation)",
+            "John nash",
+            "John Nash",
+            "Nash, John",
+        ]
+    },
+}
+VELAZQUEZ = {
+    "question": "Give a year in the life of the artist Diego Velazquez.",
+    "answer": {"aliases": ["1599-1660"]},
+}
+
+
+@pytest.mark.parametrize(
+    "line, prediction, expected",
+    [
+        (JOHN_NASH, " John Nash", 1),
+        (JOHN_NASH, " Johnny Nash", 0),
+        (VELAZQUEZ, " 1599-1660", 1),
+    ],
+)
+def test_triviaqa_exact_match_normalizes_prediction(line, prediction, expected):
+    doc = triviaqa_prompt(line, task_name="triviaqa")
+    model_response = ModelResponse(text=[prediction])
+
+    for metric in triviaqa.metrics:
+        result = metric.compute_sample(doc=doc, model_response=model_response)
+        assert result[metric.metric_name] == expected