fix(tasks): read a timeout off the message prefix, not by substring

sieval/tasks/_code_eval_msg.py

@@ -0,0 +1,142 @@
+"""How a task reads a verdict out of the code-eval service's ``msg``.
+
+Every task that grades by executing a prediction reports a ``timeouts`` count and
+has to decide, from a message string, whether the failure was the clock. That is
+one contract with six readers, and it belongs to the service
+(``vendor/code-evaluator``) rather than to any benchmark — which is what puts it
+here instead of in one of their trees.
+
+**The tail of a message is not vocabulary.** The service emits a small closed set
+of shapes, but each interpolates the failing program's own output after the
+colon, so a substring test for ``"timeout"`` matches all four of these::
+
+    failed: subprocess timeout: 3.0s             a timeout
+    failed: [CaseTimeout]                        a timeout
+    failed: [TimeoutError] deadline exceeded     NOT one: the program raised
+    failed: output ['timeout'] != expect ['ok']  NOT one: wrong answer
+
+Both false positives are reachable — a LiveCodeBench comparison failure prints
+the program's stdout verbatim, and any Python program may raise ``TimeoutError``
+— and both land in the one bucket that means the program ran and was merely slow.
+
+Only the diagnostic moves: ``correct`` comes from the service's boolean, never
+from this string. That is why six copies of a substring test survived — nothing
+they scored was wrong.
+
+AI-Generated Code - Claude Opus 5 (1M context) (Anthropic)
+"""
+
+#: **Every prefix is lowercased**, and a caller comparing against these tuples
+#: must lowercase the message first. Only ``failed: [casetimeout]`` actually
+#: depends on it, so a raw-message comparison still passes against the other
+#: five and silently drops that one. :func:`is_timeout_message` normalizes for
+#: you; reach for a tuple directly only to keep the two apart, as
+#: ``multipl_e/_base.py`` does.
+#:
+#: A wall the service hit BEFORE the program ran:
+#:
+#: * ``failed: build timeout`` — a compile wall (``exec_lang``)
+#: * ``failed: compile timeout:`` — LiveCodeBench's per-case compile budget
+#:   (``exec_py_test``)
+CODE_EVAL_BUILD_TIMEOUT_PREFIXES: tuple[str, ...] = (
+    "failed: build timeout",
+    "failed: compile timeout",
+)
+
+#: A wall the service hit while the program was RUNNING. Lowercased, as above.
+#:
+#: * ``failed: subprocess timeout:`` — the whole-submission wall
+#:   (``exec_py_code``, ``exec_py_test``)
+#: * ``failed: case timeout:`` — LiveCodeBench's per-case budget
+#:   (``exec_py_test``)
+#: * ``failed: [CaseTimeout]`` — that same per-case wall, arriving late. A
+#:   delivered signal cannot be un-delivered, so an alarm landing past
+#:   ``_unsafe_execute``'s own handler is formatted by the worker's outer
+#:   ``except`` as a class name. **The one entry a program can forge**: the
+#:   service's class is a ``BaseException``, which is why that handler names it
+#:   explicitly, but a submitted program defining ``class
+#:   CaseTimeout(Exception)`` is caught by ``_unsafe_execute_fn_call``'s
+#:   ``except Exception`` and formatted into the same shape. Kept regardless —
+#:   dropping it loses a real wall, the substring test this replaced counted the
+#:   forgery too, and what the forgery costs is one diagnostic count.
+#: * ``failed: timeout`` — the run wall for a non-Python ``lang`` (``exec_js``,
+#:   ``exec_ts``, ``exec_lang``)
+CODE_EVAL_RUN_TIMEOUT_PREFIXES: tuple[str, ...] = (
+    "failed: subprocess timeout",
+    "failed: case timeout",
+    "failed: [casetimeout]",
+    "failed: timeout",
+)
+
+#: Every prefix that means the SERVICE stopped the program on a clock.
+#:
+#: A union across executors — the spelling depends on which one answered — and
+#: each is emitted only by a timeout path, so carrying one a task cannot
+#: currently produce is harmless. This module's readers are all Python and see
+#: neither ``failed: timeout`` nor ``failed: build timeout``; MultiPL-E sees
+#: only those two, since each of its 24 languages routes through ``exec_js`` /
+#: ``exec_ts`` / ``exec_lang`` and none of them is Python.
+#:
+#: Excluded, and each a live false positive under a substring test:
+#: ``failed: [TimeoutError] ...`` (the program raised) and any
+#: ``failed: output ... != expect ...`` quoting program output.
+#:
+#: **"The service stopped the clock" is not "belongs in ``timeouts``."** This
+#: answers the first question only; the counter is the caller's call. MultiPL-E
+#: routes ``failed: build timeout`` to ``n_build_errors`` rather than to
+#: ``timeouts``, deliberately — the run never started, and build-versus-run is
+#: the split its three keys exist to carry (``multipl_e/_base.py``). That is
+#: what the two groups above are for: an adopting task takes the vocabulary and
+#: still owes its own bucketing, where a bare :func:`is_timeout_message` would
+#: move its build walls.
+CODE_EVAL_TIMEOUT_PREFIXES: tuple[str, ...] = (
+    CODE_EVAL_BUILD_TIMEOUT_PREFIXES + CODE_EVAL_RUN_TIMEOUT_PREFIXES
+)
+
+
+def is_timeout_message(msg: str | None) -> bool:
+    """Whether *msg* says the service stopped the program on a clock.
+
+    ``None`` is False rather than an error: a null ``msg`` is absent on disk, so
+    a resumed record legitimately arrives without one.
+    """
+    if not msg:
+        return False
+    return msg.strip().lower().startswith(CODE_EVAL_TIMEOUT_PREFIXES)
+
+
+#: How the service reports an uncaught exception: ``failed: [<Class>] <str(e)>``.
+_EXC_MSG_PREFIX = "failed: ["
+
+
+def exception_class_name(msg: str | None) -> str | None:
+    """The exception class the service formatted into *msg*, lowercased.
+
+    The class is a structured SLOT; everything after it is the program's own
+    output. Reading the slot is what separates ``[MemoryError] ...`` from a
+    program that merely printed the word. ``None`` when *msg* is not that shape.
+
+    Returns the name rather than comparing it, so a caller can match a *family*.
+    That matters: the counters reading this want subclasses too, and the class
+    the service names is not always the builtin — ``ZipImportError`` is an
+    ``ImportError``, ``OutOfMemoryError`` a ``MemoryError``. Testing for the
+    exact builtin would drop them, which a bare substring test did not.
+    """
+    if not msg:
+        return None
+    text = msg.strip()
+    if not text.lower().startswith(_EXC_MSG_PREFIX):
+        return None
+    end = text.find("]", len(_EXC_MSG_PREFIX))
+    if end == -1:
+        return None
+    return text[len(_EXC_MSG_PREFIX) : end].lower()
+
+
+__all__ = [
+    "CODE_EVAL_BUILD_TIMEOUT_PREFIXES",
+    "CODE_EVAL_RUN_TIMEOUT_PREFIXES",
+    "CODE_EVAL_TIMEOUT_PREFIXES",
+    "exception_class_name",
+    "is_timeout_message",
+]

sieval/tasks/human_eval_0shot_base_gen.py

@@ -51,6 +51,8 @@
 )
 from sieval.datasets import HumanEvalDatasetSample
 
+from ._code_eval_msg import is_timeout_message
+
 STOP_SEQUENCES = ("\nclass", "\ndef", "\n#", "\nif", "\nprint")
 
 
@@ -219,8 +221,7 @@ async def report(self, finals, fails):
             1
             for f in finals
             for r in f.feedback_result["rollouts"]
-            # A null msg from the evaluator is absent on disk -- default it.
-            if "timeout" in (r["extra"].get("msg") or "").lower()
+            if is_timeout_message(r["extra"].get("msg"))
         )
         # `votes=False`: two correct programs are not one answer, so there is
         # nothing well-defined to take a majority over (RFC #74).

sieval/tasks/human_eval_0shot_gen.py

@@ -32,6 +32,8 @@
 )
 from sieval.datasets import HumanEvalDatasetSample
 
+from ._code_eval_msg import is_timeout_message
+
 
 @sieval_task(
     name="human_eval_0shot_gen",
@@ -182,8 +184,7 @@ async def report(self, finals, fails):
             1
             for f in finals
             for r in f.feedback_result["rollouts"]
-            # A null msg from the evaluator is absent on disk -- default it.
-            if "timeout" in (r["extra"].get("msg") or "").lower()
+            if is_timeout_message(r["extra"].get("msg"))
         )
         # `votes=False`: two correct programs are not one answer, so there is
         # nothing well-defined to take a majority over (RFC #74).

sieval/tasks/livecodebench_code_generation_0shot_gen.py

@@ -39,6 +39,8 @@
 )
 from sieval.datasets import LiveCodeBenchDatasetSample
 
+from ._code_eval_msg import is_timeout_message
+
 
 @sieval_task(
     name="livecodebench_code_generation_0shot_gen",
@@ -254,8 +256,7 @@ async def report(self, finals, fails):
             1
             for f in finals
             for r in f.feedback_result["rollouts"]
-            # A null msg from the evaluator is absent on disk -- default it.
-            if "timeout" in (r["extra"].get("msg") or "").lower()
+            if is_timeout_message(r["extra"].get("msg"))
         )
         # `votes=False`: two correct programs are not one answer, so there is
         # nothing well-defined to take a majority over (RFC #74).

sieval/tasks/livecodebench_code_generation_kshot_base_gen.py

@@ -83,6 +83,8 @@
 )
 from sieval.datasets import LiveCodeBenchDatasetSample
 
+from ._code_eval_msg import is_timeout_message
+
 N_SHOT = 3
 STOP_SEQUENCES = ("###",)
 
@@ -312,8 +314,7 @@ async def report(self, finals, fails):
             1
             for f in finals
             for r in f.feedback_result["rollouts"]
-            # A null msg from the evaluator is absent on disk -- default it.
-            if "timeout" in (r["extra"].get("msg") or "").lower()
+            if is_timeout_message(r["extra"].get("msg"))
         )
         # `votes=False`: two correct programs are not one answer, so there is
         # nothing well-defined to take a majority over (RFC #74).

sieval/tasks/mbpp_kshot_base_gen.py

@@ -44,6 +44,8 @@
 )
 from sieval.datasets import MBPPDatasetSample
 
+from ._code_eval_msg import is_timeout_message
+
 DEFAULT_NUM_SHOTS = 3
 STOP_SEQUENCES = ("[DONE]",)
 
@@ -268,8 +270,7 @@ async def report(
             1
             for f in finals
             for r in f.feedback_result["rollouts"]
-            # A null msg from the evaluator is absent on disk -- default it.
-            if "timeout" in (r["extra"].get("msg") or "").lower()
+            if is_timeout_message(r["extra"].get("msg"))
         )
         # `votes=False`: two correct programs are not one answer, so there is
         # nothing well-defined to take a majority over (RFC #74).

sieval/tasks/multipl_e/_base.py

@@ -80,6 +80,10 @@
     sampling_report,
     ungated_intervals,
 )
+from sieval.tasks._code_eval_msg import (
+    CODE_EVAL_BUILD_TIMEOUT_PREFIXES,
+    CODE_EVAL_RUN_TIMEOUT_PREFIXES,
+)
 
 # Commit-pinned: `main` moves, and a divergence is argued against the code that
 # was actually read. `main.py` is where the graded program is assembled.
@@ -626,8 +630,9 @@ def _failure_buckets(self, finals) -> dict[str, float]:
         / ``ReferenceError`` / ``Exception``); the code-eval API answers with one
         boolean and a message, so what survives is the split a caller cannot
         reconstruct from the verdict — build versus run. Read off the message
-        prefixes the service documents, the same way the HumanEval tasks read
-        theirs for `timeouts`.
+        prefixes the service documents, out of the same shared vocabulary the
+        HumanEval tasks read theirs for `timeouts` from
+        (``sieval/tasks/_code_eval_msg.py``).
 
         ``n_execution_errors`` keeps the name it has in every other task that
         executes a prediction; the build bucket is new because no other task has
@@ -641,7 +646,17 @@ def _failure_buckets(self, finals) -> dict[str, float]:
         test, in the one bucket that suggests the model's program was slow
         rather than broken. A build that exceeds ITS wall lands in the build
         bucket rather than under ``timeouts``: the run never started, and
-        build-versus-run is the split these three keys exist to carry.
+        build-versus-run is the split these three keys exist to carry. That is
+        why this takes the two prefix GROUPS and not ``is_timeout_message``,
+        which answers "did the service stop the clock" and would therefore move
+        a build wall into ``timeouts``.
+
+        Each group carries prefixes this task cannot see — none of its 24
+        languages is Python, so ``exec_py_code`` / ``exec_py_test`` never
+        answer it. Routing them anyway costs nothing and is what keeps the
+        vocabulary in one place: were a Python row ever added, a subprocess
+        wall would already land in ``timeouts`` rather than silently in
+        ``n_execution_errors``.
         """
         timeouts = build_errors = execution_errors = 0
         for final in finals:
@@ -652,10 +667,16 @@ def _failure_buckets(self, finals) -> dict[str, float]:
                 if rollout["correct"]:
                     continue
                 # A null msg from the evaluator is absent on disk -- default it.
+                # Lowercased because the shared prefixes are: `[casetimeout]` is
+                # the one entry whose match depends on it.
                 msg = (rollout["extra"].get("msg") or "").lower()
-                if msg.startswith(("failed: build timeout", "failed [build exit")):
+                # `failed [build exit` is a build failure but not a timeout, so
+                # it is not the shared module's vocabulary and stays here.
+                if msg.startswith(
+                    (*CODE_EVAL_BUILD_TIMEOUT_PREFIXES, "failed [build exit")
+                ):
                     build_errors += 1
-                elif msg.startswith("failed: timeout"):
+                elif msg.startswith(CODE_EVAL_RUN_TIMEOUT_PREFIXES):
                     timeouts += 1
                 else:
                     execution_errors += 1

sieval/tasks/scicode_0shot_gen.py

@@ -76,6 +76,8 @@
 from sieval.core.utils.meta import build_stage_meta
 from sieval.datasets import SciCodeDatasetSample
 
+from ._code_eval_msg import exception_class_name, is_timeout_message
+
 
 class StepCode(TypedDict):
     step_number: str
@@ -574,14 +576,35 @@ async def report(self, finals, fails):
                 1 for fb in feedbacks if fb.get("empty_extraction")
             )
             messages = [str(fb.get("msg", "")).lower() for fb in feedbacks]
-            timeouts += sum("timeout" in msg for msg in messages)
-            memory_errors += sum("memoryerror" in msg for msg in messages)
+            # All three counters read the exception CLASS SLOT, never the tail:
+            # `failed: [{type}] {e}` interpolates the program's own output after
+            # the class, so a bare `"memoryerror" in msg` also counted
+            # `[ValueError] simulated memoryerror path`. Matching the family
+            # (`endswith`) rather than the builtin keeps the subclasses the bare
+            # test happened to catch -- `ZipImportError` is an `ImportError`,
+            # `OutOfMemoryError` a `MemoryError` -- so this drops false
+            # positives only, and never a class the service actually named.
+            exc_classes = [exception_class_name(msg) for msg in messages]
+            # `timeouts` alone keeps a second reading: the service's own wall is
+            # a message prefix, not an exception at all.
+            timeouts += sum(
+                is_timeout_message(msg)
+                or (cls is not None and cls.endswith("timeouterror"))
+                for msg, cls in zip(messages, exc_classes, strict=True)
+            )
+            memory_errors += sum(
+                cls is not None and cls.endswith("memoryerror") for cls in exc_classes
+            )
             # ModuleNotFoundError is an ImportError subclass, but the evaluator
-            # reports the concrete class name, which does not contain
+            # reports the concrete class name, which does not END in
             # "importerror". It is the signature of a package missing from the
-            # code-eval image, so it must not read as import_errors=0.
+            # code-eval image, so it must not read as import_errors=0. Its own
+            # family is matched the same way the other two are, rather than by
+            # equality: the rule here is "the service named a class in this
+            # family", and a subclass is what the service actually reports.
             import_errors += sum(
-                "importerror" in msg or "modulenotfounderror" in msg for msg in messages
+                cls is not None and cls.endswith(("importerror", "modulenotfounderror"))
+                for cls in exc_classes
             )
 
         # A pipeline failure is an unsolved problem in BOTH accuracies. Its tested