diff --git a/sieval/tasks/_code_eval_msg.py b/sieval/tasks/_code_eval_msg.py new file mode 100644 index 00000000..32bf449f --- /dev/null +++ b/sieval/tasks/_code_eval_msg.py @@ -0,0 +1,142 @@ +"""How a task reads a verdict out of the code-eval service's ``msg``. + +Every task that grades by executing a prediction reports a ``timeouts`` count and +has to decide, from a message string, whether the failure was the clock. That is +one contract with six readers, and it belongs to the service +(``vendor/code-evaluator``) rather than to any benchmark — which is what puts it +here instead of in one of their trees. + +**The tail of a message is not vocabulary.** The service emits a small closed set +of shapes, but each interpolates the failing program's own output after the +colon, so a substring test for ``"timeout"`` matches all four of these:: + + failed: subprocess timeout: 3.0s a timeout + failed: [CaseTimeout] a timeout + failed: [TimeoutError] deadline exceeded NOT one: the program raised + failed: output ['timeout'] != expect ['ok'] NOT one: wrong answer + +Both false positives are reachable — a LiveCodeBench comparison failure prints +the program's stdout verbatim, and any Python program may raise ``TimeoutError`` +— and both land in the one bucket that means the program ran and was merely slow. + +Only the diagnostic moves: ``correct`` comes from the service's boolean, never +from this string. That is why six copies of a substring test survived — nothing +they scored was wrong. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +#: **Every prefix is lowercased**, and a caller comparing against these tuples +#: must lowercase the message first. Only ``failed: [casetimeout]`` actually +#: depends on it, so a raw-message comparison still passes against the other +#: five and silently drops that one. :func:`is_timeout_message` normalizes for +#: you; reach for a tuple directly only to keep the two apart, as +#: ``multipl_e/_base.py`` does. +#: +#: A wall the service hit BEFORE the program ran: +#: +#: * ``failed: build timeout`` — a compile wall (``exec_lang``) +#: * ``failed: compile timeout:`` — LiveCodeBench's per-case compile budget +#: (``exec_py_test``) +CODE_EVAL_BUILD_TIMEOUT_PREFIXES: tuple[str, ...] = ( + "failed: build timeout", + "failed: compile timeout", +) + +#: A wall the service hit while the program was RUNNING. Lowercased, as above. +#: +#: * ``failed: subprocess timeout:`` — the whole-submission wall +#: (``exec_py_code``, ``exec_py_test``) +#: * ``failed: case timeout:`` — LiveCodeBench's per-case budget +#: (``exec_py_test``) +#: * ``failed: [CaseTimeout]`` — that same per-case wall, arriving late. A +#: delivered signal cannot be un-delivered, so an alarm landing past +#: ``_unsafe_execute``'s own handler is formatted by the worker's outer +#: ``except`` as a class name. **The one entry a program can forge**: the +#: service's class is a ``BaseException``, which is why that handler names it +#: explicitly, but a submitted program defining ``class +#: CaseTimeout(Exception)`` is caught by ``_unsafe_execute_fn_call``'s +#: ``except Exception`` and formatted into the same shape. Kept regardless — +#: dropping it loses a real wall, the substring test this replaced counted the +#: forgery too, and what the forgery costs is one diagnostic count. +#: * ``failed: timeout`` — the run wall for a non-Python ``lang`` (``exec_js``, +#: ``exec_ts``, ``exec_lang``) +CODE_EVAL_RUN_TIMEOUT_PREFIXES: tuple[str, ...] = ( + "failed: subprocess timeout", + "failed: case timeout", + "failed: [casetimeout]", + "failed: timeout", +) + +#: Every prefix that means the SERVICE stopped the program on a clock. +#: +#: A union across executors — the spelling depends on which one answered — and +#: each is emitted only by a timeout path, so carrying one a task cannot +#: currently produce is harmless. This module's readers are all Python and see +#: neither ``failed: timeout`` nor ``failed: build timeout``; MultiPL-E sees +#: only those two, since each of its 24 languages routes through ``exec_js`` / +#: ``exec_ts`` / ``exec_lang`` and none of them is Python. +#: +#: Excluded, and each a live false positive under a substring test: +#: ``failed: [TimeoutError] ...`` (the program raised) and any +#: ``failed: output ... != expect ...`` quoting program output. +#: +#: **"The service stopped the clock" is not "belongs in ``timeouts``."** This +#: answers the first question only; the counter is the caller's call. MultiPL-E +#: routes ``failed: build timeout`` to ``n_build_errors`` rather than to +#: ``timeouts``, deliberately — the run never started, and build-versus-run is +#: the split its three keys exist to carry (``multipl_e/_base.py``). That is +#: what the two groups above are for: an adopting task takes the vocabulary and +#: still owes its own bucketing, where a bare :func:`is_timeout_message` would +#: move its build walls. +CODE_EVAL_TIMEOUT_PREFIXES: tuple[str, ...] = ( + CODE_EVAL_BUILD_TIMEOUT_PREFIXES + CODE_EVAL_RUN_TIMEOUT_PREFIXES +) + + +def is_timeout_message(msg: str | None) -> bool: + """Whether *msg* says the service stopped the program on a clock. + + ``None`` is False rather than an error: a null ``msg`` is absent on disk, so + a resumed record legitimately arrives without one. + """ + if not msg: + return False + return msg.strip().lower().startswith(CODE_EVAL_TIMEOUT_PREFIXES) + + +#: How the service reports an uncaught exception: ``failed: [] ``. +_EXC_MSG_PREFIX = "failed: [" + + +def exception_class_name(msg: str | None) -> str | None: + """The exception class the service formatted into *msg*, lowercased. + + The class is a structured SLOT; everything after it is the program's own + output. Reading the slot is what separates ``[MemoryError] ...`` from a + program that merely printed the word. ``None`` when *msg* is not that shape. + + Returns the name rather than comparing it, so a caller can match a *family*. + That matters: the counters reading this want subclasses too, and the class + the service names is not always the builtin — ``ZipImportError`` is an + ``ImportError``, ``OutOfMemoryError`` a ``MemoryError``. Testing for the + exact builtin would drop them, which a bare substring test did not. + """ + if not msg: + return None + text = msg.strip() + if not text.lower().startswith(_EXC_MSG_PREFIX): + return None + end = text.find("]", len(_EXC_MSG_PREFIX)) + if end == -1: + return None + return text[len(_EXC_MSG_PREFIX) : end].lower() + + +__all__ = [ + "CODE_EVAL_BUILD_TIMEOUT_PREFIXES", + "CODE_EVAL_RUN_TIMEOUT_PREFIXES", + "CODE_EVAL_TIMEOUT_PREFIXES", + "exception_class_name", + "is_timeout_message", +] diff --git a/sieval/tasks/human_eval_0shot_base_gen.py b/sieval/tasks/human_eval_0shot_base_gen.py index 24fcff39..a854cc6f 100644 --- a/sieval/tasks/human_eval_0shot_base_gen.py +++ b/sieval/tasks/human_eval_0shot_base_gen.py @@ -51,6 +51,8 @@ ) from sieval.datasets import HumanEvalDatasetSample +from ._code_eval_msg import is_timeout_message + STOP_SEQUENCES = ("\nclass", "\ndef", "\n#", "\nif", "\nprint") @@ -219,8 +221,7 @@ async def report(self, finals, fails): 1 for f in finals for r in f.feedback_result["rollouts"] - # A null msg from the evaluator is absent on disk -- default it. - if "timeout" in (r["extra"].get("msg") or "").lower() + if is_timeout_message(r["extra"].get("msg")) ) # `votes=False`: two correct programs are not one answer, so there is # nothing well-defined to take a majority over (RFC #74). diff --git a/sieval/tasks/human_eval_0shot_gen.py b/sieval/tasks/human_eval_0shot_gen.py index 75e8caaf..7da2406c 100644 --- a/sieval/tasks/human_eval_0shot_gen.py +++ b/sieval/tasks/human_eval_0shot_gen.py @@ -32,6 +32,8 @@ ) from sieval.datasets import HumanEvalDatasetSample +from ._code_eval_msg import is_timeout_message + @sieval_task( name="human_eval_0shot_gen", @@ -182,8 +184,7 @@ async def report(self, finals, fails): 1 for f in finals for r in f.feedback_result["rollouts"] - # A null msg from the evaluator is absent on disk -- default it. - if "timeout" in (r["extra"].get("msg") or "").lower() + if is_timeout_message(r["extra"].get("msg")) ) # `votes=False`: two correct programs are not one answer, so there is # nothing well-defined to take a majority over (RFC #74). diff --git a/sieval/tasks/livecodebench_code_generation_0shot_gen.py b/sieval/tasks/livecodebench_code_generation_0shot_gen.py index f76fbd47..881079e6 100644 --- a/sieval/tasks/livecodebench_code_generation_0shot_gen.py +++ b/sieval/tasks/livecodebench_code_generation_0shot_gen.py @@ -39,6 +39,8 @@ ) from sieval.datasets import LiveCodeBenchDatasetSample +from ._code_eval_msg import is_timeout_message + @sieval_task( name="livecodebench_code_generation_0shot_gen", @@ -254,8 +256,7 @@ async def report(self, finals, fails): 1 for f in finals for r in f.feedback_result["rollouts"] - # A null msg from the evaluator is absent on disk -- default it. - if "timeout" in (r["extra"].get("msg") or "").lower() + if is_timeout_message(r["extra"].get("msg")) ) # `votes=False`: two correct programs are not one answer, so there is # nothing well-defined to take a majority over (RFC #74). diff --git a/sieval/tasks/livecodebench_code_generation_kshot_base_gen.py b/sieval/tasks/livecodebench_code_generation_kshot_base_gen.py index a8763425..ab335fef 100644 --- a/sieval/tasks/livecodebench_code_generation_kshot_base_gen.py +++ b/sieval/tasks/livecodebench_code_generation_kshot_base_gen.py @@ -83,6 +83,8 @@ ) from sieval.datasets import LiveCodeBenchDatasetSample +from ._code_eval_msg import is_timeout_message + N_SHOT = 3 STOP_SEQUENCES = ("###",) @@ -312,8 +314,7 @@ async def report(self, finals, fails): 1 for f in finals for r in f.feedback_result["rollouts"] - # A null msg from the evaluator is absent on disk -- default it. - if "timeout" in (r["extra"].get("msg") or "").lower() + if is_timeout_message(r["extra"].get("msg")) ) # `votes=False`: two correct programs are not one answer, so there is # nothing well-defined to take a majority over (RFC #74). diff --git a/sieval/tasks/mbpp_kshot_base_gen.py b/sieval/tasks/mbpp_kshot_base_gen.py index cf988631..8d658ca4 100644 --- a/sieval/tasks/mbpp_kshot_base_gen.py +++ b/sieval/tasks/mbpp_kshot_base_gen.py @@ -44,6 +44,8 @@ ) from sieval.datasets import MBPPDatasetSample +from ._code_eval_msg import is_timeout_message + DEFAULT_NUM_SHOTS = 3 STOP_SEQUENCES = ("[DONE]",) @@ -268,8 +270,7 @@ async def report( 1 for f in finals for r in f.feedback_result["rollouts"] - # A null msg from the evaluator is absent on disk -- default it. - if "timeout" in (r["extra"].get("msg") or "").lower() + if is_timeout_message(r["extra"].get("msg")) ) # `votes=False`: two correct programs are not one answer, so there is # nothing well-defined to take a majority over (RFC #74). diff --git a/sieval/tasks/multipl_e/_base.py b/sieval/tasks/multipl_e/_base.py index 4e201666..9b150a22 100644 --- a/sieval/tasks/multipl_e/_base.py +++ b/sieval/tasks/multipl_e/_base.py @@ -80,6 +80,10 @@ sampling_report, ungated_intervals, ) +from sieval.tasks._code_eval_msg import ( + CODE_EVAL_BUILD_TIMEOUT_PREFIXES, + CODE_EVAL_RUN_TIMEOUT_PREFIXES, +) # Commit-pinned: `main` moves, and a divergence is argued against the code that # was actually read. `main.py` is where the graded program is assembled. @@ -626,8 +630,9 @@ def _failure_buckets(self, finals) -> dict[str, float]: / ``ReferenceError`` / ``Exception``); the code-eval API answers with one boolean and a message, so what survives is the split a caller cannot reconstruct from the verdict — build versus run. Read off the message - prefixes the service documents, the same way the HumanEval tasks read - theirs for `timeouts`. + prefixes the service documents, out of the same shared vocabulary the + HumanEval tasks read theirs for `timeouts` from + (``sieval/tasks/_code_eval_msg.py``). ``n_execution_errors`` keeps the name it has in every other task that executes a prediction; the build bucket is new because no other task has @@ -641,7 +646,17 @@ def _failure_buckets(self, finals) -> dict[str, float]: test, in the one bucket that suggests the model's program was slow rather than broken. A build that exceeds ITS wall lands in the build bucket rather than under ``timeouts``: the run never started, and - build-versus-run is the split these three keys exist to carry. + build-versus-run is the split these three keys exist to carry. That is + why this takes the two prefix GROUPS and not ``is_timeout_message``, + which answers "did the service stop the clock" and would therefore move + a build wall into ``timeouts``. + + Each group carries prefixes this task cannot see — none of its 24 + languages is Python, so ``exec_py_code`` / ``exec_py_test`` never + answer it. Routing them anyway costs nothing and is what keeps the + vocabulary in one place: were a Python row ever added, a subprocess + wall would already land in ``timeouts`` rather than silently in + ``n_execution_errors``. """ timeouts = build_errors = execution_errors = 0 for final in finals: @@ -652,10 +667,16 @@ def _failure_buckets(self, finals) -> dict[str, float]: if rollout["correct"]: continue # A null msg from the evaluator is absent on disk -- default it. + # Lowercased because the shared prefixes are: `[casetimeout]` is + # the one entry whose match depends on it. msg = (rollout["extra"].get("msg") or "").lower() - if msg.startswith(("failed: build timeout", "failed [build exit")): + # `failed [build exit` is a build failure but not a timeout, so + # it is not the shared module's vocabulary and stays here. + if msg.startswith( + (*CODE_EVAL_BUILD_TIMEOUT_PREFIXES, "failed [build exit") + ): build_errors += 1 - elif msg.startswith("failed: timeout"): + elif msg.startswith(CODE_EVAL_RUN_TIMEOUT_PREFIXES): timeouts += 1 else: execution_errors += 1 diff --git a/sieval/tasks/scicode_0shot_gen.py b/sieval/tasks/scicode_0shot_gen.py index f84a145f..53c70c81 100644 --- a/sieval/tasks/scicode_0shot_gen.py +++ b/sieval/tasks/scicode_0shot_gen.py @@ -76,6 +76,8 @@ from sieval.core.utils.meta import build_stage_meta from sieval.datasets import SciCodeDatasetSample +from ._code_eval_msg import exception_class_name, is_timeout_message + class StepCode(TypedDict): step_number: str @@ -574,14 +576,35 @@ async def report(self, finals, fails): 1 for fb in feedbacks if fb.get("empty_extraction") ) messages = [str(fb.get("msg", "")).lower() for fb in feedbacks] - timeouts += sum("timeout" in msg for msg in messages) - memory_errors += sum("memoryerror" in msg for msg in messages) + # All three counters read the exception CLASS SLOT, never the tail: + # `failed: [{type}] {e}` interpolates the program's own output after + # the class, so a bare `"memoryerror" in msg` also counted + # `[ValueError] simulated memoryerror path`. Matching the family + # (`endswith`) rather than the builtin keeps the subclasses the bare + # test happened to catch -- `ZipImportError` is an `ImportError`, + # `OutOfMemoryError` a `MemoryError` -- so this drops false + # positives only, and never a class the service actually named. + exc_classes = [exception_class_name(msg) for msg in messages] + # `timeouts` alone keeps a second reading: the service's own wall is + # a message prefix, not an exception at all. + timeouts += sum( + is_timeout_message(msg) + or (cls is not None and cls.endswith("timeouterror")) + for msg, cls in zip(messages, exc_classes, strict=True) + ) + memory_errors += sum( + cls is not None and cls.endswith("memoryerror") for cls in exc_classes + ) # ModuleNotFoundError is an ImportError subclass, but the evaluator - # reports the concrete class name, which does not contain + # reports the concrete class name, which does not END in # "importerror". It is the signature of a package missing from the - # code-eval image, so it must not read as import_errors=0. + # code-eval image, so it must not read as import_errors=0. Its own + # family is matched the same way the other two are, rather than by + # equality: the rule here is "the service named a class in this + # family", and a subclass is what the service actually reports. import_errors += sum( - "importerror" in msg or "modulenotfounderror" in msg for msg in messages + cls is not None and cls.endswith(("importerror", "modulenotfounderror")) + for cls in exc_classes ) # A pipeline failure is an unsolved problem in BOTH accuracies. Its tested diff --git a/tests/unit/tasks/test__code_eval_msg.py b/tests/unit/tasks/test__code_eval_msg.py new file mode 100644 index 00000000..99763fea --- /dev/null +++ b/tests/unit/tasks/test__code_eval_msg.py @@ -0,0 +1,166 @@ +"""The timeout predicate: the false positives a substring test lets through, and +the one true timeout a prefix set is easy to leave out. + +Every `NOT_TIMEOUTS` case contains the word, so this file passes trivially +against `"timeout" in msg.lower()` -- rejecting them is what the assertions are +for. `failed: [CaseTimeout]` is the mirror case: a real wall that arrives +formatted as a class name, which the substring test caught by accident. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import pytest + +from sieval.tasks._code_eval_msg import ( + CODE_EVAL_BUILD_TIMEOUT_PREFIXES, + CODE_EVAL_RUN_TIMEOUT_PREFIXES, + CODE_EVAL_TIMEOUT_PREFIXES, + exception_class_name, + is_timeout_message, +) + +# Every timeout spelling the code-eval service emits, verbatim from +# `vendor/code-evaluator/app/exec_*.py`. +TIMEOUTS = [ + "failed: subprocess timeout: 3.0s", # exec_py_code / exec_py_test wall + "failed: case timeout: 6.0s", # exec_py_test, per case + "failed: compile timeout: 6.0s", # exec_py_test, per-case compile + # The per-case wall landing past `_unsafe_execute`'s own handler, so the + # worker's outer `except (Exception, CaseTimeout)` formats it as a class + # name. The trailing space is the empty `{e}`; the message is verbatim. + "failed: [CaseTimeout] ", + "failed: build timeout", # exec_lang compile wall (MultiPL-E) + "failed: timeout", # exec_js / exec_ts / exec_lang run wall +] + +# Messages that are NOT the service stopping a clock, but DO contain the word -- +# so a substring test counts every one of them as a timeout. +NOT_TIMEOUTS = [ + # The program raised; the class name lands in the interpolated tail. + "failed: [TimeoutError] deadline exceeded", + "failed: [ValueError] timeout must be positive", + # LiveCodeBench prints the program's own stdout next to the expectation. + "failed: output ['timeout'] != expect ['ok']", + "failed: output mismatch: got 'timeout' expected 'done'", + # A c++ diagnostic quoting an identifier (exec_lang / MultiPL-E). + "failed [build exit 1]: error: no member named 'timeout' in 'Config'", + # A non-zero exit whose stderr mentions it. + "failed [exit 1]: Traceback ...\nTimeoutError: x", +] + + +@pytest.mark.parametrize("msg", TIMEOUTS) +def test_service_timeouts_are_recognized(msg): + assert is_timeout_message(msg) is True + + +@pytest.mark.parametrize("msg", NOT_TIMEOUTS) +def test_the_word_elsewhere_in_a_message_is_not_a_timeout(msg): + assert "timeout" in msg.lower(), "case must be a substring-test false positive" + assert is_timeout_message(msg) is False + + +def test_a_late_case_timeout_is_not_confused_with_a_raised_one(): + """The two class-name shapes differ only by which class, and split opposite ways. + + `CaseTimeout` is the service's own wall; `TimeoutError` is the program's. + Getting this pair backwards is the whole reason the predicate is not a + substring test in either direction. + + The split is by which class the service named, NOT by who could have named + it: a submitted program defining `class CaseTimeout(Exception)` reaches the + same shape through `_unsafe_execute_fn_call`'s `except Exception`. That + forgery is accepted deliberately -- see the note on the prefix group. + """ + assert is_timeout_message("failed: [CaseTimeout] ") is True + assert is_timeout_message("failed: [TimeoutError] deadline exceeded") is False + + +@pytest.mark.parametrize("msg", ["", None, "failed: compile error", "ok"]) +def test_absent_and_unrelated_messages(msg): + assert is_timeout_message(msg) is False + + +def test_case_and_surrounding_whitespace_do_not_matter(): + assert is_timeout_message(" FAILED: Subprocess Timeout: 3.0s ") is True + + +def test_every_prefix_is_itself_recognized(): + """No prefix is shadowed by another, and each one is reachable.""" + for prefix in CODE_EVAL_TIMEOUT_PREFIXES: + assert is_timeout_message(prefix) is True + + +def test_the_two_groups_partition_the_union(): + """A caller that splits the groups must still see every prefix exactly once. + + `multipl_e/_base.py` routes the two groups to different counters, so a + prefix in neither is silently bucketed as an execution error, and one in + both is decided by which branch is tested first. + """ + build = set(CODE_EVAL_BUILD_TIMEOUT_PREFIXES) + run = set(CODE_EVAL_RUN_TIMEOUT_PREFIXES) + + assert build & run == set(), "a prefix in both groups is routed by branch order" + assert build | run == set(CODE_EVAL_TIMEOUT_PREFIXES) + # The union is the concatenation, so neither group may carry a duplicate. + assert len(CODE_EVAL_TIMEOUT_PREFIXES) == len(build) + len(run) + + +def test_every_prefix_is_lowercase(): + """The documented invariant callers rely on when comparing themselves. + + `is_timeout_message` lowercases first, so it would pass either way; a caller + reaching for a tuple directly is what this protects. Only + `failed: [casetimeout]` can actually break -- the other five are lowercase + by accident of the service's own wording, which is why this is asserted + rather than left to reading. + """ + for prefix in CODE_EVAL_TIMEOUT_PREFIXES: + assert prefix == prefix.lower(), prefix + + +@pytest.mark.parametrize( + ("msg", "expected"), + [ + ("failed: [MemoryError] unable to allocate 8 GiB", "memoryerror"), + ( + "failed: [ModuleNotFoundError] No module named 'numba'", + "modulenotfounderror", + ), + # A subclass: the service names the concrete class, not the builtin. + ("failed: [ZipImportError] bad local file header", "zipimporterror"), + (" FAILED: [AssertionError] x ", "assertionerror"), + ("failed: [CaseTimeout] ", "casetimeout"), + # Not the exception shape at all. + ("failed: subprocess timeout: 3.0s", None), + ("failed: compile error", None), + # The word in the TAIL, which is the program's own output. + ("failed: output ['memoryerror'] != expect ['ok']", None), + # Malformed: opened but never closed. + ("failed: [MemoryError unable to allocate", None), + ("", None), + (None, None), + ], +) +def test_exception_class_name_reads_only_the_slot(msg, expected): + assert exception_class_name(msg) == expected + + +def test_class_families_are_matched_not_just_the_builtin(): + """The counters match a family, so a subclass the service names still counts. + + A bare `"importerror" in msg` caught `ZipImportError` by accident; testing + the slot for equality with the builtin would silently drop it, turning a + precision fix into a completeness regression. + """ + zip_err = exception_class_name("failed: [ZipImportError] bad local file header") + assert zip_err is not None and zip_err.endswith("importerror") + + oom = exception_class_name("failed: [OutOfMemoryError] cuda ran out") + assert oom is not None and oom.endswith("memoryerror") + + # ...while the tail no longer reaches either counter. + tail = exception_class_name("failed: [ValueError] simulated memoryerror path") + assert tail == "valueerror" + assert not tail.endswith("memoryerror") diff --git a/tests/unit/tasks/test_human_eval_0shot_base_gen.py b/tests/unit/tasks/test_human_eval_0shot_base_gen.py index f74efdb5..14b0f358 100644 --- a/tests/unit/tasks/test_human_eval_0shot_base_gen.py +++ b/tests/unit/tasks/test_human_eval_0shot_base_gen.py @@ -128,12 +128,23 @@ async def test_report_counts_finals_and_fails_like_chat_human_eval_task(): TaskContext( sample_id=0, raw_sample=_sample(), - feedback_result=_judgement((True, "passed"), (False, "timeout")), + # The evaluator's own wording (`exec_py_code`), not the word + # on its own: `timeouts` is read off the message PREFIX, and + # a fixture the service cannot emit would pass against any + # implementation. + feedback_result=_judgement( + (True, "passed"), (False, "failed: subprocess timeout: 3.0s") + ), ), TaskContext( sample_id=1, raw_sample=_sample(), - feedback_result=_judgement((False, "failed"), (False, "failed")), + # Contains the word and is NOT a timeout -- the program + # raised, which a substring test used to charge to the clock. + feedback_result=_judgement( + (False, "failed: [TimeoutError] x"), + (False, "failed: [AssertionError] y"), + ), ), ], [TaskContext(sample_id=2, raw_sample=_sample())], diff --git a/tests/unit/tasks/test_mbpp_kshot_base_gen.py b/tests/unit/tasks/test_mbpp_kshot_base_gen.py index c06829ef..6ef6a4bd 100644 --- a/tests/unit/tasks/test_mbpp_kshot_base_gen.py +++ b/tests/unit/tasks/test_mbpp_kshot_base_gen.py @@ -151,7 +151,10 @@ async def test_report_pass_at_k_and_timeouts(): task = MBPPFewShotBaseGenTask(_dataset(), _CapturingGenModel(), n_shot=0, k=2, n=2) finals = [ # 1 of 2 samples correct → pass@1 = 0.5, pass@2 = 1.0 - _final(_judgement((True, "ok"), (False, "Timeout exceeded"))), + # The failing rollout carries the evaluator's own wording + # (`exec_py_code`): `timeouts` is read off the message PREFIX, so a + # fixture the service cannot emit would pass against any implementation. + _final(_judgement((True, "ok"), (False, "failed: subprocess timeout: 3.0s"))), ] report = await task.report(finals, []) @@ -164,6 +167,23 @@ async def test_report_pass_at_k_and_timeouts(): assert report["timeouts"] == 1 +@pytest.mark.anyio +async def test_a_raised_timeout_error_is_not_charged_to_the_clock(): + """`failed: [TimeoutError] ...` is the program failing, not the wall. + + The substring test this replaced counted it as a timeout, pointing a reader + at efficiency instead of at the exception in front of them. + """ + task = MBPPFewShotBaseGenTask(_dataset(), _CapturingGenModel(), n_shot=0, k=1, n=1) + finals = [_final(_judgement((False, "failed: [TimeoutError] deadline exceeded")))] + + report = await task.report(finals, []) + await task.shutdown() + + assert report["timeouts"] == 0 + assert report["score"] == pytest.approx(0.0) + + @pytest.mark.anyio async def test_report_empty_returns_zero_with_populated_schema(): task = MBPPFewShotBaseGenTask(_dataset(), _CapturingGenModel(), n_shot=0) diff --git a/tests/unit/tasks/test_scicode_0shot_gen.py b/tests/unit/tasks/test_scicode_0shot_gen.py index f614ab6b..b8fdc54f 100644 --- a/tests/unit/tasks/test_scicode_0shot_gen.py +++ b/tests/unit/tasks/test_scicode_0shot_gen.py @@ -806,6 +806,59 @@ async def test_report_surfaces_step_execution_failures(): assert report["fails"] == 0 +@pytest.mark.anyio +async def test_step_counters_read_the_class_slot_not_the_message_tail(): + """The tail is the program's own output; only the class slot is vocabulary. + + Each message below contains the counter's word somewhere, so all three + counters would be non-zero under the substring tests these replaced. + """ + task = _task(_ScriptedChatModel([])) + finals = [ + _final( + 0, + [False, False, False], + messages=[ + "failed: [ValueError] simulated memoryerror path", + "failed: output ['importerror'] != expect ['ok']", + "failed: [AssertionError] timeout must be positive", + ], + ) + ] + + report = await task.report(finals, fails=[]) + + assert report["memory_errors"] == 0 + assert report["import_errors"] == 0 + assert report["timeouts"] == 0 + + +@pytest.mark.anyio +async def test_step_counters_still_match_subclasses_of_the_builtin(): + """Anchoring on the slot must not narrow to the builtin name. + + The service reports the concrete class, so a subclass is what actually + arrives. A bare substring test caught these; equality with the builtin + would drop them, trading a false positive for a false negative. + """ + task = _task(_ScriptedChatModel([])) + finals = [ + _final( + 0, + [False, False], + messages=[ + "failed: [OutOfMemoryError] cuda ran out", + "failed: [ZipImportError] bad local file header", + ], + ) + ] + + report = await task.report(finals, fails=[]) + + assert report["memory_errors"] == 1 + assert report["import_errors"] == 1 + + @pytest.mark.anyio async def test_report_counts_pipeline_fails_as_unsolved_problems(): model = _ScriptedChatModel([]) diff --git a/tests/unit/tasks/test_timeout_bucketing_convention.py b/tests/unit/tasks/test_timeout_bucketing_convention.py new file mode 100644 index 00000000..d1121341 --- /dev/null +++ b/tests/unit/tasks/test_timeout_bucketing_convention.py @@ -0,0 +1,92 @@ +"""No task decides a timeout by substring-matching the bare word. + +The defect is argued in `sieval/tasks/_code_eval_msg.py`: a code-eval message +interpolates the failing program's own output, so `"timeout" in msg` counts a +`[TimeoutError]`, a compiler diagnostic naming an identifier, and any comparison +failure printing the word. Six tasks carried that test independently, and nothing +they scored was wrong (`correct` comes from the service's boolean), so no metric +test would ever have caught the drift — which is why this is a survey and not a +hand-kept list. + +Deliberately narrow: it forbids the BARE literal only. A test that reads the +exception CLASS is a different test with a different meaning and stays allowed — +`scicode` counts a raised `TimeoutError` into its `timeouts` on purpose, beside +`memory_errors` and `import_errors`, which read the class slot the same way. +That reading now goes through `exception_class_name`, so no in-tree site spells +a bracketed literal today; the carve-out is kept because the NEXT such counter +should not have to argue with this test to exist. + +AI-Generated Code - Claude Opus 5 (1M context) (Anthropic) +""" + +import ast +import pathlib + +import sieval + +#: Resolved from the imported package, so a worktree cannot survey the primary +#: checkout's `sieval/` by accident. +TASKS_DIR = pathlib.Path(sieval.__file__).parent / "tasks" + +#: A floor, not a count: it needs raising only when readers are *removed*, and it +#: is what stops a broken scan from passing as "no offenders found". +MIN_EXPECTED_READERS = 6 + + +def _task_sources() -> list[pathlib.Path]: + return sorted(TASKS_DIR.rglob("*.py")) + + +def test_no_task_substring_matches_the_bare_word(): + offenders: list[str] = [] + for path in _task_sources(): + tree = ast.parse(path.read_text()) + for node in ast.walk(tree): + if not isinstance(node, ast.Compare): + continue + # `n` ops against `n + 1` operands: each op pairs with its own left + # side and the trailing comparator has none, so the mismatch is the + # point — `strict=True` would raise on every chained compare. + for op, left in zip(node.ops, [node.left, *node.comparators], strict=False): + if not isinstance(op, ast.In): + continue + if ( + isinstance(left, ast.Constant) + and isinstance(left.value, str) + and left.value.strip().lower() == "timeout" + ): + offenders.append( + f"{path.relative_to(TASKS_DIR.parent)}:{node.lineno}" + ) + assert offenders == [], ( + "these sites decide a timeout by substring-matching the bare word; use " + "`_code_eval_msg.is_timeout_message`, which tests the message PREFIX: " + + ", ".join(offenders) + ) + + +def test_the_shared_predicate_actually_has_readers(): + """Guards the other direction: a correct helper nobody calls fixes nothing. + + Counts CALL nodes, not occurrences of the name — a file whose only mention + is the `import` line reads the predicate nowhere, and that is exactly the + shape that got past the first version of this test. + """ + readers: list[str] = [] + for path in _task_sources(): + if path.name == "_code_eval_msg.py": + continue + tree = ast.parse(path.read_text()) + calls = sum( + 1 + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "is_timeout_message" + ) + if calls: + readers.append(f"{path.relative_to(TASKS_DIR.parent).as_posix()} x{calls}") + assert len(readers) >= MIN_EXPECTED_READERS, ( + f"expected at least {MIN_EXPECTED_READERS} tasks to CALL the shared " + f"predicate, found {len(readers)}: {readers}" + )