From db6386a411694fa655ba4d7a449220e2b236f488 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sun, 25 Oct 2026 20:01:59 -0700 Subject: [PATCH 1/3] fix(tasks): do not iterate a non-list tests or questions value in tasks inspect Signed-off-by: Alexander Rybkin --- ...20261025_200000_xaleryb_tasks-inspect-non-list.md | 6 ++++++ dokimi/dokimi/task_formats/configs.py | 3 ++- dokimi/dokimi/task_formats/datasets.py | 3 ++- dokimi/tests/test_task_format_detect.py | 12 ++++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md diff --git a/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md b/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md new file mode 100644 index 000000000..a30e0c90b --- /dev/null +++ b/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md @@ -0,0 +1,6 @@ +### Fixed + +- **`dokimi tasks inspect` no longer crashes on a non-list `tests` or `questions`** — a + promptfoo config with `tests: 5` or a dokimi questions file with `"questions": null` raised + `TypeError` and ended the scan, so no other file under the path was reported. The value is + now read as no items and the scan goes on. diff --git a/dokimi/dokimi/task_formats/configs.py b/dokimi/dokimi/task_formats/configs.py index 17dcc0101..5f05b6f06 100644 --- a/dokimi/dokimi/task_formats/configs.py +++ b/dokimi/dokimi/task_formats/configs.py @@ -79,7 +79,8 @@ def describe(self, source: FileSource) -> Report: if source.data is not None and not data: report.problems.append("not a mapping") asserts = isinstance(data.get("defaultTest"), dict) and "assert" in data["defaultTest"] - asserts = asserts or any(isinstance(t, dict) and t.get("assert") for t in tests or [] if isinstance(tests, list)) + asserts = asserts or any(isinstance(t, dict) and t.get("assert") + for t in (tests if isinstance(tests, list) else [])) report.set_parts({ "instruction": bool(data.get("prompts")), "environment": bool(data.get("providers") or data.get("targets")), diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py index f7e8071d1..d2011fba6 100644 --- a/dokimi/dokimi/task_formats/datasets.py +++ b/dokimi/dokimi/task_formats/datasets.py @@ -333,7 +333,8 @@ def describe(self, source: FileSource) -> Report: from dokimi.questions.artifact import load_questions data = source.data if isinstance(source.data, dict) else {} - rows = [q for q in data.get("questions", []) if isinstance(q, dict)] + questions = data.get("questions") + rows = [q for q in (questions if isinstance(questions, list) else []) if isinstance(q, dict)] kind = KIND_TRIGGER if any("expected_skill" in q for q in rows) else KIND_ANSWER_ONLY report = _base(source, self.name, kind, rows) has_text = all(q.get("question") or q.get("text") for q in rows) and bool(rows) diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py index fae2168f6..a413c8a1a 100644 --- a/dokimi/tests/test_task_format_detect.py +++ b/dokimi/tests/test_task_format_detect.py @@ -568,6 +568,18 @@ def test_skill_reach_keys_yaml_reads_as_bool_or_int_are_linted_not_a_crash(tmp_p assert reports["s.jsonl"].format == "inspect-samples" +@pytest.mark.parametrize("name, data", [ + ("promptfooconfig.json", {"prompts": ["p"], "providers": ["x"], "tests": 5}), + ("q.json", {"schema_version": "questions.v1", "questions": None}), +]) +def test_a_non_list_tests_or_questions_value_is_not_iterated(tmp_path, name, data): + _write(tmp_path / name, json.dumps(data)) + _jsonl(tmp_path / "ok" / "s.jsonl", [{"input": "q", "target": "a"}]) + reports = {r.path.name: r for r in inspect_path(tmp_path)} + assert reports[name].items in (None, 0) + assert reports["s.jsonl"].format == "inspect-samples" + + def test_cli_inspect_out_writes_yaml_dates_sets_and_nan_as_plain_json(tmp_path, capsys): task = _tb1(tmp_path) _write(task / "task.yaml", "instruction: do it\ndifficulty: 2024-01-01\n" From d68e563c2ce7c79101c0395832aa15d21cedfea8 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sun, 25 Oct 2026 20:09:10 -0700 Subject: [PATCH 2/3] fix(tasks): a non-string promptfoo row key or an infinite explore line does not stop tasks inspect Signed-off-by: Alexander Rybkin --- ...20261025_200000_xaleryb_tasks-inspect-non-list.md | 9 +++++---- dokimi/dokimi/explore/tasks.py | 2 +- dokimi/dokimi/task_formats/configs.py | 2 +- dokimi/tests/test_task_format_detect.py | 12 ++++++++++++ 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md b/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md index a30e0c90b..7a1c31835 100644 --- a/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md +++ b/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md @@ -1,6 +1,7 @@ ### Fixed -- **`dokimi tasks inspect` no longer crashes on a non-list `tests` or `questions`** — a - promptfoo config with `tests: 5` or a dokimi questions file with `"questions": null` raised - `TypeError` and ended the scan, so no other file under the path was reported. The value is - now read as no items and the scan goes on. +- **`dokimi tasks inspect` no longer stops on one malformed file** — a promptfoo config with + `tests: 5`, a dokimi questions file with `"questions": null`, a promptfoo tests file with a + non-string row key, or an explore task with `lines: [.inf]` raised and ended the scan, so no + other file under the path was reported. Each file is now reported on its own and the scan + goes on. diff --git a/dokimi/dokimi/explore/tasks.py b/dokimi/dokimi/explore/tasks.py index 57dc1d0d4..2d24b8ba4 100644 --- a/dokimi/dokimi/explore/tasks.py +++ b/dokimi/dokimi/explore/tasks.py @@ -98,7 +98,7 @@ def _parse_lines(value) -> tuple[int, int] | None: raise ValueError(f"'lines' must be [start, end] or [n], got {value!r}") try: nums = [int(v) for v in value] - except (TypeError, ValueError) as exc: + except (TypeError, ValueError, OverflowError) as exc: # OverflowError: int(float('inf')) raise ValueError(f"'lines' must contain integers, got {value!r}") from exc if any(n < 1 for n in nums): raise ValueError(f"'lines' values must be >= 1 (1-based), got {value!r}") diff --git a/dokimi/dokimi/task_formats/configs.py b/dokimi/dokimi/task_formats/configs.py index 5f05b6f06..f26c8c62c 100644 --- a/dokimi/dokimi/task_formats/configs.py +++ b/dokimi/dokimi/task_formats/configs.py @@ -125,7 +125,7 @@ def describe(self, source: FileSource) -> Report: rows = source.rows or [] report = Report(source.path, self.name, KIND_ANSWER_ONLY, items=len(rows)) report.problems.extend(source.problems) - expected = any(k.startswith("__expected") for r in rows[:1] for k in r) + expected = any(str(k).startswith("__expected") for r in rows[:1] for k in r) report.set_parts({ "instruction": bool(rows), "verifier": expected or any(r.get("assert") for r in rows), diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py index a413c8a1a..50fc3da3d 100644 --- a/dokimi/tests/test_task_format_detect.py +++ b/dokimi/tests/test_task_format_detect.py @@ -580,6 +580,18 @@ def test_a_non_list_tests_or_questions_value_is_not_iterated(tmp_path, name, dat assert reports["s.jsonl"].format == "inspect-samples" +@pytest.mark.parametrize("name, text, fmt", [ + ("cases.yaml", "- vars: {a: 1}\n assert: [{type: equals, value: x}]\n 1: y\n", "promptfoo-tests"), + ("e.yaml", "id: e\nquery: where?\nreferences: [{path: a.py, lines: [.inf]}]\n", "dokimi-explore-task"), +]) +def test_a_non_string_key_or_an_infinite_line_does_not_stop_the_scan(tmp_path, name, text, fmt): + _write(tmp_path / name, text) + _jsonl(tmp_path / "ok" / "s.jsonl", [{"input": "q", "target": "a"}]) + reports = {r.path.name: r for r in inspect_path(tmp_path)} + assert reports[name].format == fmt + assert reports["s.jsonl"].format == "inspect-samples" + + def test_cli_inspect_out_writes_yaml_dates_sets_and_nan_as_plain_json(tmp_path, capsys): task = _tb1(tmp_path) _write(task / "task.yaml", "instruction: do it\ndifficulty: 2024-01-01\n" From 403cf7a25f51c1bcabe08e26e275e676b1c669c4 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Sun, 25 Oct 2026 22:21:51 -0700 Subject: [PATCH 3/3] fix(tasks): report a promptfoo tests value that is neither a list nor a path Review on #227: tests: 5 no longer crashed but passed with no problem, while a bad questions value gets one. The non-list test cases now assert exact items and the problem, and tests: true is a case of its own. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/task_formats/configs.py | 2 ++ dokimi/tests/test_task_format_detect.py | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/task_formats/configs.py b/dokimi/dokimi/task_formats/configs.py index f26c8c62c..3606a481e 100644 --- a/dokimi/dokimi/task_formats/configs.py +++ b/dokimi/dokimi/task_formats/configs.py @@ -96,6 +96,8 @@ def describe(self, source: FileSource) -> Report: report.details["references"] = [reference(local, FORMAT_PROMPTFOO_TESTS)] elif data and tests is None and not any(k in data for k in ("scenarios", "redteam")): report.problems.append("no tests") + elif tests is not None and not isinstance(tests, list): + report.problems.append(f"tests is {type(tests).__name__}, expected a list or a path") if data and not data.get("prompts") and "redteam" not in data: report.problems.append("no prompts") return report diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py index 50fc3da3d..52055d267 100644 --- a/dokimi/tests/test_task_format_detect.py +++ b/dokimi/tests/test_task_format_detect.py @@ -568,15 +568,17 @@ def test_skill_reach_keys_yaml_reads_as_bool_or_int_are_linted_not_a_crash(tmp_p assert reports["s.jsonl"].format == "inspect-samples" -@pytest.mark.parametrize("name, data", [ - ("promptfooconfig.json", {"prompts": ["p"], "providers": ["x"], "tests": 5}), - ("q.json", {"schema_version": "questions.v1", "questions": None}), +@pytest.mark.parametrize("name, data, items, problem", [ + ("promptfooconfig.json", {"prompts": ["p"], "providers": ["x"], "tests": 5}, None, "tests is int"), + ("promptfooconfig.json", {"prompts": ["p"], "providers": ["x"], "tests": True}, None, "tests is bool"), + ("q.json", {"schema_version": "questions.v1", "questions": None}, 0, "NoneType"), ]) -def test_a_non_list_tests_or_questions_value_is_not_iterated(tmp_path, name, data): +def test_a_non_list_tests_or_questions_value_is_not_iterated(tmp_path, name, data, items, problem): _write(tmp_path / name, json.dumps(data)) _jsonl(tmp_path / "ok" / "s.jsonl", [{"input": "q", "target": "a"}]) reports = {r.path.name: r for r in inspect_path(tmp_path)} - assert reports[name].items in (None, 0) + assert reports[name].items == items + assert any(problem in p for p in reports[name].problems), reports[name].problems assert reports["s.jsonl"].format == "inspect-samples"