diff --git a/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md b/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md new file mode 100644 index 000000000..7a1c31835 --- /dev/null +++ b/dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md @@ -0,0 +1,7 @@ +### Fixed + +- **`dokimi tasks inspect` no longer stops on one malformed file** — a promptfoo config with + `tests: 5`, a dokimi questions file with `"questions": null`, a promptfoo tests file with a + non-string row key, or an explore task with `lines: [.inf]` raised and ended the scan, so no + other file under the path was reported. Each file is now reported on its own and the scan + goes on. diff --git a/dokimi/dokimi/explore/tasks.py b/dokimi/dokimi/explore/tasks.py index 57dc1d0d4..2d24b8ba4 100644 --- a/dokimi/dokimi/explore/tasks.py +++ b/dokimi/dokimi/explore/tasks.py @@ -98,7 +98,7 @@ def _parse_lines(value) -> tuple[int, int] | None: raise ValueError(f"'lines' must be [start, end] or [n], got {value!r}") try: nums = [int(v) for v in value] - except (TypeError, ValueError) as exc: + except (TypeError, ValueError, OverflowError) as exc: # OverflowError: int(float('inf')) raise ValueError(f"'lines' must contain integers, got {value!r}") from exc if any(n < 1 for n in nums): raise ValueError(f"'lines' values must be >= 1 (1-based), got {value!r}") diff --git a/dokimi/dokimi/task_formats/configs.py b/dokimi/dokimi/task_formats/configs.py index 17dcc0101..3606a481e 100644 --- a/dokimi/dokimi/task_formats/configs.py +++ b/dokimi/dokimi/task_formats/configs.py @@ -79,7 +79,8 @@ def describe(self, source: FileSource) -> Report: if source.data is not None and not data: report.problems.append("not a mapping") asserts = isinstance(data.get("defaultTest"), dict) and "assert" in data["defaultTest"] - asserts = asserts or any(isinstance(t, dict) and t.get("assert") for t in tests or [] if isinstance(tests, list)) + asserts = asserts or any(isinstance(t, dict) and t.get("assert") + for t in (tests if isinstance(tests, list) else [])) report.set_parts({ "instruction": bool(data.get("prompts")), "environment": bool(data.get("providers") or data.get("targets")), @@ -95,6 +96,8 @@ def describe(self, source: FileSource) -> Report: report.details["references"] = [reference(local, FORMAT_PROMPTFOO_TESTS)] elif data and tests is None and not any(k in data for k in ("scenarios", "redteam")): report.problems.append("no tests") + elif tests is not None and not isinstance(tests, list): + report.problems.append(f"tests is {type(tests).__name__}, expected a list or a path") if data and not data.get("prompts") and "redteam" not in data: report.problems.append("no prompts") return report @@ -124,7 +127,7 @@ def describe(self, source: FileSource) -> Report: rows = source.rows or [] report = Report(source.path, self.name, KIND_ANSWER_ONLY, items=len(rows)) report.problems.extend(source.problems) - expected = any(k.startswith("__expected") for r in rows[:1] for k in r) + expected = any(str(k).startswith("__expected") for r in rows[:1] for k in r) report.set_parts({ "instruction": bool(rows), "verifier": expected or any(r.get("assert") for r in rows), diff --git a/dokimi/dokimi/task_formats/datasets.py b/dokimi/dokimi/task_formats/datasets.py index f7e8071d1..d2011fba6 100644 --- a/dokimi/dokimi/task_formats/datasets.py +++ b/dokimi/dokimi/task_formats/datasets.py @@ -333,7 +333,8 @@ def describe(self, source: FileSource) -> Report: from dokimi.questions.artifact import load_questions data = source.data if isinstance(source.data, dict) else {} - rows = [q for q in data.get("questions", []) if isinstance(q, dict)] + questions = data.get("questions") + rows = [q for q in (questions if isinstance(questions, list) else []) if isinstance(q, dict)] kind = KIND_TRIGGER if any("expected_skill" in q for q in rows) else KIND_ANSWER_ONLY report = _base(source, self.name, kind, rows) has_text = all(q.get("question") or q.get("text") for q in rows) and bool(rows) diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py index fae2168f6..52055d267 100644 --- a/dokimi/tests/test_task_format_detect.py +++ b/dokimi/tests/test_task_format_detect.py @@ -568,6 +568,32 @@ def test_skill_reach_keys_yaml_reads_as_bool_or_int_are_linted_not_a_crash(tmp_p assert reports["s.jsonl"].format == "inspect-samples" +@pytest.mark.parametrize("name, data, items, problem", [ + ("promptfooconfig.json", {"prompts": ["p"], "providers": ["x"], "tests": 5}, None, "tests is int"), + ("promptfooconfig.json", {"prompts": ["p"], "providers": ["x"], "tests": True}, None, "tests is bool"), + ("q.json", {"schema_version": "questions.v1", "questions": None}, 0, "NoneType"), +]) +def test_a_non_list_tests_or_questions_value_is_not_iterated(tmp_path, name, data, items, problem): + _write(tmp_path / name, json.dumps(data)) + _jsonl(tmp_path / "ok" / "s.jsonl", [{"input": "q", "target": "a"}]) + reports = {r.path.name: r for r in inspect_path(tmp_path)} + assert reports[name].items == items + assert any(problem in p for p in reports[name].problems), reports[name].problems + assert reports["s.jsonl"].format == "inspect-samples" + + +@pytest.mark.parametrize("name, text, fmt", [ + ("cases.yaml", "- vars: {a: 1}\n assert: [{type: equals, value: x}]\n 1: y\n", "promptfoo-tests"), + ("e.yaml", "id: e\nquery: where?\nreferences: [{path: a.py, lines: [.inf]}]\n", "dokimi-explore-task"), +]) +def test_a_non_string_key_or_an_infinite_line_does_not_stop_the_scan(tmp_path, name, text, fmt): + _write(tmp_path / name, text) + _jsonl(tmp_path / "ok" / "s.jsonl", [{"input": "q", "target": "a"}]) + reports = {r.path.name: r for r in inspect_path(tmp_path)} + assert reports[name].format == fmt + assert reports["s.jsonl"].format == "inspect-samples" + + def test_cli_inspect_out_writes_yaml_dates_sets_and_nan_as_plain_json(tmp_path, capsys): task = _tb1(tmp_path) _write(task / "task.yaml", "instruction: do it\ndifficulty: 2024-01-01\n"