fix(tasks): one malformed file no longer stops tasks inspect

dokimi/changelog.d/20261025_200000_xaleryb_tasks-inspect-non-list.md

@@ -0,0 +1,7 @@
+### Fixed
+
+- **`dokimi tasks inspect` no longer stops on one malformed file** — a promptfoo config with
+  `tests: 5`, a dokimi questions file with `"questions": null`, a promptfoo tests file with a
+  non-string row key, or an explore task with `lines: [.inf]` raised and ended the scan, so no
+  other file under the path was reported. Each file is now reported on its own and the scan
+  goes on.

dokimi/dokimi/explore/tasks.py

@@ -98,7 +98,7 @@ def _parse_lines(value) -> tuple[int, int] | None:
         raise ValueError(f"'lines' must be [start, end] or [n], got {value!r}")
     try:
         nums = [int(v) for v in value]
-    except (TypeError, ValueError) as exc:
+    except (TypeError, ValueError, OverflowError) as exc:  # OverflowError: int(float('inf'))
         raise ValueError(f"'lines' must contain integers, got {value!r}") from exc
     if any(n < 1 for n in nums):
         raise ValueError(f"'lines' values must be >= 1 (1-based), got {value!r}")

dokimi/dokimi/task_formats/configs.py

@@ -79,7 +79,8 @@ def describe(self, source: FileSource) -> Report:
         if source.data is not None and not data:
             report.problems.append("not a mapping")
         asserts = isinstance(data.get("defaultTest"), dict) and "assert" in data["defaultTest"]
-        asserts = asserts or any(isinstance(t, dict) and t.get("assert") for t in tests or [] if isinstance(tests, list))
+        asserts = asserts or any(isinstance(t, dict) and t.get("assert")
+                                 for t in (tests if isinstance(tests, list) else []))
         report.set_parts({
             "instruction": bool(data.get("prompts")),
             "environment": bool(data.get("providers") or data.get("targets")),
@@ -95,6 +96,8 @@ def describe(self, source: FileSource) -> Report:
                 report.details["references"] = [reference(local, FORMAT_PROMPTFOO_TESTS)]
         elif data and tests is None and not any(k in data for k in ("scenarios", "redteam")):
             report.problems.append("no tests")
+        elif tests is not None and not isinstance(tests, list):
+            report.problems.append(f"tests is {type(tests).__name__}, expected a list or a path")
         if data and not data.get("prompts") and "redteam" not in data:
             report.problems.append("no prompts")
         return report
@@ -124,7 +127,7 @@ def describe(self, source: FileSource) -> Report:
         rows = source.rows or []
         report = Report(source.path, self.name, KIND_ANSWER_ONLY, items=len(rows))
         report.problems.extend(source.problems)
-        expected = any(k.startswith("__expected") for r in rows[:1] for k in r)
+        expected = any(str(k).startswith("__expected") for r in rows[:1] for k in r)
         report.set_parts({
             "instruction": bool(rows),
             "verifier": expected or any(r.get("assert") for r in rows),

dokimi/dokimi/task_formats/datasets.py

@@ -333,7 +333,8 @@ def describe(self, source: FileSource) -> Report:
         from dokimi.questions.artifact import load_questions
 
         data = source.data if isinstance(source.data, dict) else {}
-        rows = [q for q in data.get("questions", []) if isinstance(q, dict)]
+        questions = data.get("questions")
+        rows = [q for q in (questions if isinstance(questions, list) else []) if isinstance(q, dict)]
         kind = KIND_TRIGGER if any("expected_skill" in q for q in rows) else KIND_ANSWER_ONLY
         report = _base(source, self.name, kind, rows)
         has_text = all(q.get("question") or q.get("text") for q in rows) and bool(rows)

dokimi/tests/test_task_format_detect.py

@@ -568,6 +568,32 @@ def test_skill_reach_keys_yaml_reads_as_bool_or_int_are_linted_not_a_crash(tmp_p
     assert reports["s.jsonl"].format == "inspect-samples"
 
 
+@pytest.mark.parametrize("name, data, items, problem", [
+    ("promptfooconfig.json", {"prompts": ["p"], "providers": ["x"], "tests": 5}, None, "tests is int"),
+    ("promptfooconfig.json", {"prompts": ["p"], "providers": ["x"], "tests": True}, None, "tests is bool"),
+    ("q.json", {"schema_version": "questions.v1", "questions": None}, 0, "NoneType"),
+])
+def test_a_non_list_tests_or_questions_value_is_not_iterated(tmp_path, name, data, items, problem):
+    _write(tmp_path / name, json.dumps(data))
+    _jsonl(tmp_path / "ok" / "s.jsonl", [{"input": "q", "target": "a"}])
+    reports = {r.path.name: r for r in inspect_path(tmp_path)}
+    assert reports[name].items == items
+    assert any(problem in p for p in reports[name].problems), reports[name].problems
+    assert reports["s.jsonl"].format == "inspect-samples"
+
+
+@pytest.mark.parametrize("name, text, fmt", [
+    ("cases.yaml", "- vars: {a: 1}\n  assert: [{type: equals, value: x}]\n  1: y\n", "promptfoo-tests"),
+    ("e.yaml", "id: e\nquery: where?\nreferences: [{path: a.py, lines: [.inf]}]\n", "dokimi-explore-task"),
+])
+def test_a_non_string_key_or_an_infinite_line_does_not_stop_the_scan(tmp_path, name, text, fmt):
+    _write(tmp_path / name, text)
+    _jsonl(tmp_path / "ok" / "s.jsonl", [{"input": "q", "target": "a"}])
+    reports = {r.path.name: r for r in inspect_path(tmp_path)}
+    assert reports[name].format == fmt
+    assert reports["s.jsonl"].format == "inspect-samples"
+
+
 def test_cli_inspect_out_writes_yaml_dates_sets_and_nan_as_plain_json(tmp_path, capsys):
     task = _tb1(tmp_path)
     _write(task / "task.yaml", "instruction: do it\ndifficulty: 2024-01-01\n"