From 28a360931969d70b9dba9172bc7e7f6392b34889 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 17:45:40 -0700 Subject: [PATCH 01/36] feat(eval): red-team grading blocks against wrong, empty, hedged and stuffed answers The redteam command grades five synthetic answers against each question's grading block: a canary right answer (built from one literal check) must pass; a planted wrong answer, an empty answer, a hedge and a keyword-stuffed list must fail. An accepted attack or a rejected canary blocks the run (exit 1). Judge checks are unscored unless --judge is given. Nothing here changes a score. Signed-off-by: Nikolay Petrov --- ...8_120100_napetrov_feat-verifier-redteam.md | 3 + dokimi/docs/answer-formats.md | 20 ++ dokimi/dokimi/artifacts.py | 1 + dokimi/dokimi/commands/evaluate.py | 4 + dokimi/dokimi/commands/redteam.py | 59 ++++++ dokimi/dokimi/eval/redteam.py | 180 ++++++++++++++++++ dokimi/dokimi/schemas/redteam.v1.json | 79 ++++++++ .../tests/fixtures/artifacts/redteam.v1.json | 40 ++++ dokimi/tests/test_judge_provider_default.py | 2 +- dokimi/tests/test_redteam.py | 133 +++++++++++++ 10 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md create mode 100644 dokimi/dokimi/commands/redteam.py create mode 100644 dokimi/dokimi/eval/redteam.py create mode 100644 dokimi/dokimi/schemas/redteam.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/redteam.v1.json create mode 100644 dokimi/tests/test_redteam.py diff --git a/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md b/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md new file mode 100644 index 000000000..a823af134 --- /dev/null +++ b/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md @@ -0,0 +1,3 @@ +### Added + +- **`eval redteam` checks that grading blocks reject wrong answers** — grades a planted wrong, an empty, a hedged and a keyword-stuffed answer against each question's `grading` block, and a canary right answer built from its one literal check. A wrong answer the grader accepts, or a canary it rejects, is a violation: the command prints pass rates, exits 1 and, with `-o`, writes a `redteam.v1` artifact. Judge checks are unscored unless `--judge` is given, so the default run makes no model call. diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 80482f95c..a8671e285 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -85,6 +85,26 @@ artifact for an `arms run` file. Each block carries `scorer: reference`, and the run's pass rates and mean scores are under `run_metadata.summary` (or `reference_scoring`). +## Red-teaming the grading + +`eval redteam QUESTIONS.json` grades synthetic answers against each question's +`grading` block and reports how often the grader accepts them: + +| Attack | Answer | The grader must | +|---|---|---| +| `canary` | The right answer, built from the one literal check (`match`, `fuzzy`, `exact`, `choice`, `json`); a question with several checks, a `regex`, a `judge` or a `negate` has none | pass it | +| `wrong` | A planted wrong answer | fail it | +| `empty` | `""` | fail it | +| `hedge` | A non-committal sentence with no check value in it | fail it | +| `stuffed` | Two or more literal check values, listed with no sentence around them (a single value is the right answer, not stuffing) | fail it | + +An attack the grader accepts, or a canary it rejects, is a violation, and any +violation makes the run `blocked`: the command prints the pass rates and the +violations and exits 1 (`-o` writes `redteam.v1`). A probe the grader cannot +score is unscored and does not block. `judge` checks are unscored unless +`--judge` is given, so the default run makes no model call. Nothing here +changes a score; it tells you whether a grading block is worth trusting. + ## What does not carry over | Format | Lossy | diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 5731c0f46..c0b9e2746 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -57,6 +57,7 @@ "result": "result.v1", "repo_budget": "repo_budget.v1", "task_inspect": "task_inspect.v1", + "redteam": "redteam.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 9d3602f31..74a8abe08 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -565,6 +565,10 @@ def register(sub, positive_int) -> None: answer_formats.register_eval(eval_sub, positive_int) # eval reference, eval native-import + from dokimi.commands import redteam + + redteam.register_eval(eval_sub) # eval redteam + # eval panel-score panel_p = eval_sub.add_parser("panel-score", help="Score answers using a multi-judge panel") panel_p.add_argument("--answers", required=True, help="Path to answers JSON file") diff --git a/dokimi/dokimi/commands/redteam.py b/dokimi/dokimi/commands/redteam.py new file mode 100644 index 000000000..7e823fbb7 --- /dev/null +++ b/dokimi/dokimi/commands/redteam.py @@ -0,0 +1,59 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``eval redteam``: check that a question set's grading rejects the answers it should. + +Reads a questions set (with ``grading`` blocks), grades synthetic attacks against it and prints +the pass rates. Exits 1 when any attack is accepted or a canary is rejected. Judge checks are not +scored unless ``--judge`` is given, so the default run makes no model call. +""" + +from __future__ import annotations + +import argparse +import sys + + +def _ask(args: argparse.Namespace): + if not args.judge: + return None + from dokimi.eval.judge import Judge + + llm = Judge(model=args.judge_model, provider=args.judge_provider).llm + return lambda prompt, draw=0, cache_if=None: llm.invoke(prompt, draw=draw, cache_if=cache_if).content + + +def cmd_eval_redteam(args: argparse.Namespace) -> None: + from dokimi.artifacts import save_artifact + from dokimi.eval.redteam import format_table, run + from dokimi.questions.artifact import load_questions + + try: + questions = load_questions(args.questions).artifact.get("questions") or [] + result = run(questions, ask=_ask(args)) + except (OSError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(2) + result["questions_path"] = str(args.questions) + print(format_table(result)) + if args.output: + save_artifact("redteam", result, args.output) + print(f"Wrote redteam artifact to {args.output}") + if result["status"] != "pass": + sys.exit(1) + + +def register_eval(eval_sub) -> None: + from dokimi.defaults import DEFAULT_JUDGE_PROVIDER, JUDGE_MODEL_FOR_PROVIDER, JUDGE_MODEL_HELP + from dokimi.eval.judge import JUDGE_PROVIDERS + + r = eval_sub.add_parser( + "redteam", + help="Check grading blocks reject wrong, empty, hedged and stuffed answers and accept a canary") + r.add_argument("questions", help="questions JSON with grading blocks (from `questions import`)") + r.add_argument("--judge", action="store_true", + help="Grade judge checks with the judge model (model calls); default leaves them unscored") + r.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, help=JUDGE_MODEL_HELP) + r.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS) + r.add_argument("-o", "--output", default=None, help="redteam.v1 JSON to write") + r.set_defaults(func=cmd_eval_redteam) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py new file mode 100644 index 000000000..001160259 --- /dev/null +++ b/dokimi/dokimi/eval/redteam.py @@ -0,0 +1,180 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Verifier red team: do a question's grading checks reject the answers they should? + +Each question with a ``grading`` block is graded against five synthetic answers: + +- ``canary``: the right answer, built from the one literal check; it must pass. +- ``wrong``: a planted wrong answer; it must fail. +- ``empty``: the empty answer; it must fail. +- ``hedge``: a non-committal answer that carries no check value; it must fail. +- ``stuffed``: every literal check value listed with no sentence around it; it must fail. + +An answer the grader accepts when it should reject it (or rejects a canary) is a violation, and +any violation blocks the run. A probe the grader cannot score (a judge check with no judge model, +a regex past its timeout) is reported as unscored and does not block. The canary is only built for +a question whose grading is one literal check; other questions get the four attacks alone. + +Nothing here calls a model unless the caller passes ``ask`` (the judge of judge checks). +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Mapping, Optional + +from dokimi.eval.reference_checks import Ask, grade +from dokimi.eval.reference_run import grading_by_id + +ATTACKS = ("canary", "wrong", "empty", "hedge", "stuffed") +EXPECTED = {"canary": True, "wrong": False, "empty": False, "hedge": False, "stuffed": False} +WRONG = "PLANTED-WRONG-ANSWER-7731" +HEDGE = ("I am not certain. It could be several things, depending on the setup, " + "and I cannot say which one applies here.") +BLOCKED, PASS = "blocked", "pass" + +_LITERAL = frozenset({"match", "fuzzy", "exact", "choice", "json"}) + + +def _literal_values(check: Mapping[str, Any]) -> list[Any]: + raw = check.get("values", check.get("value")) + if raw is None: + return [] + return list(raw) if isinstance(raw, (list, tuple)) else [raw] + + +def canary_for(grading: Mapping[str, Any]) -> Optional[str]: + """The right answer for a grading block of one literal check, or None when none is built.""" + checks = grading.get("checks", []) + if len(checks) != 1 or checks[0].get("type") not in _LITERAL or checks[0].get("negate"): + return None + check = checks[0] + kind, values = check["type"], _literal_values(check) + mode, location = str(check.get("mode", "any")), str(check.get("location", "end")) + if not values or (len(values) > 1 and mode == "all"): + return None + text = str(values[0]) + if kind == "choice": + return "ANSWER: " + ", ".join(sorted(str(v).strip().upper() for v in values)) + if kind == "json": + return json.dumps(values[0]) + if kind == "exact" or (kind == "match" and location == "exact"): + return text + if kind == "match" and location == "begin": + return f"{text} is the answer." + return f"The answer is {text}." + + +def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: + """Every literal check value, listed with no sentence around them; None unless there are two or more. + + One value listed alone is the right answer itself, not stuffing, so it is not an attack. A choice + check is left out: its answer is a bare letter, which is a legitimate answer. + """ + values = [str(v) for c in grading.get("checks", []) if c.get("type") in _LITERAL - {"choice"} + for v in _literal_values(c)] + return " ".join(values) if len(values) >= 2 else None + + +def attacks(grading: Mapping[str, Any]) -> dict[str, Optional[str]]: + """The synthetic answer for each attack; None where the attack does not apply to the block.""" + return {"canary": canary_for(grading), "wrong": WRONG, "empty": "", "hedge": HEDGE, + "stuffed": stuffed_for(grading)} + + +@dataclass(frozen=True) +class Probe: + attack: str + passed: Optional[bool] + detail: str = "" + + @property + def expected(self) -> bool: + return EXPECTED[self.attack] + + @property + def violation(self) -> bool: + return self.passed is not None and self.passed != self.expected + + def as_dict(self) -> dict[str, Any]: + return {"passed": self.passed, "expected": self.expected, "violation": self.violation, + "detail": self.detail} + + +def probe(grading: Mapping[str, Any], question: str, ask: Optional[Ask] = None) -> dict[str, Probe]: + """Grade every applicable attack against one grading block.""" + out: dict[str, Probe] = {} + for name, answer in attacks(grading).items(): + if answer is None: + continue + g = grade(grading, question, answer, ask) + out[name] = Probe(name, g.passed, _detail(g)) + return out + + +def _detail(g: Any) -> str: + return "; ".join(c.detail for c in g.checks if c.detail) + + +def run(questions: list[Mapping[str, Any]], *, ask: Optional[Ask] = None) -> dict[str, Any]: + """Red-team every question with a grading block. Raises ValueError for a block that cannot be graded.""" + gradings = grading_by_id(questions) + rows: list[dict[str, Any]] = [] + for q in questions: + qid = str(q.get("id")) + if qid not in gradings: + continue + text = str(q.get("question") or q.get("text") or "") + probes = probe(gradings[qid], text, ask) + rows.append({"question_id": qid, "canary": "canary" in probes, + "probes": {name: p.as_dict() for name, p in probes.items()}}) + violations = [{"question_id": r["question_id"], "attack": name, "passed": p["passed"]} + for r in rows for name, p in r["probes"].items() if p["violation"]] + return { + "status": BLOCKED if violations else PASS, + "attacks": _attack_rates(rows), + "violations": violations, + "questions": rows, + "summary": _summary(rows), + } + + +def _attack_rates(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for name in ATTACKS: + scored = [r["probes"][name]["passed"] for r in rows if name in r["probes"]] + scored = [p for p in scored if p is not None] + accepted = sum(1 for p in scored if p) + out[name] = {"expected": "pass" if EXPECTED[name] else "fail", "scored": len(scored), + "accepted": accepted, "rate": round(accepted / len(scored), 4) if scored else None} + return out + + +def _summary(rows: list[dict[str, Any]]) -> dict[str, int]: + probes = [p for r in rows for p in r["probes"].values()] + return { + "questions": len(rows), + "probed": sum(1 for r in rows if any(p["passed"] is not None for p in r["probes"].values())), + "no_canary": sum(1 for r in rows if not r["canary"]), + "unscored": sum(1 for p in probes if p["passed"] is None), + "violations": sum(1 for p in probes if p["violation"]), + } + + +def format_table(result: dict[str, Any]) -> str: + lines = [f"{'attack':<8} {'expected':<9} {'scored':>6} {'accepted':>8} {'rate':>6}"] + for name, a in result["attacks"].items(): + rate = "-" if a["rate"] is None else f"{a['rate']:.2f}" + lines.append(f"{name:<8} {a['expected']:<9} {a['scored']:>6} {a['accepted']:>8} {rate:>6}") + for v in result["violations"][:20]: + verb = "accepted" if v["attack"] != "canary" else "rejected" + lines.append(f" violation: question {v['question_id']}: {v['attack']} answer {verb}") + if len(result["violations"]) > 20: + lines.append(f" ... {len(result['violations']) - 20} more violations in the artifact") + s = result["summary"] + lines.append(f"status {result['status']}: {s['questions']} questions, {s['probed']} probed, " + f"{s['no_canary']} without a canary, {s['unscored']} probes unscored, " + f"{s['violations']} violations") + return "\n".join(lines) diff --git a/dokimi/dokimi/schemas/redteam.v1.json b/dokimi/dokimi/schemas/redteam.v1.json new file mode 100644 index 000000000..a6049f942 --- /dev/null +++ b/dokimi/dokimi/schemas/redteam.v1.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/redteam.v1.json", + "title": "Verifier red team v1", + "description": "Whether a question set's grading blocks reject the synthetic wrong, empty, hedged and stuffed answers they should, and accept a canary right answer. Written by `eval redteam`; status is blocked when any attack is accepted or a canary is rejected.", + "type": "object", + "required": ["schema_version", "status", "questions_path", "attacks", "violations", "questions", "summary"], + "additionalProperties": false, + "properties": { + "schema_version": {"const": "redteam.v1"}, + "status": {"enum": ["pass", "blocked"]}, + "questions_path": {"type": "string"}, + "attacks": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["expected", "scored", "accepted", "rate"], + "additionalProperties": false, + "properties": { + "expected": {"enum": ["pass", "fail"], "description": "What the grader must do with this attack."}, + "scored": {"type": "integer", "minimum": 0}, + "accepted": {"type": "integer", "minimum": 0, "description": "Probes the grader passed."}, + "rate": {"type": ["number", "null"], "minimum": 0, "maximum": 1, "description": "accepted / scored; null when nothing was scored."} + } + } + }, + "violations": { + "type": "array", + "items": { + "type": "object", + "required": ["question_id", "attack", "passed"], + "additionalProperties": false, + "properties": { + "question_id": {"type": "string"}, + "attack": {"enum": ["canary", "wrong", "empty", "hedge", "stuffed"]}, + "passed": {"type": "boolean"} + } + } + }, + "questions": { + "type": "array", + "items": { + "type": "object", + "required": ["question_id", "canary", "probes"], + "additionalProperties": false, + "properties": { + "question_id": {"type": "string"}, + "canary": {"type": "boolean", "description": "True when a right answer could be built from the grading block."}, + "probes": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["passed", "expected", "violation", "detail"], + "additionalProperties": false, + "properties": { + "passed": {"type": ["boolean", "null"], "description": "Null when the grader could not score the probe."}, + "expected": {"type": "boolean"}, + "violation": {"type": "boolean"}, + "detail": {"type": "string"} + } + } + } + } + } + }, + "summary": { + "type": "object", + "required": ["questions", "probed", "no_canary", "unscored", "violations"], + "additionalProperties": false, + "properties": { + "questions": {"type": "integer", "minimum": 0}, + "probed": {"type": "integer", "minimum": 0}, + "no_canary": {"type": "integer", "minimum": 0}, + "unscored": {"type": "integer", "minimum": 0}, + "violations": {"type": "integer", "minimum": 0} + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/redteam.v1.json b/dokimi/tests/fixtures/artifacts/redteam.v1.json new file mode 100644 index 000000000..ce9a3d1ac --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/redteam.v1.json @@ -0,0 +1,40 @@ +{ + "schema_version": "redteam.v1", + "status": "blocked", + "questions_path": "questions.json", + "attacks": { + "canary": {"expected": "pass", "scored": 2, "accepted": 1, "rate": 0.5}, + "wrong": {"expected": "fail", "scored": 1, "accepted": 0, "rate": 0.0}, + "empty": {"expected": "fail", "scored": 2, "accepted": 0, "rate": 0.0}, + "hedge": {"expected": "fail", "scored": 2, "accepted": 0, "rate": 0.0}, + "stuffed": {"expected": "fail", "scored": 1, "accepted": 1, "rate": 1.0} + }, + "violations": [ + {"question_id": "1", "attack": "stuffed", "passed": true}, + {"question_id": "2", "attack": "canary", "passed": false} + ], + "questions": [ + { + "question_id": "1", + "canary": true, + "probes": { + "canary": {"passed": true, "expected": true, "violation": false, "detail": ""}, + "wrong": {"passed": false, "expected": false, "violation": false, "detail": ""}, + "empty": {"passed": false, "expected": false, "violation": false, "detail": ""}, + "hedge": {"passed": false, "expected": false, "violation": false, "detail": ""}, + "stuffed": {"passed": true, "expected": false, "violation": true, "detail": ""} + } + }, + { + "question_id": "2", + "canary": true, + "probes": { + "canary": {"passed": false, "expected": true, "violation": true, "detail": "not a match"}, + "wrong": {"passed": null, "expected": false, "violation": false, "detail": "not judged: no judge model"}, + "empty": {"passed": false, "expected": false, "violation": false, "detail": ""}, + "hedge": {"passed": false, "expected": false, "violation": false, "detail": ""} + } + } + ], + "summary": {"questions": 2, "probed": 2, "no_canary": 0, "unscored": 1, "violations": 2} +} diff --git a/dokimi/tests/test_judge_provider_default.py b/dokimi/tests/test_judge_provider_default.py index e1d8fc4d4..31d8d8c63 100644 --- a/dokimi/tests/test_judge_provider_default.py +++ b/dokimi/tests/test_judge_provider_default.py @@ -49,7 +49,7 @@ def test_the_commands_that_take_a_judge_provider_are_all_covered(): assert set(COMMANDS) == { "evaluate", "eval score", "eval judge-validity", "eval checklist", "eval judge-wording", "eval pairwise", "benchmark run", "benchmark preflight", "benchmark batch", "arms run", - "arms matrix-run", "subjects run", "eval reference", + "arms matrix-run", "subjects run", "eval reference", "eval redteam", } diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py new file mode 100644 index 000000000..d2eb9cfd1 --- /dev/null +++ b/dokimi/tests/test_redteam.py @@ -0,0 +1,133 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for ``dokimi.eval.redteam`` and ``eval redteam``: attacks against grading blocks. + +Gradings are built in memory; judge checks are never sent to a model (no ``ask`` is passed). +""" + +from __future__ import annotations + +import argparse +import json +from types import SimpleNamespace + +import pytest + +from dokimi.commands.redteam import cmd_eval_redteam +from dokimi.eval import redteam +from dokimi.eval.redteam import canary_for, format_table, run +from dokimi.eval.reference_checks import CheckResult, Grade + + +def _q(qid, grading, text="What is the capital of France?"): + return {"id": qid, "question": text, "grading": grading} + + +def _match(*values, mode="any"): + return {"checks": [{"type": "match", "values": list(values), "mode": mode}], "threshold": None} + + +def test_a_grader_that_rejects_every_attack_passes_the_red_team(): + result = run([_q("1", _match("Paris"))]) + assert result["status"] == "pass" and result["violations"] == [] + assert set(result["questions"][0]["probes"]) == {"canary", "wrong", "empty", "hedge"} + assert result["attacks"]["canary"] == {"expected": "pass", "scored": 1, "accepted": 1, "rate": 1.0} + + +def test_keyword_stuffing_that_the_grader_accepts_blocks_the_run(): + # Two alternative values under "any": the bare list ends with one of them, so the match accepts it. + result = run([_q("1", _match("Paris", "Lyon"))]) + assert result["status"] == "blocked" + assert result["violations"] == [{"question_id": "1", "attack": "stuffed", "passed": True}] + + +def test_a_grader_that_accepts_anything_is_blocked_on_every_attack(): + grading = {"checks": [{"type": "regex", "values": [".*"]}]} + result = run([_q("1", grading)]) + attacks = {v["attack"] for v in result["violations"]} + assert result["status"] == "blocked" and attacks == {"wrong", "empty", "hedge"} + assert result["questions"][0]["canary"] is False + + +def test_a_rejected_canary_is_a_violation(monkeypatch): + def reject_all(grading, question, answer, ask=None): + return Grade(checks=[CheckResult("match", False, 0.0)]) + + monkeypatch.setattr(redteam, "grade", reject_all) + result = run([_q("1", _match("Paris"))]) + assert result["violations"] == [{"question_id": "1", "attack": "canary", "passed": False}] + assert result["status"] == "blocked" + + +def test_a_judge_check_without_a_judge_is_unscored_and_does_not_block(): + grading = {"checks": [{"type": "judge", "criterion": "names the capital"}]} + result = run([_q("1", grading)]) + assert result["status"] == "pass" + assert result["summary"]["unscored"] == 3 # wrong, empty and hedge; no canary, no stuffing + assert result["summary"]["probed"] == 0 + + +@pytest.mark.parametrize("grading", [ + {"checks": [{"type": "match", "values": ["Paris", "Lyon"], "mode": "all"}]}, + {"checks": [{"type": "match", "values": ["Paris"], "negate": True}]}, + {"checks": [{"type": "regex", "values": ["Paris"]}]}, + {"checks": [{"type": "match", "values": ["Paris"]}, {"type": "match", "values": ["Lyon"]}]}, +]) +def test_no_canary_when_the_block_is_not_one_literal_check(grading): + assert canary_for(grading) is None + + +def test_canary_text_follows_the_check_kind(): + assert canary_for(_match("Paris")) == "The answer is Paris." + assert canary_for({"checks": [{"type": "match", "values": ["Paris"], "location": "begin"}]}) == "Paris is the answer." + assert canary_for({"checks": [{"type": "exact", "value": "Paris"}]}) == "Paris" + assert canary_for({"checks": [{"type": "choice", "values": ["b", "a"]}]}) == "ANSWER: A, B" + assert canary_for({"checks": [{"type": "json", "value": 3}]}) == "3" + + +def test_a_block_that_cannot_be_graded_is_refused(): + with pytest.raises(ValueError, match="cannot be graded"): + run([_q("1", {"checks": [{"type": "nope"}]})]) + + +def test_table_names_each_violation_and_the_status(): + result = run([_q("7", _match("Paris", "Lyon"))]) + table = format_table(result) + assert "violation: question 7: stuffed answer accepted" in table + assert "status blocked: 1 questions" in table + + +def _args(path, **overrides): + base = {"questions": str(path), "judge": False, "judge_model": "m", "judge_provider": "anthropic", + "output": None} + return argparse.Namespace(**{**base, **overrides}) + + +def _patch_questions(monkeypatch, questions): + monkeypatch.setattr("dokimi.questions.artifact.load_questions", + lambda path: SimpleNamespace(artifact={"questions": questions})) + + +def test_command_passes_and_writes_a_redteam_artifact(tmp_path, monkeypatch, capsys): + _patch_questions(monkeypatch, [_q("1", _match("Paris"))]) + out = tmp_path / "redteam.json" + cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["schema_version"] == "redteam.v1" and saved["status"] == "pass" + assert saved["questions_path"].endswith("questions.json") + assert f"Wrote redteam artifact to {out}" in capsys.readouterr().out + + +def test_command_exits_1_when_the_run_is_blocked(tmp_path, monkeypatch): + _patch_questions(monkeypatch, [_q("1", _match("Paris", "Lyon"))]) + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json")) + assert exc.value.code == 1 + + +def test_command_exits_2_on_an_unreadable_question_set(tmp_path, capsys): + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "missing.json")) + assert exc.value.code == 2 + assert capsys.readouterr().err.startswith("Error:") From 71cc8e6aa1b9e2390b4603d5f55b95aa02c489e1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 17:56:19 -0700 Subject: [PATCH 02/36] fix(eval): redteam skips the attacks a grading block passes by design Self-review found two probes that blocked a sound grading block: a negated check passes an empty, wrong or hedged answer, and the fixed hedge sentence is accepted by a check whose value occurs in it (for example a match on "one"). Both are now skipped; stuffed still runs for a negated check, and the hedge is skipped only on a clash. Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 6 ++++-- dokimi/dokimi/eval/redteam.py | 17 +++++++++++++++-- dokimi/tests/test_redteam.py | 24 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index a8671e285..1cdf158ce 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -95,12 +95,14 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or | `canary` | The right answer, built from the one literal check (`match`, `fuzzy`, `exact`, `choice`, `json`); a question with several checks, a `regex`, a `judge` or a `negate` has none | pass it | | `wrong` | A planted wrong answer | fail it | | `empty` | `""` | fail it | -| `hedge` | A non-committal sentence with no check value in it | fail it | +| `hedge` | A fixed non-committal sentence; skipped when a check value occurs in it | fail it | | `stuffed` | Two or more literal check values, listed with no sentence around them (a single value is the right answer, not stuffing) | fail it | An attack the grader accepts, or a canary it rejects, is a violation, and any violation makes the run `blocked`: the command prints the pass rates and the -violations and exits 1 (`-o` writes `redteam.v1`). A probe the grader cannot +violations and exits 1 (`-o` writes `redteam.v1`). A `negate` check passes an +answer that lacks its value, so a block with one gets no `wrong`, `empty` or +`hedge` probe; `stuffed` still runs. A probe the grader cannot score is unscored and does not block. `judge` checks are unscored unless `--judge` is given, so the default run makes no model call. Nothing here changes a score; it tells you whether a grading block is worth trusting. diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 001160259..3a38dc854 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -79,8 +79,21 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: def attacks(grading: Mapping[str, Any]) -> dict[str, Optional[str]]: - """The synthetic answer for each attack; None where the attack does not apply to the block.""" - return {"canary": canary_for(grading), "wrong": WRONG, "empty": "", "hedge": HEDGE, + """The synthetic answer for each attack; None where the attack does not apply to the block. + + A ``negate`` check passes an answer that lacks its value, so a wrong or empty answer passes it + by design; ``stuffed`` still runs, since it carries the negated value and must fail. The hedge is + fixed text, so it is skipped when a check value occurs in it (the grader would accept it). + """ + checks = grading.get("checks", []) + negated = any(c.get("negate") for c in checks) + values = [str(v).strip().lower() for c in checks if c.get("type") in _LITERAL - {"choice"} + for v in _literal_values(c)] + clashes = any(v and v in HEDGE.lower() for v in values) + return {"canary": canary_for(grading), + "wrong": None if negated else WRONG, + "empty": None if negated else "", + "hedge": None if negated or clashes else HEDGE, "stuffed": stuffed_for(grading)} diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index d2eb9cfd1..c4b605aaf 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -131,3 +131,27 @@ def test_command_exits_2_on_an_unreadable_question_set(tmp_path, capsys): cmd_eval_redteam(_args(tmp_path / "missing.json")) assert exc.value.code == 2 assert capsys.readouterr().err.startswith("Error:") + + +def test_a_negated_check_is_not_blocked_by_the_answers_it_passes_by_design(): + # "must not say Paris": a wrong, empty or hedged answer lacks Paris and so passes the check. + grading = {"checks": [{"type": "match", "values": ["Paris"], "negate": True}]} + probes = redteam.probe(grading, "Capital of France?") + assert "wrong" not in probes and "empty" not in probes and "hedge" not in probes + assert "canary" not in probes + + +@pytest.mark.parametrize("value", ["one", "SETUP"]) +def test_a_check_value_that_occurs_in_the_hedge_skips_the_hedge_probe(value): + # The hedge sentence says "which one" and "the setup"; a grader that accepts the word would + # be blamed for a hedge it was never asked to reject. + grading = {"checks": [{"type": "fuzzy", "values": [value]}]} + assert redteam.attacks(grading)["hedge"] is None + assert redteam.attacks({"checks": [{"type": "match", "values": ["Paris"]}]})["hedge"] == redteam.HEDGE + + +def test_a_negated_check_still_fails_the_stuffed_answer_that_carries_its_value(): + grading = {"checks": [{"type": "match", "values": ["Paris", "Lyon"], "negate": True}]} + assert redteam.attacks(grading)["stuffed"] == "Paris Lyon" + probes = redteam.probe(grading, "Q?") + assert probes["stuffed"].passed is False and not probes["stuffed"].violation From 068abc8ea377818ac882dbc39354ee91c3efcd96 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:23:38 -0700 Subject: [PATCH 03/36] fix(redteam): inconclusive status, single-literal stuffing, hedge collisions Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 6 ++-- dokimi/dokimi/commands/redteam.py | 9 ++++-- dokimi/dokimi/eval/redteam.py | 41 ++++++++++++++++++-------- dokimi/dokimi/schemas/redteam.v1.json | 4 +-- dokimi/tests/test_redteam.py | 42 ++++++++++++++++++++++++++- 5 files changed, 83 insertions(+), 19 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 1cdf158ce..cf8955955 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -96,11 +96,13 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or | `wrong` | A planted wrong answer | fail it | | `empty` | `""` | fail it | | `hedge` | A fixed non-committal sentence; skipped when a check value occurs in it | fail it | -| `stuffed` | Two or more literal check values, listed with no sentence around them (a single value is the right answer, not stuffing) | fail it | +| `stuffed` | Two or more values of a block with one literal check (a single value is the right answer, not stuffing); not built for a block that needs every value (`mode: all`) or for several checks, which the stuffed answer satisfies by design | fail it | An attack the grader accepts, or a canary it rejects, is a violation, and any violation makes the run `blocked`: the command prints the pass rates and the -violations and exits 1 (`-o` writes `redteam.v1`). A `negate` check passes an +violations and exits 1 (`-o` writes `redteam.v1`). A run that probed no question +(nothing graded, or only judge checks without `--judge`) is `inconclusive` and +also exits 1. An unwritable `-o` exits 2. A `negate` check passes an answer that lacks its value, so a block with one gets no `wrong`, `empty` or `hedge` probe; `stuffed` still runs. A probe the grader cannot score is unscored and does not block. `judge` checks are unscored unless diff --git a/dokimi/dokimi/commands/redteam.py b/dokimi/dokimi/commands/redteam.py index 7e823fbb7..ba4142f3b 100644 --- a/dokimi/dokimi/commands/redteam.py +++ b/dokimi/dokimi/commands/redteam.py @@ -4,7 +4,8 @@ """``eval redteam``: check that a question set's grading rejects the answers it should. Reads a questions set (with ``grading`` blocks), grades synthetic attacks against it and prints -the pass rates. Exits 1 when any attack is accepted or a canary is rejected. Judge checks are not +the pass rates. Exits 1 when any attack is accepted, a canary is rejected, or nothing was graded +(``inconclusive``); exits 2 on an unreadable input or an unwritable ``-o``. Judge checks are not scored unless ``--judge`` is given, so the default run makes no model call. """ @@ -37,7 +38,11 @@ def cmd_eval_redteam(args: argparse.Namespace) -> None: result["questions_path"] = str(args.questions) print(format_table(result)) if args.output: - save_artifact("redteam", result, args.output) + try: + save_artifact("redteam", result, args.output) + except (OSError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(2) print(f"Wrote redteam artifact to {args.output}") if result["status"] != "pass": sys.exit(1) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 3a38dc854..fecac6f66 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -9,12 +9,16 @@ - ``wrong``: a planted wrong answer; it must fail. - ``empty``: the empty answer; it must fail. - ``hedge``: a non-committal answer that carries no check value; it must fail. -- ``stuffed``: every literal check value listed with no sentence around it; it must fail. +- ``stuffed``: every literal check value listed with no sentence around it; it must fail. Built only + for a block of one literal check (or one negated check), since a block that needs every value, or + several checks that each accept one value, is satisfied by the stuffed answer by design. An answer the grader accepts when it should reject it (or rejects a canary) is a violation, and -any violation blocks the run. A probe the grader cannot score (a judge check with no judge model, -a regex past its timeout) is reported as unscored and does not block. The canary is only built for -a question whose grading is one literal check; other questions get the four attacks alone. +any violation blocks the run. A run that probed no question at all (nothing graded, or only judge +checks without a judge) is ``inconclusive``: it is not a pass, and it exits 1 like a blocked run. +A probe the grader cannot score (a judge check with no judge model, a regex past its timeout) is +reported as unscored. The canary is only built for a question whose grading is one literal check; +other questions get the other attacks alone. Nothing here calls a model unless the caller passes ``ask`` (the judge of judge checks). """ @@ -33,7 +37,7 @@ WRONG = "PLANTED-WRONG-ANSWER-7731" HEDGE = ("I am not certain. It could be several things, depending on the setup, " "and I cannot say which one applies here.") -BLOCKED, PASS = "blocked", "pass" +BLOCKED, PASS, INCONCLUSIVE = "blocked", "pass", "inconclusive" _LITERAL = frozenset({"match", "fuzzy", "exact", "choice", "json"}) @@ -68,13 +72,21 @@ def canary_for(grading: Mapping[str, Any]) -> Optional[str]: def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: - """Every literal check value, listed with no sentence around them; None unless there are two or more. + """Every value of the block's one literal check, listed with no sentence around them. - One value listed alone is the right answer itself, not stuffing, so it is not an attack. A choice - check is left out: its answer is a bare letter, which is a legitimate answer. + Only a block of one literal check is stuffed, and a positive check must accept any one of its + values (mode ``any``): a block that needs every value, or several checks, is satisfied by the + stuffed answer by design, so accepting it is not a violation. A negated check fails the stuffed + answer that carries its value, so it is always a real attack. One value listed alone is the right + answer itself, not stuffing. A choice check is left out: its answer is a bare letter. """ - values = [str(v) for c in grading.get("checks", []) if c.get("type") in _LITERAL - {"choice"} - for v in _literal_values(c)] + checks = grading.get("checks", []) + if len(checks) != 1 or checks[0].get("type") not in _LITERAL - {"choice"}: + return None + check = checks[0] + if not check.get("negate") and str(check.get("mode", "any")) == "all": + return None + values = [str(v) for v in _literal_values(check)] return " ".join(values) if len(values) >= 2 else None @@ -145,12 +157,17 @@ def run(questions: list[Mapping[str, Any]], *, ask: Optional[Ask] = None) -> dic "probes": {name: p.as_dict() for name, p in probes.items()}}) violations = [{"question_id": r["question_id"], "attack": name, "passed": p["passed"]} for r in rows for name, p in r["probes"].items() if p["violation"]] + summary = _summary(rows) + if violations: + status = BLOCKED + else: + status = PASS if summary["probed"] else INCONCLUSIVE return { - "status": BLOCKED if violations else PASS, + "status": status, "attacks": _attack_rates(rows), "violations": violations, "questions": rows, - "summary": _summary(rows), + "summary": summary, } diff --git a/dokimi/dokimi/schemas/redteam.v1.json b/dokimi/dokimi/schemas/redteam.v1.json index a6049f942..908f9c973 100644 --- a/dokimi/dokimi/schemas/redteam.v1.json +++ b/dokimi/dokimi/schemas/redteam.v1.json @@ -2,13 +2,13 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/intel/dokimi/schemas/redteam.v1.json", "title": "Verifier red team v1", - "description": "Whether a question set's grading blocks reject the synthetic wrong, empty, hedged and stuffed answers they should, and accept a canary right answer. Written by `eval redteam`; status is blocked when any attack is accepted or a canary is rejected.", + "description": "Whether a question set's grading blocks reject the synthetic wrong, empty, hedged and stuffed answers they should, and accept a canary right answer. Written by `eval redteam`; status is blocked when any attack is accepted or a canary is rejected, and inconclusive when no question was probed (nothing graded, or only judge checks without a judge).", "type": "object", "required": ["schema_version", "status", "questions_path", "attacks", "violations", "questions", "summary"], "additionalProperties": false, "properties": { "schema_version": {"const": "redteam.v1"}, - "status": {"enum": ["pass", "blocked"]}, + "status": {"enum": ["pass", "blocked", "inconclusive"]}, "questions_path": {"type": "string"}, "attacks": { "type": "object", diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index c4b605aaf..783221753 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -63,11 +63,34 @@ def reject_all(grading, question, answer, ask=None): def test_a_judge_check_without_a_judge_is_unscored_and_does_not_block(): grading = {"checks": [{"type": "judge", "criterion": "names the capital"}]} result = run([_q("1", grading)]) - assert result["status"] == "pass" + assert result["status"] == "inconclusive" # nothing was graded, so nothing was verified + assert result["violations"] == [] assert result["summary"]["unscored"] == 3 # wrong, empty and hedge; no canary, no stuffing assert result["summary"]["probed"] == 0 +@pytest.mark.parametrize("questions", [ + [{"id": "1", "question": "Capital of France?"}], # no grading block at all + [_q("1", {"checks": [{"type": "judge", "criterion": "names the capital"}]})], +]) +def test_a_run_that_probed_nothing_is_inconclusive_not_pass(questions): + result = run(questions) + assert result["status"] == "inconclusive" + assert result["summary"]["probed"] == 0 + + +def test_stuffing_is_not_an_attack_on_a_block_that_needs_every_value(): + # "Paris Lyon" satisfies a block that requires both values, so the grader is right to accept it. + result = run([_q("1", _match("Paris", "Lyon", mode="all"))]) + assert result["status"] == "pass" + assert "stuffed" not in result["questions"][0]["probes"] + + +def test_stuffing_is_not_an_attack_on_several_checks_that_each_accept_a_value(): + grading = {"checks": [{"type": "match", "values": ["Paris"]}, {"type": "match", "values": ["Lyon"]}]} + assert redteam.attacks(grading)["stuffed"] is None + + @pytest.mark.parametrize("grading", [ {"checks": [{"type": "match", "values": ["Paris", "Lyon"], "mode": "all"}]}, {"checks": [{"type": "match", "values": ["Paris"], "negate": True}]}, @@ -126,6 +149,23 @@ def test_command_exits_1_when_the_run_is_blocked(tmp_path, monkeypatch): assert exc.value.code == 1 +def test_command_exits_1_when_nothing_was_graded(tmp_path, monkeypatch): + _patch_questions(monkeypatch, [{"id": "1", "question": "Capital of France?"}]) + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json")) + assert exc.value.code == 1 + + +def test_command_exits_2_when_the_artifact_cannot_be_written(tmp_path, monkeypatch, capsys): + _patch_questions(monkeypatch, [_q("1", _match("Paris"))]) + blocker = tmp_path / "a_file" + blocker.write_text("not a directory", encoding="utf-8") + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json", output=blocker / "out.json")) + assert exc.value.code == 2 # not 1: CI must not read an unwritable -o as a blocked run + assert capsys.readouterr().err.startswith("Error:") + + def test_command_exits_2_on_an_unreadable_question_set(tmp_path, capsys): with pytest.raises(SystemExit) as exc: cmd_eval_redteam(_args(tmp_path / "missing.json")) From 04f21424c5dfd891a2a4e9e0502cabf7f1792ee9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:29:00 -0700 Subject: [PATCH 04/36] fix(redteam): negated checks no longer silence wrong, empty and hedge A negate check passes a wrong, empty or hedged answer by design, but a block that also has a positive check still rejects them. The red team skipped all three for any negated check, so a gameable block such as regex ".*" plus "must not say Lyon" reported pass with nothing probed. - wrong, empty and hedge grade the block's non-negated checks alone. A block of only negated checks gets none of them and is reported as negated_only (row field, summary count, table line, schema). - The hedge clash test also uses the grader's normalisation (squad_normalize), so a value the grader finds in the hedge no longer blames the grader for accepting it. - A match at location end with ignore_punctuation false gets a canary without a full stop, which the grader accepts. Known limits: with a threshold, wrong/empty/hedge do not see the negated checks' share of the score, so a block can accept the wrong answer and still report pass. A choice value of several letters ("AB") is never accepted by the grader, so its canary is rejected and blocks the run; that is a grader issue for validate_grading. Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 7 +- dokimi/dokimi/eval/redteam.py | 64 +++++++++++++++---- dokimi/dokimi/schemas/redteam.v1.json | 6 +- .../tests/fixtures/artifacts/redteam.v1.json | 4 +- dokimi/tests/test_redteam.py | 45 +++++++++++++ 5 files changed, 108 insertions(+), 18 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index cf8955955..bf1ac3716 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -103,8 +103,11 @@ violation makes the run `blocked`: the command prints the pass rates and the violations and exits 1 (`-o` writes `redteam.v1`). A run that probed no question (nothing graded, or only judge checks without `--judge`) is `inconclusive` and also exits 1. An unwritable `-o` exits 2. A `negate` check passes an -answer that lacks its value, so a block with one gets no `wrong`, `empty` or -`hedge` probe; `stuffed` still runs. A probe the grader cannot +answer that lacks its value, so `wrong`, `empty` and `hedge` grade the block's +non-negated checks alone; a block of only negated checks gets none of the three +and is reported as `negated_only`, and `stuffed` still runs. A `threshold` block +is graded the same way, so a negated check's share of the score is not counted. +A probe the grader cannot score is unscored and does not block. `judge` checks are unscored unless `--judge` is given, so the default run makes no model call. Nothing here changes a score; it tells you whether a grading block is worth trusting. diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index fecac6f66..69a9a097b 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -20,6 +20,10 @@ reported as unscored. The canary is only built for a question whose grading is one literal check; other questions get the other attacks alone. +A ``negate`` check passes a wrong, empty or hedged answer by design, so those three attacks grade +the block's non-negated checks alone. A block of only negated checks gets none of them and is +reported as ``negated_only``; its stuffed answer still runs. + Nothing here calls a model unless the caller passes ``ask`` (the judge of judge checks). """ @@ -29,7 +33,7 @@ from dataclasses import dataclass from typing import Any, Mapping, Optional -from dokimi.eval.reference_checks import Ask, grade +from dokimi.eval.reference_checks import Ask, grade, squad_normalize from dokimi.eval.reference_run import grading_by_id ATTACKS = ("canary", "wrong", "empty", "hedge", "stuffed") @@ -38,6 +42,8 @@ HEDGE = ("I am not certain. It could be several things, depending on the setup, " "and I cannot say which one applies here.") BLOCKED, PASS, INCONCLUSIVE = "blocked", "pass", "inconclusive" +# The attacks a negated check makes unfit for a block: the answers it passes by design. +NEGATION_SKIPPED = ("wrong", "empty", "hedge") _LITERAL = frozenset({"match", "fuzzy", "exact", "choice", "json"}) @@ -68,6 +74,8 @@ def canary_for(grading: Mapping[str, Any]) -> Optional[str]: return text if kind == "match" and location == "begin": return f"{text} is the answer." + if kind == "match" and location == "end" and not check.get("ignore_punctuation", True): + return f"The answer is {text}" # the full stop would be the raw end of the answer, not the value return f"The answer is {text}." @@ -90,22 +98,45 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: return " ".join(values) if len(values) >= 2 else None +def _positive_checks(grading: Mapping[str, Any]) -> list[Mapping[str, Any]]: + return [c for c in grading.get("checks", []) if not c.get("negate")] + + +def _positive_only(grading: Mapping[str, Any]) -> Mapping[str, Any]: + """The grading block with its negated checks removed; the threshold and the rest are kept.""" + return {**grading, "checks": _positive_checks(grading)} + + +def negated_only(grading: Mapping[str, Any]) -> bool: + """Whether every check of the block is negated, so no answer is wrong by its positive checks.""" + checks = grading.get("checks", []) + return bool(checks) and all(c.get("negate") for c in checks) + + +def _clashes_with_hedge(value: str) -> bool: + """Whether the hedge carries the value: a raw substring, or one after the grader's normalisation.""" + raw = value.strip().lower() + norm = squad_normalize(value) + return (bool(raw) and raw in HEDGE.lower()) or (bool(norm) and norm in squad_normalize(HEDGE)) + + def attacks(grading: Mapping[str, Any]) -> dict[str, Optional[str]]: """The synthetic answer for each attack; None where the attack does not apply to the block. - A ``negate`` check passes an answer that lacks its value, so a wrong or empty answer passes it - by design; ``stuffed`` still runs, since it carries the negated value and must fail. The hedge is - fixed text, so it is skipped when a check value occurs in it (the grader would accept it). + The wrong, empty and hedge answers apply when the block has a non-negated check: a negated + check passes them by design, but a positive check still rejects them, so they run and the probe + grades the non-negated checks alone. A block of only negated checks gets none of the three. + ``stuffed`` still runs, since it carries the negated value and must fail. The hedge is fixed + text, so it is skipped when a non-negated check value occurs in it (the grader would accept it). """ - checks = grading.get("checks", []) - negated = any(c.get("negate") for c in checks) - values = [str(v).strip().lower() for c in checks if c.get("type") in _LITERAL - {"choice"} - for v in _literal_values(c)] - clashes = any(v and v in HEDGE.lower() for v in values) + positive = _positive_checks(grading) + clashes = any(_clashes_with_hedge(str(v)) for c in positive if c.get("type") in _LITERAL - {"choice"} + for v in _literal_values(c)) + applies = bool(positive) return {"canary": canary_for(grading), - "wrong": None if negated else WRONG, - "empty": None if negated else "", - "hedge": None if negated or clashes else HEDGE, + "wrong": WRONG if applies else None, + "empty": "" if applies else None, + "hedge": None if not applies or clashes else HEDGE, "stuffed": stuffed_for(grading)} @@ -131,10 +162,12 @@ def as_dict(self) -> dict[str, Any]: def probe(grading: Mapping[str, Any], question: str, ask: Optional[Ask] = None) -> dict[str, Probe]: """Grade every applicable attack against one grading block.""" out: dict[str, Probe] = {} + positive = _positive_only(grading) for name, answer in attacks(grading).items(): if answer is None: continue - g = grade(grading, question, answer, ask) + block = positive if name in NEGATION_SKIPPED else grading + g = grade(block, question, answer, ask) out[name] = Probe(name, g.passed, _detail(g)) return out @@ -154,6 +187,7 @@ def run(questions: list[Mapping[str, Any]], *, ask: Optional[Ask] = None) -> dic text = str(q.get("question") or q.get("text") or "") probes = probe(gradings[qid], text, ask) rows.append({"question_id": qid, "canary": "canary" in probes, + "negated_only": negated_only(gradings[qid]), "probes": {name: p.as_dict() for name, p in probes.items()}}) violations = [{"question_id": r["question_id"], "attack": name, "passed": p["passed"]} for r in rows for name, p in r["probes"].items() if p["violation"]] @@ -188,6 +222,7 @@ def _summary(rows: list[dict[str, Any]]) -> dict[str, int]: "questions": len(rows), "probed": sum(1 for r in rows if any(p["passed"] is not None for p in r["probes"].values())), "no_canary": sum(1 for r in rows if not r["canary"]), + "negated_only": sum(1 for r in rows if r["negated_only"]), "unscored": sum(1 for p in probes if p["passed"] is None), "violations": sum(1 for p in probes if p["violation"]), } @@ -204,6 +239,9 @@ def format_table(result: dict[str, Any]) -> str: if len(result["violations"]) > 20: lines.append(f" ... {len(result['violations']) - 20} more violations in the artifact") s = result["summary"] + if s["negated_only"]: + lines.append(f" {s['negated_only']} questions with only negated checks: " + f"{', '.join(NEGATION_SKIPPED[:-1])} and {NEGATION_SKIPPED[-1]} not run") lines.append(f"status {result['status']}: {s['questions']} questions, {s['probed']} probed, " f"{s['no_canary']} without a canary, {s['unscored']} probes unscored, " f"{s['violations']} violations") diff --git a/dokimi/dokimi/schemas/redteam.v1.json b/dokimi/dokimi/schemas/redteam.v1.json index 908f9c973..be650e835 100644 --- a/dokimi/dokimi/schemas/redteam.v1.json +++ b/dokimi/dokimi/schemas/redteam.v1.json @@ -41,11 +41,12 @@ "type": "array", "items": { "type": "object", - "required": ["question_id", "canary", "probes"], + "required": ["question_id", "canary", "negated_only", "probes"], "additionalProperties": false, "properties": { "question_id": {"type": "string"}, "canary": {"type": "boolean", "description": "True when a right answer could be built from the grading block."}, + "negated_only": {"type": "boolean", "description": "True when every check is negated: no wrong, empty or hedged probe is run."}, "probes": { "type": "object", "additionalProperties": { @@ -65,12 +66,13 @@ }, "summary": { "type": "object", - "required": ["questions", "probed", "no_canary", "unscored", "violations"], + "required": ["questions", "probed", "no_canary", "negated_only", "unscored", "violations"], "additionalProperties": false, "properties": { "questions": {"type": "integer", "minimum": 0}, "probed": {"type": "integer", "minimum": 0}, "no_canary": {"type": "integer", "minimum": 0}, + "negated_only": {"type": "integer", "minimum": 0}, "unscored": {"type": "integer", "minimum": 0}, "violations": {"type": "integer", "minimum": 0} } diff --git a/dokimi/tests/fixtures/artifacts/redteam.v1.json b/dokimi/tests/fixtures/artifacts/redteam.v1.json index ce9a3d1ac..51fcaf4d5 100644 --- a/dokimi/tests/fixtures/artifacts/redteam.v1.json +++ b/dokimi/tests/fixtures/artifacts/redteam.v1.json @@ -17,6 +17,7 @@ { "question_id": "1", "canary": true, + "negated_only": false, "probes": { "canary": {"passed": true, "expected": true, "violation": false, "detail": ""}, "wrong": {"passed": false, "expected": false, "violation": false, "detail": ""}, @@ -28,6 +29,7 @@ { "question_id": "2", "canary": true, + "negated_only": false, "probes": { "canary": {"passed": false, "expected": true, "violation": true, "detail": "not a match"}, "wrong": {"passed": null, "expected": false, "violation": false, "detail": "not judged: no judge model"}, @@ -36,5 +38,5 @@ } } ], - "summary": {"questions": 2, "probed": 2, "no_canary": 0, "unscored": 1, "violations": 2} + "summary": {"questions": 2, "probed": 2, "no_canary": 0, "negated_only": 0, "unscored": 1, "violations": 2} } diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 783221753..ca7253672 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -195,3 +195,48 @@ def test_a_negated_check_still_fails_the_stuffed_answer_that_carries_its_value() assert redteam.attacks(grading)["stuffed"] == "Paris Lyon" probes = redteam.probe(grading, "Q?") assert probes["stuffed"].passed is False and not probes["stuffed"].violation + + +def test_a_negated_check_does_not_silence_the_attacks_on_the_positive_check(): + # regex ".*" accepts any answer, and "must not say Lyon" accepts the wrong one too: the block is + # gameable, so the run is blocked rather than passing with nothing probed. + grading = {"checks": [{"type": "regex", "values": [".*"]}, {"type": "match", "values": ["Lyon"], "negate": True}]} + result = run([_q("1", grading)]) + assert result["status"] == "blocked" + assert {v["attack"] for v in result["violations"]} == {"wrong", "empty", "hedge"} + assert result["summary"]["probed"] == 1 + + +def test_a_negated_check_beside_a_positive_check_leaves_the_positive_check_to_reject_the_attacks(): + grading = {"checks": [{"type": "match", "values": ["Paris"]}, {"type": "match", "values": ["Lyon"], "negate": True}]} + result = run([_q("1", grading)]) + probes = result["questions"][0]["probes"] + assert result["status"] == "pass" + assert all(probes[name]["passed"] is False for name in ("wrong", "empty", "hedge")) + + +def test_a_block_of_only_negated_checks_runs_no_wrong_empty_or_hedge_probe_and_reports_it(): + grading = {"checks": [{"type": "match", "values": ["Paris"], "negate": True}]} + result = run([_q("1", grading)]) + row = result["questions"][0] + assert row["negated_only"] is True and result["summary"]["negated_only"] == 1 + assert not {"wrong", "empty", "hedge"} & set(row["probes"]) + assert result["status"] == "inconclusive" + assert "1 questions with only negated checks: wrong, empty and hedge not run" in format_table(result) + + +@pytest.mark.parametrize("check", [ + {"type": "match", "values": ["several things depending on the setup"], "location": "any"}, # comma is punctuation + {"type": "fuzzy", "values": ["the setup and I"]}, # articles drop out of the grader's normalised text +]) +def test_a_value_the_grader_finds_in_the_hedge_after_normalising_skips_the_hedge_probe(check): + assert redteam.attacks({"checks": [check]})["hedge"] is None + result = run([_q("1", {"checks": [check]})]) + assert "hedge" not in {v["attack"] for v in result["violations"]} + + +def test_a_match_without_punctuation_stripping_gets_a_canary_the_grader_accepts(): + grading = {"checks": [{"type": "match", "values": ["Paris"], "ignore_punctuation": False}]} + assert canary_for(grading) == "The answer is Paris" # a full stop would end the answer, not the value + result = run([_q("1", grading)]) + assert result["status"] == "pass" and result["questions"][0]["canary"] is True From 6e16c6bbf7693f305df545ff4a7509beadc97181 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:41:36 -0700 Subject: [PATCH 05/36] fix(redteam): clear wrong answer of check values, stuff negated checks - wrong: the planted answer is picked per block from a few candidates so no positive literal value occurs in it; skipped only when none is clean. Stops valid graders (match "31", match "1", fuzzy "on") being blocked. - stuffed: not built for a negated exact, json or location-exact match, which the listed values never equal; built for one negated value so a block of a single negated check is probed with the forbidden value. - summary: unprobed count, in the table and redteam.v1 (schema, fixture). - docs: answer-formats.md and the module docstring match the code. Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 9 ++- dokimi/dokimi/eval/redteam.py | 76 +++++++++++++------ dokimi/dokimi/schemas/redteam.v1.json | 3 +- .../tests/fixtures/artifacts/redteam.v1.json | 2 +- dokimi/tests/test_redteam.py | 53 ++++++++++++- 5 files changed, 113 insertions(+), 30 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index bf1ac3716..31e9e5461 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -93,10 +93,10 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or | Attack | Answer | The grader must | |---|---|---| | `canary` | The right answer, built from the one literal check (`match`, `fuzzy`, `exact`, `choice`, `json`); a question with several checks, a `regex`, a `judge` or a `negate` has none | pass it | -| `wrong` | A planted wrong answer | fail it | +| `wrong` | A planted wrong answer, picked per block so that no positive check value occurs in it (the probe is skipped only when no candidate is clean) | fail it | | `empty` | `""` | fail it | | `hedge` | A fixed non-committal sentence; skipped when a check value occurs in it | fail it | -| `stuffed` | Two or more values of a block with one literal check (a single value is the right answer, not stuffing); not built for a block that needs every value (`mode: all`) or for several checks, which the stuffed answer satisfies by design | fail it | +| `stuffed` | The values of a block with one literal check, listed with no sentence around them: two or more values of a positive check (a single value is the right answer, not stuffing), or one or more values of a `negate` check, whose stuffed answer is the forbidden value; not built for a positive block that needs every value (`mode: all`), for several checks, or for a negated `exact`, `json` or `match` with `location: exact`, which the listed values never equal | fail it | An attack the grader accepts, or a canary it rejects, is a violation, and any violation makes the run `blocked`: the command prints the pass rates and the @@ -105,7 +105,10 @@ violations and exits 1 (`-o` writes `redteam.v1`). A run that probed no question also exits 1. An unwritable `-o` exits 2. A `negate` check passes an answer that lacks its value, so `wrong`, `empty` and `hedge` grade the block's non-negated checks alone; a block of only negated checks gets none of the three -and is reported as `negated_only`, and `stuffed` still runs. A `threshold` block +and is reported as `negated_only`, and `stuffed` still runs: its probe is the negated value itself, so a +block of one negated value is probed and a run whose only probes are those passes when they are rejected +(`negated_only` in the table is the sign that the block is checked for that value alone). A question with no +probe at all is counted as `unprobed`. A `threshold` block is graded the same way, so a negated check's share of the score is not counted. A probe the grader cannot score is unscored and does not block. `judge` checks are unscored unless diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 69a9a097b..021e2adca 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -6,12 +6,16 @@ Each question with a ``grading`` block is graded against five synthetic answers: - ``canary``: the right answer, built from the one literal check; it must pass. -- ``wrong``: a planted wrong answer; it must fail. +- ``wrong``: a planted wrong answer; it must fail. Its text is picked so that no positive literal + check value occurs in it; the probe is skipped only when no candidate is clean. - ``empty``: the empty answer; it must fail. - ``hedge``: a non-committal answer that carries no check value; it must fail. -- ``stuffed``: every literal check value listed with no sentence around it; it must fail. Built only - for a block of one literal check (or one negated check), since a block that needs every value, or - several checks that each accept one value, is satisfied by the stuffed answer by design. +- ``stuffed``: every literal check value listed with no sentence around it; it must fail. Built for + a positive check of two or more values (mode ``any``), since a block that needs every value, or + several checks that each accept one value, is satisfied by the stuffed answer by design. Also built + for a negated check of one or more values, whose stuffed answer carries the forbidden value; not + built for a negated ``exact``, ``json`` or location-``exact`` match, which the listed values never + equal, so the check passes the stuffed answer by design. An answer the grader accepts when it should reject it (or rejects a canary) is a violation, and any violation blocks the run. A run that probed no question at all (nothing graded, or only judge @@ -38,7 +42,8 @@ ATTACKS = ("canary", "wrong", "empty", "hedge", "stuffed") EXPECTED = {"canary": True, "wrong": False, "empty": False, "hedge": False, "stuffed": False} -WRONG = "PLANTED-WRONG-ANSWER-7731" +# Planted wrong answers, tried in order; the first that carries no positive check value is used. +WRONG_CANDIDATES = ("PLANTED-WRONG-ANSWER-7731", "QZX-BOGUS-REPLY-4096", "ZKV-MISMATCH-2508", "XJQ-NONSENSE-6349") HEDGE = ("I am not certain. It could be several things, depending on the setup, " "and I cannot say which one applies here.") BLOCKED, PASS, INCONCLUSIVE = "blocked", "pass", "inconclusive" @@ -79,23 +84,33 @@ def canary_for(grading: Mapping[str, Any]) -> Optional[str]: return f"The answer is {text}." -def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: - """Every value of the block's one literal check, listed with no sentence around them. +def _whole_answer(check: Mapping[str, Any]) -> bool: + """Whether the check compares the whole answer, so a listed value never equals it.""" + kind = check.get("type") + return kind in ("exact", "json") or (kind == "match" and str(check.get("location", "end")) == "exact") + - Only a block of one literal check is stuffed, and a positive check must accept any one of its - values (mode ``any``): a block that needs every value, or several checks, is satisfied by the - stuffed answer by design, so accepting it is not a violation. A negated check fails the stuffed - answer that carries its value, so it is always a real attack. One value listed alone is the right - answer itself, not stuffing. A choice check is left out: its answer is a bare letter. +def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: + """The block's one literal check's values, listed with no sentence around them; None when not built. + + A positive check needs two or more values and must accept any one of them (mode ``any``): a block + that needs every value, or several checks, is satisfied by the stuffed answer by design, so + accepting it is not a violation. A single positive value listed alone is the right answer itself, + not stuffing. A negated check is stuffed with its one or more values: the stuffed answer carries + the forbidden value, so it must fail, unless the check compares the whole answer (see + ``_whole_answer``), which the listed values never equal. A choice check is left out: its answer is + a bare letter. """ checks = grading.get("checks", []) if len(checks) != 1 or checks[0].get("type") not in _LITERAL - {"choice"}: return None check = checks[0] - if not check.get("negate") and str(check.get("mode", "any")) == "all": - return None values = [str(v) for v in _literal_values(check)] - return " ".join(values) if len(values) >= 2 else None + if check.get("negate"): + return " ".join(values) if values and not _whole_answer(check) else None + if len(values) < 2 or str(check.get("mode", "any")) == "all": + return None + return " ".join(values) def _positive_checks(grading: Mapping[str, Any]) -> list[Mapping[str, Any]]: @@ -113,11 +128,23 @@ def negated_only(grading: Mapping[str, Any]) -> bool: return bool(checks) and all(c.get("negate") for c in checks) -def _clashes_with_hedge(value: str) -> bool: - """Whether the hedge carries the value: a raw substring, or one after the grader's normalisation.""" +def _contains(value: str, text: str) -> bool: + """Whether the text carries the value: a raw substring, or one after the grader's normalisation.""" raw = value.strip().lower() norm = squad_normalize(value) - return (bool(raw) and raw in HEDGE.lower()) or (bool(norm) and norm in squad_normalize(HEDGE)) + return (bool(raw) and raw in text.lower()) or (bool(norm) and norm in squad_normalize(text)) + + +def _positive_values(grading: Mapping[str, Any]) -> list[str]: + """The values of the block's non-negated literal checks, bare choice letters left out.""" + return [str(v) for c in _positive_checks(grading) if c.get("type") in _LITERAL - {"choice"} + for v in _literal_values(c)] + + +def wrong_for(grading: Mapping[str, Any]) -> Optional[str]: + """The first planted wrong answer that carries none of the block's positive values, or None.""" + values = _positive_values(grading) + return next((w for w in WRONG_CANDIDATES if not any(_contains(v, w) for v in values)), None) def attacks(grading: Mapping[str, Any]) -> dict[str, Optional[str]]: @@ -126,15 +153,15 @@ def attacks(grading: Mapping[str, Any]) -> dict[str, Optional[str]]: The wrong, empty and hedge answers apply when the block has a non-negated check: a negated check passes them by design, but a positive check still rejects them, so they run and the probe grades the non-negated checks alone. A block of only negated checks gets none of the three. - ``stuffed`` still runs, since it carries the negated value and must fail. The hedge is fixed - text, so it is skipped when a non-negated check value occurs in it (the grader would accept it). + ``stuffed`` still runs, since it carries the negated value and must fail. The planted wrong answer + is chosen per block so that no positive value occurs in it, and the hedge is fixed text, so it is + skipped when a positive value occurs in it (the grader would accept it). """ positive = _positive_checks(grading) - clashes = any(_clashes_with_hedge(str(v)) for c in positive if c.get("type") in _LITERAL - {"choice"} - for v in _literal_values(c)) + clashes = any(_contains(v, HEDGE) for v in _positive_values(grading)) applies = bool(positive) return {"canary": canary_for(grading), - "wrong": WRONG if applies else None, + "wrong": wrong_for(grading) if applies else None, "empty": "" if applies else None, "hedge": None if not applies or clashes else HEDGE, "stuffed": stuffed_for(grading)} @@ -221,6 +248,7 @@ def _summary(rows: list[dict[str, Any]]) -> dict[str, int]: return { "questions": len(rows), "probed": sum(1 for r in rows if any(p["passed"] is not None for p in r["probes"].values())), + "unprobed": sum(1 for r in rows if not any(p["passed"] is not None for p in r["probes"].values())), "no_canary": sum(1 for r in rows if not r["canary"]), "negated_only": sum(1 for r in rows if r["negated_only"]), "unscored": sum(1 for p in probes if p["passed"] is None), @@ -243,6 +271,6 @@ def format_table(result: dict[str, Any]) -> str: lines.append(f" {s['negated_only']} questions with only negated checks: " f"{', '.join(NEGATION_SKIPPED[:-1])} and {NEGATION_SKIPPED[-1]} not run") lines.append(f"status {result['status']}: {s['questions']} questions, {s['probed']} probed, " - f"{s['no_canary']} without a canary, {s['unscored']} probes unscored, " + f"{s['unprobed']} unprobed, {s['no_canary']} without a canary, {s['unscored']} probes unscored, " f"{s['violations']} violations") return "\n".join(lines) diff --git a/dokimi/dokimi/schemas/redteam.v1.json b/dokimi/dokimi/schemas/redteam.v1.json index be650e835..5d841bf1d 100644 --- a/dokimi/dokimi/schemas/redteam.v1.json +++ b/dokimi/dokimi/schemas/redteam.v1.json @@ -66,11 +66,12 @@ }, "summary": { "type": "object", - "required": ["questions", "probed", "no_canary", "negated_only", "unscored", "violations"], + "required": ["questions", "probed", "unprobed", "no_canary", "negated_only", "unscored", "violations"], "additionalProperties": false, "properties": { "questions": {"type": "integer", "minimum": 0}, "probed": {"type": "integer", "minimum": 0}, + "unprobed": {"type": "integer", "minimum": 0}, "no_canary": {"type": "integer", "minimum": 0}, "negated_only": {"type": "integer", "minimum": 0}, "unscored": {"type": "integer", "minimum": 0}, diff --git a/dokimi/tests/fixtures/artifacts/redteam.v1.json b/dokimi/tests/fixtures/artifacts/redteam.v1.json index 51fcaf4d5..50a800141 100644 --- a/dokimi/tests/fixtures/artifacts/redteam.v1.json +++ b/dokimi/tests/fixtures/artifacts/redteam.v1.json @@ -38,5 +38,5 @@ } } ], - "summary": {"questions": 2, "probed": 2, "no_canary": 0, "negated_only": 0, "unscored": 1, "violations": 2} + "summary": {"questions": 2, "probed": 2, "unprobed": 0, "no_canary": 0, "negated_only": 0, "unscored": 1, "violations": 2} } diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index ca7253672..df6e8c138 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -221,10 +221,61 @@ def test_a_block_of_only_negated_checks_runs_no_wrong_empty_or_hedge_probe_and_r row = result["questions"][0] assert row["negated_only"] is True and result["summary"]["negated_only"] == 1 assert not {"wrong", "empty", "hedge"} & set(row["probes"]) - assert result["status"] == "inconclusive" + assert set(row["probes"]) == {"stuffed"} # the forbidden value itself, which must fail + assert result["status"] == "pass" assert "1 questions with only negated checks: wrong, empty and hedge not run" in format_table(result) +def test_a_negated_single_value_is_stuffed_with_that_value_and_must_fail(): + grading = {"checks": [{"type": "match", "values": ["Paris"], "negate": True}]} + assert redteam.attacks(grading)["stuffed"] == "Paris" + probes = redteam.probe(grading, "Capital of France?") + assert set(probes) == {"stuffed"} and probes["stuffed"].passed is False and not probes["stuffed"].violation + + +@pytest.mark.parametrize("check", [ + {"type": "exact", "values": ["Paris", "Lyon"], "negate": True}, + {"type": "match", "values": ["Paris", "Lyon"], "location": "exact", "negate": True}, + {"type": "json", "values": [1, 2], "negate": True}, +]) +def test_a_negated_whole_answer_check_is_not_stuffed(check): + # "Paris Lyon" is neither "Paris" nor "Lyon", so the negated check passes it: that is not an attack. + grading = {"checks": [check]} + assert redteam.attacks(grading)["stuffed"] is None + result = run([_q("1", grading)]) + assert result["violations"] == [] and "stuffed" not in result["questions"][0]["probes"] + + +@pytest.mark.parametrize("check", [ + {"type": "match", "values": ["31"]}, # the planted answer ends in 7731 + {"type": "match", "values": ["1"]}, # the planted answer ends in 1 + {"type": "fuzzy", "values": ["on"]}, # the planted answer contains "on" in "wrong" +]) +def test_a_check_value_in_the_planted_wrong_answer_does_not_block_a_grader_that_rejects_it(check): + result = run([_q("1", {"checks": [check]})]) + assert result["status"] == "pass" and result["violations"] == [] + assert result["questions"][0]["probes"]["wrong"]["passed"] is False # still probed, on a clean wrong answer + + +def test_the_planted_wrong_answer_carries_none_of_the_positive_check_values(): + grading = {"checks": [{"type": "match", "values": ["1", "on"]}]} + wrong = redteam.attacks(grading)["wrong"] + assert wrong is not None and "1" not in wrong and "on" not in wrong.lower() + + +def test_the_wrong_attack_is_skipped_when_every_candidate_carries_a_check_value(monkeypatch): + monkeypatch.setattr(redteam, "WRONG_CANDIDATES", ("PLANTED-WRONG-ANSWER-7731",)) + assert redteam.attacks({"checks": [{"type": "match", "values": ["31"]}]})["wrong"] is None + + +def test_the_summary_counts_the_questions_that_were_not_probed(): + questions = [_q("1", _match("Paris")), _q("2", {"checks": [{"type": "judge", "criterion": "names the capital"}]})] + result = run(questions) + assert result["summary"]["probed"] == 1 and result["summary"]["unprobed"] == 1 + assert result["status"] == "pass" + assert "1 unprobed" in format_table(result) + + @pytest.mark.parametrize("check", [ {"type": "match", "values": ["several things depending on the setup"], "location": "any"}, # comma is punctuation {"type": "fuzzy", "values": ["the setup and I"]}, # articles drop out of the grader's normalised text From e881ab5722d3a8754c75707a742ce6dd9d2eace7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 21:26:20 -0700 Subject: [PATCH 06/36] fix(redteam): skip the stuffed probe for a negated mode all match A negated match with mode all and the default location end gets the stuffed answer "Paris Lyon". That answer ends with one value only, so the all-match is false and the negation passes it: a correct grader was reported as accepting a stuffed answer and the run was blocked. Skip the probe for negated mode all, as the positive rule already does for mode all. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/redteam.py | 15 ++++++++++----- dokimi/tests/test_redteam.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 021e2adca..d9e16d467 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -15,7 +15,9 @@ several checks that each accept one value, is satisfied by the stuffed answer by design. Also built for a negated check of one or more values, whose stuffed answer carries the forbidden value; not built for a negated ``exact``, ``json`` or location-``exact`` match, which the listed values never - equal, so the check passes the stuffed answer by design. + equal, nor for a negated ``mode: all`` match, whose stuffed answer can pass the negation by its + location (``Paris Lyon`` ends with one value only). The skip covers every negated ``mode: all`` + match, which is wider than that case and never a false block. An answer the grader accepts when it should reject it (or rejects a canary) is a violation, and any violation blocks the run. A run that probed no question at all (nothing graded, or only judge @@ -98,17 +100,20 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: accepting it is not a violation. A single positive value listed alone is the right answer itself, not stuffing. A negated check is stuffed with its one or more values: the stuffed answer carries the forbidden value, so it must fail, unless the check compares the whole answer (see - ``_whole_answer``), which the listed values never equal. A choice check is left out: its answer is - a bare letter. + ``_whole_answer``), which the listed values never equal, or is ``mode: all``, which the stuffed + answer can pass by its location (the skip is kept for every such check, for simplicity). A choice + check is left out: its answer is a bare letter. """ checks = grading.get("checks", []) if len(checks) != 1 or checks[0].get("type") not in _LITERAL - {"choice"}: return None check = checks[0] values = [str(v) for v in _literal_values(check)] + mode = str(check.get("mode", "any")) if check.get("negate"): - return " ".join(values) if values and not _whole_answer(check) else None - if len(values) < 2 or str(check.get("mode", "any")) == "all": + built = bool(values) and not _whole_answer(check) and mode != "all" + return " ".join(values) if built else None + if len(values) < 2 or mode == "all": return None return " ".join(values) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index df6e8c138..4edadd460 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -246,6 +246,19 @@ def test_a_negated_whole_answer_check_is_not_stuffed(check): assert result["violations"] == [] and "stuffed" not in result["questions"][0]["probes"] +@pytest.mark.parametrize("check", [ + {"type": "match", "values": ["Paris", "Lyon"], "mode": "all", "negate": True}, # location end + {"type": "match", "values": ["Paris", "Lyon"], "mode": "all", "location": "begin", "negate": True}, +]) +def test_a_negated_all_mode_check_is_not_stuffed(check): + # "Paris Lyon" ends with only one value, so the "all" match is false and the negated check passes + # it: a correct grader would be blamed for accepting it. The positive rule skips mode all alike. + grading = {"checks": [check]} + assert redteam.attacks(grading)["stuffed"] is None + result = run([_q("1", grading)]) + assert result["violations"] == [] and "stuffed" not in result["questions"][0]["probes"] + + @pytest.mark.parametrize("check", [ {"type": "match", "values": ["31"]}, # the planted answer ends in 7731 {"type": "match", "values": ["1"]}, # the planted answer ends in 1 From d96f87dd8c5103d5427b88ebe89fe4382d158745 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:40:35 -0700 Subject: [PATCH 07/36] fix(redteam): choose the planted wrong answer and the hedge probe by the check itself The wrong candidate and the hedge skip were decided by a substring rule, so a numeric match that reads 7731 out of the planted answer blocked a correct grader, and a short value such as "one" dropped the hedge probe although an end match does not find it there. Each is now asked of the block's own positive literal checks. Thread: PRRT_kwDOVBg7g86xgzKG Thread: PRRT_kwDOVBg7g86xgzKN Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 4 ++-- dokimi/dokimi/eval/redteam.py | 34 +++++++++++++--------------------- dokimi/tests/test_redteam.py | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index ffcae9e60..130648205 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -98,9 +98,9 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or | Attack | Answer | The grader must | |---|---|---| | `canary` | The right answer, built from the one literal check (`match`, `fuzzy`, `exact`, `choice`, `json`); a question with several checks, a `regex`, a `judge` or a `negate` has none | pass it | -| `wrong` | A planted wrong answer, picked per block so that no positive check value occurs in it (the probe is skipped only when no candidate is clean) | fail it | +| `wrong` | A planted wrong answer, picked per block so that no positive literal check accepts it, asked with the check's own rules (the probe is skipped only when no candidate is clean) | fail it | | `empty` | `""` | fail it | -| `hedge` | A fixed non-committal sentence; skipped when a check value occurs in it | fail it | +| `hedge` | A fixed non-committal sentence; skipped when a positive literal check accepts it | fail it | | `stuffed` | The values of a block with one literal check, listed with no sentence around them: two or more values of a positive check (a single value is the right answer, not stuffing), or one or more values of a `negate` check, whose stuffed answer is the forbidden value; not built for a positive block that needs every value (`mode: all`), for several checks, or for a negated `exact`, `json` or `match` with `location: exact`, which the listed values never equal | fail it | An attack the grader accepts, or a canary it rejects, is a violation, and any diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index d9e16d467..738eeb992 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -7,7 +7,7 @@ - ``canary``: the right answer, built from the one literal check; it must pass. - ``wrong``: a planted wrong answer; it must fail. Its text is picked so that no positive literal - check value occurs in it; the probe is skipped only when no candidate is clean. + check accepts it; the probe is skipped only when no candidate is clean. - ``empty``: the empty answer; it must fail. - ``hedge``: a non-committal answer that carries no check value; it must fail. - ``stuffed``: every literal check value listed with no sentence around it; it must fail. Built for @@ -39,7 +39,7 @@ from dataclasses import dataclass from typing import Any, Mapping, Optional -from dokimi.eval.reference_checks import Ask, grade, squad_normalize +from dokimi.eval.reference_checks import Ask, grade, grade_check from dokimi.eval.reference_run import grading_by_id ATTACKS = ("canary", "wrong", "empty", "hedge", "stuffed") @@ -133,23 +133,14 @@ def negated_only(grading: Mapping[str, Any]) -> bool: return bool(checks) and all(c.get("negate") for c in checks) -def _contains(value: str, text: str) -> bool: - """Whether the text carries the value: a raw substring, or one after the grader's normalisation.""" - raw = value.strip().lower() - norm = squad_normalize(value) - return (bool(raw) and raw in text.lower()) or (bool(norm) and norm in squad_normalize(text)) - - -def _positive_values(grading: Mapping[str, Any]) -> list[str]: - """The values of the block's non-negated literal checks, bare choice letters left out.""" - return [str(v) for c in _positive_checks(grading) if c.get("type") in _LITERAL - {"choice"} - for v in _literal_values(c)] +def _accepts(grading: Mapping[str, Any], text: str) -> bool: + """Whether a non-negated literal check of the block accepts the text on its own, by the grader's rules.""" + return any(grade_check(c, "", text, None).passed for c in _positive_checks(grading) if c.get("type") in _LITERAL) def wrong_for(grading: Mapping[str, Any]) -> Optional[str]: - """The first planted wrong answer that carries none of the block's positive values, or None.""" - values = _positive_values(grading) - return next((w for w in WRONG_CANDIDATES if not any(_contains(v, w) for v in values)), None) + """The first planted wrong answer that no positive literal check of the block accepts, or None.""" + return next((w for w in WRONG_CANDIDATES if not _accepts(grading, w)), None) def attacks(grading: Mapping[str, Any]) -> dict[str, Optional[str]]: @@ -159,12 +150,13 @@ def attacks(grading: Mapping[str, Any]) -> dict[str, Optional[str]]: check passes them by design, but a positive check still rejects them, so they run and the probe grades the non-negated checks alone. A block of only negated checks gets none of the three. ``stuffed`` still runs, since it carries the negated value and must fail. The planted wrong answer - is chosen per block so that no positive value occurs in it, and the hedge is fixed text, so it is - skipped when a positive value occurs in it (the grader would accept it). + is chosen per block so that no positive check accepts it, and the hedge is fixed text, so it is + skipped when a positive check accepts it (each is asked with the grader's own rules, so a numeric + match that reads a number out of the planted answer, or a short value the check does not find in the + hedge, are judged as the grader judges them). """ - positive = _positive_checks(grading) - clashes = any(_contains(v, HEDGE) for v in _positive_values(grading)) - applies = bool(positive) + applies = bool(_positive_checks(grading)) + clashes = _accepts(grading, HEDGE) return {"canary": canary_for(grading), "wrong": wrong_for(grading) if applies else None, "empty": "" if applies else None, diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 4edadd460..bf83ef185 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -304,3 +304,21 @@ def test_a_match_without_punctuation_stripping_gets_a_canary_the_grader_accepts( assert canary_for(grading) == "The answer is Paris" # a full stop would end the answer, not the value result = run([_q("1", grading)]) assert result["status"] == "pass" and result["questions"][0]["canary"] is True + + +def test_a_numeric_check_does_not_block_on_a_planted_answer_it_reads_as_its_number(): + # The planted answer ends in 7731 and a numeric match reads that number: a clean candidate is chosen by + # asking the check itself, not by looking for the value's text in it. + check = {"type": "match", "values": ["-7731.0"], "numeric": True} + wrong = redteam.attacks({"checks": [check]})["wrong"] + assert wrong is not None and "7731" not in wrong + result = run([_q("1", {"checks": [check]})]) + assert result["status"] == "pass" and result["violations"] == [] + + +def test_a_short_value_the_check_does_not_find_in_the_hedge_keeps_the_hedge_probe(): + # "one" occurs in "which one applies", but an end match reads the last words of the hedge only. + grading = {"checks": [{"type": "match", "values": ["one"]}]} + assert redteam.attacks(grading)["hedge"] == redteam.HEDGE + result = run([_q("1", grading)]) + assert result["questions"][0]["probes"]["hedge"]["passed"] is False From 6b6aa2f41dbf1a4b6210afc2b6f7f371a3338b09 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:00 -0700 Subject: [PATCH 08/36] fix(redteam): grade wrong, empty and hedge against the whole block, threshold included Dropping the negated checks from those three probes hid the one case where they matter: a threshold block whose negated check lifts a wrong answer to the line passed the red team while the real grader accepted it. Thread: PRRT_kwDOVBg7g86xgzJ2 Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 9 +++++---- dokimi/dokimi/eval/redteam.py | 21 +++++++-------------- dokimi/tests/test_redteam.py | 10 ++++++++++ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 130648205..f56b6c7a8 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -108,13 +108,14 @@ violation makes the run `blocked`: the command prints the pass rates and the violations and exits 1 (`-o` writes `redteam.v1`). A run that probed no question (nothing graded, or only judge checks without `--judge`) is `inconclusive` and also exits 1. An unwritable `-o` exits 2. A `negate` check passes an -answer that lacks its value, so `wrong`, `empty` and `hedge` grade the block's -non-negated checks alone; a block of only negated checks gets none of the three +answer that lacks its value, so a block of only negated checks gets none of +`wrong`, `empty` and `hedge` and is reported as `negated_only`, and `stuffed` still runs: its probe is the negated value itself, so a block of one negated value is probed and a run whose only probes are those passes when they are rejected (`negated_only` in the table is the sign that the block is checked for that value alone). A question with no -probe at all is counted as `unprobed`. A `threshold` block -is graded the same way, so a negated check's share of the score is not counted. +probe at all is counted as `unprobed`. A block with a positive check +is graded as written, `threshold` included, so a negated check's share of the score +counts: a block it lifts over the line is blocked. A probe the grader cannot score is unscored and does not block. `judge` checks are unscored unless `--judge` is given, so the default run makes no model call. Nothing here diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 738eeb992..2efa07eb5 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -26,9 +26,10 @@ reported as unscored. The canary is only built for a question whose grading is one literal check; other questions get the other attacks alone. -A ``negate`` check passes a wrong, empty or hedged answer by design, so those three attacks grade -the block's non-negated checks alone. A block of only negated checks gets none of them and is -reported as ``negated_only``; its stuffed answer still runs. +A ``negate`` check passes a wrong, empty or hedged answer by design, so a block of only negated +checks gets none of those three attacks and is reported as ``negated_only``; its stuffed answer still +runs. A block with a positive check is graded as written, ``threshold`` included: a negated check's +share of the score counts, so a block whose negated checks lift a wrong answer over the line is blocked. Nothing here calls a model unless the caller passes ``ask`` (the judge of judge checks). """ @@ -122,11 +123,6 @@ def _positive_checks(grading: Mapping[str, Any]) -> list[Mapping[str, Any]]: return [c for c in grading.get("checks", []) if not c.get("negate")] -def _positive_only(grading: Mapping[str, Any]) -> Mapping[str, Any]: - """The grading block with its negated checks removed; the threshold and the rest are kept.""" - return {**grading, "checks": _positive_checks(grading)} - - def negated_only(grading: Mapping[str, Any]) -> bool: """Whether every check of the block is negated, so no answer is wrong by its positive checks.""" checks = grading.get("checks", []) @@ -146,9 +142,8 @@ def wrong_for(grading: Mapping[str, Any]) -> Optional[str]: def attacks(grading: Mapping[str, Any]) -> dict[str, Optional[str]]: """The synthetic answer for each attack; None where the attack does not apply to the block. - The wrong, empty and hedge answers apply when the block has a non-negated check: a negated - check passes them by design, but a positive check still rejects them, so they run and the probe - grades the non-negated checks alone. A block of only negated checks gets none of the three. + The wrong, empty and hedge answers apply when the block has a non-negated check, and are graded + against the whole block as written. A block of only negated checks gets none of the three. ``stuffed`` still runs, since it carries the negated value and must fail. The planted wrong answer is chosen per block so that no positive check accepts it, and the hedge is fixed text, so it is skipped when a positive check accepts it (each is asked with the grader's own rules, so a numeric @@ -186,12 +181,10 @@ def as_dict(self) -> dict[str, Any]: def probe(grading: Mapping[str, Any], question: str, ask: Optional[Ask] = None) -> dict[str, Probe]: """Grade every applicable attack against one grading block.""" out: dict[str, Probe] = {} - positive = _positive_only(grading) for name, answer in attacks(grading).items(): if answer is None: continue - block = positive if name in NEGATION_SKIPPED else grading - g = grade(block, question, answer, ask) + g = grade(grading, question, answer, ask) out[name] = Probe(name, g.passed, _detail(g)) return out diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index bf83ef185..c6c54b752 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -322,3 +322,13 @@ def test_a_short_value_the_check_does_not_find_in_the_hedge_keeps_the_hedge_prob assert redteam.attacks(grading)["hedge"] == redteam.HEDGE result = run([_q("1", grading)]) assert result["questions"][0]["probes"]["hedge"]["passed"] is False + + +def test_a_threshold_block_that_a_negated_check_lifts_over_the_line_is_blocked(): + # Half the score comes from "must not say London", which every wrong, empty or hedged answer earns: + # the real grader passes them at 0.5, so the block is gameable. + grading = {"threshold": 0.5, "checks": [{"type": "match", "values": ["Paris"]}, + {"type": "match", "values": ["London"], "negate": True}]} + result = run([_q("1", grading)]) + assert result["status"] == "blocked" + assert {v["attack"] for v in result["violations"]} == {"wrong", "empty", "hedge"} From 19f30efe6d3e875e0ddd3a097ddac60d13537b69 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:52 -0700 Subject: [PATCH 09/36] fix(redteam): a set with a question that was not measured is inconclusive, not pass One probed question made the whole set pass while five of six judge questions were never scored, and a block of only negated checks counted as probed through its stuffed probe even though the real grader accepts any answer without the forbidden value. Both are now unprobed; any unprobed question makes the run inconclusive and the table names each of them. Thread: PRRT_kwDOVBg7g86xgzJ6 Thread: PRRT_kwDOVBg7g86xgzKC Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 20 +++++++++--------- dokimi/dokimi/eval/redteam.py | 29 ++++++++++++++++++++++----- dokimi/dokimi/schemas/redteam.v1.json | 2 +- dokimi/tests/test_redteam.py | 29 +++++++++++++++++++++++++-- 4 files changed, 63 insertions(+), 17 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index f56b6c7a8..5fb7ab4ac 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -105,15 +105,17 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or An attack the grader accepts, or a canary it rejects, is a violation, and any violation makes the run `blocked`: the command prints the pass rates and the -violations and exits 1 (`-o` writes `redteam.v1`). A run that probed no question -(nothing graded, or only judge checks without `--judge`) is `inconclusive` and -also exits 1. An unwritable `-o` exits 2. A `negate` check passes an -answer that lacks its value, so a block of only negated checks gets none of -`wrong`, `empty` and `hedge` -and is reported as `negated_only`, and `stuffed` still runs: its probe is the negated value itself, so a -block of one negated value is probed and a run whose only probes are those passes when they are rejected -(`negated_only` in the table is the sign that the block is checked for that value alone). A question with no -probe at all is counted as `unprobed`. A block with a positive check +violations and exits 1 (`-o` writes `redteam.v1`). A run in which a graded +question was not measured is `inconclusive` and also exits 1: a question whose +`judge` checks have no verdict without `--judge` (no probe scored), or a block +of only negated checks. A `negate` check passes an answer that lacks its value, +so a block of only negated checks gets none of `wrong`, `empty` and `hedge`: it +is reported as `negated_only`, its `stuffed` probe (the negated value itself) +still runs, but the grader would accept any answer without that value, so +nothing was verified about it. The table names every question that was not +measured and counts it as `unprobed`; only a run that measured every graded +question and found no violation is `pass`. An unwritable `-o` exits 2. +A block with a positive check is graded as written, `threshold` included, so a negated check's share of the score counts: a block it lifts over the line is blocked. A probe the grader cannot diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 2efa07eb5..5790eeb54 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -20,8 +20,9 @@ match, which is wider than that case and never a false block. An answer the grader accepts when it should reject it (or rejects a canary) is a violation, and -any violation blocks the run. A run that probed no question at all (nothing graded, or only judge -checks without a judge) is ``inconclusive``: it is not a pass, and it exits 1 like a blocked run. +any violation blocks the run. A run with a question whose correctness no probe measured (a judge check without a judge, a block of +only negated checks) is ``inconclusive``, and so is one with nothing graded: it is not a pass, and it exits 1 +like a blocked run. Only a run that measured every graded question and found no violation is ``pass``. A probe the grader cannot score (a judge check with no judge model, a regex past its timeout) is reported as unscored. The canary is only built for a question whose grading is one literal check; other questions get the other attacks alone. @@ -212,7 +213,7 @@ def run(questions: list[Mapping[str, Any]], *, ask: Optional[Ask] = None) -> dic if violations: status = BLOCKED else: - status = PASS if summary["probed"] else INCONCLUSIVE + status = PASS if rows and not summary["unprobed"] else INCONCLUSIVE return { "status": status, "attacks": _attack_rates(rows), @@ -233,12 +234,26 @@ def _attack_rates(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: return out +def _unmeasured(row: Mapping[str, Any]) -> str: + """Why a question's correctness was not measured, or "" when a probe scored it. + + A block of only negated checks gets no wrong, empty or hedge probe: the real grader accepts any + answer without the forbidden value, so nothing was verified about it, whatever its stuffed probe said. + """ + if row["negated_only"]: + return "only negated checks" + if not any(p["passed"] is not None for p in row["probes"].values()): + return "no probe scored" + return "" + + def _summary(rows: list[dict[str, Any]]) -> dict[str, int]: probes = [p for r in rows for p in r["probes"].values()] + unprobed = sum(1 for r in rows if _unmeasured(r)) return { "questions": len(rows), - "probed": sum(1 for r in rows if any(p["passed"] is not None for p in r["probes"].values())), - "unprobed": sum(1 for r in rows if not any(p["passed"] is not None for p in r["probes"].values())), + "probed": len(rows) - unprobed, + "unprobed": unprobed, "no_canary": sum(1 for r in rows if not r["canary"]), "negated_only": sum(1 for r in rows if r["negated_only"]), "unscored": sum(1 for p in probes if p["passed"] is None), @@ -260,6 +275,10 @@ def format_table(result: dict[str, Any]) -> str: if s["negated_only"]: lines.append(f" {s['negated_only']} questions with only negated checks: " f"{', '.join(NEGATION_SKIPPED[:-1])} and {NEGATION_SKIPPED[-1]} not run") + unmeasured = [f"question {r['question_id']} ({why})" for r in result["questions"] if (why := _unmeasured(r))] + if unmeasured: + lines.append(f" not measured: {', '.join(unmeasured[:20])}" + + (f", and {len(unmeasured) - 20} more" if len(unmeasured) > 20 else "")) lines.append(f"status {result['status']}: {s['questions']} questions, {s['probed']} probed, " f"{s['unprobed']} unprobed, {s['no_canary']} without a canary, {s['unscored']} probes unscored, " f"{s['violations']} violations") diff --git a/dokimi/dokimi/schemas/redteam.v1.json b/dokimi/dokimi/schemas/redteam.v1.json index 5d841bf1d..a52475201 100644 --- a/dokimi/dokimi/schemas/redteam.v1.json +++ b/dokimi/dokimi/schemas/redteam.v1.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/intel/dokimi/schemas/redteam.v1.json", "title": "Verifier red team v1", - "description": "Whether a question set's grading blocks reject the synthetic wrong, empty, hedged and stuffed answers they should, and accept a canary right answer. Written by `eval redteam`; status is blocked when any attack is accepted or a canary is rejected, and inconclusive when no question was probed (nothing graded, or only judge checks without a judge).", + "description": "Whether a question set's grading blocks reject the synthetic wrong, empty, hedged and stuffed answers they should, and accept a canary right answer. Written by `eval redteam`; status is blocked when any attack is accepted or a canary is rejected, and inconclusive when nothing was graded or a graded question was not measured (judge checks without a judge, or only negated checks).", "type": "object", "required": ["schema_version", "status", "questions_path", "attacks", "violations", "questions", "summary"], "additionalProperties": false, diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index c6c54b752..e44394cb7 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -222,7 +222,8 @@ def test_a_block_of_only_negated_checks_runs_no_wrong_empty_or_hedge_probe_and_r assert row["negated_only"] is True and result["summary"]["negated_only"] == 1 assert not {"wrong", "empty", "hedge"} & set(row["probes"]) assert set(row["probes"]) == {"stuffed"} # the forbidden value itself, which must fail - assert result["status"] == "pass" + assert result["status"] == "inconclusive" # the real grader accepts any wrong answer: nothing was verified + assert result["summary"]["probed"] == 0 and result["summary"]["unprobed"] == 1 assert "1 questions with only negated checks: wrong, empty and hedge not run" in format_table(result) @@ -285,7 +286,7 @@ def test_the_summary_counts_the_questions_that_were_not_probed(): questions = [_q("1", _match("Paris")), _q("2", {"checks": [{"type": "judge", "criterion": "names the capital"}]})] result = run(questions) assert result["summary"]["probed"] == 1 and result["summary"]["unprobed"] == 1 - assert result["status"] == "pass" + assert result["status"] == "inconclusive" assert "1 unprobed" in format_table(result) @@ -332,3 +333,27 @@ def test_a_threshold_block_that_a_negated_check_lifts_over_the_line_is_blocked() result = run([_q("1", grading)]) assert result["status"] == "blocked" assert {v["attack"] for v in result["violations"]} == {"wrong", "empty", "hedge"} + + +def test_one_probed_question_does_not_make_a_set_pass_when_the_others_were_not_measured(): + judge = {"checks": [{"type": "judge", "criterion": "names the capital"}]} + questions = [_q(str(i), judge) for i in range(5)] + [_q("m", _match("Paris"))] + result = run(questions) + assert result["status"] == "inconclusive" and result["violations"] == [] + assert result["summary"]["probed"] == 1 and result["summary"]["unprobed"] == 5 + table = format_table(result) + assert "not measured: question 0 (no probe scored)" in table + assert "question m" not in table.split("not measured:")[1] + + +def test_a_question_with_only_negated_checks_is_named_in_the_table_as_not_measured(): + only_negated = {"checks": [{"type": "match", "values": ["error"], "negate": True}]} + result = run([_q("g", only_negated), _q("m", _match("Paris"))]) + assert result["status"] == "inconclusive" + assert "not measured: question g (only negated checks)" in format_table(result) + + +def test_a_violation_still_blocks_a_set_with_unmeasured_questions(): + judge = {"checks": [{"type": "judge", "criterion": "names the capital"}]} + result = run([_q("j", judge), _q("1", {"checks": [{"type": "regex", "values": [".*"]}]})]) + assert result["status"] == "blocked" From 3b4c70185572c3fdc9ced5822c8857931f644214 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:42:51 -0700 Subject: [PATCH 10/36] fix(redteam): bury aliases between decoys instead of listing them for the stuffed probe The bare list "4 four" for values ["4", "four"] ends with, begins with or contains an accepted value, so match and fuzzy checks with two or more aliases were blocked on a correct grader. A match at begin or end now gets the list between two decoys the check rejects; match at any, numeric match and fuzzy find the value anywhere, so no probe is built for them. Thread: PRRT_kwDOVBg7g86xgzJz Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 2 +- dokimi/dokimi/eval/redteam.py | 32 ++++++++++++++++++++------- dokimi/tests/test_redteam.py | 41 +++++++++++++++++++++++++++++++---- 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 5fb7ab4ac..7d4f271ab 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -101,7 +101,7 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or | `wrong` | A planted wrong answer, picked per block so that no positive literal check accepts it, asked with the check's own rules (the probe is skipped only when no candidate is clean) | fail it | | `empty` | `""` | fail it | | `hedge` | A fixed non-committal sentence; skipped when a positive literal check accepts it | fail it | -| `stuffed` | The values of a block with one literal check, listed with no sentence around them: two or more values of a positive check (a single value is the right answer, not stuffing), or one or more values of a `negate` check, whose stuffed answer is the forbidden value; not built for a positive block that needs every value (`mode: all`), for several checks, or for a negated `exact`, `json` or `match` with `location: exact`, which the listed values never equal | fail it | +| `stuffed` | The values of a block with one literal check, listed with no sentence around them: two or more values of a positive check (a single value is the right answer, not stuffing), bare for a check on the whole answer and between two decoys for a `match` at `begin` or `end`, or one or more values of a `negate` check, whose stuffed answer is the forbidden value; not built for a positive block that needs every value (`mode: all`), for several checks, for `fuzzy`, numeric or `location: any` checks (they find a listed value wherever it is, so accepting it is right), or for a negated `exact`, `json` or `match` with `location: exact`, which the listed values never equal | fail it | An attack the grader accepts, or a canary it rejects, is a violation, and any violation makes the run `blocked`: the command prints the pass rates and the diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 5790eeb54..797828bbb 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -11,8 +11,11 @@ - ``empty``: the empty answer; it must fail. - ``hedge``: a non-committal answer that carries no check value; it must fail. - ``stuffed``: every literal check value listed with no sentence around it; it must fail. Built for - a positive check of two or more values (mode ``any``), since a block that needs every value, or - several checks that each accept one value, is satisfied by the stuffed answer by design. Also built + a positive check of two or more values (mode ``any``) that does not read the list where it stands: + bare for ``exact``, ``json`` and location-``exact`` checks, between two decoys for a ``match`` at + ``begin`` or ``end``. It is not built for ``fuzzy``, numeric or location-``any`` checks, a block that + needs every value, or several checks that each accept one value: the grader is right to accept the + stuffed answer there. Also built for a negated check of one or more values, whose stuffed answer carries the forbidden value; not built for a negated ``exact``, ``json`` or location-``exact`` match, which the listed values never equal, nor for a negated ``mode: all`` match, whose stuffed answer can pass the negation by its @@ -100,11 +103,19 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: A positive check needs two or more values and must accept any one of them (mode ``any``): a block that needs every value, or several checks, is satisfied by the stuffed answer by design, so accepting it is not a violation. A single positive value listed alone is the right answer itself, - not stuffing. A negated check is stuffed with its one or more values: the stuffed answer carries - the forbidden value, so it must fail, unless the check compares the whole answer (see - ``_whole_answer``), which the listed values never equal, or is ``mode: all``, which the stuffed - answer can pass by its location (the skip is kept for every such check, for simplicity). A choice - check is left out: its answer is a bare letter. + not stuffing. The list is an attack only where the check does not read it: + + - a check on the whole answer (``exact``, ``json``, ``match`` at ``exact``): the bare list; + - a ``match`` at ``begin`` or ``end``: the list between two decoys the check rejects, so no accepted + value is at the place the check reads; + - ``match`` at ``any``, a numeric ``match`` and ``fuzzy`` find a listed value wherever it is, so the + grader is right to accept it and no probe is built. + + A negated check is stuffed with its one or more values: the stuffed answer carries the forbidden + value, so it must fail, unless the check compares the whole answer (see ``_whole_answer``), which the + listed values never equal, or is ``mode: all``, which the stuffed answer can pass by its location + (the skip is kept for every such check, for simplicity). A choice check is left out: its answer is + a bare letter. """ checks = grading.get("checks", []) if len(checks) != 1 or checks[0].get("type") not in _LITERAL - {"choice"}: @@ -117,7 +128,12 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: return " ".join(values) if built else None if len(values) < 2 or mode == "all": return None - return " ".join(values) + if _whole_answer(check): + return " ".join(values) + if check.get("type") != "match" or check.get("numeric") or str(check.get("location", "end")) == "any": + return None + decoys = [w for w in WRONG_CANDIDATES if not grade_check(check, "", w, None).passed] + return f"{decoys[0]} {' '.join(values)} {decoys[1]}" if len(decoys) >= 2 else None def _positive_checks(grading: Mapping[str, Any]) -> list[Mapping[str, Any]]: diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index e44394cb7..4dbd5eb48 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -35,13 +35,45 @@ def test_a_grader_that_rejects_every_attack_passes_the_red_team(): assert result["attacks"]["canary"] == {"expected": "pass", "scored": 1, "accepted": 1, "rate": 1.0} -def test_keyword_stuffing_that_the_grader_accepts_blocks_the_run(): - # Two alternative values under "any": the bare list ends with one of them, so the match accepts it. +def _accept_stuffed(monkeypatch): + """A grader that accepts the stuffed answer and rejects every other attack.""" + real = redteam.grade + + def grade(grading, question, answer, ask=None): + if answer == redteam.attacks(grading)["stuffed"]: + return Grade(checks=[CheckResult("match", True, 1.0)]) + return real(grading, question, answer, ask) + + monkeypatch.setattr(redteam, "grade", grade) + + +def test_keyword_stuffing_that_the_grader_accepts_blocks_the_run(monkeypatch): + _accept_stuffed(monkeypatch) result = run([_q("1", _match("Paris", "Lyon"))]) assert result["status"] == "blocked" assert result["violations"] == [{"question_id": "1", "attack": "stuffed", "passed": True}] +@pytest.mark.parametrize("check", [ + {"type": "match", "values": ["4", "four"], "location": "any"}, + {"type": "match", "values": ["Paris", "Lyon"]}, # location end + {"type": "match", "values": ["Paris", "Lyon"], "location": "begin"}, + {"type": "fuzzy", "values": ["Barack Obama", "Obama"]}, +]) +def test_a_correct_grader_with_aliases_is_not_blocked_by_the_stuffed_answer(check): + # "4 four" ends with, begins with or contains an accepted value, so accepting it is right, not a violation. + result = run([_q("1", {"checks": [check]})]) + assert result["status"] == "pass" and result["violations"] == [] + + +def test_the_stuffed_answer_buries_the_aliases_between_decoys_for_an_end_or_begin_check(): + stuffed = redteam.attacks(_match("Paris", "Lyon"))["stuffed"] + middle = stuffed.split() + assert middle[1:3] == ["Paris", "Lyon"] and len(middle) == 4 + assert not stuffed.endswith(("Paris", "Lyon")) and not stuffed.startswith(("Paris", "Lyon")) + assert redteam.attacks({"checks": [{"type": "match", "values": ["a", "b"], "location": "any"}]})["stuffed"] is None + + def test_a_grader_that_accepts_anything_is_blocked_on_every_attack(): grading = {"checks": [{"type": "regex", "values": [".*"]}]} result = run([_q("1", grading)]) @@ -114,7 +146,8 @@ def test_a_block_that_cannot_be_graded_is_refused(): run([_q("1", {"checks": [{"type": "nope"}]})]) -def test_table_names_each_violation_and_the_status(): +def test_table_names_each_violation_and_the_status(monkeypatch): + _accept_stuffed(monkeypatch) result = run([_q("7", _match("Paris", "Lyon"))]) table = format_table(result) assert "violation: question 7: stuffed answer accepted" in table @@ -143,7 +176,7 @@ def test_command_passes_and_writes_a_redteam_artifact(tmp_path, monkeypatch, cap def test_command_exits_1_when_the_run_is_blocked(tmp_path, monkeypatch): - _patch_questions(monkeypatch, [_q("1", _match("Paris", "Lyon"))]) + _patch_questions(monkeypatch, [_q("1", {"checks": [{"type": "regex", "values": [".*"]}]})]) with pytest.raises(SystemExit) as exc: cmd_eval_redteam(_args(tmp_path / "questions.json")) assert exc.value.code == 1 From 1b645082867dc7efe314343faa90f129b9633fd3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:43:06 -0700 Subject: [PATCH 11/36] fix(redteam): grade each question with its own block when an id repeats run looked each block up by question id, and the dict keeps one block per id, so two rows with the same id were both graded with the last block and a weak grader got pass. The blocks are now read from the question itself. Thread: PRRT_kwDOVBg7g86xhWCZ Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/redteam.py | 10 +++++----- dokimi/tests/test_redteam.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 797828bbb..ffb01d22f 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -212,16 +212,16 @@ def _detail(g: Any) -> str: def run(questions: list[Mapping[str, Any]], *, ask: Optional[Ask] = None) -> dict[str, Any]: """Red-team every question with a grading block. Raises ValueError for a block that cannot be graded.""" - gradings = grading_by_id(questions) + grading_by_id(questions) # refuses a set with a block that cannot be graded; the blocks are read per question rows: list[dict[str, Any]] = [] for q in questions: - qid = str(q.get("id")) - if qid not in gradings: + if "grading" not in q: continue + qid, grading = str(q.get("id")), q["grading"] # not looked up by id: ids may repeat text = str(q.get("question") or q.get("text") or "") - probes = probe(gradings[qid], text, ask) + probes = probe(grading, text, ask) rows.append({"question_id": qid, "canary": "canary" in probes, - "negated_only": negated_only(gradings[qid]), + "negated_only": negated_only(grading), "probes": {name: p.as_dict() for name, p in probes.items()}}) violations = [{"question_id": r["question_id"], "attack": name, "passed": p["passed"]} for r in rows for name, p in r["probes"].items() if p["violation"]] diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 4dbd5eb48..a347c4f79 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -390,3 +390,13 @@ def test_a_violation_still_blocks_a_set_with_unmeasured_questions(): judge = {"checks": [{"type": "judge", "criterion": "names the capital"}]} result = run([_q("j", judge), _q("1", {"checks": [{"type": "regex", "values": [".*"]}]})]) assert result["status"] == "blocked" + + +def test_a_repeated_question_id_grades_each_row_with_its_own_block(): + weak = {"checks": [{"type": "match", "values": ["Paris"]}], "threshold": 0.0} # accepts everything + strong = {"checks": [{"type": "exact", "value": "Madrid"}]} + result = run([_q("a", weak), _q("a", strong)]) + assert result["status"] == "blocked" + assert {v["attack"] for v in result["violations"]} == {"wrong", "empty", "hedge"} + assert [r["canary"] for r in result["questions"]] == [True, True] + assert result["summary"]["questions"] == 2 and result["summary"]["violations"] == 3 From 4d7cef6c58de482550bff1648d82331df7e339f3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:43:44 -0700 Subject: [PATCH 12/36] feat(redteam): record the judge model and provider in the redteam.v1 artifact With --judge, a reader of the artifact could not tell whether the judge checks were scored by a model or which one; eval reference stores both. judge_model and judge_provider are required keys, null without --judge, while the schema is still new. Thread: PRRT_kwDOVBg7g86xgzKR Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 3 ++- dokimi/dokimi/commands/redteam.py | 2 ++ dokimi/dokimi/schemas/redteam.v1.json | 4 +++- .../tests/fixtures/artifacts/redteam.v1.json | 2 ++ dokimi/tests/test_redteam.py | 18 ++++++++++++++++++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 7d4f271ab..604c7f461 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -120,7 +120,8 @@ is graded as written, `threshold` included, so a negated check's share of the sc counts: a block it lifts over the line is blocked. A probe the grader cannot score is unscored and does not block. `judge` checks are unscored unless -`--judge` is given, so the default run makes no model call. Nothing here +`--judge` is given, so the default run makes no model call; with `--judge` the +artifact records `judge_model` and `judge_provider` (both `null` without it). Nothing here changes a score; it tells you whether a grading block is worth trusting. ## What does not carry over diff --git a/dokimi/dokimi/commands/redteam.py b/dokimi/dokimi/commands/redteam.py index ba4142f3b..5ca9b8803 100644 --- a/dokimi/dokimi/commands/redteam.py +++ b/dokimi/dokimi/commands/redteam.py @@ -36,6 +36,8 @@ def cmd_eval_redteam(args: argparse.Namespace) -> None: print(f"Error: {e}", file=sys.stderr) sys.exit(2) result["questions_path"] = str(args.questions) + result["judge_model"] = args.judge_model if args.judge else None + result["judge_provider"] = args.judge_provider if args.judge else None print(format_table(result)) if args.output: try: diff --git a/dokimi/dokimi/schemas/redteam.v1.json b/dokimi/dokimi/schemas/redteam.v1.json index a52475201..711fc2ce0 100644 --- a/dokimi/dokimi/schemas/redteam.v1.json +++ b/dokimi/dokimi/schemas/redteam.v1.json @@ -4,12 +4,14 @@ "title": "Verifier red team v1", "description": "Whether a question set's grading blocks reject the synthetic wrong, empty, hedged and stuffed answers they should, and accept a canary right answer. Written by `eval redteam`; status is blocked when any attack is accepted or a canary is rejected, and inconclusive when nothing was graded or a graded question was not measured (judge checks without a judge, or only negated checks).", "type": "object", - "required": ["schema_version", "status", "questions_path", "attacks", "violations", "questions", "summary"], + "required": ["schema_version", "status", "questions_path", "judge_model", "judge_provider", "attacks", "violations", "questions", "summary"], "additionalProperties": false, "properties": { "schema_version": {"const": "redteam.v1"}, "status": {"enum": ["pass", "blocked", "inconclusive"]}, "questions_path": {"type": "string"}, + "judge_model": {"type": ["string", "null"], "description": "The judge model that graded the judge checks; null without --judge, when they were left unscored."}, + "judge_provider": {"type": ["string", "null"], "description": "The provider of the judge model; null without --judge."}, "attacks": { "type": "object", "additionalProperties": { diff --git a/dokimi/tests/fixtures/artifacts/redteam.v1.json b/dokimi/tests/fixtures/artifacts/redteam.v1.json index 50a800141..cb4d552d0 100644 --- a/dokimi/tests/fixtures/artifacts/redteam.v1.json +++ b/dokimi/tests/fixtures/artifacts/redteam.v1.json @@ -2,6 +2,8 @@ "schema_version": "redteam.v1", "status": "blocked", "questions_path": "questions.json", + "judge_model": null, + "judge_provider": null, "attacks": { "canary": {"expected": "pass", "scored": 2, "accepted": 1, "rate": 0.5}, "wrong": {"expected": "fail", "scored": 1, "accepted": 0, "rate": 0.0}, diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index a347c4f79..15fdf8cbd 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -400,3 +400,21 @@ def test_a_repeated_question_id_grades_each_row_with_its_own_block(): assert {v["attack"] for v in result["violations"]} == {"wrong", "empty", "hedge"} assert [r["canary"] for r in result["questions"]] == [True, True] assert result["summary"]["questions"] == 2 and result["summary"]["violations"] == 3 + + +def test_the_artifact_names_the_judge_that_graded_the_judge_checks(tmp_path, monkeypatch): + _patch_questions(monkeypatch, [_q("1", _match("Paris"))]) + monkeypatch.setattr("dokimi.eval.judge.Judge", lambda **kw: SimpleNamespace(llm=None)) + out = tmp_path / "judged.json" + cmd_eval_redteam(_args(tmp_path / "questions.json", output=out, judge=True, + judge_model="judge-x", judge_provider="openai")) + saved = json.loads(out.read_text(encoding="utf-8")) + assert (saved["judge_model"], saved["judge_provider"]) == ("judge-x", "openai") + + +def test_the_artifact_has_no_judge_without_the_judge_flag(tmp_path, monkeypatch): + _patch_questions(monkeypatch, [_q("1", _match("Paris"))]) + out = tmp_path / "plain.json" + cmd_eval_redteam(_args(tmp_path / "questions.json", output=out, judge_model="judge-x")) + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["judge_model"] is None and saved["judge_provider"] is None From 70e66bd066d56d0ab6fd98b092103e0c63097801 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:46:22 -0700 Subject: [PATCH 13/36] refactor(redteam): read check values and build the judge ask through the grader's own helpers The red team had a copy of reference_checks._values and eval redteam a copy of the judge Ask builder of eval reference, so either could drift from the grader it tests. check_values is public now and judge_ask in eval.judge is the one builder both commands call. Thread: PRRT_kwDOVBg7g86xgzKW Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/answer_formats.py | 5 ++--- dokimi/dokimi/commands/redteam.py | 5 ++--- dokimi/dokimi/eval/judge.py | 6 ++++++ dokimi/dokimi/eval/redteam.py | 13 +++---------- dokimi/dokimi/eval/reference_checks.py | 8 ++++---- dokimi/tests/test_redteam.py | 19 +++++++++++++++++++ 6 files changed, 36 insertions(+), 20 deletions(-) diff --git a/dokimi/dokimi/commands/answer_formats.py b/dokimi/dokimi/commands/answer_formats.py index 035b262b6..7c808355b 100644 --- a/dokimi/dokimi/commands/answer_formats.py +++ b/dokimi/dokimi/commands/answer_formats.py @@ -35,10 +35,9 @@ def cmd_questions_import(args: argparse.Namespace) -> None: def _ask(args: argparse.Namespace): if args.no_judge: return None - from dokimi.eval.judge import Judge + from dokimi.eval.judge import judge_ask - llm = Judge(model=args.judge_model, provider=args.judge_provider).llm - return lambda prompt, draw=0, cache_if=None: llm.invoke(prompt, draw=draw, cache_if=cache_if).content + return judge_ask(args.judge_model, args.judge_provider) def cmd_eval_reference(args: argparse.Namespace) -> None: diff --git a/dokimi/dokimi/commands/redteam.py b/dokimi/dokimi/commands/redteam.py index 5ca9b8803..017de1a05 100644 --- a/dokimi/dokimi/commands/redteam.py +++ b/dokimi/dokimi/commands/redteam.py @@ -18,10 +18,9 @@ def _ask(args: argparse.Namespace): if not args.judge: return None - from dokimi.eval.judge import Judge + from dokimi.eval.judge import judge_ask - llm = Judge(model=args.judge_model, provider=args.judge_provider).llm - return lambda prompt, draw=0, cache_if=None: llm.invoke(prompt, draw=draw, cache_if=cache_if).content + return judge_ask(args.judge_model, args.judge_provider) def cmd_eval_redteam(args: argparse.Namespace) -> None: diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py index f07c9cc4d..77d5ddc68 100644 --- a/dokimi/dokimi/eval/judge.py +++ b/dokimi/dokimi/eval/judge.py @@ -679,3 +679,9 @@ def load_evaluations(input_path: Path) -> Dict[str, Any]: def _get_timestamp() -> str: from datetime import datetime, timezone return datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z" + + +def judge_ask(model: str, provider: str) -> Callable[..., str]: + """The ``Ask`` of the reference checks (one prompt in, the reply text out) backed by a ``Judge``'s model.""" + llm = Judge(model=model, provider=provider).llm + return lambda prompt, draw=0, cache_if=None: llm.invoke(prompt, draw=draw, cache_if=cache_if).content diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index ffb01d22f..aadc6d671 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -44,7 +44,7 @@ from dataclasses import dataclass from typing import Any, Mapping, Optional -from dokimi.eval.reference_checks import Ask, grade, grade_check +from dokimi.eval.reference_checks import Ask, check_values, grade, grade_check from dokimi.eval.reference_run import grading_by_id ATTACKS = ("canary", "wrong", "empty", "hedge", "stuffed") @@ -60,20 +60,13 @@ _LITERAL = frozenset({"match", "fuzzy", "exact", "choice", "json"}) -def _literal_values(check: Mapping[str, Any]) -> list[Any]: - raw = check.get("values", check.get("value")) - if raw is None: - return [] - return list(raw) if isinstance(raw, (list, tuple)) else [raw] - - def canary_for(grading: Mapping[str, Any]) -> Optional[str]: """The right answer for a grading block of one literal check, or None when none is built.""" checks = grading.get("checks", []) if len(checks) != 1 or checks[0].get("type") not in _LITERAL or checks[0].get("negate"): return None check = checks[0] - kind, values = check["type"], _literal_values(check) + kind, values = check["type"], check_values(check) mode, location = str(check.get("mode", "any")), str(check.get("location", "end")) if not values or (len(values) > 1 and mode == "all"): return None @@ -121,7 +114,7 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: if len(checks) != 1 or checks[0].get("type") not in _LITERAL - {"choice"}: return None check = checks[0] - values = [str(v) for v in _literal_values(check)] + values = [str(v) for v in check_values(check)] mode = str(check.get("mode", "any")) if check.get("negate"): built = bool(values) and not _whole_answer(check) and mode != "all" diff --git a/dokimi/dokimi/eval/reference_checks.py b/dokimi/dokimi/eval/reference_checks.py index 3ad4e3f70..45ee55262 100644 --- a/dokimi/dokimi/eval/reference_checks.py +++ b/dokimi/dokimi/eval/reference_checks.py @@ -247,7 +247,7 @@ def parse_json(answer: str, *, contains: bool = False) -> tuple[bool, Any]: # --- one check --------------------------------------------------------------------------------- -def _values(check: Mapping[str, Any]) -> list[Any]: +def check_values(check: Mapping[str, Any]) -> list[Any]: raw = check.get("values", check.get("value")) if raw is None: return [] @@ -280,7 +280,7 @@ def _regex(check: Mapping[str, Any], values: list[Any], answer: str, mode: str) def _deterministic(check: Mapping[str, Any], answer: str) -> tuple[Optional[bool], str]: kind = check["type"] - values = _values(check) + values = check_values(check) mode = str(check.get("mode", "any")) if kind == "match": location = str(check.get("location", "end")) @@ -391,7 +391,7 @@ def _check_problems(kind: str, c: Mapping[str, Any]) -> list[str]: out = ["weight is not a number"] if not _is_number(weight) else ( ["weight must not be negative"] if weight < 0 else []) if kind in DETERMINISTIC and kind != "json": - values = _values(c) + values = check_values(c) if not values: out.append(f"{kind} has no values") elif any(blank_value(kind, c, v) for v in values): @@ -402,7 +402,7 @@ def _check_problems(kind: str, c: Mapping[str, Any]) -> list[str]: if kind == "match" and str(c.get("location", "end")) not in _LOCATIONS: out.append(f"match location must be one of {', '.join(_LOCATIONS)}, not {c.get('location')!r}") if kind == "regex": - for v in _values(c): + for v in check_values(c): try: regex.compile(str(v)) except regex.error as exc: diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 15fdf8cbd..47af78bf1 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -418,3 +418,22 @@ def test_the_artifact_has_no_judge_without_the_judge_flag(tmp_path, monkeypatch) cmd_eval_redteam(_args(tmp_path / "questions.json", output=out, judge_model="judge-x")) saved = json.loads(out.read_text(encoding="utf-8")) assert saved["judge_model"] is None and saved["judge_provider"] is None + + +def test_the_judge_ask_sends_the_prompt_to_the_named_model_and_returns_the_reply(monkeypatch): + from dokimi.eval.judge import judge_ask + + made, sent = {}, {} + + class FakeLLM: + def invoke(self, prompt, draw=0, cache_if=None): + sent.update(prompt=prompt, draw=draw) + return SimpleNamespace(content="YES") + + def fake_judge(**kw): + made.update(kw) + return SimpleNamespace(llm=FakeLLM()) + + monkeypatch.setattr("dokimi.eval.judge.Judge", fake_judge) + assert judge_ask("judge-x", "openai")("is it covered?", draw=2) == "YES" + assert made == {"model": "judge-x", "provider": "openai"} and sent == {"prompt": "is it covered?", "draw": 2} From ef1ed29f3877072f1661d0f96b184b34b6e4c112 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:46:38 -0700 Subject: [PATCH 14/36] fix(redteam): name -o in the cut-violations line when no artifact is written The table said "N more violations in the artifact" even without -o, when no artifact exists. Thread: PRRT_kwDOVBg7g86xgzKY Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/redteam.py | 2 +- dokimi/dokimi/eval/redteam.py | 7 +++++-- dokimi/tests/test_redteam.py | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/redteam.py b/dokimi/dokimi/commands/redteam.py index 017de1a05..02b580c2f 100644 --- a/dokimi/dokimi/commands/redteam.py +++ b/dokimi/dokimi/commands/redteam.py @@ -37,7 +37,7 @@ def cmd_eval_redteam(args: argparse.Namespace) -> None: result["questions_path"] = str(args.questions) result["judge_model"] = args.judge_model if args.judge else None result["judge_provider"] = args.judge_provider if args.judge else None - print(format_table(result)) + print(format_table(result, output=args.output)) if args.output: try: save_artifact("redteam", result, args.output) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index aadc6d671..d518324ca 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -270,7 +270,8 @@ def _summary(rows: list[dict[str, Any]]) -> dict[str, int]: } -def format_table(result: dict[str, Any]) -> str: +def format_table(result: dict[str, Any], output: Any = None) -> str: + """The pass-rate table; ``output`` is the ``-o`` path when an artifact is written, else None.""" lines = [f"{'attack':<8} {'expected':<9} {'scored':>6} {'accepted':>8} {'rate':>6}"] for name, a in result["attacks"].items(): rate = "-" if a["rate"] is None else f"{a['rate']:.2f}" @@ -279,7 +280,9 @@ def format_table(result: dict[str, Any]) -> str: verb = "accepted" if v["attack"] != "canary" else "rejected" lines.append(f" violation: question {v['question_id']}: {v['attack']} answer {verb}") if len(result["violations"]) > 20: - lines.append(f" ... {len(result['violations']) - 20} more violations in the artifact") + more = len(result["violations"]) - 20 + lines.append(f" ... {more} more violations in the artifact" if output + else f" ... {more} more violations; rerun with -o FILE to list them all") s = result["summary"] if s["negated_only"]: lines.append(f" {s['negated_only']} questions with only negated checks: " diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 47af78bf1..83a896a50 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -437,3 +437,26 @@ def fake_judge(**kw): monkeypatch.setattr("dokimi.eval.judge.Judge", fake_judge) assert judge_ask("judge-x", "openai")("is it covered?", draw=2) == "YES" assert made == {"model": "judge-x", "provider": "openai"} and sent == {"prompt": "is it covered?", "draw": 2} + + +def _many_violations(): + return run([_q(str(i), {"checks": [{"type": "regex", "values": [".*"]}]}) for i in range(8)]) # 24 violations + + +def test_the_table_names_dash_o_when_it_cuts_the_violations_and_no_artifact_is_written(): + table = format_table(_many_violations()) + assert "... 4 more violations; rerun with -o FILE to list them all" in table + assert "in the artifact" not in table + + +def test_the_table_points_to_the_artifact_when_dash_o_was_given(tmp_path): + table = format_table(_many_violations(), output=tmp_path / "r.json") + assert "... 4 more violations in the artifact" in table + assert "rerun with -o" not in table + + +def test_the_command_prints_the_cut_violations_with_the_output_path_it_wrote(tmp_path, monkeypatch, capsys): + _patch_questions(monkeypatch, [_q(str(i), {"checks": [{"type": "regex", "values": [".*"]}]}) for i in range(8)]) + with pytest.raises(SystemExit): + cmd_eval_redteam(_args(tmp_path / "questions.json", output=tmp_path / "r.json")) + assert "4 more violations in the artifact" in capsys.readouterr().out From f108f53ea280170c64f44bc63a0fa0e844c2a935 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:46:59 -0700 Subject: [PATCH 15/36] test(redteam): pin the canary of a whole-answer match and of a string json value Dropping the location exact clause of canary_for, or replacing json.dumps with str, passed the whole file (43 tests) while the grader rejected the mutated canary. Each canary kind is now graded by the real grader; both mutations turn the new tests red. Thread: PRRT_kwDOVBg7g86xhWCh Signed-off-by: Nikolay Petrov --- dokimi/tests/test_redteam.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 83a896a50..68be645d7 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -460,3 +460,23 @@ def test_the_command_prints_the_cut_violations_with_the_output_path_it_wrote(tmp with pytest.raises(SystemExit): cmd_eval_redteam(_args(tmp_path / "questions.json", output=tmp_path / "r.json")) assert "4 more violations in the artifact" in capsys.readouterr().out + + +@pytest.mark.parametrize("check", [ + {"type": "match", "value": "Paris", "location": "exact"}, + {"type": "json", "value": "Paris"}, + {"type": "json", "value": 3}, + {"type": "exact", "value": "Paris"}, + {"type": "match", "value": "Paris", "location": "begin"}, + {"type": "choice", "values": ["b", "a"]}, +]) +def test_the_canary_for_each_literal_check_is_accepted_by_the_grader(check): + from dokimi.eval.reference_checks import grade + + grading = {"checks": [check]} + assert grade(grading, "Q?", canary_for(grading)).passed is True + + +def test_the_canary_is_the_bare_value_for_a_whole_answer_match_and_a_json_string(): + assert canary_for({"checks": [{"type": "match", "value": "Paris", "location": "exact"}]}) == "Paris" + assert canary_for({"checks": [{"type": "json", "value": "Paris"}]}) == '"Paris"' From 89641b79cda6021f0c92cba3cc4a814d6f18b9da Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:47:28 -0700 Subject: [PATCH 16/36] docs(redteam): list the negated mode all exclusion and say where negated_only appears stuffed_for builds no stuffed answer for a negated match with mode all, which the table row did not say. format_table never prints the key negated_only: only the artifact has it, the table prints a line about the questions with only negated checks. Thread: PRRT_kwDOVBg7g86xlqTg Thread: PRRT_kwDOVBg7g86xlqTu Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 604c7f461..48da304ea 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -101,7 +101,7 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or | `wrong` | A planted wrong answer, picked per block so that no positive literal check accepts it, asked with the check's own rules (the probe is skipped only when no candidate is clean) | fail it | | `empty` | `""` | fail it | | `hedge` | A fixed non-committal sentence; skipped when a positive literal check accepts it | fail it | -| `stuffed` | The values of a block with one literal check, listed with no sentence around them: two or more values of a positive check (a single value is the right answer, not stuffing), bare for a check on the whole answer and between two decoys for a `match` at `begin` or `end`, or one or more values of a `negate` check, whose stuffed answer is the forbidden value; not built for a positive block that needs every value (`mode: all`), for several checks, for `fuzzy`, numeric or `location: any` checks (they find a listed value wherever it is, so accepting it is right), or for a negated `exact`, `json` or `match` with `location: exact`, which the listed values never equal | fail it | +| `stuffed` | The values of a block with one literal check, listed with no sentence around them: two or more values of a positive check (a single value is the right answer, not stuffing), bare for a check on the whole answer and between two decoys for a `match` at `begin` or `end`, or one or more values of a `negate` check, whose stuffed answer is the forbidden value; not built for a positive block that needs every value (`mode: all`), for several checks, for `fuzzy`, numeric or `location: any` checks (they find a listed value wherever it is, so accepting it is right), or for a negated check that the listed values cannot fail: a negated `exact`, `json` or `match` with `location: exact` (they never equal the list) and a negated `match` with `mode: all` (the list passes it by its location) | fail it | An attack the grader accepts, or a canary it rejects, is a violation, and any violation makes the run `blocked`: the command prints the pass rates and the @@ -110,7 +110,7 @@ question was not measured is `inconclusive` and also exits 1: a question whose `judge` checks have no verdict without `--judge` (no probe scored), or a block of only negated checks. A `negate` check passes an answer that lacks its value, so a block of only negated checks gets none of `wrong`, `empty` and `hedge`: it -is reported as `negated_only`, its `stuffed` probe (the negated value itself) +is marked `negated_only` in the artifact (the table prints a line "N questions with only negated checks: wrong, empty and hedge not run"), its `stuffed` probe (the negated value itself) still runs, but the grader would accept any answer without that value, so nothing was verified about it. The table names every question that was not measured and counts it as `unprobed`; only a run that measured every graded From 0918eea1343c3bb9afcbef0b409ff58986180d58 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:47:59 -0700 Subject: [PATCH 17/36] test(redteam): run judge checks through run(ask=) and through the --judge path Every command test set judge=False and none passed an ask, so dropping ask from run, ignoring --judge or swapping the model and provider all passed the file. A stub judge now scores the judge checks (rejecting and accepting), and the command test patches Judge and checks the arguments it receives. Each of the three mutations turns a new test red. Thread: PRRT_kwDOVBg7g86xlqT9 Signed-off-by: Nikolay Petrov --- dokimi/tests/test_redteam.py | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 68be645d7..ebed89398 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -480,3 +480,61 @@ def test_the_canary_for_each_literal_check_is_accepted_by_the_grader(check): def test_the_canary_is_the_bare_value_for_a_whole_answer_match_and_a_json_string(): assert canary_for({"checks": [{"type": "match", "value": "Paris", "location": "exact"}]}) == "Paris" assert canary_for({"checks": [{"type": "json", "value": "Paris"}]}) == '"Paris"' + + +def _covered(covered): + """A judge that finds the point covered (or absent) in both polarities, consistently.""" + from dokimi.eval.checklist import ASK_COVERED + + def ask(prompt, draw=0, cache_if=None): + return "YES" if (ASK_COVERED in prompt) == covered else "NO" + + return ask + + +_JUDGE_ONLY = {"checks": [{"type": "judge", "criterion": "names the capital"}]} + + +def test_a_judge_check_is_scored_when_an_ask_is_given_and_a_judge_that_rejects_every_attack_passes(): + result = run([_q("1", _JUDGE_ONLY)], ask=_covered(False)) + assert result["status"] == "pass" and result["summary"]["unscored"] == 0 + assert all(p["passed"] is False for p in result["questions"][0]["probes"].values()) + + +def test_a_judge_that_accepts_every_answer_blocks_the_run(): + result = run([_q("1", _JUDGE_ONLY)], ask=_covered(True)) + assert result["status"] == "blocked" + assert {v["attack"] for v in result["violations"]} == {"wrong", "empty", "hedge"} + + +def test_the_judge_flag_grades_judge_checks_with_the_named_model(tmp_path, monkeypatch): + made = {} + + def fake_judge(**kw): + made.update(kw) + return SimpleNamespace(llm=SimpleNamespace(invoke=lambda prompt, draw=0, cache_if=None: + SimpleNamespace(content=_covered(True)(prompt)))) + + _patch_questions(monkeypatch, [_q("1", _JUDGE_ONLY)]) + monkeypatch.setattr("dokimi.eval.judge.Judge", fake_judge) + out = tmp_path / "judged.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json", output=out, judge=True, + judge_model="judge-x", judge_provider="openai")) + assert exc.value.code == 1 # the stub judge accepts every attack + assert made == {"model": "judge-x", "provider": "openai"} + saved = json.loads(out.read_text(encoding="utf-8")) + assert saved["status"] == "blocked" and saved["summary"]["unscored"] == 0 + + +def test_without_the_judge_flag_no_judge_is_built_and_judge_checks_stay_unscored(tmp_path, monkeypatch): + def refuse(**kw): + raise AssertionError("a judge was built without --judge") + + _patch_questions(monkeypatch, [_q("1", _JUDGE_ONLY)]) + monkeypatch.setattr("dokimi.eval.judge.Judge", refuse) + out = tmp_path / "plain.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + assert exc.value.code == 1 + assert json.loads(out.read_text(encoding="utf-8"))["summary"]["unscored"] == 3 From 27828e451bc5492a259158dcf7724c63b4b1b462 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:48:09 -0700 Subject: [PATCH 18/36] test(redteam): run an ungradable block through the command and pin the empty inputs Only run raising was tested; the except (OSError, ValueError) exit 2 path of the command never ran (narrowing it to OSError left the file green). Now a block with no values exits 2 with the problem and writes nothing, and run is pinned on an empty set, values [], a blank value, checks [] and grading None. Thread: PRRT_kwDOVBg7g86xlqT0 Signed-off-by: Nikolay Petrov --- dokimi/tests/test_redteam.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index ebed89398..9cea45d3b 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -538,3 +538,36 @@ def refuse(**kw): cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) assert exc.value.code == 1 assert json.loads(out.read_text(encoding="utf-8"))["summary"]["unscored"] == 3 + + +@pytest.mark.parametrize("grading", [ + {"checks": [{"type": "match", "values": []}]}, + {"checks": [{"type": "match", "values": [""]}]}, + {"checks": []}, + None, +]) +def test_a_degenerate_grading_block_is_refused_not_passed(grading): + with pytest.raises(ValueError, match="cannot be graded"): + run([_q("1", grading)]) + + +def test_an_empty_question_list_is_inconclusive_not_pass(): + result = run([]) + assert result["status"] == "inconclusive" and result["summary"]["questions"] == 0 + + +def test_command_exits_2_with_the_problem_when_a_grading_block_cannot_be_graded(tmp_path, monkeypatch, capsys): + _patch_questions(monkeypatch, [_q("7", {"checks": [{"type": "match", "values": []}]})]) + out = tmp_path / "redteam.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + assert exc.value.code == 2 + assert "7: check 0: match has no values" in capsys.readouterr().err + assert not out.exists() + + +def test_command_exits_1_on_an_empty_question_set(tmp_path, monkeypatch): + _patch_questions(monkeypatch, []) + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json")) + assert exc.value.code == 1 From 1946f481e73c147a8bd6fb2363b34eacca445c10 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:48:17 -0700 Subject: [PATCH 19/36] test(redteam): check the artifact of a blocked and an inconclusive command run Only the passing run was checked for a written artifact; writing it only for status pass left every test green although CI needs the artifact most for a blocked run. Both runs now pass -o and assert the file and its status. Thread: PRRT_kwDOVBg7g86xlqUH Signed-off-by: Nikolay Petrov --- dokimi/tests/test_redteam.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 9cea45d3b..2e2b6ee59 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -571,3 +571,21 @@ def test_command_exits_1_on_an_empty_question_set(tmp_path, monkeypatch): with pytest.raises(SystemExit) as exc: cmd_eval_redteam(_args(tmp_path / "questions.json")) assert exc.value.code == 1 + + +def test_a_blocked_run_still_writes_the_artifact(tmp_path, monkeypatch): + _patch_questions(monkeypatch, [_q("1", {"checks": [{"type": "regex", "values": [".*"]}]})]) + out = tmp_path / "blocked.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + assert exc.value.code == 1 + assert json.loads(out.read_text(encoding="utf-8"))["status"] == "blocked" + + +def test_an_inconclusive_run_still_writes_the_artifact(tmp_path, monkeypatch): + _patch_questions(monkeypatch, [{"id": "1", "question": "Capital of France?"}]) + out = tmp_path / "inconclusive.json" + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + assert exc.value.code == 1 + assert json.loads(out.read_text(encoding="utf-8"))["status"] == "inconclusive" From c1596ef6faac9f4d207baab3bd26e54fc3df6a95 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:49:01 -0700 Subject: [PATCH 20/36] test(redteam): pin the bare stuffed answer of a whole-answer check with aliases The branch that lists the aliases of an exact, json or location-exact check without decoys had no test. Thread: PRRT_kwDOVBg7g86xgzJz Signed-off-by: Nikolay Petrov --- dokimi/tests/test_redteam.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 2e2b6ee59..3c5c7e976 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -589,3 +589,15 @@ def test_an_inconclusive_run_still_writes_the_artifact(tmp_path, monkeypatch): cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) assert exc.value.code == 1 assert json.loads(out.read_text(encoding="utf-8"))["status"] == "inconclusive" + + +@pytest.mark.parametrize("check", [ + {"type": "exact", "values": ["Paris", "Lyon"]}, + {"type": "match", "values": ["Paris", "Lyon"], "location": "exact"}, + {"type": "json", "values": ["Paris", "Lyon"]}, +]) +def test_aliases_of_a_whole_answer_check_are_stuffed_bare_and_the_grader_rejects_the_list(check): + # The list equals none of the values, so a grader that accepts it is wrong; the real one does not. + assert redteam.attacks({"checks": [check]})["stuffed"] == "Paris Lyon" + result = run([_q("1", {"checks": [check]})]) + assert result["status"] == "pass" and result["questions"][0]["probes"]["stuffed"]["passed"] is False From a296b5a4ea2d0470e758ae015a6bab3aeb806d9b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:49:13 -0700 Subject: [PATCH 21/36] docs(redteam): changelog fragment says what an inconclusive run is Thread: PRRT_kwDOVBg7g86xgzJ6 Thread: PRRT_kwDOVBg7g86xgzKC Signed-off-by: Nikolay Petrov --- .../20261028_120100_napetrov_feat-verifier-redteam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md b/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md index a823af134..285601c39 100644 --- a/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md +++ b/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md @@ -1,3 +1,3 @@ ### Added -- **`eval redteam` checks that grading blocks reject wrong answers** — grades a planted wrong, an empty, a hedged and a keyword-stuffed answer against each question's `grading` block, and a canary right answer built from its one literal check. A wrong answer the grader accepts, or a canary it rejects, is a violation: the command prints pass rates, exits 1 and, with `-o`, writes a `redteam.v1` artifact. Judge checks are unscored unless `--judge` is given, so the default run makes no model call. +- **`eval redteam` checks that grading blocks reject wrong answers** — grades a planted wrong, an empty, a hedged and a keyword-stuffed answer against each question's `grading` block, and a canary right answer built from its one literal check. A wrong answer the grader accepts, or a canary it rejects, is a violation: the command prints pass rates, exits 1 and, with `-o`, writes a `redteam.v1` artifact (it records the judge model and provider). A run in which a graded question was not measured (judge checks without `--judge`, a block of only negated checks) is `inconclusive` and also exits 1. Judge checks are unscored unless `--judge` is given, so the default run makes no model call. From 2a259de53f4ac377991056ac0c9f8835cfcb66ea Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:49:13 -0700 Subject: [PATCH 22/36] Address review: answer the review summaries and bot comments in their threads The review bodies and the bot comments restate or summarise the inline findings, each fixed in its own commit of this round. Thread: PRR_kwDOVBg7g88AAAABU4eT2Q Thread: PRR_kwDOVBg7g88AAAABU4wvyw Thread: PRR_kwDOVBg7g88AAAABU7EFzw Thread: IC_kwDOVBg7g88AAAABgrasmA Thread: IC_kwDOVBg7g88AAAABgrh3Cg Thread: IC_kwDOVBg7g88AAAABg12xOQ Signed-off-by: Nikolay Petrov From c68a1f16dc5ca8f05d6157f7f1d14c05e5f39d22 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:58:41 -0700 Subject: [PATCH 23/36] docs(redteam): refresh two stale comments on the wrong and stuffed attacks Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/redteam.py | 2 +- dokimi/tests/test_redteam.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index d518324ca..ef0d69f99 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -49,7 +49,7 @@ ATTACKS = ("canary", "wrong", "empty", "hedge", "stuffed") EXPECTED = {"canary": True, "wrong": False, "empty": False, "hedge": False, "stuffed": False} -# Planted wrong answers, tried in order; the first that carries no positive check value is used. +# Planted wrong answers, tried in order; the first that no positive literal check accepts is used. WRONG_CANDIDATES = ("PLANTED-WRONG-ANSWER-7731", "QZX-BOGUS-REPLY-4096", "ZKV-MISMATCH-2508", "XJQ-NONSENSE-6349") HEDGE = ("I am not certain. It could be several things, depending on the setup, " "and I cannot say which one applies here.") diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 3c5c7e976..afeb725f7 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -61,7 +61,8 @@ def test_keyword_stuffing_that_the_grader_accepts_blocks_the_run(monkeypatch): {"type": "fuzzy", "values": ["Barack Obama", "Obama"]}, ]) def test_a_correct_grader_with_aliases_is_not_blocked_by_the_stuffed_answer(check): - # "4 four" ends with, begins with or contains an accepted value, so accepting it is right, not a violation. + # The stuffed answer is built so that a correct grader rejects it (values between decoys) or it is + # not built at all (fuzzy, numeric, location any): a correct grader is never blocked. result = run([_q("1", {"checks": [check]})]) assert result["status"] == "pass" and result["violations"] == [] From 98ccb9620ee412575132de3224c844fe3633e216 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:53:10 -0700 Subject: [PATCH 24/36] fix(redteam): a question set nested too deep to parse exits 2, not with a traceback load_questions reads the file with json.loads, which raises RecursionError on deep nesting. eval redteam caught OSError and ValueError only, so the command died with a traceback and exit 1, which CI reads as a blocked run. arms and catalog already catch RecursionError for the same reason. Addresses review thread on dokimi/commands/redteam.py (PRRT_kwDOVBg7g86xtePO). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/redteam.py | 2 +- dokimi/tests/test_redteam.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/redteam.py b/dokimi/dokimi/commands/redteam.py index 02b580c2f..3e96002bf 100644 --- a/dokimi/dokimi/commands/redteam.py +++ b/dokimi/dokimi/commands/redteam.py @@ -31,7 +31,7 @@ def cmd_eval_redteam(args: argparse.Namespace) -> None: try: questions = load_questions(args.questions).artifact.get("questions") or [] result = run(questions, ask=_ask(args)) - except (OSError, ValueError) as e: + except (OSError, ValueError, RecursionError) as e: # RecursionError: nested too deep print(f"Error: {e}", file=sys.stderr) sys.exit(2) result["questions_path"] = str(args.questions) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index afeb725f7..07acdbfa7 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -200,6 +200,15 @@ def test_command_exits_2_when_the_artifact_cannot_be_written(tmp_path, monkeypat assert capsys.readouterr().err.startswith("Error:") +def test_command_exits_2_on_a_question_set_nested_too_deep_to_read(tmp_path, capsys): + path = tmp_path / "deep.json" + path.write_text("[" * 100_000 + "]" * 100_000, encoding="utf-8") + with pytest.raises(SystemExit) as exc: + cmd_eval_redteam(_args(path)) + assert exc.value.code == 2 # not a traceback and exit 1: CI reads exit 1 as a blocked run + assert capsys.readouterr().err.startswith("Error:") + + def test_command_exits_2_on_an_unreadable_question_set(tmp_path, capsys): with pytest.raises(SystemExit) as exc: cmd_eval_redteam(_args(tmp_path / "missing.json")) From c09f9b64ba8771106079121e1c10728b5b938d7c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:23 -0700 Subject: [PATCH 25/36] fix(redteam): a block with an unrun judge or regex is unprobed, not a pass or a block probe() took the grade's pass as the probe's verdict even when part of the block never ran: a judge check with no judge model, or a regex past its timeout, is left out of the score. Without a threshold a counted pass then reads as a pass that is not a measurement, so `[{judge}, {match, negate}]` blocked wrong, empty and hedge answers that the negated check passes by design. With a threshold the score over the counted checks alone decides the probe, so `{threshold, [{judge}, {match}]}` passed the canary and rejected the attacks with no judge at all, and `pass` exited 0. Such a probe now has no verdict when the grade passed or a threshold is set, and the run reports it inconclusive. A counted failure with no threshold stays a measured rejection, since every counted check must pass. Addresses review threads on dokimi/eval/redteam.py (PRRT_kwDOVBg7g86xtePe). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/redteam.py | 7 ++++++- dokimi/tests/test_redteam.py | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index ef0d69f99..f9ca72fc1 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -195,7 +195,12 @@ def probe(grading: Mapping[str, Any], question: str, ask: Optional[Ask] = None) if answer is None: continue g = grade(grading, question, answer, ask) - out[name] = Probe(name, g.passed, _detail(g)) + # A judge check with no judge, or a regex past its timeout, leaves part of the block unrun. A + # pass on the rest is not a measurement, and with a threshold neither is a result: the score + # is taken over the counted checks alone. + partial = bool(g.unjudged or g.unrun) + passed = None if partial and (g.passed or g.threshold is not None) else g.passed + out[name] = Probe(name, passed, _detail(g)) return out diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 07acdbfa7..dbf9cf79c 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -93,6 +93,27 @@ def reject_all(grading, question, answer, ask=None): assert result["status"] == "blocked" +def test_a_judge_check_without_a_judge_with_a_negated_check_is_unscored_not_blocked(): + # The negated check passes wrong, empty and hedge by design; the judge check was never run, so + # those passes say nothing about the block and must not count as violations. + grading = {"checks": [{"type": "judge", "criterion": "names the capital"}, + {"type": "match", "values": ["Lyon"], "negate": True}]} + result = run([_q("1", grading)]) + assert result["status"] == "inconclusive" + assert result["violations"] == [] + assert result["summary"]["probed"] == 0 + + +def test_a_judge_check_without_a_judge_under_a_threshold_is_not_a_pass(): + # With a threshold the score is taken over the counted checks alone, so a missing judge can + # turn a block that rejects nothing into one that passes the canary and rejects the attacks. + grading = {"threshold": 0.5, "checks": [{"type": "judge", "criterion": "names Paris"}, + {"type": "match", "values": ["Paris"]}]} + result = run([_q("1", grading)]) + assert result["status"] == "inconclusive" + assert result["summary"]["probed"] == 0 + + def test_a_judge_check_without_a_judge_is_unscored_and_does_not_block(): grading = {"checks": [{"type": "judge", "criterion": "names the capital"}]} result = run([_q("1", grading)]) From 5e396356a7203581799e5a7c19a15d19ef9aa834 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:28:44 -0700 Subject: [PATCH 26/36] test(redteam): a regex past its timeout leaves the probe unscored, with and without a threshold Reverting the condition that unscores a probe with an unrun check turns both cases red: the negated match passes wrong, empty and hedge and the run is blocked, and a threshold scores the counted match alone and the run passes. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_redteam.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index dbf9cf79c..384514603 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -13,6 +13,7 @@ from types import SimpleNamespace import pytest +import regex from dokimi.commands.redteam import cmd_eval_redteam from dokimi.eval import redteam @@ -114,6 +115,35 @@ def test_a_judge_check_without_a_judge_under_a_threshold_is_not_a_pass(): assert result["summary"]["probed"] == 0 +SLOW = "slow pattern" # the regex that runs past its timeout; any other pattern runs as written + + +@pytest.fixture +def regex_past_its_timeout(monkeypatch): + """Make the regex check with the SLOW pattern run past its timeout, as a catastrophic one does.""" + real_search = regex.search + + def search(pattern, string, *args, **kwargs): + if pattern == SLOW: + raise TimeoutError("regex ran past its limit") + return real_search(pattern, string, *args, **kwargs) + + monkeypatch.setattr(regex, "search", search) + + +@pytest.mark.parametrize("grading", [ + # No threshold: the negated match passes wrong, empty and hedge by design, and the regex never ran. + {"checks": [{"type": "regex", "values": [SLOW]}, {"type": "match", "values": ["Lyon"], "negate": True}]}, + # Threshold: the score over the counted match alone would pass the canary and reject the attacks. + {"threshold": 0.5, "checks": [{"type": "regex", "values": [SLOW]}, {"type": "match", "values": ["Paris"]}]}, +]) +def test_a_regex_past_its_timeout_leaves_the_probe_unscored_not_passed(regex_past_its_timeout, grading): + result = run([_q("1", grading)]) + assert result["status"] == "inconclusive" + assert result["violations"] == [] + assert result["summary"]["probed"] == 0 + + def test_a_judge_check_without_a_judge_is_unscored_and_does_not_block(): grading = {"checks": [{"type": "judge", "criterion": "names the capital"}]} result = run([_q("1", grading)]) From de35dd856ea4c4ddf01d2b30f397b47cd4aaf393 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:57:47 -0700 Subject: [PATCH 27/36] docs: keep the comment blocks this PR adds to two lines Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/redteam.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index f9ca72fc1..d0aaddc38 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -195,9 +195,8 @@ def probe(grading: Mapping[str, Any], question: str, ask: Optional[Ask] = None) if answer is None: continue g = grade(grading, question, answer, ask) - # A judge check with no judge, or a regex past its timeout, leaves part of the block unrun. A - # pass on the rest is not a measurement, and with a threshold neither is a result: the score - # is taken over the counted checks alone. + # A check left unrun (no judge, or a regex past its timeout) makes a pass on the rest no measurement; + # with a threshold it is no result either, so the score is taken over the counted checks alone. partial = bool(g.unjudged or g.unrun) passed = None if partial and (g.passed or g.threshold is not None) else g.passed out[name] = Probe(name, passed, _detail(g)) From 04c454129e20bc299d17fc3844461711501573f4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 20:00:52 -0700 Subject: [PATCH 28/36] fix(redteam): a json contains check's stuffed answer goes after a decoy A json check with contains reads the first JSON in the answer, so the bare list "[1, 2] [3, 4]" starts with a listed value and the grader accepts it, which blocked a correct grading block. The list now follows two JSON decoys the check rejects, as match at begin or end does. Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 2 +- dokimi/dokimi/eval/redteam.py | 23 ++++++++++++++++++----- dokimi/tests/test_redteam.py | 11 +++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index 48da304ea..ffd6de105 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -101,7 +101,7 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or | `wrong` | A planted wrong answer, picked per block so that no positive literal check accepts it, asked with the check's own rules (the probe is skipped only when no candidate is clean) | fail it | | `empty` | `""` | fail it | | `hedge` | A fixed non-committal sentence; skipped when a positive literal check accepts it | fail it | -| `stuffed` | The values of a block with one literal check, listed with no sentence around them: two or more values of a positive check (a single value is the right answer, not stuffing), bare for a check on the whole answer and between two decoys for a `match` at `begin` or `end`, or one or more values of a `negate` check, whose stuffed answer is the forbidden value; not built for a positive block that needs every value (`mode: all`), for several checks, for `fuzzy`, numeric or `location: any` checks (they find a listed value wherever it is, so accepting it is right), or for a negated check that the listed values cannot fail: a negated `exact`, `json` or `match` with `location: exact` (they never equal the list) and a negated `match` with `mode: all` (the list passes it by its location) | fail it | +| `stuffed` | The values of a block with one literal check, listed with no sentence around them: two or more values of a positive check (a single value is the right answer, not stuffing), bare for a check on the whole answer, between two decoys for a `match` at `begin` or `end`, and after a JSON decoy for a `json` check with `contains` (it reads the first JSON in the answer), or one or more values of a `negate` check, whose stuffed answer is the forbidden value; not built for a positive block that needs every value (`mode: all`), for several checks, for `fuzzy`, numeric or `location: any` checks (they find a listed value wherever it is, so accepting it is right), or for a negated check that the listed values cannot fail: a negated `exact`, `json` or `match` with `location: exact` (they never equal the list) and a negated `match` with `mode: all` (the list passes it by its location) | fail it | An attack the grader accepts, or a canary it rejects, is a violation, and any violation makes the run `blocked`: the command prints the pass rates and the diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index d0aaddc38..0f4dccabc 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -13,7 +13,7 @@ - ``stuffed``: every literal check value listed with no sentence around it; it must fail. Built for a positive check of two or more values (mode ``any``) that does not read the list where it stands: bare for ``exact``, ``json`` and location-``exact`` checks, between two decoys for a ``match`` at - ``begin`` or ``end``. It is not built for ``fuzzy``, numeric or location-``any`` checks, a block that + ``begin`` or ``end``, after a JSON decoy for a ``json`` check with ``contains``. It is not built for ``fuzzy``, numeric or location-``any`` checks, a block that needs every value, or several checks that each accept one value: the grader is right to accept the stuffed answer there. Also built for a negated check of one or more values, whose stuffed answer carries the forbidden value; not @@ -51,6 +51,8 @@ EXPECTED = {"canary": True, "wrong": False, "empty": False, "hedge": False, "stuffed": False} # Planted wrong answers, tried in order; the first that no positive literal check accepts is used. WRONG_CANDIDATES = ("PLANTED-WRONG-ANSWER-7731", "QZX-BOGUS-REPLY-4096", "ZKV-MISMATCH-2508", "XJQ-NONSENSE-6349") +# JSON decoys for a json check with contains, which reads the first JSON object or array in the text. +JSON_DECOYS = ('{"planted": "wrong"}', '{"planted": "other"}') HEDGE = ("I am not certain. It could be several things, depending on the setup, " "and I cannot say which one applies here.") BLOCKED, PASS, INCONCLUSIVE = "blocked", "pass", "inconclusive" @@ -90,6 +92,11 @@ def _whole_answer(check: Mapping[str, Any]) -> bool: return kind in ("exact", "json") or (kind == "match" and str(check.get("location", "end")) == "exact") +def _reads_first_json(check: Mapping[str, Any]) -> bool: + """Whether a json check with ``contains`` reads the first JSON in the answer, not the whole answer.""" + return check.get("type") == "json" and bool(check.get("contains")) + + def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: """The block's one literal check's values, listed with no sentence around them; None when not built. @@ -98,9 +105,11 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: accepting it is not a violation. A single positive value listed alone is the right answer itself, not stuffing. The list is an attack only where the check does not read it: - - a check on the whole answer (``exact``, ``json``, ``match`` at ``exact``): the bare list; + - a check on the whole answer (``exact``, ``json`` without ``contains``, ``match`` at ``exact``): the bare list; - a ``match`` at ``begin`` or ``end``: the list between two decoys the check rejects, so no accepted value is at the place the check reads; + - a ``json`` check with ``contains`` reads the first JSON in the text, so the list goes after two JSON + decoys the check rejects: a bare list would start with a listed value and be accepted; - ``match`` at ``any``, a numeric ``match`` and ``fuzzy`` find a listed value wherever it is, so the grader is right to accept it and no probe is built. @@ -121,11 +130,15 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: return " ".join(values) if built else None if len(values) < 2 or mode == "all": return None - if _whole_answer(check): + if _whole_answer(check) and not _reads_first_json(check): return " ".join(values) - if check.get("type") != "match" or check.get("numeric") or str(check.get("location", "end")) == "any": + if _reads_first_json(check): + candidates = JSON_DECOYS + elif check.get("type") != "match" or check.get("numeric") or str(check.get("location", "end")) == "any": return None - decoys = [w for w in WRONG_CANDIDATES if not grade_check(check, "", w, None).passed] + else: + candidates = WRONG_CANDIDATES + decoys = [w for w in candidates if not grade_check(check, "", w, None).passed] return f"{decoys[0]} {' '.join(values)} {decoys[1]}" if len(decoys) >= 2 else None diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 384514603..cb68b80b0 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -76,6 +76,17 @@ def test_the_stuffed_answer_buries_the_aliases_between_decoys_for_an_end_or_begi assert redteam.attacks({"checks": [{"type": "match", "values": ["a", "b"], "location": "any"}]})["stuffed"] is None +def test_a_json_contains_check_with_list_values_is_stuffed_after_a_decoy_the_grader_rejects(): + # contains reads the first JSON in the text: the bare list "[1, 2] [3, 4]" starts with the value [1, 2], + # which a correct grader accepts, so the list goes after a JSON decoy that is not one of the values. + grading = {"checks": [{"type": "json", "values": [[1, 2], [3, 4]], "contains": True}]} + stuffed = redteam.attacks(grading)["stuffed"] + assert stuffed is not None and stuffed.startswith('{"planted"') + assert redteam.grade(grading, "", stuffed).checks[0].passed is False + result = run([_q("1", grading)]) + assert result["status"] == "pass" and result["violations"] == [] + + def test_a_grader_that_accepts_anything_is_blocked_on_every_attack(): grading = {"checks": [{"type": "regex", "values": [".*"]}]} result = run([_q("1", grading)]) From 58e1a63f599e9b137587e7d6d9833d272f137ad6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 23:17:42 -0700 Subject: [PATCH 29/36] fix(redteam): type the decoy candidates as a tuple of any length (mypy) Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/redteam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 0f4dccabc..af1a7aba3 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -132,6 +132,7 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: return None if _whole_answer(check) and not _reads_first_json(check): return " ".join(values) + candidates: tuple[str, ...] if _reads_first_json(check): candidates = JSON_DECOYS elif check.get("type") != "match" or check.get("numeric") or str(check.get("location", "end")) == "any": From 992bced81559363d1350aed06574a3443a5351a9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:17:02 -0700 Subject: [PATCH 30/36] test(redteam): pin the decoy filter, the two-decoy fallback and the any-probe rule Signed-off-by: Nikolay Petrov --- dokimi/tests/test_redteam.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index cb68b80b0..ae9c33bbc 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -196,6 +196,29 @@ def test_no_canary_when_the_block_is_not_one_literal_check(grading): assert canary_for(grading) is None +def test_a_stuffed_answer_never_ends_in_a_decoy_the_check_accepts(): + # QZX is a value, so a stuffed answer that ends in it is accepted by a correct grader; the decoy + # is dropped and the answer ends in one the check rejects. + grading = _match("Lyon", "QZX-BOGUS-REPLY-4096") + stuffed = redteam.attacks(grading)["stuffed"] + assert stuffed is not None and not stuffed.endswith("QZX-BOGUS-REPLY-4096") + result = run([_q("1", grading)]) + assert result["violations"] == [] + assert result["status"] != "blocked" + + +def test_a_stuffed_answer_needs_two_decoys_the_check_rejects(): + grading = _match("PLANTED-WRONG-ANSWER-7731", "QZX-BOGUS-REPLY-4096", "ZKV-MISMATCH-2508") + assert redteam.attacks(grading)["stuffed"] is None + + +def test_a_question_with_one_scored_probe_is_measured_when_another_probe_is_not(): + # Unmeasured means no probe scored; one unscored probe next to a scored one is not a reason + # to call the question unmeasured. + row = {"negated_only": False, "probes": {"canary": {"passed": None}, "wrong": {"passed": False}}} + assert redteam._unmeasured(row) == "" + + def test_canary_text_follows_the_check_kind(): assert canary_for(_match("Paris")) == "The answer is Paris." assert canary_for({"checks": [{"type": "match", "values": ["Paris"], "location": "begin"}]}) == "Paris is the answer." From c29dd782b972b48d289e8db4061388b60aa03f7d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 08:17:02 -0700 Subject: [PATCH 31/36] docs(redteam): name the unmeasured case in the exit-code docstring Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/redteam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/redteam.py b/dokimi/dokimi/commands/redteam.py index 3e96002bf..381271a87 100644 --- a/dokimi/dokimi/commands/redteam.py +++ b/dokimi/dokimi/commands/redteam.py @@ -4,8 +4,8 @@ """``eval redteam``: check that a question set's grading rejects the answers it should. Reads a questions set (with ``grading`` blocks), grades synthetic attacks against it and prints -the pass rates. Exits 1 when any attack is accepted, a canary is rejected, or nothing was graded -(``inconclusive``); exits 2 on an unreadable input or an unwritable ``-o``. Judge checks are not +the pass rates. Exits 1 when any attack is accepted, a canary is rejected, a graded question was not +measured, or nothing was graded (``inconclusive``); exits 2 on an unreadable input or an unwritable ``-o``. Judge checks are not scored unless ``--judge`` is given, so the default run makes no model call. """ From 4f7c0fcd59a9914c6905bcbf32460176a08c320c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 11:19:12 -0700 Subject: [PATCH 32/36] docs(reference): regenerate cli-interface.json for dokimi eval redteam make cli-snapshot after the merge of intel/main: the snapshot on main does not list the redteam subcommand this branch adds. Signed-off-by: Nikolay Petrov --- dokimi/docs/reference/cli-interface.json | 111 +++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 7f68d6a09..e3c9742e5 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -5420,6 +5420,117 @@ "subcommand_required": null, "subcommands": {} }, + "redteam": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "questions", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": "claude-sonnet-5-5", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-model", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'amazon-bedrock'", + "'anthropic'", + "'google'", + "'google-vertex'", + "'openai'", + "'openrouter'" + ], + "const": null, + "default": "anthropic", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-provider", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--output/-o", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi eval redteam", + "subcommand_required": null, + "subcommands": {} + }, "reference": { "allow_abbrev": true, "params": [ From 3a4cf5b50360600ea91c516689cc453718098e7e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 13:56:08 -0700 Subject: [PATCH 33/36] fix(redteam): a numeric canary, a stuffed list the grader accepts, a decoy pair that completes a value - A numeric match canary spells the number the grader parsed: the number regex reads the "5" of "-.5" with no sign, so a correct grader rejected the canary. - A bare list the grader accepts as a value ("b a" is "b" once the article goes) is no stuffed attack; the probe is skipped. - A stuffed list between two decoys is built from the first decoy pair whose whole answer the check rejects, so a value no longer ends the answer ("Paris QZX-BOGUS-REPLY-4096"). Regression tests for each: 12 new cases in tests/test_redteam.py. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/redteam.py | 19 ++++++++++++++++--- dokimi/tests/test_redteam.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index af1a7aba3..ed5fd702d 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -42,9 +42,10 @@ import json from dataclasses import dataclass +from itertools import permutations from typing import Any, Mapping, Optional -from dokimi.eval.reference_checks import Ask, check_values, grade, grade_check +from dokimi.eval.reference_checks import Ask, _number, check_values, grade, grade_check from dokimi.eval.reference_run import grading_by_id ATTACKS = ("canary", "wrong", "empty", "hedge", "stuffed") @@ -73,6 +74,10 @@ def canary_for(grading: Mapping[str, Any]) -> Optional[str]: if not values or (len(values) > 1 and mode == "all"): return None text = str(values[0]) + if kind == "match" and check.get("numeric") and (number := _number(text)) is not None: + # The numeric matcher reads a number from the answer with a digit before the point, so "-.5" + # would be read as 5: the canary spells the number the grader parsed, in plain decimal. + text = f"{number:.12f}".rstrip("0").rstrip(".") if kind == "choice": return "ANSWER: " + ", ".join(sorted(str(v).strip().upper() for v in values)) if kind == "json": @@ -131,7 +136,9 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: if len(values) < 2 or mode == "all": return None if _whole_answer(check) and not _reads_first_json(check): - return " ".join(values) + bare = " ".join(values) + # The list is no attack where the grader's own rules accept it (``b a`` is ``b`` once the article goes). + return None if grade_check(check, "", bare, None).passed else bare candidates: tuple[str, ...] if _reads_first_json(check): candidates = JSON_DECOYS @@ -140,7 +147,13 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: else: candidates = WRONG_CANDIDATES decoys = [w for w in candidates if not grade_check(check, "", w, None).passed] - return f"{decoys[0]} {' '.join(values)} {decoys[1]}" if len(decoys) >= 2 else None + # Each decoy is rejected alone, but the whole answer can still end (or begin) with a value: the first + # decoy pair whose complete answer the check rejects is used, and none is built when no pair is. + for before, after in permutations(decoys, 2): + stuffed = f"{before} {' '.join(values)} {after}" + if not grade_check(check, "", stuffed, None).passed: + return stuffed + return None def _positive_checks(grading: Mapping[str, Any]) -> list[Mapping[str, Any]]: diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index ae9c33bbc..84a99de92 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -696,3 +696,33 @@ def test_aliases_of_a_whole_answer_check_are_stuffed_bare_and_the_grader_rejects assert redteam.attacks({"checks": [check]})["stuffed"] == "Paris Lyon" result = run([_q("1", {"checks": [check]})]) assert result["status"] == "pass" and result["questions"][0]["probes"]["stuffed"]["passed"] is False + + +@pytest.mark.parametrize("value", ["-.5", "+.5"]) +@pytest.mark.parametrize("location", ["end", "begin", "any", "exact"]) +def test_a_numeric_target_with_no_digit_before_the_point_gets_a_canary_the_numeric_matcher_reads(value, location): + # The number regex reads "-.5" as 5 (the sign has no digit after it), so the raw spelling is no canary. + from dokimi.eval.reference_checks import grade + + grading = {"checks": [{"type": "match", "value": value, "numeric": True, "location": location}]} + assert grade(grading, "Q?", canary_for(grading)).passed is True + assert run([_q("1", grading)])["status"] == "pass" + + +@pytest.mark.parametrize("check", [ + {"type": "exact", "values": ["b", "a"]}, # "b a" is "b" once the article is dropped + {"type": "fuzzy", "values": ["Paris", "Lyon"]}, +]) +def test_a_list_the_grader_accepts_as_a_value_is_no_stuffing_attack(check): + assert redteam.attacks({"checks": [check]})["stuffed"] is None + result = run([_q("1", {"checks": [check]})]) + assert result["status"] == "pass" and result["violations"] == [] + + +@pytest.mark.parametrize("strip", [True, False]) +def test_a_stuffed_list_at_the_end_picks_decoys_that_do_not_complete_a_value(strip): + # Each decoy alone is rejected, but "Paris QZX-BOGUS-REPLY-4096" ends the answer and is a value. + check = {"type": "match", "values": ["Paris QZX-BOGUS-REPLY-4096", "Paris"], "strip": strip} + result = run([_q("1", {"checks": [check]})]) + assert result["status"] == "pass" and result["violations"] == [] + assert result["questions"][0]["probes"]["stuffed"]["passed"] is False From 8f45fed126492bbd07dd516729aded38066a1086 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 17:53:53 -0700 Subject: [PATCH 34/36] feat(redteam): use the canonical CLI flags and the route table `--judge PROVIDER/MODEL` replaces the `--judge` switch with `--judge-model` and `--judge-provider`, which are banned spellings in the flag registry; `--out FILE` replaces `-o/--output`. A malformed `--judge` is a usage error (exit 2). The route table lists `eval redteam` as a paid leaf, and the CLI snapshot is regenerated. Signed-off-by: Nikolay Petrov --- dokimi/docs/answer-formats.md | 2 +- dokimi/docs/reference/cli-interface.json | 47 ++------------------- dokimi/dokimi/cli_routes.py | 1 + dokimi/dokimi/commands/redteam.py | 41 ++++++++++-------- dokimi/dokimi/eval/redteam.py | 7 ++- dokimi/tests/test_cli_routes.py | 1 + dokimi/tests/test_judge_provider_default.py | 2 +- dokimi/tests/test_redteam.py | 45 ++++++++++++-------- 8 files changed, 63 insertions(+), 83 deletions(-) diff --git a/dokimi/docs/answer-formats.md b/dokimi/docs/answer-formats.md index e7af0484d..506a1b1fe 100644 --- a/dokimi/docs/answer-formats.md +++ b/dokimi/docs/answer-formats.md @@ -113,7 +113,7 @@ the run's pass rates and mean scores are under `run_metadata.summary` (or An attack the grader accepts, or a canary it rejects, is a violation, and any violation makes the run `blocked`: the command prints the pass rates and the -violations and exits 1 (`-o` writes `redteam.v1`). A run in which a graded +violations and exits 1 (`--out FILE` writes `redteam.v1`). A run in which a graded question was not measured is `inconclusive` and also exits 1: a question whose `judge` checks have no verdict without `--judge` (no probe scored), or a block of only negated checks. A `negate` check passes an answer that lacks its value, diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 049c212a3..6433b4794 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -5571,60 +5571,21 @@ "required": false, "type": null }, - { - "action": "_StoreTrueAction", - "choices": null, - "const": true, - "default": false, - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": true, - "key": "--judge", - "kind": "option", - "nargs": 0, - "position": null, - "required": false, - "type": null - }, { "action": "_StoreAction", "choices": null, "const": null, - "default": "claude-sonnet-5-5", - "group_key": null, - "group_required": null, - "hidden": false, - "is_flag": false, - "key": "--judge-model", - "kind": "option", - "nargs": null, - "position": null, - "required": false, - "type": null - }, - { - "action": "_StoreAction", - "choices": [ - "'amazon-bedrock'", - "'anthropic'", - "'google'", - "'google-vertex'", - "'openai'", - "'openrouter'" - ], - "const": null, - "default": "anthropic", + "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--judge-provider", + "key": "--judge", "kind": "option", "nargs": null, "position": null, "required": false, - "type": null + "type": "dokimi.commands.redteam._judge_ref#a59ddb60ae31" }, { "action": "_StoreAction", @@ -5635,7 +5596,7 @@ "group_required": null, "hidden": false, "is_flag": false, - "key": "--output/-o", + "key": "--out", "kind": "option", "nargs": null, "position": null, diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index d6ae4daf2..6852010d1 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -147,6 +147,7 @@ class Tombstone: Leaf(("eval", "pairwise"), PAID), Leaf(("eval", "reference"), PAID), Leaf(("eval", "native-import"), FREE), + Leaf(("eval", "redteam"), PAID), Leaf(("eval", "panel-score"), PAID), Leaf(("dataset", "export"), FREE), Leaf(("ingest", "docling"), FREE), diff --git a/dokimi/dokimi/commands/redteam.py b/dokimi/dokimi/commands/redteam.py index 381271a87..8db16c74d 100644 --- a/dokimi/dokimi/commands/redteam.py +++ b/dokimi/dokimi/commands/redteam.py @@ -5,8 +5,9 @@ Reads a questions set (with ``grading`` blocks), grades synthetic attacks against it and prints the pass rates. Exits 1 when any attack is accepted, a canary is rejected, a graded question was not -measured, or nothing was graded (``inconclusive``); exits 2 on an unreadable input or an unwritable ``-o``. Judge checks are not -scored unless ``--judge`` is given, so the default run makes no model call. +measured, or nothing was graded (``inconclusive``); exits 2 on an unreadable input, a malformed +``--judge`` or an unwritable ``--out``. Judge checks are not scored unless ``--judge PROVIDER/MODEL`` +is given, so the default run makes no model call. """ from __future__ import annotations @@ -20,7 +21,18 @@ def _ask(args: argparse.Namespace): return None from dokimi.eval.judge import judge_ask - return judge_ask(args.judge_model, args.judge_provider) + provider, model = args.judge + return judge_ask(model, provider) + + +def _judge_ref(value: str) -> tuple[str, str]: + from dokimi.eval.judge import JUDGE_PROVIDERS + + provider, _, model = value.partition("/") + if provider not in JUDGE_PROVIDERS or not model.strip(): + raise argparse.ArgumentTypeError( + f"expected PROVIDER/MODEL with PROVIDER one of {', '.join(JUDGE_PROVIDERS)}, got {value!r}") + return provider, model.strip() def cmd_eval_redteam(args: argparse.Namespace) -> None: @@ -35,31 +47,26 @@ def cmd_eval_redteam(args: argparse.Namespace) -> None: print(f"Error: {e}", file=sys.stderr) sys.exit(2) result["questions_path"] = str(args.questions) - result["judge_model"] = args.judge_model if args.judge else None - result["judge_provider"] = args.judge_provider if args.judge else None - print(format_table(result, output=args.output)) - if args.output: + result["judge_model"] = args.judge[1] if args.judge else None + result["judge_provider"] = args.judge[0] if args.judge else None + print(format_table(result, output=args.out)) + if args.out: try: - save_artifact("redteam", result, args.output) + save_artifact("redteam", result, args.out) except (OSError, ValueError) as e: print(f"Error: {e}", file=sys.stderr) sys.exit(2) - print(f"Wrote redteam artifact to {args.output}") + print(f"Wrote redteam artifact to {args.out}") if result["status"] != "pass": sys.exit(1) def register_eval(eval_sub) -> None: - from dokimi.defaults import DEFAULT_JUDGE_PROVIDER, JUDGE_MODEL_FOR_PROVIDER, JUDGE_MODEL_HELP - from dokimi.eval.judge import JUDGE_PROVIDERS - r = eval_sub.add_parser( "redteam", help="Check grading blocks reject wrong, empty, hedged and stuffed answers and accept a canary") r.add_argument("questions", help="questions JSON with grading blocks (from `questions import`)") - r.add_argument("--judge", action="store_true", - help="Grade judge checks with the judge model (model calls); default leaves them unscored") - r.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, help=JUDGE_MODEL_HELP) - r.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS) - r.add_argument("-o", "--output", default=None, help="redteam.v1 JSON to write") + r.add_argument("--judge", metavar="PROVIDER/MODEL", type=_judge_ref, default=None, + help="Grade judge checks with this judge model (model calls); default leaves them unscored") + r.add_argument("--out", metavar="FILE", default=None, help="redteam.v1 JSON to write") r.set_defaults(func=cmd_eval_redteam) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index ed5fd702d..040588ee4 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -121,8 +121,7 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: A negated check is stuffed with its one or more values: the stuffed answer carries the forbidden value, so it must fail, unless the check compares the whole answer (see ``_whole_answer``), which the listed values never equal, or is ``mode: all``, which the stuffed answer can pass by its location - (the skip is kept for every such check, for simplicity). A choice check is left out: its answer is - a bare letter. + (the skip is kept for every such check, for simplicity). A choice check is left out: its answer is a bare letter. """ checks = grading.get("checks", []) if len(checks) != 1 or checks[0].get("type") not in _LITERAL - {"choice"}: @@ -302,7 +301,7 @@ def _summary(rows: list[dict[str, Any]]) -> dict[str, int]: def format_table(result: dict[str, Any], output: Any = None) -> str: - """The pass-rate table; ``output`` is the ``-o`` path when an artifact is written, else None.""" + """The pass-rate table; ``output`` is the ``--out`` path when an artifact is written, else None.""" lines = [f"{'attack':<8} {'expected':<9} {'scored':>6} {'accepted':>8} {'rate':>6}"] for name, a in result["attacks"].items(): rate = "-" if a["rate"] is None else f"{a['rate']:.2f}" @@ -313,7 +312,7 @@ def format_table(result: dict[str, Any], output: Any = None) -> str: if len(result["violations"]) > 20: more = len(result["violations"]) - 20 lines.append(f" ... {more} more violations in the artifact" if output - else f" ... {more} more violations; rerun with -o FILE to list them all") + else f" ... {more} more violations; rerun with --out FILE to list them all") s = result["summary"] if s["negated_only"]: lines.append(f" {s['negated_only']} questions with only negated checks: " diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index cf80f27b3..991ccccec 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -131,6 +131,7 @@ def test_help_lists_each_root_under_its_section(): ("eval", "cross-judge"), ("eval", "pairwise"), ("eval", "reference"), + ("eval", "redteam"), ("eval", "panel-score"), ("tools", "preference"), } diff --git a/dokimi/tests/test_judge_provider_default.py b/dokimi/tests/test_judge_provider_default.py index 31d8d8c63..e1d8fc4d4 100644 --- a/dokimi/tests/test_judge_provider_default.py +++ b/dokimi/tests/test_judge_provider_default.py @@ -49,7 +49,7 @@ def test_the_commands_that_take_a_judge_provider_are_all_covered(): assert set(COMMANDS) == { "evaluate", "eval score", "eval judge-validity", "eval checklist", "eval judge-wording", "eval pairwise", "benchmark run", "benchmark preflight", "benchmark batch", "arms run", - "arms matrix-run", "subjects run", "eval reference", "eval redteam", + "arms matrix-run", "subjects run", "eval reference", } diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 84a99de92..2a048ce8c 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -241,8 +241,7 @@ def test_table_names_each_violation_and_the_status(monkeypatch): def _args(path, **overrides): - base = {"questions": str(path), "judge": False, "judge_model": "m", "judge_provider": "anthropic", - "output": None} + base = {"questions": str(path), "judge": None, "out": None} return argparse.Namespace(**{**base, **overrides}) @@ -254,7 +253,7 @@ def _patch_questions(monkeypatch, questions): def test_command_passes_and_writes_a_redteam_artifact(tmp_path, monkeypatch, capsys): _patch_questions(monkeypatch, [_q("1", _match("Paris"))]) out = tmp_path / "redteam.json" - cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=out)) saved = json.loads(out.read_text(encoding="utf-8")) assert saved["schema_version"] == "redteam.v1" and saved["status"] == "pass" assert saved["questions_path"].endswith("questions.json") @@ -280,8 +279,8 @@ def test_command_exits_2_when_the_artifact_cannot_be_written(tmp_path, monkeypat blocker = tmp_path / "a_file" blocker.write_text("not a directory", encoding="utf-8") with pytest.raises(SystemExit) as exc: - cmd_eval_redteam(_args(tmp_path / "questions.json", output=blocker / "out.json")) - assert exc.value.code == 2 # not 1: CI must not read an unwritable -o as a blocked run + cmd_eval_redteam(_args(tmp_path / "questions.json", out=blocker / "out.json")) + assert exc.value.code == 2 # not 1: CI must not read an unwritable --out as a blocked run assert capsys.readouterr().err.startswith("Error:") @@ -501,8 +500,7 @@ def test_the_artifact_names_the_judge_that_graded_the_judge_checks(tmp_path, mon _patch_questions(monkeypatch, [_q("1", _match("Paris"))]) monkeypatch.setattr("dokimi.eval.judge.Judge", lambda **kw: SimpleNamespace(llm=None)) out = tmp_path / "judged.json" - cmd_eval_redteam(_args(tmp_path / "questions.json", output=out, judge=True, - judge_model="judge-x", judge_provider="openai")) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=out, judge=("openai", "judge-x"))) saved = json.loads(out.read_text(encoding="utf-8")) assert (saved["judge_model"], saved["judge_provider"]) == ("judge-x", "openai") @@ -510,7 +508,7 @@ def test_the_artifact_names_the_judge_that_graded_the_judge_checks(tmp_path, mon def test_the_artifact_has_no_judge_without_the_judge_flag(tmp_path, monkeypatch): _patch_questions(monkeypatch, [_q("1", _match("Paris"))]) out = tmp_path / "plain.json" - cmd_eval_redteam(_args(tmp_path / "questions.json", output=out, judge_model="judge-x")) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=out)) saved = json.loads(out.read_text(encoding="utf-8")) assert saved["judge_model"] is None and saved["judge_provider"] is None @@ -540,20 +538,20 @@ def _many_violations(): def test_the_table_names_dash_o_when_it_cuts_the_violations_and_no_artifact_is_written(): table = format_table(_many_violations()) - assert "... 4 more violations; rerun with -o FILE to list them all" in table + assert "... 4 more violations; rerun with --out FILE to list them all" in table assert "in the artifact" not in table def test_the_table_points_to_the_artifact_when_dash_o_was_given(tmp_path): table = format_table(_many_violations(), output=tmp_path / "r.json") assert "... 4 more violations in the artifact" in table - assert "rerun with -o" not in table + assert "rerun with --out" not in table def test_the_command_prints_the_cut_violations_with_the_output_path_it_wrote(tmp_path, monkeypatch, capsys): _patch_questions(monkeypatch, [_q(str(i), {"checks": [{"type": "regex", "values": [".*"]}]}) for i in range(8)]) with pytest.raises(SystemExit): - cmd_eval_redteam(_args(tmp_path / "questions.json", output=tmp_path / "r.json")) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=tmp_path / "r.json")) assert "4 more violations in the artifact" in capsys.readouterr().out @@ -614,8 +612,7 @@ def fake_judge(**kw): monkeypatch.setattr("dokimi.eval.judge.Judge", fake_judge) out = tmp_path / "judged.json" with pytest.raises(SystemExit) as exc: - cmd_eval_redteam(_args(tmp_path / "questions.json", output=out, judge=True, - judge_model="judge-x", judge_provider="openai")) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=out, judge=("openai", "judge-x"))) assert exc.value.code == 1 # the stub judge accepts every attack assert made == {"model": "judge-x", "provider": "openai"} saved = json.loads(out.read_text(encoding="utf-8")) @@ -630,7 +627,7 @@ def refuse(**kw): monkeypatch.setattr("dokimi.eval.judge.Judge", refuse) out = tmp_path / "plain.json" with pytest.raises(SystemExit) as exc: - cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=out)) assert exc.value.code == 1 assert json.loads(out.read_text(encoding="utf-8"))["summary"]["unscored"] == 3 @@ -655,7 +652,7 @@ def test_command_exits_2_with_the_problem_when_a_grading_block_cannot_be_graded( _patch_questions(monkeypatch, [_q("7", {"checks": [{"type": "match", "values": []}]})]) out = tmp_path / "redteam.json" with pytest.raises(SystemExit) as exc: - cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=out)) assert exc.value.code == 2 assert "7: check 0: match has no values" in capsys.readouterr().err assert not out.exists() @@ -672,7 +669,7 @@ def test_a_blocked_run_still_writes_the_artifact(tmp_path, monkeypatch): _patch_questions(monkeypatch, [_q("1", {"checks": [{"type": "regex", "values": [".*"]}]})]) out = tmp_path / "blocked.json" with pytest.raises(SystemExit) as exc: - cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=out)) assert exc.value.code == 1 assert json.loads(out.read_text(encoding="utf-8"))["status"] == "blocked" @@ -681,7 +678,7 @@ def test_an_inconclusive_run_still_writes_the_artifact(tmp_path, monkeypatch): _patch_questions(monkeypatch, [{"id": "1", "question": "Capital of France?"}]) out = tmp_path / "inconclusive.json" with pytest.raises(SystemExit) as exc: - cmd_eval_redteam(_args(tmp_path / "questions.json", output=out)) + cmd_eval_redteam(_args(tmp_path / "questions.json", out=out)) assert exc.value.code == 1 assert json.loads(out.read_text(encoding="utf-8"))["status"] == "inconclusive" @@ -726,3 +723,17 @@ def test_a_stuffed_list_at_the_end_picks_decoys_that_do_not_complete_a_value(str result = run([_q("1", {"checks": [check]})]) assert result["status"] == "pass" and result["violations"] == [] assert result["questions"][0]["probes"]["stuffed"]["passed"] is False + + + + +def test_judge_takes_a_provider_model_reference_and_a_bad_one_is_a_usage_error(capsys): + from dokimi.cli import parse_args + + args = parse_args(["eval", "redteam", "q.json", "--judge", "openai/gpt-x", "--out", "r.json"]) + assert (args.judge, args.out) == (("openai", "gpt-x"), "r.json") + assert parse_args(["eval", "redteam", "q.json"]).judge is None + for bad in ("gpt-x", "nosuch/gpt-x", "openai/"): + with pytest.raises(SystemExit) as exc: + parse_args(["eval", "redteam", "q.json", "--judge", bad]) + assert exc.value.code == 2, bad From 0aa6c16c36bc6eb21dd7a12b64ab6b30870af452 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 17:54:06 -0700 Subject: [PATCH 35/36] fix(redteam): a negated value the matcher reads as another number is not stuffed The numeric matcher reads "5" out of "-.5", so a negated check on "-.5" accepts the stuffed answer as it should, and the run reported a false `blocked`. The negated stuffed answer is now built only where the same check without `negate` accepts it. Regressions: "-.5" at end and "+.5" at begin build no probe, and "7731" is still stuffed. Thread: PRRT_kwDOVBg7g86yKAAM Thread: PRR_kwDOVBg7g88AAAABVPIvTg Thread: IC_kwDOVBg7g88AAAABhDPe6A Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/redteam.py | 9 +++++++-- dokimi/tests/test_redteam.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/eval/redteam.py b/dokimi/dokimi/eval/redteam.py index 040588ee4..0e1849719 100644 --- a/dokimi/dokimi/eval/redteam.py +++ b/dokimi/dokimi/eval/redteam.py @@ -121,7 +121,9 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: A negated check is stuffed with its one or more values: the stuffed answer carries the forbidden value, so it must fail, unless the check compares the whole answer (see ``_whole_answer``), which the listed values never equal, or is ``mode: all``, which the stuffed answer can pass by its location - (the skip is kept for every such check, for simplicity). A choice check is left out: its answer is a bare letter. + (the skip is kept for every such check, for simplicity), and the list is built only where the same + check without ``negate`` accepts it, so a value the grader reads as another one is no attack. + A choice check is left out: its answer is a bare letter. """ checks = grading.get("checks", []) if len(checks) != 1 or checks[0].get("type") not in _LITERAL - {"choice"}: @@ -130,8 +132,11 @@ def stuffed_for(grading: Mapping[str, Any]) -> Optional[str]: values = [str(v) for v in check_values(check)] mode = str(check.get("mode", "any")) if check.get("negate"): + bare = " ".join(values) built = bool(values) and not _whole_answer(check) and mode != "all" - return " ".join(values) if built else None + # Built only when the grader reads the value out of the list, as the positive check does. + carries = grade_check({**check, "negate": False}, "", bare, None).passed + return bare if built and carries else None if len(values) < 2 or mode == "all": return None if _whole_answer(check) and not _reads_first_json(check): diff --git a/dokimi/tests/test_redteam.py b/dokimi/tests/test_redteam.py index 2a048ce8c..9cdc43527 100644 --- a/dokimi/tests/test_redteam.py +++ b/dokimi/tests/test_redteam.py @@ -725,6 +725,19 @@ def test_a_stuffed_list_at_the_end_picks_decoys_that_do_not_complete_a_value(str assert result["questions"][0]["probes"]["stuffed"]["passed"] is False +@pytest.mark.parametrize("value, location", [("-.5", "end"), ("+.5", "begin")]) +def test_a_negated_numeric_value_the_matcher_reads_as_another_number_is_not_stuffed(value, location): + # The numeric matcher reads "5" out of "-.5", so the negated check accepts the stuffed answer as the + # grader should: no probe is built, and a correct grader is not blocked. + grading = {"checks": [{"type": "match", "value": value, "numeric": True, "negate": True, + "location": location}]} + assert redteam.stuffed_for(grading) is None + assert run([{"id": "1", "question": "Q?", "grading": grading}])["status"] != "blocked" + + +def test_a_negated_numeric_value_the_matcher_reads_is_still_stuffed(): + grading = {"checks": [{"type": "match", "value": "7731", "numeric": True, "negate": True}]} + assert redteam.stuffed_for(grading) == "7731" def test_judge_takes_a_provider_model_reference_and_a_bad_one_is_a_usage_error(capsys): From b9f0980f3188d0f7a0df869223d5e1c2501a5834 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 18:46:18 -0700 Subject: [PATCH 36/36] docs(changelog): eval redteam writes its artifact with --out FILE Signed-off-by: Nikolay Petrov --- .../20261028_120100_napetrov_feat-verifier-redteam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md b/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md index 285601c39..a4b20bbde 100644 --- a/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md +++ b/dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md @@ -1,3 +1,3 @@ ### Added -- **`eval redteam` checks that grading blocks reject wrong answers** — grades a planted wrong, an empty, a hedged and a keyword-stuffed answer against each question's `grading` block, and a canary right answer built from its one literal check. A wrong answer the grader accepts, or a canary it rejects, is a violation: the command prints pass rates, exits 1 and, with `-o`, writes a `redteam.v1` artifact (it records the judge model and provider). A run in which a graded question was not measured (judge checks without `--judge`, a block of only negated checks) is `inconclusive` and also exits 1. Judge checks are unscored unless `--judge` is given, so the default run makes no model call. +- **`eval redteam` checks that grading blocks reject wrong answers** — grades a planted wrong, an empty, a hedged and a keyword-stuffed answer against each question's `grading` block, and a canary right answer built from its one literal check. A wrong answer the grader accepts, or a canary it rejects, is a violation: the command prints pass rates, exits 1 and, with `--out FILE`, writes a `redteam.v1` artifact (it records the judge model and provider). A run in which a graded question was not measured (judge checks without `--judge`, a block of only negated checks) is `inconclusive` and also exits 1. Judge checks are unscored unless `--judge` is given, so the default run makes no model call.