From af07f85974b91c712f4904f4339e0baf95eacdb2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 21:58:42 -0700 Subject: [PATCH 01/42] feat(llm): claude-cli provider: run the Q&A track through a logged-in Claude Code CLI A provider in dokimi.llm_cli.BACKENDS runs one prompt through the CLI the user is signed in to instead of an API key. LiteLLMAdapter hands such a request to llm_cli, so retry, the response cache and usage totals are unchanged. Each call is isolated (no tools, no settings, no MCP, no session, never --bare), runs in one fixed empty directory and takes the prompt on stdin; the parent's Claude Code session variables are dropped. temperature/seed are reported as unsupported, cost_usd stays None (the CLI's figure is list price), and CliCallError says whether to retry. Also: cli.py backends (install/login status, no model call), benchmark preflight checks the CLI login instead of a key, claude-cli for the judge and get_llm, the --provider choices come from llm.ANSWER_PROVIDERS, and the JSON helpers move to llm_json so llm.py does not grow. Signed-off-by: Nikolay Petrov --- dokimi/benchmarks/spec.v1.yaml | 2 +- ...1026_150000_napetrov_claude-cli-backend.md | 19 ++ dokimi/docs/README.md | 2 + dokimi/docs/architecture.md | 4 + dokimi/docs/subscription-backends.md | 100 +++++++ dokimi/dokimi/cli.py | 2 + dokimi/dokimi/commands/answers.py | 3 +- dokimi/dokimi/commands/arms.py | 5 +- dokimi/dokimi/commands/backends.py | 41 +++ dokimi/dokimi/commands/benchmark.py | 27 +- dokimi/dokimi/commands/evaluate.py | 3 +- dokimi/dokimi/commands/novelty.py | 3 +- dokimi/dokimi/commands/orchestrate.py | 3 +- dokimi/dokimi/commands/personas.py | 3 +- dokimi/dokimi/commands/questions.py | 7 +- dokimi/dokimi/commands/subjects.py | 3 +- dokimi/dokimi/commands/tools.py | 3 +- dokimi/dokimi/defaults.py | 5 +- dokimi/dokimi/eval/judge.py | 6 +- dokimi/dokimi/llm.py | 95 +++---- dokimi/dokimi/llm_cli.py | 260 ++++++++++++++++++ dokimi/dokimi/llm_json.py | 63 +++++ dokimi/dokimi/utils.py | 4 +- dokimi/mkdocs.yml | 1 + dokimi/scripts/growth_baseline.json | 6 +- dokimi/tests/test_import_architecture.py | 3 +- dokimi/tests/test_llm_cli.py | 201 ++++++++++++++ 27 files changed, 783 insertions(+), 91 deletions(-) create mode 100644 dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md create mode 100644 dokimi/docs/subscription-backends.md create mode 100644 dokimi/dokimi/commands/backends.py create mode 100644 dokimi/dokimi/llm_cli.py create mode 100644 dokimi/dokimi/llm_json.py create mode 100644 dokimi/tests/test_llm_cli.py diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index 41c6e1fe4..bdefa65c0 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -80,7 +80,7 @@ future: # edit here, so raise max_docs in the same commit that adds the doc. golden_manifest: min_docs: 10 - max_docs: 32 + max_docs: 33 include: - docs/**/*.md exclude: diff --git a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md new file mode 100644 index 000000000..af7e268e5 --- /dev/null +++ b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md @@ -0,0 +1,19 @@ +### Added + +- **`--provider claude-cli` / `--judge-provider claude-cli`**: answers, judging, + question generation, personas and novelty can run through a logged-in Claude + Code CLI (`claude -p`) instead of an API key. Each call runs with no tools, no + user or project settings and no MCP servers, in one fixed empty directory, with + the prompt on stdin. A requested `temperature` or `seed` is refused, since + the CLI cannot honour them. `cost_usd` is `None`, because the CLI's figure is + list price. Rate limits, overload and timeouts are retried; an unknown model or + a missing login is not. See `docs/subscription-backends.md`. +- **`backends`**: lists the CLI backends with install, version and login status; + calls no model. `benchmark preflight` checks a CLI backend's login instead + of an API key. + +### Changed + +- The `--provider` choices of every command come from one tuple, + `dokimi.llm.ANSWER_PROVIDERS`. `tools` keeps API providers only, since it needs + tool calling. diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index c2f3fdd60..f05f135bb 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -257,6 +257,8 @@ code. verifier pass rate against a baseline. - [answer-formats.md](answer-formats.md) — run Inspect AI, OpenAI evals, skill-creator and promptfoo answer-only sets on the answers track +- [subscription-backends.md](subscription-backends.md) — run the Q&A track through a + logged-in coding-agent CLI (`--provider claude-cli`) instead of an API key - [context-window.md](context-window.md) — read a run's context window from its Claude Code trace with `cli.py context trace`: start-up load, growth, compactions, tool-result weight. No LLM call. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 8f973fa21..8ac08b358 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -401,6 +401,10 @@ prints one or more traces as a comparison table; it calls no model. Guide: adapter seam: `LiteLLMAdapter` in production, `FakeAdapter` in tests (the `fake_llm` fixture). `ask_parsed` is the redraw-until-it-parses loop. `llm_call`, `chat_completion` and `utils.get_llm` delegate to `LLM`. + `dokimi/llm_cli.py` serves the CLI providers (`claude-cli`): `LiteLLMAdapter` + hands their requests to it, and it runs the logged-in CLI through `process.run` + ([subscription-backends.md](subscription-backends.md)). `dokimi/llm_json.py` + holds the JSON extraction helpers `llm.py` re-exports. - `dokimi/caching.py` — the two caches that hang off `llm.py`: provider prompt caching (the stable part of a prompt goes first, so the provider can reuse it) and an opt-in response cache for byte-identical single-shot requests diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md new file mode 100644 index 000000000..00ffbdb54 --- /dev/null +++ b/dokimi/docs/subscription-backends.md @@ -0,0 +1,100 @@ +--- +depends_on: + - dokimi/dokimi/llm_cli.py + - dokimi/dokimi/commands/backends.py +--- + + +# Subscription backends: a logged-in CLI as the LLM provider + +The Q&A track (answers, judging, question generation, personas, novelty) can call +a model through a coding-agent CLI the machine is already signed in to, instead +of an API key. Today one backend exists, `claude-cli` (Claude Code's `claude -p`). +Pass it wherever a command takes `--provider` or `--judge-provider`. + +## Check what this machine can use + +```bash +python cli.py backends # one row per backend: installed, version, signed in, how +python cli.py backends --json +``` + +This calls no model. A backend that is not signed in prints how to fix it. On a +machine without a browser, `claude setup-token` prints a token; export it as +`CLAUDE_CODE_OAUTH_TOKEN`. `benchmark preflight` checks the login the same way, +instead of looking for an API key. + +## Run + +```bash +python cli.py answers generate --product oneTBB --questions questions/oneTBB.json \ + --output answers/oneTBB.json --provider claude-cli --model haiku +python cli.py eval score --product oneTBB --answers answers/oneTBB.json \ + --output eval/oneTBB.json --judge-provider claude-cli --judge-model sonnet +``` + +`--model` takes what `claude --model` takes: an alias (`haiku`, `sonnet`, +`opus`) or a full id (`claude-sonnet-5-5`). The default judge model works on +`claude-cli` as it does on `anthropic`. When the answerer and the judge are the +same model, the judge grades its own answers. + +## How it differs from an API provider + +| | API provider | `claude-cli` | +|---|---|---| +| Credential | `*_API_KEY` | the CLI's own login | +| `--temperature`, `--seed` | sent when set | refused before the run (`unsupported_sampling_params`) | +| `max_tokens` | output ceiling | ignored: the CLI has no flag for it | +| Hidden prompt | none | fixed CLI preamble (≈0.5k–1.3k tokens, by CLI version), plus working directory, git status, platform and date | +| Tools | `chat(tools=...)` | refused: one prompt, one reply | +| `cost_usd` | litellm price | `None`: the CLI's `total_cost_usd` is list price, not what a subscription is charged | +| Latency | model only | plus ≈2–3 s CLI start-up per call | + +What this means for results: + +- **Don't pool scores across transports.** `claude-cli:sonnet` and + `anthropic:claude-sonnet-5-5` get different prompts (the CLI adds its own text) + and different sampling. Compare arms within one backend. +- **The CLI states its environment, and nothing turns this off.** It does so + with no tools, with its own system prompt replaced, and with + `--exclude-dynamic-system-prompt-sections`. Tested and not working: + `CLAUDE_CODE_HIDE_CWD`, `CLAUDE_CODE_DISABLE_GIT_INSTRUCTIONS` (Claude Code + 2.1.304). dokimi runs every call in one fixed empty directory + (`/dokimi-cli/claude-cli`), so the stated environment is the same in the + with-docs and the without-docs arm, and the response cache still hits. A + without-docs answer may still mention it ("the working directory isn't a git + repository"). +- **Repeats differ.** With no temperature or seed, the same prompt gives a + different answer each time; measure variance with repeats, not a pinned seed. + +## Isolation + +Each call is `claude -p --input-format text --output-format json --model +--tools "" --setting-sources "" --strict-mcp-config --no-session-persistence +--system-prompt `. The prompt goes on stdin, so its size is not bounded by +the argument limit. Without these flags, a plain `claude -p` in a repository +carries about 37k tokens of agent prompt, tool schemas and `CLAUDE.md`. + +- `--bare` is never used: it reads only `ANTHROPIC_API_KEY`, so a subscription + login fails with "Authentication error". +- The child gets the caller's environment minus the variables a Claude Code + session sets for its children (`CLAUDECODE`, `CLAUDE_CODE_SESSION_ID`, …). If + dokimi itself runs inside Claude Code and they were passed on, the child would + join the parent's session. +- The CLI keeps its own config directory, because that is where the login is + stored. User settings and `CLAUDE.md` stay out because of + `--setting-sources ""`. + +## Errors and retries + +Rate limits, overload, 5xx and a timeout are retried with the same backoff as +an API provider. An unknown model, a missing login or a missing binary fail on +the first attempt. + +## Terms of use + +A consumer subscription is for personal use. Keep CI and shared benchmark +runs on API keys. diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index 32ee6e64c..e6dd9e18e 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -14,6 +14,7 @@ from dokimi.commands import ( answers, arms, + backends, baseline, benchmark, catalog, @@ -81,6 +82,7 @@ def positive_int(value: str) -> int: repo.register(sub, positive_int) # repo (repositories as subjects) tools.register(sub, positive_int) # tools (tool-choice probes) context.register(sub, positive_int) # context (context window of a run) + backends.register(sub, positive_int) # backends (CLI providers) return p diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py index 4b08abe6a..f93828616 100644 --- a/dokimi/dokimi/commands/answers.py +++ b/dokimi/dokimi/commands/answers.py @@ -9,6 +9,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands.evaluate import _require_ragas, _run_ragas_eval from dokimi.defaults import ( DEFAULT_ANSWER_MODEL, @@ -158,7 +159,7 @@ def register(sub, positive_int) -> None: "Sets output to results/{product}_{run_id}/answers/{product}.json") gen_a_p.add_argument("--output", default=None, help="Output file (default: answers/{product}.json)") gen_a_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM model for answering") - gen_a_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + gen_a_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) gen_a_p.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, help="Max tokens to retrieve per question") gen_a_p.add_argument("--top-k", type=int, default=None, help=f"Number of docs to retrieve before reranking (default from config/products.yaml retrieval.top_k or {DEFAULT_RETRIEVAL_TOP_K})") gen_a_p.add_argument("--rerank-threshold", type=float, default=0.3, help="Min relevance score (0-1) to keep docs") diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 5696708f8..1d941bda6 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -11,6 +11,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.utils import question_id from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -234,7 +235,7 @@ def register(sub, positive_int) -> None: ) arms_run_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM for answering") arms_run_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, - choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + choices=ANSWER_PROVIDERS) arms_run_p.add_argument("--harness", default="arms-runner", help="Execution harness label stamped into output (default: arms-runner)") arms_run_p.add_argument("--plugins", default="", @@ -380,7 +381,7 @@ def _add_common_run_args(parser, positive_int, *, matrix_required: bool = False) ) parser.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM for answering") parser.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, - choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + choices=ANSWER_PROVIDERS) parser.add_argument("--harness", default="arms-runner", help="Execution harness label stamped into output (default: arms-runner)") parser.add_argument("--plugins", default="", diff --git a/dokimi/dokimi/commands/backends.py b/dokimi/dokimi/commands/backends.py new file mode 100644 index 000000000..9a71495ea --- /dev/null +++ b/dokimi/dokimi/commands/backends.py @@ -0,0 +1,41 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""backends command: which coding-agent CLIs can serve as LLM providers here.""" + +from __future__ import annotations + +import argparse +import dataclasses +import json + + +def cmd_backends(args: argparse.Namespace) -> None: + """One row per CLI backend: installed, version, signed in and how. Calls no model.""" + from dokimi import llm_cli + + rows = [llm_cli.detect(name) for name in llm_cli.BACKENDS] + if args.json: + print(json.dumps([dataclasses.asdict(r) for r in rows], indent=2)) + return + print("| provider | binary | version | signed in | method | models |") + print("|---|---|---|---|---|---|") + for r in rows: + signed = {True: "yes", False: "no", None: "unknown"}[r.authenticated] if r.installed else "-" + models = ", ".join(llm_cli.BACKENDS[r.backend].aliases) + print(f"| {r.backend} | {r.binary} | {r.version or 'not installed'} | {signed} " + f"| {r.method or '-'} | {models} |") + for r in rows: + if r.hint: + print(f"{r.backend}: {r.hint}") + + +def register(sub, positive_int) -> None: + """Add the `backends` command.""" + p = sub.add_parser( + "backends", + help="Coding-agent CLIs usable as --provider (claude-cli): installed, version, signed " + "in. No model call.", + ) + p.add_argument("--json", action="store_true", help="Print the rows as JSON") + p.set_defaults(func=cmd_backends) diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index 2c50c724e..610537dbc 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, List, cast +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands.evaluate import _warn_judge_independence from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -82,6 +83,21 @@ def _credential_env_var(provider: str) -> str: }.get(provider, "OPENAI_API_KEY") +def _credential_error(provider: str) -> str | None: + """Why *provider* cannot run here, or ``None``: a missing key, or a CLI not signed in.""" + from dokimi import llm_cli + + if llm_cli.backend(provider) is None: + key = _credential_env_var(provider) + return None if os.environ.get(key) else f"missing {key} for provider {provider}" + status = llm_cli.detect(provider) + if not status.installed: + return f"{provider}: '{status.binary}' is not installed" + if status.authenticated is False: + return f"{provider}: '{status.binary}' is not signed in ({status.hint})" + return None + + def _resolved_benchmark_plan(entry, args: argparse.Namespace) -> dict: doc_source = getattr(args, "doc_source", None) or (entry.doc_sources[0] if entry.doc_sources else "context7") output_dir = getattr(args, "output_dir", None) or f"results/{entry.key}{DEFAULT_RESULTS_SUFFIX}" @@ -141,10 +157,7 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None: errors: list[str] = [] warnings: list[str] = [] - for provider in {args.provider, args.judge_provider}: - key = _credential_env_var(provider) - if not os.environ.get(key): - errors.append(f"missing {key} for provider {provider}") + errors.extend(e for e in map(_credential_error, sorted({args.provider, args.judge_provider})) if e) try: Path(plan["output_dir"]).mkdir(parents=True, exist_ok=True) @@ -364,7 +377,7 @@ def register(sub, positive_int) -> None: bench_run_p.add_argument("--output-dir", default=None, dest="output_dir", help="Output directory (default: results/{target}_final)") bench_run_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL) - bench_run_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + bench_run_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) bench_run_p.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, dest="judge_model", help=JUDGE_MODEL_HELP) bench_run_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider", @@ -400,7 +413,7 @@ def register(sub, positive_int) -> None: bench_pre_p.add_argument("--doc-source", default=None, dest="doc_source", help="Override doc/context source") bench_pre_p.add_argument("--output-dir", default=None, dest="output_dir", help="Output directory (default: results/{target}_final)") bench_pre_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL) - bench_pre_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + bench_pre_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) bench_pre_p.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, dest="judge_model", help=JUDGE_MODEL_HELP) bench_pre_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider", choices=JUDGE_PROVIDERS) @@ -422,7 +435,7 @@ def register(sub, positive_int) -> None: help="Run for all libraries in registry") bench_batch_p.add_argument("--output-dir", default="results", dest="output_dir") bench_batch_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL) - bench_batch_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + bench_batch_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) bench_batch_p.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, dest="judge_model", help=JUDGE_MODEL_HELP) bench_batch_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider", diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 9d3602f31..e5fcfddc8 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -11,6 +11,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.defaults import ( DEFAULT_CONCURRENCY, DEFAULT_JUDGE_MODEL, @@ -571,7 +572,7 @@ def register(sub, positive_int) -> None: panel_p.add_argument("--product", required=True, help="Product name (e.g., oneTBB)") panel_p.add_argument("--output", default=None, help="Output file (default: eval/{product}_panel.json)") panel_p.add_argument("--model", default=DEFAULT_JUDGE_MODEL, help="Default LLM model for all judges") - panel_p.add_argument("--provider", default=DEFAULT_JUDGE_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + panel_p.add_argument("--provider", default=DEFAULT_JUDGE_PROVIDER, choices=ANSWER_PROVIDERS) panel_p.add_argument("--roles", default=None, help="Comma-separated judge roles (default: technical_expert,developer_advocate,doc_reviewer)") panel_p.add_argument( diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index bc357242f..a6865545e 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -27,12 +27,11 @@ DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, ) +from dokimi.llm import ANSWER_PROVIDERS as PROVIDERS if TYPE_CHECKING: from dokimi.novelty.tiers import RunSettings -PROVIDERS = ["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"] - def cmd_novelty_analyze(args: argparse.Namespace) -> None: """Run the full pipeline over one artifact and write the novelty artifact + views.""" diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py index f31b9bbac..b03843517 100644 --- a/dokimi/dokimi/commands/orchestrate.py +++ b/dokimi/dokimi/commands/orchestrate.py @@ -9,6 +9,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands.evaluate import _warn_judge_independence from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -117,7 +118,7 @@ def register(sub, positive_int) -> None: "is looked up, for example a checkout of the evaluation-content " "repository: --questions-root /path/to/content-repo/questions") eval_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM model for generation/answering") - eval_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + eval_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) eval_p.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, help=JUDGE_MODEL_HELP) eval_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS) eval_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", diff --git a/dokimi/dokimi/commands/personas.py b/dokimi/dokimi/commands/personas.py index 73248afdd..ef30a23b4 100644 --- a/dokimi/dokimi/commands/personas.py +++ b/dokimi/dokimi/commands/personas.py @@ -10,6 +10,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands._output import exit_on_unwritable from dokimi.defaults import DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER from dokimi.personas.analyzer import PersonaAnalyzer @@ -133,7 +134,7 @@ def register(sub, positive_int) -> None: discover_p.add_argument("--output", default=None, help="Output file (default: personas/{product}.json)") discover_p.add_argument("--count", type=int, default=5, help="Target number of personas (5-8)") discover_p.add_argument("--model", default=DEFAULT_GENERATOR_MODEL, help="LLM model for generation") - discover_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + discover_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=ANSWER_PROVIDERS) discover_p.add_argument("--github-token", default=None, help="GitHub token (or set GITHUB_TOKEN env)") discover_p.add_argument("--save-analysis", action="store_true", help="Save intermediate analysis JSON") discover_p.set_defaults(func=cmd_personas_discover) diff --git a/dokimi/dokimi/commands/questions.py b/dokimi/dokimi/commands/questions.py index 89cc2d795..5cf3980f7 100644 --- a/dokimi/dokimi/commands/questions.py +++ b/dokimi/dokimi/commands/questions.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, Dict +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands._output import exit_on_unwritable from dokimi.defaults import DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER @@ -389,7 +390,7 @@ def register(sub, positive_int) -> None: "(sentence-transformers, no API key), or auto " "(openai when a key is set, else local)") gen_q_p.add_argument("--model", default=DEFAULT_GENERATOR_MODEL, help="LLM model for generation") - gen_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + gen_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=ANSWER_PROVIDERS) gen_q_p.add_argument("--doc-source", default="context7", help="Documentation source for topic extraction: 'context7', 'local:', 'url:'") gen_q_p.add_argument("--context7-id", default=None, dest="context7_id", @@ -402,7 +403,7 @@ def register(sub, positive_int) -> None: analyze_q_p.add_argument("--product", required=True, help="Product name (e.g., oneTBB)") analyze_q_p.add_argument("--output", default=None, help="Output report JSON (default: reports/{product}_question_quality.json)") analyze_q_p.add_argument("--model", default=DEFAULT_GENERATOR_MODEL, help="LLM model for classification") - analyze_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + analyze_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=ANSWER_PROVIDERS) analyze_q_p.add_argument("--concurrency", type=int, default=5, help="Parallel classification requests") analyze_q_p.set_defaults(func=cmd_questions_analyze) @@ -433,7 +434,7 @@ def register(sub, positive_int) -> None: panel_q_p.add_argument("--output", default=None, help="Output report JSON (default: reports/{product}_question_panel.json)") panel_q_p.add_argument("--model", default=DEFAULT_GENERATOR_MODEL) - panel_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + panel_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=ANSWER_PROVIDERS) panel_q_p.add_argument("--reviewers", default=None, help="Comma-separated reviewers (default: domain_expert,user_advocate,qa_engineer)") panel_q_p.add_argument("--concurrency", type=positive_int, default=6) diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index 9e0f6b7fe..5566b2150 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -10,6 +10,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands._output import exit_on_unwritable from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -209,7 +210,7 @@ def register(sub, positive_int) -> None: run_p.add_argument("--out-md", default=None, dest="out_md") run_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help=argparse.SUPPRESS) run_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, - choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + choices=ANSWER_PROVIDERS) run_p.add_argument("--harness", default="arms-runner", help=argparse.SUPPRESS) run_p.add_argument("--context7-id", default=None, dest="context7_id") run_p.add_argument("--judge", action="store_true") diff --git a/dokimi/dokimi/commands/tools.py b/dokimi/dokimi/commands/tools.py index 5908fab7a..0b14c7ac7 100644 --- a/dokimi/dokimi/commands/tools.py +++ b/dokimi/dokimi/commands/tools.py @@ -15,9 +15,10 @@ import time from pathlib import Path +from dokimi.llm import PROVIDERS as API_PROVIDERS from dokimi.defaults import DEFAULT_ANSWER_PROVIDER -PROVIDERS = ["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"] +PROVIDERS = list(API_PROVIDERS) # tool calling: no CLI backend class CutBeforeToolCall(RuntimeError): diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py index 71056525c..10edc9e20 100644 --- a/dokimi/dokimi/defaults.py +++ b/dokimi/dokimi/defaults.py @@ -67,7 +67,8 @@ def default_judge_model(provider: str) -> str | None: Pairing a provider with another provider's model (``openai/claude-sonnet-5-5``) fails at the first call, so a command with no default here asks for ``--judge-model``. """ - if provider == DEFAULT_JUDGE_PROVIDER or (provider == "amazon-bedrock" + # claude-cli takes Anthropic's model ids, as the anthropic provider does. + if provider in (DEFAULT_JUDGE_PROVIDER, "claude-cli") or (provider == "amazon-bedrock" and DEFAULT_JUDGE_MODEL in BEDROCK_MODEL_IDS): return DEFAULT_JUDGE_MODEL if provider == DEFAULT_GENERATOR_PROVIDER: @@ -84,7 +85,7 @@ class _JudgeModelForProvider(str): #: ``dokimi.cli.parse_args`` reads it as ``DEFAULT_JUDGE_MODEL``. JUDGE_MODEL_FOR_PROVIDER = _JudgeModelForProvider(DEFAULT_JUDGE_MODEL) JUDGE_MODEL_HELP = (f"Judge model (default: the default judge for --judge-provider: " - f"{DEFAULT_JUDGE_MODEL} on {DEFAULT_JUDGE_PROVIDER} and amazon-bedrock, " + f"{DEFAULT_JUDGE_MODEL} on {DEFAULT_JUDGE_PROVIDER}, amazon-bedrock and claude-cli, " f"{DEFAULT_GENERATOR_MODEL} on {DEFAULT_GENERATOR_PROVIDER}; other providers " f"need this flag)") diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py index f07c9cc4d..428e15773 100644 --- a/dokimi/dokimi/eval/judge.py +++ b/dokimi/dokimi/eval/judge.py @@ -18,6 +18,7 @@ from dokimi import caching from dokimi.artifacts import save_artifact from dokimi.llm import ChatModel, ask_parsed +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.eval.diagnoser import diagnose from dokimi.eval.question_text import require_question_texts @@ -161,7 +162,8 @@ MAX_GROUND_TRUTH_CHARS = 4000 #: Providers ``Judge.__init__`` accepts; CLI ``--judge-provider`` choices read this. -JUDGE_PROVIDERS = ("openai", "anthropic", "google", "google-vertex", "openrouter", "amazon-bedrock") +JUDGE_PROVIDERS = ("openai", "anthropic", "google", "google-vertex", "openrouter", "amazon-bedrock", + *CLI_BACKENDS) def _clip_ground_truth(ground_truth: Optional[str], limit: int = MAX_GROUND_TRUTH_CHARS) -> str: @@ -250,7 +252,7 @@ def __init__( self.llm = ChatAnthropic(model=model, api_key=api_key, temperature=temperature, seed=seed) elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai"): + "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): from dokimi.utils import get_llm import os fallback_key = os.environ.get("OPENROUTER_API_KEY") if provider == "openrouter" else None diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 0967a2011..5204ed1c5 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -28,7 +28,9 @@ :data:`PROVIDER_ALIASES`. :func:`resolve_model` is this module's model-name map and :func:`api_key_env` its key-variable map. A provider outside the table is passed to litellm as its own route prefix (``azure/``), reading - ``OPENAI_API_KEY``. An explicit ``api_key`` wins over the environment; either + ``OPENAI_API_KEY``. A provider in :data:`dokimi.llm_cli.BACKENDS` (``claude-cli``) + is a logged-in coding-agent CLI, not an API: no key, no tools, no sampling + control (:mod:`dokimi.llm_cli`). An explicit ``api_key`` wins over the environment; either may be ``file:``, and an unreadable file raises ``ValueError``. - **Bedrock bare names.** A name in :data:`dokimi.defaults.BEDROCK_MODEL_IDS` runs under its inference profile (:func:`dokimi.defaults.bedrock_profile`); @@ -68,7 +70,6 @@ from __future__ import annotations import contextlib -import json import logging import os import re @@ -77,8 +78,10 @@ from dataclasses import dataclass, field, replace from typing import Any, Callable, Dict, Iterator, List, Optional, Protocol, Tuple, TypeVar, cast -from dokimi import caching +from dokimi import caching, llm_cli from dokimi.defaults import bedrock_profile +from dokimi.llm_json import extract_json_array as extract_json_array +from dokimi.llm_json import extract_json_object as extract_json_object from dokimi.metrics.usage import UsageRecord logger = logging.getLogger(__name__) @@ -97,6 +100,9 @@ "openrouter": "OPENROUTER_API_KEY", } +#: ``--provider`` choices: the API providers, then the CLI backends. +ANSWER_PROVIDERS: Tuple[str, ...] = (*PROVIDERS, *llm_cli.BACKENDS) + #: Other spellings of a canonical provider. PROVIDER_ALIASES: Dict[str, str] = { "google": "google-vertex", @@ -198,6 +204,9 @@ def unsupported_sampling_params( requested = list(_sampling_kwargs(temperature=temperature, seed=seed)) if not requested: return [] + cli = llm_cli.backend(_canonical(provider)) + if cli is not None: + return [n for n in requested if n not in cli.supports] try: from litellm import get_supported_openai_params @@ -263,6 +272,9 @@ def is_retryable(exc: BaseException) -> bool: if its message happens to contain a transient-sounding word, so the non-retryable set is consulted first. """ + flag = getattr(exc, "retryable", None) # set by an error that knows (llm_cli.CliCallError) + if isinstance(flag, bool): + return flag name = type(exc).__name__.lower() if name in _NON_RETRYABLE_EXC_NAMES: return False @@ -515,10 +527,28 @@ def _stream(completion: Any, request: Request) -> Reply: return Reply(text="".join(parts), usage=usage, finish_reason=finish_reason) +def _cli_reply(request: Request) -> Reply: + """A request for a CLI provider (:mod:`dokimi.llm_cli`): one prompt, an optional system.""" + turns = [m for m in request.messages if m.get("role") != "system"] + if request.tools or len(turns) != 1: + raise llm_cli.CliCallError(f"{request.provider} takes one prompt: no tools, no conversation", + retryable=False) + system = [prompt_text(replace(request, messages=[m])) + for m in request.messages if m.get("role") == "system"] + out = llm_cli.call(request.provider, model=request.model, prompt=prompt_text(request), + system="\n\n".join(system) or None) + return Reply(text=out.text, usage=out.usage, finish_reason=out.finish_reason) + + class LiteLLMAdapter: - """The production adapter: one ``litellm.completion`` call, priced by litellm.""" + """The production adapter: one ``litellm.completion`` call, priced by litellm. + + A CLI provider (:data:`dokimi.llm_cli.BACKENDS`) goes to its CLI instead. + """ def __call__(self, request: Request) -> Reply: + if llm_cli.backend(request.provider) is not None: + return _cli_reply(request) from litellm import completion if request.stream: @@ -1059,60 +1089,3 @@ def chat_completion( llm = LLM(model, provider, api_key, max_tokens=max_tokens, temperature=temperature, seed=seed, max_retries=max_retries, retry_delay=retry_delay) return llm.chat(messages, tools=tools, tool_choice=tool_choice).response - - -# ── Robust JSON extraction ───────────────────────────────────────────────────── - - -def extract_json_object(text: str) -> dict: - """Extract a JSON object from LLM output, handling markdown fences and leading text.""" - text = (text or "").strip() - if not text: - raise ValueError("LLM returned empty response") - # Direct parse - try: - return json.loads(text) - except Exception: - pass - # Markdown fence - fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) - if fence: - try: - return json.loads(fence.group(1).strip()) - except Exception: - pass - # First { ... } block - m = re.search(r"\{[\s\S]*\}", text) - if m: - return json.loads(m.group(0)) - raise ValueError(f"No JSON object found in LLM response: {text[:200]!r}") - - -def extract_json_array(text: str) -> list: - """Extract a JSON array from LLM output, handling markdown fences and leading text.""" - text = (text or "").strip() - if not text: - raise ValueError("LLM returned empty response") - # Direct parse - try: - result = json.loads(text) - if isinstance(result, list): - return result - except Exception: - pass - # Markdown fence - fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) - if fence: - try: - result = json.loads(fence.group(1).strip()) - if isinstance(result, list): - return result - except Exception: - pass - # First [ ... ] block - m = re.search(r"\[[\s\S]*\]", text) - if m: - result = json.loads(m.group(0)) - if isinstance(result, list): - return result - raise ValueError(f"No JSON array found in LLM response: {text[:200]!r}") diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py new file mode 100644 index 000000000..e5c44f47d --- /dev/null +++ b/dokimi/dokimi/llm_cli.py @@ -0,0 +1,260 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Coding-agent CLIs as LLM providers: a completion through a locally logged-in CLI. + +A provider in :data:`BACKENDS` (``claude-cli``) runs one prompt through the CLI the +user is already signed in to (a Claude subscription, ``claude setup-token``, or +whatever ``claude auth status`` reports) instead of an API key. :mod:`dokimi.llm` +routes a :class:`~dokimi.llm.Request` for such a provider here, so retry, the +response cache and usage totals work as for any provider. + +What differs from an API provider, and what a caller can rely on: + +- **No sampling control.** The CLI takes no temperature, seed or output ceiling. + :data:`CliBackend.supports` lists none of them, so + :func:`dokimi.llm.unsupported_sampling_params` names a requested ``temperature`` + or ``seed`` and a run refuses before it starts. ``max_tokens`` is dropped: every + caller sets one, and refusing it would refuse every run. +- **Hidden context.** Even with no tools, no settings and its own system prompt, the + CLI adds a fixed preamble (about 0.5k-1.3k tokens, by version) and states the + working directory, git status, platform and date. The child runs in one fixed + empty directory per backend (:func:`work_dir`), so that text is the same for + every call of a run, with and without docs. Do not pool its scores with the same + model through an API provider. +- **Isolation.** ``--tools "" --setting-sources "" --strict-mcp-config + --no-session-persistence``: no tools, no user or project settings (so no + ``CLAUDE.md``), no MCP servers, no session written. Never ``--bare``: it reads only + ``ANTHROPIC_API_KEY``, so a subscription login fails with "Authentication error". + The parent's environment is passed on except :data:`CLAUDE_ENV_DROP`; without + that, a child started inside a Claude Code session joins the parent's session. +- **Usage.** Tokens and the resolved model id come from the CLI's JSON. + ``cost_usd`` is ``None``: ``total_cost_usd`` is a list-price estimate, not what a + subscription is charged; it is kept as :attr:`CliResult.list_cost_usd`. +- **Errors.** A failed call raises :class:`CliCallError` with ``retryable`` set: + rate limits, overload, 5xx and timeouts are retried; an unknown model, a missing + login or a missing binary are not. + +:func:`detect` reports whether a backend's binary is installed and signed in +without calling a model (``cli.py backends``). +""" + +from __future__ import annotations + +import json +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Dict, Optional, Tuple + +from dokimi import process +from dokimi.metrics.usage import UsageRecord + +#: Variables a Claude Code session sets for its own children. Dropped, so a call made +#: from inside a session is a session of its own. +CLAUDE_ENV_DROP = frozenset({ + "CLAUDECODE", "CLAUDE_CODE_SESSION_ID", "CLAUDE_CODE_CHILD_SESSION", + "CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_SSE_PORT", +}) + +#: The flags that strip the CLI to a bare completion; see the module docstring. +CLAUDE_ISOLATION = ("--tools", "", "--setting-sources", "", "--strict-mcp-config", + "--no-session-persistence") + +#: ``api_error_status`` values worth another attempt. +_RETRYABLE_STATUS = frozenset({408, 429, 500, 502, 503, 504, 529}) +_RETRYABLE_WORDS = ("rate limit", "rate_limit", "overloaded", "usage limit", "timed out") + +DEFAULT_TIMEOUT_SEC = 600.0 + + +class CliCallError(RuntimeError): + """A CLI call that produced no reply; ``retryable`` is what :func:`dokimi.llm.is_retryable` reads.""" + + def __init__(self, message: str, *, retryable: bool): + super().__init__(message) + self.retryable = retryable + + +@dataclass(frozen=True) +class CliResult: + """One CLI reply: the text, its usage, and the model id the CLI resolved.""" + + text: str + usage: UsageRecord + model_id: Optional[str] = None + list_cost_usd: Optional[float] = None + finish_reason: Optional[str] = None + + +@dataclass(frozen=True) +class AuthStatus: + """What :func:`detect` found. ``authenticated`` is ``None`` when the CLI cannot tell.""" + + backend: str + binary: str + installed: bool + version: str = "" + authenticated: Optional[bool] = None + method: str = "" + hint: str = "" + + +@dataclass(frozen=True) +class CliBackend: + """One CLI: how to call it, read its reply, and ask it whether it is signed in.""" + + name: str + binary: str + build_argv: Callable[[str, Optional[str]], list] # (model, system) -> argv; prompt goes on stdin + parse: Callable[[process.Completed, str, str], CliResult] # (completed, model, provider) + auth_argv: Tuple[str, ...] + parse_auth: Callable[[process.Completed], Tuple[Optional[bool], str]] # -> (authenticated, method) + env_drop: frozenset = frozenset() + supports: frozenset = frozenset() # of "temperature", "seed", "max_tokens" + login_hint: str = "" + aliases: Tuple[str, ...] = () + + +# ── claude-cli ──────────────────────────────────────────────────────────────── + + +def _claude_argv(model: str, system: Optional[str]) -> list: + argv = ["claude", "-p", "--input-format", "text", "--output-format", "json", + "--model", model, *CLAUDE_ISOLATION] + # Without --system-prompt the CLI sends its own agent prompt (~37k tokens with tools). + return argv + ["--system-prompt", system or "You are a helpful assistant."] + + +def _claude_error(payload: Dict[str, Any], done: process.Completed) -> CliCallError: + message = str(payload.get("result") or done.stderr.strip() or f"exit {done.returncode}")[:500] + status = payload.get("api_error_status") + lowered = message.lower() + if any(w in lowered for w in ("authentication", "not logged in", "/login")): + return CliCallError(f"claude-cli: {message} (sign in: claude setup-token, or claude /login)", + retryable=False) + retryable = status in _RETRYABLE_STATUS or any(w in lowered for w in _RETRYABLE_WORDS) + return CliCallError(f"claude-cli: {message}", retryable=retryable) + + +def _claude_parse(done: process.Completed, model: str, provider: str) -> CliResult: + if done.timed_out: + raise CliCallError(f"claude-cli: no reply in {done.elapsed_sec:.0f}s", retryable=True) + try: + payload = json.loads(done.stdout) + except ValueError: + payload = None + if not isinstance(payload, dict): + raise CliCallError(f"claude-cli: exit {done.returncode}, no JSON reply: " + f"{(done.stderr or done.stdout).strip()[:300]}", retryable=False) + if payload.get("is_error") or done.returncode != 0: + raise _claude_error(payload, done) + u = payload.get("usage") or {} + cache_write = int(u.get("cache_creation_input_tokens") or 0) + cache_read = int(u.get("cache_read_input_tokens") or 0) + prompt = int(u.get("input_tokens") or 0) + cache_write + cache_read + completion = int(u.get("output_tokens") or 0) + model_usage = payload.get("modelUsage") or {} + list_cost = payload.get("total_cost_usd") + usage = UsageRecord( + prompt_tokens=prompt, completion_tokens=completion, total_tokens=prompt + completion, + cache_read_tokens=cache_read, cache_write_tokens=cache_write, + cost_usd=None, latency_sec=done.elapsed_sec, model=model, provider=provider, + n_calls=1, cost_known_calls=0, + ) + return CliResult( + text=str(payload.get("result") or ""), + usage=usage, + model_id=next(iter(model_usage), None), + list_cost_usd=float(list_cost) if isinstance(list_cost, (int, float)) else None, + finish_reason="length" if payload.get("stop_reason") == "max_tokens" else "stop", + ) + + +def _claude_auth(done: process.Completed) -> Tuple[Optional[bool], str]: + try: + status = json.loads(done.stdout) + except ValueError: + return None, "" + if not isinstance(status, dict): + return None, "" + method = str(status.get("authMethod") or "") + if status.get("apiProvider"): + method = f"{method} ({status['apiProvider']})" if method else str(status["apiProvider"]) + return bool(status.get("loggedIn")), method + + +CLAUDE_CLI = CliBackend( + name="claude-cli", + binary="claude", + build_argv=_claude_argv, + parse=_claude_parse, + auth_argv=("claude", "auth", "status"), + parse_auth=_claude_auth, + env_drop=CLAUDE_ENV_DROP, + login_hint="claude setup-token (headless) or claude, then /login", + aliases=("haiku", "sonnet", "opus"), +) + +#: CLI backends by provider name. +BACKENDS: Dict[str, CliBackend] = {CLAUDE_CLI.name: CLAUDE_CLI} + + +def backend(provider: str) -> Optional[CliBackend]: + """The CLI backend for *provider*, or ``None`` for an API provider.""" + return BACKENDS.get(provider) + + +def work_dir(name: str) -> Path: + """The fixed empty working directory a backend's calls run in. + + Fixed, not fresh per call: the CLI states its working directory in the prompt, and + a random path would make every request, and its response-cache key, differ. + """ + path = Path(tempfile.gettempdir()) / "dokimi-cli" / name + path.mkdir(parents=True, exist_ok=True) + return path + + +def _env(cli: CliBackend) -> process.Env: + return process.Env.only([k for k in os.environ if k not in cli.env_drop]) + + +def call(provider: str, *, model: str, prompt: str, system: Optional[str] = None, + timeout: float = DEFAULT_TIMEOUT_SEC, runner: Optional[process.Runner] = None) -> CliResult: + """One completion through *provider*'s CLI; the prompt goes on stdin (no ``ARG_MAX``). + + Raises: + CliCallError: the CLI is missing, failed, timed out or returned no reply. + KeyError: *provider* is not a CLI backend. + """ + cli = BACKENDS[provider] + try: + done = (runner or process.run)(cli.build_argv(model, system), timeout=timeout, env=_env(cli), + cwd=work_dir(cli.name), input=prompt, capture_limit=None) + except FileNotFoundError as exc: + raise CliCallError(f"{provider}: '{cli.binary}' is not installed or not on PATH ({exc})", + retryable=False) from exc + return cli.parse(done, model, provider) + + +def detect(provider: str, runner: Optional[process.Runner] = None) -> AuthStatus: + """Whether *provider*'s binary is installed and signed in. Calls no model.""" + cli = BACKENDS[provider] + runner = runner or process.run + env = _env(cli) + try: + version = runner([cli.binary, "--version"], timeout=30, env=env) + except OSError: + return AuthStatus(provider, cli.binary, installed=False, hint=f"install '{cli.binary}'") + try: + auth = runner(list(cli.auth_argv), timeout=30, env=env) + authenticated, method = cli.parse_auth(auth) + except OSError: + authenticated, method = None, "" + return AuthStatus( + provider, cli.binary, installed=True, version=version.stdout.strip().splitlines()[0] + if version.stdout.strip() else "", authenticated=authenticated, method=method, + hint="" if authenticated else cli.login_hint, + ) diff --git a/dokimi/dokimi/llm_json.py b/dokimi/dokimi/llm_json.py new file mode 100644 index 000000000..397b013b5 --- /dev/null +++ b/dokimi/dokimi/llm_json.py @@ -0,0 +1,63 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Robust JSON extraction from LLM output; re-exported by :mod:`dokimi.llm`.""" + +from __future__ import annotations + +import json +import re + + +def extract_json_object(text: str) -> dict: + """Extract a JSON object from LLM output, handling markdown fences and leading text.""" + text = (text or "").strip() + if not text: + raise ValueError("LLM returned empty response") + # Direct parse + try: + return json.loads(text) + except Exception: + pass + # Markdown fence + fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + if fence: + try: + return json.loads(fence.group(1).strip()) + except Exception: + pass + # First { ... } block + m = re.search(r"\{[\s\S]*\}", text) + if m: + return json.loads(m.group(0)) + raise ValueError(f"No JSON object found in LLM response: {text[:200]!r}") + + +def extract_json_array(text: str) -> list: + """Extract a JSON array from LLM output, handling markdown fences and leading text.""" + text = (text or "").strip() + if not text: + raise ValueError("LLM returned empty response") + # Direct parse + try: + result = json.loads(text) + if isinstance(result, list): + return result + except Exception: + pass + # Markdown fence + fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + if fence: + try: + result = json.loads(fence.group(1).strip()) + if isinstance(result, list): + return result + except Exception: + pass + # First [ ... ] block + m = re.search(r"\[[\s\S]*\]", text) + if m: + result = json.loads(m.group(0)) + if isinstance(result, list): + return result + raise ValueError(f"No JSON array found in LLM response: {text[:200]!r}") diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index cda2ff68c..6f9a2b379 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -161,8 +161,10 @@ def get_llm(provider: str, model: str, api_key: str | None = None, ``max_tokens=2000``, Bedrock reads only its bearer token) and its cache keys, until the judge, generator and validator move to ``LLM``. """ - from dokimi.llm import _GetLLMCompat, _Invoker + from dokimi.llm import LLM, _GetLLMCompat, _Invoker, llm_cli + if llm_cli.backend(provider) is not None: # a logged-in CLI: no route, no key + return _Invoker(LLM(model, provider, temperature=temperature, seed=seed)) api_key = _read_key(api_key) if api_key else api_key max_tokens: int | None = None key: str | None diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 35e376c32..2344cab1c 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -114,6 +114,7 @@ nav: - evaluating-treatments.md - tool-preference.md - coding-harnesses.md + - subscription-backends.md - answer-formats.md - context-window.md - Content novelty: diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index cbab8598c..1e6b0ecd6 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,9 +1,9 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 999, "rationale": "estimate-only projection (expected_usd, arm scaling, extrapolation flag) in novelty CLI, on top of main's W-series and #173 lines, plus W1 routing cost output (#164); file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 998, "rationale": "estimate-only projection (expected_usd, arm scaling, extrapolation flag) in novelty CLI, on top of main's W-series and #173 lines, plus W1 routing cost output (#164); file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 900, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, + "dokimi/llm.py": {"lines": 1091, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1209, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, @@ -19,7 +19,7 @@ "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13}, "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, - "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 21}, + "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 19}, "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 15}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 30, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index e543356fc..cf9a36004 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -69,7 +69,8 @@ # process chokepoints, console output. ("foundation", ( "dokimi", "dokimi._version", "dokimi.artifacts", "dokimi.caching", "dokimi.config", - "dokimi.console", "dokimi.defaults", "dokimi.intents", "dokimi.llm", "dokimi.llm_pricing", + "dokimi.console", "dokimi.defaults", "dokimi.intents", "dokimi.llm", "dokimi.llm_cli", + "dokimi.llm_json", "dokimi.llm_pricing", "dokimi.process", "dokimi.records", "dokimi.registry", "dokimi.utils", # Token/cost record the LLM chokepoint fills; it lives under metrics/ but # depends on nothing and is read by every LLM caller. diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py new file mode 100644 index 000000000..68c16ab50 --- /dev/null +++ b/dokimi/tests/test_llm_cli.py @@ -0,0 +1,201 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for dokimi.llm_cli: a logged-in coding-agent CLI as an LLM provider. + +The payloads are trimmed from real ``claude -p --output-format json`` replies +(Claude Code 2.1.3xx): a success, an unknown model, a failed login. +""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from dokimi import llm, llm_cli, process +from dokimi.process import FakeRunner, reply + + +def _ok(text: str = "PONG", **over) -> str: + payload = { + "type": "result", "is_error": False, "result": text, "stop_reason": "end_turn", + "total_cost_usd": 0.005394, + "usage": {"input_tokens": 2, "cache_creation_input_tokens": 1340, + "cache_read_input_tokens": 10, "output_tokens": 3}, + "modelUsage": {"claude-sonnet-5-5": {"costUSD": 0.005394, "costBasis": "list"}}, + } + payload.update(over) + return json.dumps(payload) + + +def _err(result: str, status) -> str: + return json.dumps({"type": "result", "is_error": True, "result": result, + "api_error_status": status, "usage": {}, "modelUsage": {}}) + + +UNKNOWN_MODEL = _err("There's an issue with the selected model (claude-nope-1). " + "It may not exist or you may not have access to it.", 404) +NO_LOGIN = _err("Authentication error · This may be a temporary network issue, please try again", None) + + +@pytest.fixture +def runner(monkeypatch, tmp_path): + """Every CLI call answers from ``runner.answers`` (a list, consumed in order).""" + monkeypatch.setattr(llm_cli.tempfile, "gettempdir", lambda: str(tmp_path)) + fake = FakeRunner(lambda call: fake.answers.pop(0)) + fake.answers = [] # type: ignore[attr-defined] + monkeypatch.setattr(llm_cli.process, "run", fake) + return fake + + +def test_a_reply_reads_text_tokens_and_model_and_leaves_cost_unknown(runner): + runner.answers.append(reply(_ok())) + out = llm_cli.call("claude-cli", model="sonnet", prompt="ping", system="Be terse.") + assert out.text == "PONG" + assert out.model_id == "claude-sonnet-5-5" + u = out.usage + assert (u.prompt_tokens, u.completion_tokens, u.total_tokens) == (1352, 3, 1355) + assert (u.cache_write_tokens, u.cache_read_tokens) == (1340, 10) + # A subscription is not billed per call: the CLI's figure is list price, kept apart. + assert u.cost_usd is None and u.cost_known_calls == 0 + assert out.list_cost_usd == pytest.approx(0.005394) + assert (u.model, u.provider) == ("sonnet", "claude-cli") + + +def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): + monkeypatch.setenv("CLAUDECODE", "1") + monkeypatch.setenv("CLAUDE_CODE_SESSION_ID", "parent") + monkeypatch.setenv("DOKIMI_TEST_KEEP", "kept") + runner.answers.extend([reply(_ok()), reply(_ok())]) + llm_cli.call("claude-cli", model="haiku", prompt="x" * 300_000, system="Grade it.") + llm_cli.call("claude-cli", model="haiku", prompt="again") + first, second = runner.calls + argv = list(first.argv) + assert argv[:2] == ["claude", "-p"] and "--bare" not in argv + for flag in ("--strict-mcp-config", "--no-session-persistence"): + assert flag in argv + assert argv[argv.index("--tools") + 1] == "" and argv[argv.index("--setting-sources") + 1] == "" + assert argv[argv.index("--system-prompt") + 1] == "Grade it." + assert first.input == b"x" * 300_000 and "x" * 100 not in " ".join(argv) + assert "CLAUDECODE" not in first.env and "CLAUDE_CODE_SESSION_ID" not in first.env + assert first.env["DOKIMI_TEST_KEEP"] == "kept" + # One fixed, empty directory: the cwd the CLI states is the same in every call. + assert first.cwd == second.cwd and list(first.cwd.iterdir()) == [] + # No system prompt is still a system prompt: never the CLI's own agent prompt. + assert "--system-prompt" in second.argv + + +@pytest.mark.parametrize("answer, retryable, words", [ + (reply(UNKNOWN_MODEL, returncode=1), False, "selected model"), + (reply(NO_LOGIN, returncode=1), False, "setup-token"), + (reply(_err("Rate limit reached", 429), returncode=1), True, "Rate limit"), + (reply(_err("Overloaded", 529), returncode=1), True, "Overloaded"), + (reply("", timed_out=True), True, "no reply"), + (reply("segfault", stderr="boom", returncode=139), False, "no JSON reply"), +]) +def test_a_failed_call_says_whether_another_attempt_can_help(runner, answer, retryable, words): + runner.answers.append(answer) + with pytest.raises(llm_cli.CliCallError, match=words) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is retryable + assert llm.is_retryable(info.value) is retryable + + +def test_a_missing_binary_is_a_permanent_error(monkeypatch): + def missing(call): + raise FileNotFoundError("claude") + monkeypatch.setattr(llm_cli.process, "run", FakeRunner(missing)) + with pytest.raises(llm_cli.CliCallError, match="not installed") as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False + + +def test_llm_routes_a_cli_provider_retries_it_and_meters_it(runner): + runner.answers.extend([reply(_err("Rate limit reached", 429), returncode=1), reply(_ok("4"))]) + model = llm.LLM("sonnet", provider="claude-cli", max_tokens=50, retry_delay=0) + out = model.complete("2+2?", system="Answer with a number.") + assert out.text == "4" and len(runner.calls) == 2 + assert runner.calls[1].input == b"2+2?" + assert model.metrics().completion_tokens == 3 and model.metrics().cost_usd is None + + +def test_tools_or_a_conversation_are_refused_before_any_call(runner): + model = llm.LLM("sonnet", provider="claude-cli") + with pytest.raises(llm_cli.CliCallError, match="one prompt"): + model.chat([{"role": "user", "content": "hi"}], + tools=[{"type": "function", "function": {"name": "f"}}]) + with pytest.raises(llm_cli.CliCallError, match="one prompt"): + model.chat([{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}, + {"role": "user", "content": "c"}]) + assert runner.calls == [] + + +def test_sampling_the_cli_cannot_honour_is_named(): + assert llm.unsupported_sampling_params("sonnet", "claude-cli", temperature=0.0, seed=7) == [ + "temperature", "seed"] + assert llm.unsupported_sampling_params("sonnet", "claude-cli") == [] + + +def test_judge_and_get_llm_reach_the_cli_backend(runner): + from dokimi.eval.judge import JUDGE_PROVIDERS, Judge + from dokimi.utils import get_llm + + assert "claude-cli" in JUDGE_PROVIDERS and "claude-cli" in llm.ANSWER_PROVIDERS + runner.answers.extend([reply(_ok("judged")), reply(_ok("generated"))]) + assert Judge(model="sonnet", provider="claude-cli").llm.invoke("grade").content == "judged" + resp = get_llm("claude-cli", "haiku").invoke("write") + assert resp.content == "generated" and resp.usage.provider == "claude-cli" + + +def _auth_runner(status: dict | None, version: str = "2.1.304 (Claude Code)"): + def handle(call): + if call.argv[-1] == "--version": + if status is None: + raise FileNotFoundError("claude") + return reply(version + "\n") + return reply(json.dumps(status)) + return FakeRunner(handle) + + +def test_detect_reports_install_login_and_method_without_a_model_call(): + fake = _auth_runner({"loggedIn": True, "authMethod": "oauth_token"}) + got = llm_cli.detect("claude-cli", runner=fake) + assert (got.installed, got.authenticated, got.method, got.version) == ( + True, True, "oauth_token", "2.1.304 (Claude Code)") + assert got.hint == "" and [c.argv[1:] for c in fake.calls] == [("--version",), ("auth", "status")] + out = llm_cli.detect("claude-cli", runner=_auth_runner({"loggedIn": False})) + assert out.authenticated is False and "setup-token" in out.hint + gone = llm_cli.detect("claude-cli", runner=_auth_runner(None)) + assert gone.installed is False and gone.authenticated is None + + +def test_backends_command_prints_one_row_per_backend(monkeypatch, capsys): + from dokimi.commands.backends import cmd_backends + + monkeypatch.setattr(process, "run", _auth_runner( + {"loggedIn": True, "authMethod": "third_party", "apiProvider": "bedrock"})) + cmd_backends(argparse.Namespace(json=False)) + text = capsys.readouterr().out + assert "| claude-cli | claude | 2.1.304 (Claude Code) | yes | third_party (bedrock) |" in text + cmd_backends(argparse.Namespace(json=True)) + (row,) = json.loads(capsys.readouterr().out) + assert row["backend"] == "claude-cli" and row["authenticated"] is True + + +def test_benchmark_preflight_checks_the_login_not_a_key(monkeypatch): + from dokimi.commands.benchmark import _credential_error + + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": False})) + assert "not signed in" in _credential_error("claude-cli") + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) + assert _credential_error("claude-cli") is None + monkeypatch.setattr(process, "run", _auth_runner(None)) + assert "not installed" in _credential_error("claude-cli") + + +def test_the_default_judge_runs_on_the_cli(): + from dokimi.defaults import DEFAULT_JUDGE_MODEL, default_judge_model + + assert default_judge_model("claude-cli") == DEFAULT_JUDGE_MODEL From c3c88abb7f0d3b388006a6f53119ce945cbc3201 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 22:43:00 -0700 Subject: [PATCH 02/42] fix(llm): claude-cli reaches the answerer and generators, refuses sampling, drops the API key Answerer, PersonaGenerator and QuestionGenerator kept their own provider lists, so answers generate / personas discover / questions generate --provider claude-cli raised "Unsupported provider". A judge built with --judge-temperature or a seed now fails instead of recording a value the CLI never applied. ANTHROPIC_API_KEY is no longer passed to the child: set, it wins over the login and the run bills the key. Signed-off-by: Nikolay Petrov --- dokimi/docs/subscription-backends.md | 3 ++- dokimi/dokimi/eval/answerer.py | 5 +++-- dokimi/dokimi/llm.py | 6 +++--- dokimi/dokimi/llm_cli.py | 6 ++++-- dokimi/dokimi/personas/generator.py | 5 +++-- dokimi/dokimi/questions/llm_gen.py | 5 +++-- dokimi/tests/test_llm_cli.py | 18 +++++++++++++++++- 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 00ffbdb54..387558590 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -83,7 +83,8 @@ carries about 37k tokens of agent prompt, tool schemas and `CLAUDE.md`. - The child gets the caller's environment minus the variables a Claude Code session sets for its children (`CLAUDECODE`, `CLAUDE_CODE_SESSION_ID`, …). If dokimi itself runs inside Claude Code and they were passed on, the child would - join the parent's session. + join the parent's session. `ANTHROPIC_API_KEY` is dropped too: set, it would + win over the login and the run would bill the key. - The CLI keeps its own config directory, because that is where the login is stored. User settings and `CLAUDE.md` stay out because of `--setting-sources ""`. diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py index 1dbdc0612..5e2dca750 100644 --- a/dokimi/dokimi/eval/answerer.py +++ b/dokimi/dokimi/eval/answerer.py @@ -16,6 +16,7 @@ from dokimi.utils import question_id from dokimi.artifacts import save_artifact from dokimi.llm import ChatModel +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS from dokimi.llm import llm_call_with_usage, ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER from dokimi.metrics.usage import UsageRecord, to_record @@ -101,12 +102,12 @@ def __init__( elif provider == "anthropic": self.llm = ChatAnthropic(model=model, api_key=api_key) elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai"): + "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): self.llm = None # llm_call_with_usage handles these natively else: raise ValueError( f"Unsupported provider: '{provider}'. " - "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'." + "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) self.top_k = top_k self.debug_retrieval = debug_retrieval diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 5204ed1c5..9271bab61 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -530,9 +530,9 @@ def _stream(completion: Any, request: Request) -> Reply: def _cli_reply(request: Request) -> Reply: """A request for a CLI provider (:mod:`dokimi.llm_cli`): one prompt, an optional system.""" turns = [m for m in request.messages if m.get("role") != "system"] - if request.tools or len(turns) != 1: - raise llm_cli.CliCallError(f"{request.provider} takes one prompt: no tools, no conversation", - retryable=False) + if request.tools or len(turns) != 1 or request.temperature is not None or request.seed is not None: + raise llm_cli.CliCallError(f"{request.provider} takes one prompt: no tools, no conversation, " + "no temperature or seed", retryable=False) system = [prompt_text(replace(request, messages=[m])) for m in request.messages if m.get("role") == "system"] out = llm_cli.call(request.provider, model=request.model, prompt=prompt_text(request), diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index e5c44f47d..05b1e55b4 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -51,11 +51,13 @@ from dokimi import process from dokimi.metrics.usage import UsageRecord -#: Variables a Claude Code session sets for its own children. Dropped, so a call made -#: from inside a session is a session of its own. +#: Variables a Claude Code session sets for its own children, plus the API key. Dropped, so +#: a call made from inside a session is a session of its own and runs on the CLI's login. CLAUDE_ENV_DROP = frozenset({ "CLAUDECODE", "CLAUDE_CODE_SESSION_ID", "CLAUDE_CODE_CHILD_SESSION", "CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_SSE_PORT", + # Set, it wins over the CLI's login: the run would bill the key, not the subscription. + "ANTHROPIC_API_KEY", }) #: The flags that strip the CLI to a bare completion; see the module docstring. diff --git a/dokimi/dokimi/personas/generator.py b/dokimi/dokimi/personas/generator.py index d7f00f9d1..2936cd3c1 100644 --- a/dokimi/dokimi/personas/generator.py +++ b/dokimi/dokimi/personas/generator.py @@ -9,6 +9,7 @@ import json from dokimi.llm import ChatModel +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.defaults import DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER @@ -105,13 +106,13 @@ def __init__( elif provider == "anthropic": self.llm = ChatAnthropic(model=model, api_key=api_key) elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai"): + "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): from dokimi.utils import get_llm self.llm = get_llm(provider, model, api_key) else: raise ValueError( f"Unsupported provider: '{provider}'. " - "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'." + "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) logger.info(f"Initialized PersonaGenerator with {provider}/{model}") diff --git a/dokimi/dokimi/questions/llm_gen.py b/dokimi/dokimi/questions/llm_gen.py index 9cfcb6680..2ca5a397b 100644 --- a/dokimi/dokimi/questions/llm_gen.py +++ b/dokimi/dokimi/questions/llm_gen.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) from dokimi.llm import ChatModel +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.defaults import DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER @@ -98,7 +99,7 @@ def __init__( elif provider == "anthropic": self.llm = ChatAnthropic(model=model, api_key=api_key) elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai"): + "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): from dokimi.utils import get_llm import os fallback_key = os.environ.get("OPENROUTER_API_KEY") if provider == "openrouter" else None @@ -106,7 +107,7 @@ def __init__( else: raise ValueError( f"Unsupported provider: '{provider}'. " - "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'." + "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) logger.info(f"QuestionGenerator initialized: {provider}/{model}") diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 68c16ab50..a759118c9 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -67,6 +67,7 @@ def test_a_reply_reads_text_tokens_and_model_and_leaves_cost_unknown(runner): def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): monkeypatch.setenv("CLAUDECODE", "1") monkeypatch.setenv("CLAUDE_CODE_SESSION_ID", "parent") + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-would-bill") monkeypatch.setenv("DOKIMI_TEST_KEEP", "kept") runner.answers.extend([reply(_ok()), reply(_ok())]) llm_cli.call("claude-cli", model="haiku", prompt="x" * 300_000, system="Grade it.") @@ -79,7 +80,8 @@ def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): assert argv[argv.index("--tools") + 1] == "" and argv[argv.index("--setting-sources") + 1] == "" assert argv[argv.index("--system-prompt") + 1] == "Grade it." assert first.input == b"x" * 300_000 and "x" * 100 not in " ".join(argv) - assert "CLAUDECODE" not in first.env and "CLAUDE_CODE_SESSION_ID" not in first.env + for name in ("CLAUDECODE", "CLAUDE_CODE_SESSION_ID", "ANTHROPIC_API_KEY"): + assert name not in first.env assert first.env["DOKIMI_TEST_KEEP"] == "kept" # One fixed, empty directory: the cwd the CLI states is the same in every call. assert first.cwd == second.cwd and list(first.cwd.iterdir()) == [] @@ -129,6 +131,9 @@ def test_tools_or_a_conversation_are_refused_before_any_call(runner): with pytest.raises(llm_cli.CliCallError, match="one prompt"): model.chat([{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}, {"role": "user", "content": "c"}]) + from dokimi.eval.judge import Judge + with pytest.raises(llm_cli.CliCallError, match="temperature"): # --judge-temperature + Judge(model="sonnet", provider="claude-cli", temperature=0.0).llm.invoke("grade") assert runner.calls == [] @@ -199,3 +204,14 @@ def test_the_default_judge_runs_on_the_cli(): from dokimi.defaults import DEFAULT_JUDGE_MODEL, default_judge_model assert default_judge_model("claude-cli") == DEFAULT_JUDGE_MODEL + + +def test_answerer_and_generators_accept_the_cli_provider(runner): + from dokimi.eval.answerer import Answerer + from dokimi.personas.generator import PersonaGenerator + from dokimi.questions.llm_gen import QuestionGenerator + + Answerer(model="haiku", provider="claude-cli") + runner.answers.extend([reply(_ok("persona")), reply(_ok("question"))]) + assert PersonaGenerator(model="haiku", provider="claude-cli").llm.invoke("p").content == "persona" + assert QuestionGenerator(model="haiku", provider="claude-cli").llm.invoke("q").content == "question" From aab69f7b92dfd6ded358f68a02ec5756ab05a543 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 23:00:11 -0700 Subject: [PATCH 03/42] fix(judge): refuse --judge-temperature and --seed for a CLI judge before the run The refusal was per call, so eval score exited 0 with an error row per answer and recorded a judge_temperature the CLI never applied. Tests now also pin --output-format json and the whole env drop list. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/judge.py | 4 +++- dokimi/tests/test_llm_cli.py | 15 ++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py index 428e15773..e769b0bde 100644 --- a/dokimi/dokimi/eval/judge.py +++ b/dokimi/dokimi/eval/judge.py @@ -244,6 +244,8 @@ def __init__( "redrawn_judgements": 0, } + if provider in CLI_BACKENDS and (temperature is not None or seed is not None): + raise ValueError(f"{provider} takes no temperature or seed: drop --judge-temperature/--seed") self.llm: ChatModel if provider == "openai": self.llm = ChatOpenAI(model=model, api_key=api_key, @@ -261,7 +263,7 @@ def __init__( else: raise ValueError( f"Unsupported provider: '{provider}'. " - "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'." + "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) logger.info(f"Judge initialized: {provider}/{model}") diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index a759118c9..d438580a9 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -66,7 +66,8 @@ def test_a_reply_reads_text_tokens_and_model_and_leaves_cost_unknown(runner): def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): monkeypatch.setenv("CLAUDECODE", "1") - monkeypatch.setenv("CLAUDE_CODE_SESSION_ID", "parent") + for name in llm_cli.CLAUDE_ENV_DROP: + monkeypatch.setenv(name, "parent") monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-would-bill") monkeypatch.setenv("DOKIMI_TEST_KEEP", "kept") runner.answers.extend([reply(_ok()), reply(_ok())]) @@ -77,11 +78,12 @@ def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): assert argv[:2] == ["claude", "-p"] and "--bare" not in argv for flag in ("--strict-mcp-config", "--no-session-persistence"): assert flag in argv + assert argv[argv.index("--output-format") + 1] == "json" assert argv[argv.index("--tools") + 1] == "" and argv[argv.index("--setting-sources") + 1] == "" assert argv[argv.index("--system-prompt") + 1] == "Grade it." assert first.input == b"x" * 300_000 and "x" * 100 not in " ".join(argv) - for name in ("CLAUDECODE", "CLAUDE_CODE_SESSION_ID", "ANTHROPIC_API_KEY"): - assert name not in first.env + assert not llm_cli.CLAUDE_ENV_DROP & set(first.env) + assert {"CLAUDE_CODE_ENTRYPOINT", "ANTHROPIC_API_KEY"} <= llm_cli.CLAUDE_ENV_DROP assert first.env["DOKIMI_TEST_KEEP"] == "kept" # One fixed, empty directory: the cwd the CLI states is the same in every call. assert first.cwd == second.cwd and list(first.cwd.iterdir()) == [] @@ -132,8 +134,11 @@ def test_tools_or_a_conversation_are_refused_before_any_call(runner): model.chat([{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}, {"role": "user", "content": "c"}]) from dokimi.eval.judge import Judge - with pytest.raises(llm_cli.CliCallError, match="temperature"): # --judge-temperature - Judge(model="sonnet", provider="claude-cli", temperature=0.0).llm.invoke("grade") + for sampling in ({"temperature": 0.0}, {"seed": 7}): # --judge-temperature / --seed: before the run + with pytest.raises(ValueError, match="temperature or seed"): + Judge(model="sonnet", provider="claude-cli", **sampling) + with pytest.raises(llm_cli.CliCallError, match="temperature"): + llm.LLM("sonnet", provider="claude-cli", seed=1).complete("x") assert runner.calls == [] From e9f759a52920eccbba29d79ace86448d8abb2932 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 26 Oct 2026 23:06:44 -0700 Subject: [PATCH 04/42] fix(judge): the CLI sampling refusal covers eval panel-score too JudgePanel builds its judges without Judge.__init__, so panel-score with --judge-temperature still ran and recorded the value. The check moves to llm_cli.check_sampling, called by both. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/eval/judge.py | 5 ++--- dokimi/dokimi/eval/panel.py | 3 +++ dokimi/dokimi/llm_cli.py | 6 ++++++ dokimi/tests/test_llm_cli.py | 4 ++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py index e769b0bde..4cc4d8eeb 100644 --- a/dokimi/dokimi/eval/judge.py +++ b/dokimi/dokimi/eval/judge.py @@ -18,7 +18,7 @@ from dokimi import caching from dokimi.artifacts import save_artifact from dokimi.llm import ChatModel, ask_parsed -from dokimi.llm_cli import BACKENDS as CLI_BACKENDS +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS, check_sampling from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.eval.diagnoser import diagnose from dokimi.eval.question_text import require_question_texts @@ -244,8 +244,7 @@ def __init__( "redrawn_judgements": 0, } - if provider in CLI_BACKENDS and (temperature is not None or seed is not None): - raise ValueError(f"{provider} takes no temperature or seed: drop --judge-temperature/--seed") + check_sampling(provider, temperature, seed) self.llm: ChatModel if provider == "openai": self.llm = ChatOpenAI(model=model, api_key=api_key, diff --git a/dokimi/dokimi/eval/panel.py b/dokimi/dokimi/eval/panel.py index 4aa0b0ed4..d107c3882 100644 --- a/dokimi/dokimi/eval/panel.py +++ b/dokimi/dokimi/eval/panel.py @@ -27,6 +27,7 @@ from dokimi import caching from dokimi.llm import ask_parsed, llm_call +from dokimi.llm_cli import check_sampling from dokimi.defaults import DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER from dokimi.eval.question_text import require_question_texts # The panel had no numeric-claims criterion (see `eval/rubric.py`) until it was @@ -203,6 +204,8 @@ def __init__( temperature=temperature, seed=seed) for role in DEFAULT_PANEL ] + for j in self.judges: + check_sampling(j.provider, j.temperature, j.seed) self.concurrency = concurrency @property diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 05b1e55b4..df80b9cc9 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -208,6 +208,12 @@ def backend(provider: str) -> Optional[CliBackend]: return BACKENDS.get(provider) +def check_sampling(provider: str, temperature: Optional[float], seed: Optional[int]) -> None: + """Raise ``ValueError`` before a run when a CLI *provider* is given a temperature or seed.""" + if provider in BACKENDS and (temperature is not None or seed is not None): + raise ValueError(f"{provider} takes no temperature or seed: drop --judge-temperature/--seed") + + def work_dir(name: str) -> Path: """The fixed empty working directory a backend's calls run in. diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index d438580a9..02fffa6c0 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -134,9 +134,13 @@ def test_tools_or_a_conversation_are_refused_before_any_call(runner): model.chat([{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}, {"role": "user", "content": "c"}]) from dokimi.eval.judge import Judge + from dokimi.eval.panel import JudgePanel for sampling in ({"temperature": 0.0}, {"seed": 7}): # --judge-temperature / --seed: before the run with pytest.raises(ValueError, match="temperature or seed"): Judge(model="sonnet", provider="claude-cli", **sampling) + with pytest.raises(ValueError, match="temperature or seed"): # eval panel-score + JudgePanel(default_provider="claude-cli", **sampling) + JudgePanel(default_provider="claude-cli") with pytest.raises(llm_cli.CliCallError, match="temperature"): llm.LLM("sonnet", provider="claude-cli", seed=1).complete("x") assert runner.calls == [] From ea38db4bf32da18169f2c8ff26cd96539810cf82 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 27 Oct 2026 10:38:19 +0000 Subject: [PATCH 05/42] test(llm_cli): keep the missing-binary test's work dir in tmp_path Signed-off-by: Anton Volkov --- dokimi/tests/test_llm_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 02fffa6c0..9fe08ddaf 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -107,7 +107,9 @@ def test_a_failed_call_says_whether_another_attempt_can_help(runner, answer, ret assert llm.is_retryable(info.value) is retryable -def test_a_missing_binary_is_a_permanent_error(monkeypatch): +def test_a_missing_binary_is_a_permanent_error(monkeypatch, tmp_path): + monkeypatch.setattr(llm_cli.tempfile, "gettempdir", lambda: str(tmp_path)) + def missing(call): raise FileNotFoundError("claude") monkeypatch.setattr(llm_cli.process, "run", FakeRunner(missing)) From cb986286a9ef5eb923214aa380eb79748d27e7b6 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 27 Oct 2026 10:38:21 +0000 Subject: [PATCH 06/42] docs(spec): update the measured doc count to 33 Signed-off-by: Anton Volkov --- dokimi/benchmarks/spec.v1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index bdefa65c0..253aae53a 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,7 +76,7 @@ future: # on the include/exclude globs, not a target: min_docs catches a glob that # matched almost nothing, max_docs catches one that escaped into the tree (8 # markdown files live outside docs/). Measured selection at the time of -# writing: 32 files. Adding a doc past the ceiling is meant to be a deliberate +# writing: 33 files. Adding a doc past the ceiling is meant to be a deliberate # edit here, so raise max_docs in the same commit that adds the doc. golden_manifest: min_docs: 10 From 35c5219b0b403a3578c304c628eeab7f69bf4467 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 00:11:10 -0700 Subject: [PATCH 07/42] fix(llm): claude-cli review: key, work dir, sampling, model, login warning - A CLI provider never reads an API key: OPENAI_API_KEY (the fallback for an unknown provider) no longer fails a call with a missing key file. - The work dir moves from the shared temp dir to the user's cache dir (0700); a dir that cannot be made is a permanent CliCallError, not an OSError. - orchestrate evaluate refuses --judge-temperature for a CLI judge before any persona, question or answer is paid for. - parse_args refuses a non-Claude --model for --provider claude-cli (the default answer model is not a Claude model). - Benchmark preflight warns when `claude auth status` gives no answer, instead of passing silently. - The env drop list also takes ANTHROPIC_AUTH_TOKEN and the Claude Code messaging, pid, effort and attended variables. - Tests: each retryable status and word alone, is_error and the exit code each alone, a judge panel with explicit judges, the work-dir failure. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/cli.py | 7 ++- dokimi/dokimi/commands/benchmark.py | 13 +++++ dokimi/dokimi/commands/orchestrate.py | 6 +++ dokimi/dokimi/llm.py | 4 +- dokimi/dokimi/llm_cli.py | 25 +++++++-- dokimi/tests/test_llm_cli.py | 77 +++++++++++++++++++++++++-- 6 files changed, 121 insertions(+), 11 deletions(-) diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index e6dd9e18e..8b90c0d83 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -36,7 +36,7 @@ tasks, tools, ) -from dokimi import __version__, process +from dokimi import __version__, llm_cli, process from dokimi.console import tolerate_console_encoding from dokimi.defaults import JUDGE_MODEL_FOR_PROVIDER, default_judge_model @@ -101,6 +101,11 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.error(f"--judge-provider {args.judge_provider} has no default judge model; " f"pass --judge-model") args.judge_model = model + provider, model = getattr(args, "provider", None), getattr(args, "model", None) + if provider in llm_cli.BACKENDS and model is not None and not llm_cli.accepts_model(provider, model): + # The default answer model is not a Claude model: left out, every call would fail. + parser.error(f"--provider {provider} takes a Claude model (sonnet, haiku, opus or a claude-* id), " + f"not {model!r}: pass --model") return args diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index 610537dbc..474623b5c 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -98,6 +98,18 @@ def _credential_error(provider: str) -> str | None: return None +def _login_warning(provider: str) -> str | None: + """A warning when a CLI provider's login could not be checked (no answer from ``auth status``).""" + from dokimi import llm_cli + + if llm_cli.backend(provider) is None: + return None + status = llm_cli.detect(provider) + if status.installed and status.authenticated is None: + return f"{provider}: could not check the login ('{status.binary} auth status' gave no answer)" + return None + + def _resolved_benchmark_plan(entry, args: argparse.Namespace) -> dict: doc_source = getattr(args, "doc_source", None) or (entry.doc_sources[0] if entry.doc_sources else "context7") output_dir = getattr(args, "output_dir", None) or f"results/{entry.key}{DEFAULT_RESULTS_SUFFIX}" @@ -158,6 +170,7 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None: warnings: list[str] = [] errors.extend(e for e in map(_credential_error, sorted({args.provider, args.judge_provider})) if e) + warnings.extend(w for w in map(_login_warning, sorted({args.provider, args.judge_provider})) if w) try: Path(plan["output_dir"]).mkdir(parents=True, exist_ok=True) diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py index b03843517..b037d6854 100644 --- a/dokimi/dokimi/commands/orchestrate.py +++ b/dokimi/dokimi/commands/orchestrate.py @@ -10,6 +10,7 @@ from pathlib import Path from dokimi.llm import ANSWER_PROVIDERS +from dokimi.llm_cli import check_sampling from dokimi.commands.evaluate import _warn_judge_independence from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -28,6 +29,11 @@ def cmd_evaluate(args: argparse.Namespace) -> None: if not getattr(args, "repo", None) and not getattr(args, "description", None): print("Error: either --repo or --description must be provided.", file=sys.stderr) sys.exit(1) + try: # before any persona, question or answer is paid for; the judge would refuse later + check_sampling(args.judge_provider, getattr(args, "judge_temperature", None), None) + except ValueError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) print(f"Starting full evaluation pipeline for {args.product}") if args.repo: diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 9271bab61..983c19933 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -122,12 +122,12 @@ def api_key_env(provider: str) -> str: def resolve_api_key(provider: str, api_key: Optional[str] = None) -> Optional[str]: """*api_key*, else the provider's environment variable; a ``file:`` value is read. - Returns ``None`` when there is no key. + Returns ``None`` when there is no key; a CLI provider takes its login, never a key. Raises: ValueError: a ``file:`` key is unreadable. """ - key = api_key or os.environ.get(api_key_env(provider), "") + key = "" if llm_cli.backend(provider) else (api_key or os.environ.get(api_key_env(provider), "")) if key.startswith("file:"): path = os.path.expanduser(key[len("file:"):]) try: diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index df80b9cc9..530c5df4e 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -43,7 +43,6 @@ import json import os -import tempfile from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Dict, Optional, Tuple @@ -56,8 +55,10 @@ CLAUDE_ENV_DROP = frozenset({ "CLAUDECODE", "CLAUDE_CODE_SESSION_ID", "CLAUDE_CODE_CHILD_SESSION", "CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_SSE_PORT", - # Set, it wins over the CLI's login: the run would bill the key, not the subscription. - "ANTHROPIC_API_KEY", + "CLAUDE_CODE_MESSAGING_SOCKET", "CLAUDE_CODE_MESSAGING_TOKEN", "CLAUDE_PID", + "CLAUDE_EFFORT", "CLAUDE_CODE_SESSION_ATTENDED", + # Set, they win over the CLI's login: the run would bill the key or token, not the subscription. + "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", }) #: The flags that strip the CLI to a bare completion; see the module docstring. @@ -117,6 +118,7 @@ class CliBackend: supports: frozenset = frozenset() # of "temperature", "seed", "max_tokens" login_hint: str = "" aliases: Tuple[str, ...] = () + model_prefixes: Tuple[str, ...] = () # full model ids the CLI takes, by prefix # ── claude-cli ──────────────────────────────────────────────────────────────── @@ -197,6 +199,7 @@ def _claude_auth(done: process.Completed) -> Tuple[Optional[bool], str]: env_drop=CLAUDE_ENV_DROP, login_hint="claude setup-token (headless) or claude, then /login", aliases=("haiku", "sonnet", "opus"), + model_prefixes=("claude-",), ) #: CLI backends by provider name. @@ -208,6 +211,12 @@ def backend(provider: str) -> Optional[CliBackend]: return BACKENDS.get(provider) +def accepts_model(provider: str, model: str) -> bool: + """Whether *provider*'s CLI takes *model*: one of its aliases, or a full id it names.""" + cli = BACKENDS[provider] + return model in cli.aliases or model.startswith(cli.model_prefixes) + + def check_sampling(provider: str, temperature: Optional[float], seed: Optional[int]) -> None: """Raise ``ValueError`` before a run when a CLI *provider* is given a temperature or seed.""" if provider in BACKENDS and (temperature is not None or seed is not None): @@ -219,9 +228,15 @@ def work_dir(name: str) -> Path: Fixed, not fresh per call: the CLI states its working directory in the prompt, and a random path would make every request, and its response-cache key, differ. + Under the user's cache dir, not the shared temp dir: another account there could + pre-create it, and the CLI reads the git status of its working directory into the prompt. """ - path = Path(tempfile.gettempdir()) / "dokimi-cli" / name - path.mkdir(parents=True, exist_ok=True) + base = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") + path = base / "dokimi" / "cli" / name + try: + path.mkdir(parents=True, exist_ok=True, mode=0o700) + except OSError as exc: + raise CliCallError(f"{name}: cannot create the work dir {path} ({exc})", retryable=False) from exc return path diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 9fe08ddaf..3955191c8 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -43,7 +43,7 @@ def _err(result: str, status) -> str: @pytest.fixture def runner(monkeypatch, tmp_path): """Every CLI call answers from ``runner.answers`` (a list, consumed in order).""" - monkeypatch.setattr(llm_cli.tempfile, "gettempdir", lambda: str(tmp_path)) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) fake = FakeRunner(lambda call: fake.answers.pop(0)) fake.answers = [] # type: ignore[attr-defined] monkeypatch.setattr(llm_cli.process, "run", fake) @@ -83,7 +83,8 @@ def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): assert argv[argv.index("--system-prompt") + 1] == "Grade it." assert first.input == b"x" * 300_000 and "x" * 100 not in " ".join(argv) assert not llm_cli.CLAUDE_ENV_DROP & set(first.env) - assert {"CLAUDE_CODE_ENTRYPOINT", "ANTHROPIC_API_KEY"} <= llm_cli.CLAUDE_ENV_DROP + assert {"CLAUDE_CODE_ENTRYPOINT", "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_MESSAGING_TOKEN", "CLAUDE_PID"} <= llm_cli.CLAUDE_ENV_DROP assert first.env["DOKIMI_TEST_KEEP"] == "kept" # One fixed, empty directory: the cwd the CLI states is the same in every call. assert first.cwd == second.cwd and list(first.cwd.iterdir()) == [] @@ -107,8 +108,51 @@ def test_a_failed_call_says_whether_another_attempt_can_help(runner, answer, ret assert llm.is_retryable(info.value) is retryable +@pytest.mark.parametrize("status", sorted(llm_cli._RETRYABLE_STATUS)) +def test_each_retryable_status_is_retried_whatever_the_message(runner, status): + runner.answers.append(reply(_err("something went wrong", status), returncode=1)) + with pytest.raises(llm_cli.CliCallError) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is True + + +@pytest.mark.parametrize("word", llm_cli._RETRYABLE_WORDS) +def test_each_retryable_word_is_retried_without_a_status(runner, word): + runner.answers.append(reply(_err(f"API said: {word}", None), returncode=1)) + with pytest.raises(llm_cli.CliCallError) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is True + + +@pytest.mark.parametrize("answer", [ + reply(_ok("bad", is_error=True), returncode=0), # is_error alone, exit 0 + reply(_ok("boom"), returncode=1), # exit 1 alone, is_error false +]) +def test_is_error_and_the_exit_code_each_mark_a_failure(runner, answer): + runner.answers.append(answer) + with pytest.raises(llm_cli.CliCallError) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False + + +def test_an_unusable_work_dir_is_a_permanent_cli_error(runner, monkeypatch, tmp_path): + blocker = tmp_path / "a-file" + blocker.write_text("not a directory", encoding="utf-8") + monkeypatch.setenv("XDG_CACHE_HOME", str(blocker)) + with pytest.raises(llm_cli.CliCallError, match="work dir") as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False and runner.calls == [] + + +def test_a_cli_provider_never_reads_an_api_key_file(runner, monkeypatch, tmp_path): + monkeypatch.setenv("OPENAI_API_KEY", f"file:{tmp_path / 'missing'}") + assert llm.resolve_api_key("claude-cli") is None + runner.answers.append(reply(_ok("4"))) + assert llm.LLM("sonnet", provider="claude-cli", max_tokens=50).complete("2+2?").text == "4" + + def test_a_missing_binary_is_a_permanent_error(monkeypatch, tmp_path): - monkeypatch.setattr(llm_cli.tempfile, "gettempdir", lambda: str(tmp_path)) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) def missing(call): raise FileNotFoundError("claude") @@ -211,12 +255,39 @@ def test_benchmark_preflight_checks_the_login_not_a_key(monkeypatch): assert "not installed" in _credential_error("claude-cli") +def test_a_judge_panel_refuses_sampling_on_the_cli_with_explicit_judges(): + from dokimi.eval.panel import JudgeConfig, JudgePanel + + with pytest.raises(ValueError, match="takes no temperature"): + JudgePanel(judges=[JudgeConfig(role="a", model="sonnet", provider="claude-cli", temperature=0.0)]) + + +def test_an_unanswered_login_check_is_a_warning_not_a_pass(monkeypatch): + from dokimi.commands.benchmark import _credential_error, _login_warning + + monkeypatch.setattr(process, "run", _auth_runner("no answer")) + assert _credential_error("claude-cli") is None + assert "could not check the login" in _login_warning("claude-cli") + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) + assert _login_warning("claude-cli") is None + + def test_the_default_judge_runs_on_the_cli(): from dokimi.defaults import DEFAULT_JUDGE_MODEL, default_judge_model assert default_judge_model("claude-cli") == DEFAULT_JUDGE_MODEL +def test_a_cli_provider_needs_a_claude_model_on_the_command_line(): + from dokimi.cli import parse_args + + base = ["answers", "generate", "--product", "p", "--questions", "q.json", "--provider", "claude-cli"] + with pytest.raises(SystemExit): + parse_args(base) # the default answer model is not a Claude model + assert parse_args([*base, "--model", "haiku"]).model == "haiku" + assert parse_args([*base, "--model", "claude-sonnet-5-5"]).model == "claude-sonnet-5-5" + + def test_answerer_and_generators_accept_the_cli_provider(runner): from dokimi.eval.answerer import Answerer from dokimi.personas.generator import PersonaGenerator From fff2f195857583764edd9689ed7acab2a67daea7 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 28 Oct 2026 10:30:06 +0000 Subject: [PATCH 08/42] docs(subscription-backends): name the per-user work dir Signed-off-by: Anton Volkov --- dokimi/docs/subscription-backends.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 387558590..521af5d34 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -63,7 +63,8 @@ What this means for results: `--exclude-dynamic-system-prompt-sections`. Tested and not working: `CLAUDE_CODE_HIDE_CWD`, `CLAUDE_CODE_DISABLE_GIT_INSTRUCTIONS` (Claude Code 2.1.304). dokimi runs every call in one fixed empty directory - (`/dokimi-cli/claude-cli`), so the stated environment is the same in the + (`$XDG_CACHE_HOME/dokimi/cli/claude-cli`, default + `~/.cache/dokimi/cli/claude-cli`), so the stated environment is the same in the with-docs and the without-docs arm, and the response cache still hits. A without-docs answer may still mention it ("the working directory isn't a git repository"). From 9147dfa48aea869b2feb31320e15cee96d31d3bc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 16:32:57 -0700 Subject: [PATCH 09/42] fix(llm): claude-cli refuses what it cannot run, and keeps its env to itself - Agentic arms (mcp_agent, skill_agent, skill_catalog) and --session-turns are refused on a CLI provider in arms run/matrix-run before any answer. - The child env drops ANTHROPIC_*, CLAUDE_* and CLAUDECODE by prefix, keeping CLAUDE_CONFIG_DIR and CLAUDE_CODE_OAUTH_TOKEN. - An alias and its full id are one model for the independence warning and the novelty self-judge refusal; the novelty default judge on a CLI is another family. - A usage-limit error is not retried; the reply names the main model the CLI used. - Preflight checks the login through _provider_checks; tests pin the retry lists. Signed-off-by: Nikolay Petrov --- ...1026_150000_napetrov_claude-cli-backend.md | 8 + dokimi/dokimi/commands/arms.py | 1 + dokimi/dokimi/commands/arms_refusals.py | 45 +++++ dokimi/dokimi/commands/benchmark.py | 14 +- dokimi/dokimi/commands/evaluate.py | 5 +- dokimi/dokimi/commands/novelty.py | 4 +- dokimi/dokimi/llm.py | 4 +- dokimi/dokimi/llm_cli.py | 72 +++++++- dokimi/dokimi/novelty/analyze.py | 4 +- dokimi/dokimi/novelty/client.py | 26 ++- dokimi/dokimi/novelty/measure.py | 4 +- dokimi/scripts/growth_baseline.json | 6 +- dokimi/tests/test_llm_cli.py | 163 +++++++++++++++++- 13 files changed, 330 insertions(+), 26 deletions(-) create mode 100644 dokimi/dokimi/commands/arms_refusals.py diff --git a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md index af7e268e5..c610906aa 100644 --- a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md +++ b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md @@ -17,3 +17,11 @@ - The `--provider` choices of every command come from one tuple, `dokimi.llm.ANSWER_PROVIDERS`. `tools` keeps API providers only, since it needs tool calling. +- A CLI provider refuses an agentic arm (`mcp_agent`, `skill_agent`, `skill_catalog`) and + `--session-turns` in `arms run` and `arms matrix-run`, before any answer is paid for: the CLI + runs one prompt with no tools and no conversation. +- Independence checks treat an alias and its full id as one model (`opus` and + `claude-opus-…`). The novelty default judge on `claude-cli` is another Claude family than the answerer. +- The CLI child process drops `ANTHROPIC_*`, `CLAUDE_*` and `CLAUDECODE` variables from the + environment (except `CLAUDE_CONFIG_DIR` and `CLAUDE_CODE_OAUTH_TOKEN`), so a billing or + session setting in the caller does not reroute the call. diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 1d941bda6..55e0370d3 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -627,6 +627,7 @@ def _build_arms_output(args: argparse.Namespace, specs: list[str], questions: li plugins = create_plugins([s.strip() for s in args.plugins.split(",") if s.strip()]) validate_plugins_for_harness(plugins, args.harness) treatments = wrap_treatments(treatments, plugins) + refuse_cli_arms(args, treatments) plugin_set = plugin_set_metadata(plugins) print(f"Arms: {', '.join(t.name for t in treatments)}") if args.matrix_cell: diff --git a/dokimi/dokimi/commands/arms_refusals.py b/dokimi/dokimi/commands/arms_refusals.py new file mode 100644 index 000000000..e0da8ae4f --- /dev/null +++ b/dokimi/dokimi/commands/arms_refusals.py @@ -0,0 +1,45 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Refusals for the arms commands on a CLI provider, raised before any answer is paid for.""" + +from __future__ import annotations + +import argparse + + +def _is_agentic_arm(treatment) -> bool: + """True when the arm offers tools, so it runs the agentic loop (see ``AgentConfig.tools``). + + The tools are added in ``prepare()``, which needs a question and may retrieve, so the + arm types that offer tools are named here instead of probed per question. + """ + from dokimi.plugins import PluginWrappedTreatment + from dokimi.treatments.arms import ( + CombinedTreatment, + MCPAgentTreatment, + SkillAgentTreatment, + SkillCatalogTreatment, + ) + + inner = treatment.inner if isinstance(treatment, PluginWrappedTreatment) else treatment + if isinstance(inner, CombinedTreatment): + return any(_is_agentic_arm(sub) for sub in inner.treatments) + return isinstance(inner, (MCPAgentTreatment, SkillAgentTreatment, SkillCatalogTreatment)) + + +def refuse_cli_arms(args: argparse.Namespace, treatments: list) -> None: + """Refused before any answer: a CLI provider runs one prompt, with no tools and no conversation.""" + from dokimi import llm_cli + + if llm_cli.backend(args.provider) is None: + return + agentic = [t.name for t in treatments if _is_agentic_arm(t)] + if agentic: + raise ValueError(f"--provider {args.provider} runs one prompt with no tools, so it cannot run " + f"the agentic arm(s) {', '.join(agentic)}: use an API provider for them") + if getattr(args, "session_turns", None) is not None: + raise ValueError(f"--session-turns needs a conversation, which --provider {args.provider} " + "does not have: use an API provider for a session") + + diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index 474623b5c..bddb78fa6 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -98,6 +98,14 @@ def _credential_error(provider: str) -> str | None: return None +def _provider_checks(providers: list[str]) -> tuple[list[str], list[str]]: + """The errors (a missing key, a CLI not installed or signed in) and warnings for *providers*.""" + names = sorted(set(providers)) + errors = [e for e in map(_credential_error, names) if e] + warnings = [w for w in map(_login_warning, names) if w] + return errors, warnings + + def _login_warning(provider: str) -> str | None: """A warning when a CLI provider's login could not be checked (no answer from ``auth status``).""" from dokimi import llm_cli @@ -166,11 +174,7 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None: plan = _resolved_benchmark_plan(entry, args) _print_plan(plan) - errors: list[str] = [] - warnings: list[str] = [] - - errors.extend(e for e in map(_credential_error, sorted({args.provider, args.judge_provider})) if e) - warnings.extend(w for w in map(_login_warning, sorted({args.provider, args.judge_provider})) if w) + errors, warnings = _provider_checks([args.provider, args.judge_provider]) try: Path(plan["output_dir"]).mkdir(parents=True, exist_ok=True) diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index e5fcfddc8..0652e0101 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -32,7 +32,10 @@ def _warn_judge_independence( context: str = "run", ) -> bool: """Emit a non-fatal warning when answer and judge use the same model/provider.""" - same = normalize_model_ref(answer_provider, answer_model) == normalize_model_ref(judge_provider, judge_model) + from dokimi.llm_cli import same_model + + same = (normalize_model_ref(answer_provider, answer_model) == normalize_model_ref(judge_provider, judge_model) + or same_model(answer_provider, answer_model, judge_provider, judge_model)) if same: print( "⚠ Evaluator independence warning " diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 804fa396a..61a73f426 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -1006,8 +1006,8 @@ def _role_models(args: argparse.Namespace) -> dict[str, tuple[str, str]]: if args.judge_model: judge_model, judge_provider = args.judge_model, args.judge_provider or args.provider else: - try: - judge_model, judge_provider = default_judge(args.model, args.judge_provider) + try: # a CLI answerer with no judge provider judges on the same CLI, another family + judge_model, judge_provider = default_judge(args.model, args.judge_provider, args.provider) except ValueError as exc: raise SystemExit(str(exc)) from None extract_model = args.extract_model or judge_model diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 983c19933..22da74e3f 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -452,6 +452,8 @@ class Reply: finish_reason: Optional[str] = None tool_calls: Tuple[ToolCall, ...] = () response: Any = field(default=None, compare=False, repr=False) + #: The model the provider resolved to, when it says (a CLI names the version behind an alias). + model_id: Optional[str] = None @property def truncated(self) -> bool: @@ -537,7 +539,7 @@ def _cli_reply(request: Request) -> Reply: for m in request.messages if m.get("role") == "system"] out = llm_cli.call(request.provider, model=request.model, prompt=prompt_text(request), system="\n\n".join(system) or None) - return Reply(text=out.text, usage=out.usage, finish_reason=out.finish_reason) + return Reply(text=out.text, usage=out.usage, finish_reason=out.finish_reason, model_id=out.model_id) class LiteLLMAdapter: diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 530c5df4e..335c374ad 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -60,6 +60,13 @@ # Set, they win over the CLI's login: the run would bill the key or token, not the subscription. "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", }) +#: Dropped by prefix, so a variable not yet listed (an endpoint, a model alias override, a +#: provider switch such as CLAUDE_CODE_USE_BEDROCK) cannot change who is billed or which +#: session the child joins. +CLAUDE_ENV_DROP_PREFIXES = ("ANTHROPIC_", "CLAUDE_", "CLAUDECODE") +#: Kept despite a prefix: where the account's login lives, and the token ``claude setup-token`` +#: makes for a headless login. +CLAUDE_ENV_KEEP = frozenset({"CLAUDE_CONFIG_DIR", "CLAUDE_CODE_OAUTH_TOKEN"}) #: The flags that strip the CLI to a bare completion; see the module docstring. CLAUDE_ISOLATION = ("--tools", "", "--setting-sources", "", "--strict-mcp-config", @@ -115,6 +122,8 @@ class CliBackend: auth_argv: Tuple[str, ...] parse_auth: Callable[[process.Completed], Tuple[Optional[bool], str]] # -> (authenticated, method) env_drop: frozenset = frozenset() + env_drop_prefixes: Tuple[str, ...] = () + env_keep: frozenset = frozenset() supports: frozenset = frozenset() # of "temperature", "seed", "max_tokens" login_hint: str = "" aliases: Tuple[str, ...] = () @@ -138,10 +147,30 @@ def _claude_error(payload: Dict[str, Any], done: process.Completed) -> CliCallEr if any(w in lowered for w in ("authentication", "not logged in", "/login")): return CliCallError(f"claude-cli: {message} (sign in: claude setup-token, or claude /login)", retryable=False) + if "usage limit" in lowered: # a subscription cap: it resets in hours, a retry cannot help + return CliCallError(f"claude-cli: {message}", retryable=False) retryable = status in _RETRYABLE_STATUS or any(w in lowered for w in _RETRYABLE_WORDS) return CliCallError(f"claude-cli: {message}", retryable=retryable) +def _main_model(model_usage: Dict[str, Any], requested: str) -> Optional[str]: + """The model the CLI answered with: the requested family if listed, else the one with most output. + + ``modelUsage`` also lists the helper models a call used (a title or a classifier), so + its first key is not always the answering model. + """ + names = [n for n in model_usage if isinstance(n, str)] + if not names: + return None + for name in names: + if _family(name) == _family(requested): + return name + def output(name: str) -> int: + stats = model_usage.get(name) + return int(stats.get("outputTokens") or 0) if isinstance(stats, dict) else 0 + return max(names, key=output) + + def _claude_parse(done: process.Completed, model: str, provider: str) -> CliResult: if done.timed_out: raise CliCallError(f"claude-cli: no reply in {done.elapsed_sec:.0f}s", retryable=True) @@ -170,7 +199,7 @@ def _claude_parse(done: process.Completed, model: str, provider: str) -> CliResu return CliResult( text=str(payload.get("result") or ""), usage=usage, - model_id=next(iter(model_usage), None), + model_id=_main_model(model_usage, model), list_cost_usd=float(list_cost) if isinstance(list_cost, (int, float)) else None, finish_reason="length" if payload.get("stop_reason") == "max_tokens" else "stop", ) @@ -197,6 +226,8 @@ def _claude_auth(done: process.Completed) -> Tuple[Optional[bool], str]: auth_argv=("claude", "auth", "status"), parse_auth=_claude_auth, env_drop=CLAUDE_ENV_DROP, + env_drop_prefixes=CLAUDE_ENV_DROP_PREFIXES, + env_keep=CLAUDE_ENV_KEEP, login_hint="claude setup-token (headless) or claude, then /login", aliases=("haiku", "sonnet", "opus"), model_prefixes=("claude-",), @@ -217,6 +248,37 @@ def accepts_model(provider: str, model: str) -> bool: return model in cli.aliases or model.startswith(cli.model_prefixes) +def _family(model: str) -> Optional[str]: + """The alias a Claude model name belongs to: ``sonnet``, or of ``claude-sonnet-5-5``.""" + name = model.strip().lower() + if name in CLAUDE_CLI.aliases: + return name + if name.startswith("claude-"): + for part in name.split("-"): + if part in CLAUDE_CLI.aliases: + return part + return None + + +def same_model(provider_a: str, model_a: str, provider_b: str, model_b: str) -> bool: + """Whether two model refs name one model, for the independence and self-judge checks. + + An alias the CLI resolves (``sonnet``) is the same model as a full id of its family + (``claude-sonnet-5-5``): which version the alias is today is not known here, so the + check errs toward "same", and a warning is the cost of being wrong. Refs that involve + no CLI provider are left to the caller's own comparison (this returns ``False``). + """ + if provider_a not in BACKENDS and provider_b not in BACKENDS: + return False + a, b = model_a.strip().lower(), model_b.strip().lower() + if a == b: + return True + if a not in CLAUDE_CLI.aliases and b not in CLAUDE_CLI.aliases: + return False # two full ids that differ are two models + fa, fb = _family(a), _family(b) + return fa is not None and fa == fb + + def check_sampling(provider: str, temperature: Optional[float], seed: Optional[int]) -> None: """Raise ``ValueError`` before a run when a CLI *provider* is given a temperature or seed.""" if provider in BACKENDS and (temperature is not None or seed is not None): @@ -240,8 +302,14 @@ def work_dir(name: str) -> Path: return path +def _passed_on(name: str, cli: CliBackend) -> bool: + if name in cli.env_keep: + return True + return name not in cli.env_drop and not name.startswith(cli.env_drop_prefixes) + + def _env(cli: CliBackend) -> process.Env: - return process.Env.only([k for k in os.environ if k not in cli.env_drop]) + return process.Env.only([k for k in os.environ if _passed_on(k, cli)]) def call(provider: str, *, model: str, prompt: str, system: Optional[str] = None, diff --git a/dokimi/dokimi/novelty/analyze.py b/dokimi/dokimi/novelty/analyze.py index aa5afab29..bfa4659d6 100644 --- a/dokimi/dokimi/novelty/analyze.py +++ b/dokimi/dokimi/novelty/analyze.py @@ -239,7 +239,9 @@ def analyze( concurrency=config.concurrency, allow_self_judge=config.allow_self_judge, placebo_text=config.placebo_text, seed=config.seed, ) - if config.allow_self_judge and judge.model == answerer.model: + from dokimi.novelty.client import same_client + + if config.allow_self_judge and same_client(judge, answerer): warnings.append( "judge and answerer are the same model (--allow-self-judge): scores are inflated" ) diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index 386467ee4..e92bd03b4 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -260,7 +260,15 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]: } -def default_judge(model: str, provider: str | None = None) -> tuple[str, str]: +def same_client(a: LLMClient, b: LLMClient) -> bool: + """Whether two clients run one model: the same name, or a CLI alias and a full id of its family.""" + from dokimi.llm_cli import same_model + + return a.model == b.model or same_model(a.provider, a.model, b.provider, b.model) + + +def default_judge(model: str, provider: str | None = None, + answerer_provider: str | None = None) -> tuple[str, str]: """A (model, provider) judge that is not the answerer, when the caller did not name one. The default judge, unless it is the model under measurement (by bare name, so a @@ -269,12 +277,26 @@ def default_judge(model: str, provider: str | None = None) -> tuple[str, str]: only a default that runs there is taken: its own provider, or Bedrock for a name in ``BEDROCK_MODEL_IDS``. Pairing the provider with a model of another provider would send e.g. ``bedrock/gpt-6.1-sol``, so with no such default this raises instead. + A CLI ``answerer_provider`` with no ``provider`` judges on the same CLI. Raises: ValueError: ``provider`` runs no default judge other than ``model``. """ + from dokimi.llm_cli import backend + bare = model.rsplit("/", 1)[-1].rsplit("anthropic.", 1)[-1] - candidates = [(m, p) for m, p in ((DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER), + if provider is None and answerer_provider is not None and backend(answerer_provider) is not None: + provider = answerer_provider + if provider is not None and provider == "claude-cli": + # The CLI's aliases: the first one not of the answerer's family, so the judge is never the answerer. + from dokimi.llm_cli import CLAUDE_CLI, same_model + + for alias in ("opus", "sonnet", "haiku"): + if alias in CLAUDE_CLI.aliases and not same_model("claude-cli", alias, "claude-cli", bare): + return alias, provider + raise ValueError(f"--judge-provider {provider}: no judge of another family than {model}; " + "pass --judge-model as well") + candidates =[(m, p) for m, p in ((DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER), (DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER)) if m != bare] if provider is None: diff --git a/dokimi/dokimi/novelty/measure.py b/dokimi/dokimi/novelty/measure.py index 2143fcb58..678a8e250 100644 --- a/dokimi/dokimi/novelty/measure.py +++ b/dokimi/dokimi/novelty/measure.py @@ -255,7 +255,9 @@ def run_measurement( irrelevant context actively harms them. seed: Seed for the ``shuffled`` control, so the control is reproducible. """ - if judge.model == answerer.model and not allow_self_judge: + from dokimi.novelty.client import same_client + + if same_client(judge, answerer) and not allow_self_judge: raise SelfJudgeError( f"judge model {judge.model!r} is the answering model; grading a response with " "the model that produced it inflates the score. Pass a different --judge-model, " diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index b0823eec7..e0589ac02 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,9 +1,9 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 1192, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --repeats refused under --all-vs-all; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 1191, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --repeats refused under --all-vs-all; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, + "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1209, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, @@ -19,7 +19,7 @@ "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13}, "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, - "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 21}, + "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 19}, "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 15}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 30, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 3955191c8..de0bf3b66 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -108,7 +108,9 @@ def test_a_failed_call_says_whether_another_attempt_can_help(runner, answer, ret assert llm.is_retryable(info.value) is retryable -@pytest.mark.parametrize("status", sorted(llm_cli._RETRYABLE_STATUS)) +# The status and word lists are written out here, not read from llm_cli: dropping a +# member from the module must fail a test, not drop the test with it. +@pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504, 529]) def test_each_retryable_status_is_retried_whatever_the_message(runner, status): runner.answers.append(reply(_err("something went wrong", status), returncode=1)) with pytest.raises(llm_cli.CliCallError) as info: @@ -116,7 +118,7 @@ def test_each_retryable_status_is_retried_whatever_the_message(runner, status): assert info.value.retryable is True -@pytest.mark.parametrize("word", llm_cli._RETRYABLE_WORDS) +@pytest.mark.parametrize("word", ["rate limit", "rate_limit", "overloaded", "timed out"]) def test_each_retryable_word_is_retried_without_a_status(runner, word): runner.answers.append(reply(_err(f"API said: {word}", None), returncode=1)) with pytest.raises(llm_cli.CliCallError) as info: @@ -124,6 +126,15 @@ def test_each_retryable_word_is_retried_without_a_status(runner, word): assert info.value.retryable is True +@pytest.mark.parametrize("status", [None, 429]) +def test_a_subscription_usage_limit_is_not_retried(runner, status): + # It resets after hours: three more CLI starts would only delay the same failure. + runner.answers.append(reply(_err("Claude AI usage limit reached|1761660000", status), returncode=1)) + with pytest.raises(llm_cli.CliCallError, match="usage limit") as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False + + @pytest.mark.parametrize("answer", [ reply(_ok("bad", is_error=True), returncode=0), # is_error alone, exit 0 reply(_ok("boom"), returncode=1), # exit 1 alone, is_error false @@ -245,14 +256,21 @@ def test_backends_command_prints_one_row_per_backend(monkeypatch, capsys): def test_benchmark_preflight_checks_the_login_not_a_key(monkeypatch): - from dokimi.commands.benchmark import _credential_error + from dokimi.commands.benchmark import _provider_checks + monkeypatch.setenv("ANTHROPIC_API_KEY", "") # no key anywhere: the CLI login is what counts monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": False})) - assert "not signed in" in _credential_error("claude-cli") + errors, _ = _provider_checks(["claude-cli"]) + assert any("not signed in" in e for e in errors) monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) - assert _credential_error("claude-cli") is None + assert _provider_checks(["claude-cli"]) == ([], []) monkeypatch.setattr(process, "run", _auth_runner(None)) - assert "not installed" in _credential_error("claude-cli") + errors, _ = _provider_checks(["claude-cli"]) + assert any("not installed" in e for e in errors) + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + errors, _ = _provider_checks(["openai"]) + assert any("missing OPENAI_API_KEY" in e for e in errors) def test_a_judge_panel_refuses_sampling_on_the_cli_with_explicit_judges(): @@ -263,10 +281,11 @@ def test_a_judge_panel_refuses_sampling_on_the_cli_with_explicit_judges(): def test_an_unanswered_login_check_is_a_warning_not_a_pass(monkeypatch): - from dokimi.commands.benchmark import _credential_error, _login_warning + from dokimi.commands.benchmark import _login_warning, _provider_checks monkeypatch.setattr(process, "run", _auth_runner("no answer")) - assert _credential_error("claude-cli") is None + assert _provider_checks(["claude-cli"]) == ([], [ + "claude-cli: could not check the login ('claude auth status' gave no answer)"]) assert "could not check the login" in _login_warning("claude-cli") monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) assert _login_warning("claude-cli") is None @@ -297,3 +316,131 @@ def test_answerer_and_generators_accept_the_cli_provider(runner): runner.answers.extend([reply(_ok("persona")), reply(_ok("question"))]) assert PersonaGenerator(model="haiku", provider="claude-cli").llm.invoke("p").content == "persona" assert QuestionGenerator(model="haiku", provider="claude-cli").llm.invoke("q").content == "question" + + +def test_the_child_env_drops_billing_and_session_variables_by_prefix(runner, monkeypatch): + # Named in review: an endpoint, a model alias override and a provider switch each + # change who is billed; a messaging socket joins the parent's session. + for name, value in { + "ANTHROPIC_BASE_URL": "https://gateway.example", "ANTHROPIC_DEFAULT_SONNET_MODEL": "other", + "ANTHROPIC_CUSTOM_HEADERS": "x: y", "CLAUDE_CODE_USE_BEDROCK": "1", + "CLAUDE_CODE_MESSAGING_SOCKET": "/tmp/sock", "CLAUDE_CODE_SSE_PORT": "1234", + "CLAUDE_CODE_OAUTH_TOKEN": "headless-token", "CLAUDE_CONFIG_DIR": "/accounts/work", + }.items(): + monkeypatch.setenv(name, value) + runner.answers.append(reply(_ok())) + llm_cli.call("claude-cli", model="sonnet", prompt="p") + env = runner.calls[0].env + for name in ("ANTHROPIC_BASE_URL", "ANTHROPIC_DEFAULT_SONNET_MODEL", "ANTHROPIC_CUSTOM_HEADERS", + "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_MESSAGING_SOCKET", "CLAUDE_CODE_SSE_PORT"): + assert name not in env, name + # The login is kept: the headless token and the account's config dir. + assert env["CLAUDE_CODE_OAUTH_TOKEN"] == "headless-token" + assert env["CLAUDE_CONFIG_DIR"] == "/accounts/work" + + +def test_the_work_dir_is_under_the_users_cache_not_the_shared_temp_dir(monkeypatch, tmp_path): + import tempfile + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path / "shared-tmp")) + path = llm_cli.work_dir("claude-cli") + assert path == tmp_path / "cache" / "dokimi" / "cli" / "claude-cli" and path.is_dir() + assert not (tmp_path / "shared-tmp").exists() + + +def test_a_reply_names_the_main_model_the_cli_resolved(runner): + # modelUsage lists a helper model too; the main one has the output tokens. + payload = json.loads(_ok(modelUsage={"claude-haiku-5-5": {"outputTokens": 9}, + "claude-sonnet-5-5": {"outputTokens": 3}})) + runner.answers.append(reply(json.dumps(payload))) + out = llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert out.model_id == "claude-sonnet-5-5" + + +def test_the_llm_reply_carries_the_resolved_model_id(runner): + runner.answers.append(reply(_ok())) + request = llm.Request(model="sonnet", provider="claude-cli", litellm_model="sonnet", + messages=[{"role": "user", "content": "ping"}], max_tokens=50) + assert llm._cli_reply(request).model_id == "claude-sonnet-5-5" + + +def test_an_alias_and_its_full_id_are_one_model_for_the_identity_checks(): + same = llm_cli.same_model + assert same("claude-cli", "sonnet", "claude-cli", "claude-sonnet-5-5") + assert same("anthropic", "claude-sonnet-5-5", "claude-cli", "sonnet") + assert not same("claude-cli", "sonnet", "claude-cli", "claude-opus-5-5") + assert not same("claude-cli", "claude-sonnet-5-5", "claude-cli", "claude-sonnet-4-5") + assert not same("openai", "gpt-6.1-sol", "claude-cli", "sonnet") + + +def test_the_independence_warning_fires_for_an_alias_against_its_full_id(capsys): + from dokimi.commands.evaluate import _warn_judge_independence + + assert _warn_judge_independence("claude-cli", "sonnet", "claude-cli", "claude-sonnet-5-5") + assert "independence warning" in capsys.readouterr().err + assert not _warn_judge_independence("claude-cli", "sonnet", "claude-cli", "claude-opus-5-5") + + +def test_the_novelty_default_judge_on_the_cli_is_another_family(): + from dokimi.novelty.client import default_judge + + judge, provider = default_judge("sonnet", "claude-cli") + assert provider == "claude-cli" and not llm_cli.same_model("claude-cli", judge, "claude-cli", "sonnet") + + +def test_a_novelty_judge_of_the_answerers_family_counts_as_the_answerer(): + # run_measurement refuses a judge that same_client matches with the answerer, unless allowed. + from dokimi.novelty.client import LLMClient, same_client + + judge = LLMClient(model="claude-sonnet-5-5", provider="anthropic") + answerer = LLMClient(model="sonnet", provider="claude-cli") + assert same_client(judge, answerer) and same_client(answerer, judge) + assert not same_client(LLMClient(model="opus", provider="claude-cli"), answerer) + + +def test_an_agentic_arm_or_a_session_is_refused_on_the_cli_before_any_answer(): + from dokimi.commands import arms_refusals + from dokimi.treatments import create_treatments + + plain = create_treatments(["baseline", "docs"], top_k=3, rerank_threshold=0.3) + agentic = create_treatments(["baseline", "agent"], top_k=3, rerank_threshold=0.3) + cli = argparse.Namespace(provider="claude-cli", session_turns=None) + arms_refusals.refuse_cli_arms(cli, plain) + with pytest.raises(ValueError, match="agentic arm"): + arms_refusals.refuse_cli_arms(cli, agentic) + with pytest.raises(ValueError, match="--session-turns"): + arms_refusals.refuse_cli_arms(argparse.Namespace(provider="claude-cli", session_turns=3), plain) + arms_refusals.refuse_cli_arms(argparse.Namespace(provider="openai", session_turns=3), agentic) + + +def test_a_cli_judge_given_a_temperature_stops_the_full_pipeline_before_any_work(monkeypatch, capsys): + from dokimi.commands import orchestrate + from dokimi import orchestrator + + def pipeline_must_not_start(*args, **kwargs): + raise AssertionError("the pipeline started before the sampling refusal") + + monkeypatch.setattr(orchestrator, "EvaluationPipeline", pipeline_must_not_start) + args = argparse.Namespace(repo="owner/repo", description=None, product="demo", + judge_provider="claude-cli", judge_temperature=0.0) + with pytest.raises(SystemExit): + orchestrate.cmd_evaluate(args) + assert "takes no temperature" in capsys.readouterr().err + + +def test_every_command_with_a_hidden_or_default_model_refuses_the_cli_without_a_claude_model(): + from dokimi.cli import parse_args + + cases = [ + ["questions", "generate", "--product", "p", "--provider", "claude-cli"], + ["questions", "analyze", "--product", "p", "--provider", "claude-cli"], + ["questions", "panel-review", "--product", "p", "--provider", "claude-cli"], + ["personas", "discover", "--product", "p", "--provider", "claude-cli"], + ["subjects", "run", "subject.toml", "--provider", "claude-cli"], + ] + for argv in cases: + with pytest.raises(SystemExit): + parse_args(argv) + assert parse_args(["subjects", "run", "subject.toml", "--provider", "claude-cli", + "--model", "haiku"]).model == "haiku" From 2c491ec0751f1b83a020a8777f0e202d55684714 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 16:38:43 -0700 Subject: [PATCH 10/42] fix(llm): review round 1 on claude-cli: import, identity, env, tests - arms: import refuse_cli_arms (a NameError on every arms run before this). - Model identity strips a Bedrock or LiteLLM prefix before the family match, so an alias and its prefixed full id are one model. The main-model fallback is not taken for a model with no family. - default_judge takes the answerer's provider: a CLI answerer with no judge provider judges on the same CLI, another family. Drops the redundant alias check. - Tests: every agentic arm kind through the wrapper and the combined arm, the arms entry point refuses before any answer, the login check uses the reduced env, the model gate names --model, and the refusal cases pass required args. - Changelog: a Bedrock or Vertex route is dropped with the rest of ANTHROPIC_/CLAUDE_. Signed-off-by: Nikolay Petrov --- ...1026_150000_napetrov_claude-cli-backend.md | 4 +- dokimi/dokimi/commands/arms.py | 1 + dokimi/dokimi/commands/arms_refusals.py | 2 - dokimi/dokimi/llm_cli.py | 5 +- dokimi/dokimi/novelty/client.py | 8 +- dokimi/tests/test_llm_cli.py | 75 ++++++++++++++++++- 6 files changed, 82 insertions(+), 13 deletions(-) diff --git a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md index c610906aa..4e6f8885f 100644 --- a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md +++ b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md @@ -24,4 +24,6 @@ `claude-opus-…`). The novelty default judge on `claude-cli` is another Claude family than the answerer. - The CLI child process drops `ANTHROPIC_*`, `CLAUDE_*` and `CLAUDECODE` variables from the environment (except `CLAUDE_CONFIG_DIR` and `CLAUDE_CODE_OAUTH_TOKEN`), so a billing or - session setting in the caller does not reroute the call. + session setting in the caller does not reroute the call. A Bedrock or Vertex route set + in that environment is dropped too: the CLI then uses its own login, and `backends` reports + that login, not the route. diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 55e0370d3..e424aa9d0 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -11,6 +11,7 @@ import sys from pathlib import Path +from dokimi.commands.arms_refusals import refuse_cli_arms from dokimi.llm import ANSWER_PROVIDERS from dokimi.utils import question_id from dokimi.eval.judge import JUDGE_PROVIDERS diff --git a/dokimi/dokimi/commands/arms_refusals.py b/dokimi/dokimi/commands/arms_refusals.py index e0da8ae4f..9d5af6abf 100644 --- a/dokimi/dokimi/commands/arms_refusals.py +++ b/dokimi/dokimi/commands/arms_refusals.py @@ -41,5 +41,3 @@ def refuse_cli_arms(args: argparse.Namespace, treatments: list) -> None: if getattr(args, "session_turns", None) is not None: raise ValueError(f"--session-turns needs a conversation, which --provider {args.provider} " "does not have: use an API provider for a session") - - diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 335c374ad..4ff92d6b0 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -162,8 +162,9 @@ def _main_model(model_usage: Dict[str, Any], requested: str) -> Optional[str]: names = [n for n in model_usage if isinstance(n, str)] if not names: return None + family = _family(requested) for name in names: - if _family(name) == _family(requested): + if family is not None and _family(name) == family: return name def output(name: str) -> int: stats = model_usage.get(name) @@ -250,7 +251,7 @@ def accepts_model(provider: str, model: str) -> bool: def _family(model: str) -> Optional[str]: """The alias a Claude model name belongs to: ``sonnet``, or of ``claude-sonnet-5-5``.""" - name = model.strip().lower() + name = model.strip().lower().rsplit("/", 1)[-1].rsplit("anthropic.", 1)[-1] if name in CLAUDE_CLI.aliases: return name if name.startswith("claude-"): diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index e92bd03b4..7c442879e 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -289,14 +289,14 @@ def default_judge(model: str, provider: str | None = None, provider = answerer_provider if provider is not None and provider == "claude-cli": # The CLI's aliases: the first one not of the answerer's family, so the judge is never the answerer. - from dokimi.llm_cli import CLAUDE_CLI, same_model + from dokimi.llm_cli import same_model for alias in ("opus", "sonnet", "haiku"): - if alias in CLAUDE_CLI.aliases and not same_model("claude-cli", alias, "claude-cli", bare): + if not same_model("claude-cli", alias, "claude-cli", bare): return alias, provider - raise ValueError(f"--judge-provider {provider}: no judge of another family than {model}; " + raise ValueError(f"judge provider {provider}: no judge of another family than {model}; " "pass --judge-model as well") - candidates =[(m, p) for m, p in ((DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER), + candidates = [(m, p) for m, p in ((DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER), (DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER)) if m != bare] if provider is None: diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index de0bf3b66..383d83343 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -429,18 +429,85 @@ def pipeline_must_not_start(*args, **kwargs): assert "takes no temperature" in capsys.readouterr().err -def test_every_command_with_a_hidden_or_default_model_refuses_the_cli_without_a_claude_model(): +def test_every_command_with_a_hidden_or_default_model_refuses_the_cli_without_a_claude_model(capsys): from dokimi.cli import parse_args cases = [ - ["questions", "generate", "--product", "p", "--provider", "claude-cli"], - ["questions", "analyze", "--product", "p", "--provider", "claude-cli"], - ["questions", "panel-review", "--product", "p", "--provider", "claude-cli"], + ["questions", "generate", "--product", "p", "--personas", "p.json", "--provider", "claude-cli"], + ["questions", "analyze", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], + ["questions", "panel-review", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], ["personas", "discover", "--product", "p", "--provider", "claude-cli"], ["subjects", "run", "subject.toml", "--provider", "claude-cli"], ] for argv in cases: with pytest.raises(SystemExit): parse_args(argv) + # the refusal is the model gate, not an unrelated argparse error + assert "takes a Claude model" in capsys.readouterr().err, argv assert parse_args(["subjects", "run", "subject.toml", "--provider", "claude-cli", "--model", "haiku"]).model == "haiku" + + +def test_every_agentic_arm_kind_is_refused_on_the_cli_including_wrapped_and_combined(demo_skill): + from dokimi.skills import load_skill + + from dokimi.commands import arms_refusals + from dokimi.plugins import PluginWrappedTreatment + from dokimi.treatments.arms import ( + BaselineTreatment, + CombinedTreatment, + MCPAgentTreatment, + SkillAgentTreatment, + SkillCatalogTreatment, + ) + + cli = argparse.Namespace(provider="claude-cli", session_turns=None) + skill = load_skill(demo_skill) + base = BaselineTreatment() + agentic = { + "mcp_agent": MCPAgentTreatment(mcp_client=None), + "skill_agent": SkillAgentTreatment(skill), + "skill_catalog": SkillCatalogTreatment([skill]), + "combined_with_agent": CombinedTreatment([base, SkillAgentTreatment(skill)]), + "plugin_wrapped_agent": PluginWrappedTreatment(SkillAgentTreatment(skill), []), + } + for kind, arm in agentic.items(): + with pytest.raises(ValueError, match="agentic arm"): + arms_refusals.refuse_cli_arms(cli, [arm]) + arms_refusals.refuse_cli_arms( + cli, [base, CombinedTreatment([base, base]), PluginWrappedTreatment(base, [])]) + + +def test_the_arms_entry_point_refuses_a_cli_agentic_arm_before_any_answer(monkeypatch): + from dokimi.commands import arms + + def answer_must_not_run(*args, **kwargs): + raise AssertionError("an answer was paid for before the CLI refusal") + + monkeypatch.setattr(arms, "_resolve_doc_library_id", lambda *a, **k: None) + monkeypatch.setattr(arms, "_validate_expected_skills", lambda *a, **k: None) + args = argparse.Namespace(provider="claude-cli", model="sonnet", judge=False, session_turns=None, + top_k=3, rerank_threshold=0.3, product="p", context7_id=None, + plugins="", harness="arms-runner", matrix_cell=None, judge_provider=None) + with pytest.raises(ValueError, match="agentic arm"): + arms._build_arms_output(args, ["baseline", "agent"], []) + + +def test_an_alias_and_a_bedrock_or_prefixed_full_id_are_one_model_for_the_identity_checks(): + assert llm_cli.same_model("claude-cli", "sonnet", "anthropic", "us.anthropic.claude-sonnet-5-5-v1:0") + assert llm_cli.same_model("claude-cli", "sonnet", "litellm", "anthropic/claude-sonnet-5-5") + assert not llm_cli.same_model("claude-cli", "sonnet", "anthropic", "us.anthropic.claude-opus-5-1-v1:0") + + +def test_the_main_model_falls_back_to_most_output_when_the_requested_family_is_unknown(): + usage = {"helper-a": {"outputTokens": 5}, "helper-b": {"outputTokens": 500}} + assert llm_cli._main_model(usage, "claude-fable-5-1") == "helper-b" + + +def test_the_login_check_runs_with_the_same_reduced_env_as_the_call(monkeypatch): + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://gateway.example") + fake = _auth_runner({"loggedIn": True, "authMethod": "oauth_token"}) + llm_cli.detect("claude-cli", runner=fake) + assert fake.calls + for call in fake.calls: + assert "ANTHROPIC_BASE_URL" not in call.env From d14a7625960923f28170b99b1768f839d7d69923 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 16:40:28 -0700 Subject: [PATCH 11/42] test(llm): pin the arms entry point, the session default, the login env - The entry-point test makes any arm-runner start fail, so the refusal is checked to come before any answer machinery. - A test pins --session-turns to None by default (parse_args on arms run). - The login check test pins the reduced env (already the case at head). - A comment on the claude-cli alias list for a second CLI backend. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/novelty/client.py | 1 + dokimi/tests/test_llm_cli.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index 7c442879e..765c6fedd 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -287,6 +287,7 @@ def default_judge(model: str, provider: str | None = None, bare = model.rsplit("/", 1)[-1].rsplit("anthropic.", 1)[-1] if provider is None and answerer_provider is not None and backend(answerer_provider) is not None: provider = answerer_provider + # Hard-coded for claude-cli; a second CLI backend (codex-cli) needs its own alias list here. if provider is not None and provider == "claude-cli": # The CLI's aliases: the first one not of the answerer's family, so the judge is never the answerer. from dokimi.llm_cli import same_model diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 383d83343..6585784fd 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -480,10 +480,13 @@ def test_every_agentic_arm_kind_is_refused_on_the_cli_including_wrapped_and_comb def test_the_arms_entry_point_refuses_a_cli_agentic_arm_before_any_answer(monkeypatch): from dokimi.commands import arms + from dokimi.eval import arm_runner - def answer_must_not_run(*args, **kwargs): - raise AssertionError("an answer was paid for before the CLI refusal") + class RunnerMustNotStart: + def __init__(self, *args, **kwargs): + raise AssertionError("the arm runner started before the CLI refusal") + monkeypatch.setattr(arm_runner, "ArmRunner", RunnerMustNotStart) monkeypatch.setattr(arms, "_resolve_doc_library_id", lambda *a, **k: None) monkeypatch.setattr(arms, "_validate_expected_skills", lambda *a, **k: None) args = argparse.Namespace(provider="claude-cli", model="sonnet", judge=False, session_turns=None, @@ -511,3 +514,11 @@ def test_the_login_check_runs_with_the_same_reduced_env_as_the_call(monkeypatch) assert fake.calls for call in fake.calls: assert "ANTHROPIC_BASE_URL" not in call.env + + +def test_the_session_turns_flag_defaults_to_none_so_only_a_session_run_is_refused(): + from dokimi.cli import parse_args + + args = parse_args(["arms", "run", "--product", "p", "--questions", "q.json", "--arms", "baseline", + "--provider", "claude-cli", "--model", "sonnet"]) + assert args.session_turns is None From 4e33134fdb3ac20694b15e25af8e35928b6a7bbc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 16:41:45 -0700 Subject: [PATCH 12/42] test(llm): the entry-point refusal is observed through the runner start The arms entry point test now fails with 'the arm runner started before the CLI refusal' when the refusal is removed; before, it failed on a missing Namespace field. The session default test is renamed to what it checks. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_llm_cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 6585784fd..d72bce649 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -491,7 +491,8 @@ def __init__(self, *args, **kwargs): monkeypatch.setattr(arms, "_validate_expected_skills", lambda *a, **k: None) args = argparse.Namespace(provider="claude-cli", model="sonnet", judge=False, session_turns=None, top_k=3, rerank_threshold=0.3, product="p", context7_id=None, - plugins="", harness="arms-runner", matrix_cell=None, judge_provider=None) + plugins="", harness="arms-runner", matrix_cell=None, judge_provider=None, + judge_model=None, max_iterations=5) with pytest.raises(ValueError, match="agentic arm"): arms._build_arms_output(args, ["baseline", "agent"], []) @@ -516,7 +517,7 @@ def test_the_login_check_runs_with_the_same_reduced_env_as_the_call(monkeypatch) assert "ANTHROPIC_BASE_URL" not in call.env -def test_the_session_turns_flag_defaults_to_none_so_only_a_session_run_is_refused(): +def test_the_session_turns_flag_defaults_to_none(): from dokimi.cli import parse_args args = parse_args(["arms", "run", "--product", "p", "--questions", "q.json", "--arms", "baseline", From 8d65d1071ab1a7c9735545de2beafaeaf2bee244 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 16:49:41 -0700 Subject: [PATCH 13/42] test(llm): lint and bandit fixes in the new tests Drop an unused loop variable, and a literal temp path in the env-drop test (bandit B108), whose value does not matter to the drop rule. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_llm_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index d72bce649..e487ab4fa 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -324,7 +324,7 @@ def test_the_child_env_drops_billing_and_session_variables_by_prefix(runner, mon for name, value in { "ANTHROPIC_BASE_URL": "https://gateway.example", "ANTHROPIC_DEFAULT_SONNET_MODEL": "other", "ANTHROPIC_CUSTOM_HEADERS": "x: y", "CLAUDE_CODE_USE_BEDROCK": "1", - "CLAUDE_CODE_MESSAGING_SOCKET": "/tmp/sock", "CLAUDE_CODE_SSE_PORT": "1234", + "CLAUDE_CODE_MESSAGING_SOCKET": "messaging.sock", "CLAUDE_CODE_SSE_PORT": "1234", "CLAUDE_CODE_OAUTH_TOKEN": "headless-token", "CLAUDE_CONFIG_DIR": "/accounts/work", }.items(): monkeypatch.setenv(name, value) @@ -471,7 +471,7 @@ def test_every_agentic_arm_kind_is_refused_on_the_cli_including_wrapped_and_comb "combined_with_agent": CombinedTreatment([base, SkillAgentTreatment(skill)]), "plugin_wrapped_agent": PluginWrappedTreatment(SkillAgentTreatment(skill), []), } - for kind, arm in agentic.items(): + for arm in agentic.values(): with pytest.raises(ValueError, match="agentic arm"): arms_refusals.refuse_cli_arms(cli, [arm]) arms_refusals.refuse_cli_arms( From 4937a07823b669e3415709a14e3c4e2e96384bd0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:01:59 -0700 Subject: [PATCH 14/42] fix(llm): record the model a CLI alias resolved to, and key the cache on the CLI version - Usage rows carry resolved_model, the model the CLI resolved its alias to. Sums keep every name they mix, sorted and comma-separated; as_metrics_dict shows it when set, so non-CLI rows are unchanged. - The judge record lists judge_resolved_models. - The response-cache key takes the CLI version for a CLI provider. A new release can move an alias such as sonnet to another model, and the resolved id is only known after the call, so the version stands in for it. A move that ships with no CLI release is not caught. Signed-off-by: Nikolay Petrov --- ...1026_150000_napetrov_claude-cli-backend.md | 5 ++ dokimi/dokimi/eval/judge.py | 6 +++ dokimi/dokimi/llm.py | 2 +- dokimi/dokimi/llm_cli.py | 33 +++++++++++- dokimi/dokimi/metrics/usage.py | 13 +++++ dokimi/tests/test_judge_sampling.py | 10 ++++ dokimi/tests/test_llm_cli.py | 54 ++++++++++++++++++- 7 files changed, 119 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md index 4e6f8885f..f0c78654e 100644 --- a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md +++ b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md @@ -20,6 +20,11 @@ - A CLI provider refuses an agentic arm (`mcp_agent`, `skill_agent`, `skill_catalog`) and `--session-turns` in `arms run` and `arms matrix-run`, before any answer is paid for: the CLI runs one prompt with no tools and no conversation. +- A CLI reply records the model the CLI resolved its alias to: `resolved_model` on each usage + row (answers, judge calls), `judge_resolved_models` in the judge record. The response-cache + key takes the CLI version, so a new release that moves `sonnet` does not replay the old + model's replies. The resolved id is known only after a call, so a move the CLI does not + ship with is not caught. - Independence checks treat an alias and its full id as one model (`opus` and `claude-opus-…`). The novelty default judge on `claude-cli` is another Claude family than the answerer. - The CLI child process drops `ANTHROPIC_*`, `CLAUDE_*` and `CLAUDECODE` variables from the diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py index 4cc4d8eeb..2f035df55 100644 --- a/dokimi/dokimi/eval/judge.py +++ b/dokimi/dokimi/eval/judge.py @@ -243,6 +243,8 @@ def __init__( "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "redrawn_judgements": 0, } + # The model names a CLI judge's aliases resolved to, for the judge record. + self._resolved_models: set = set() check_sampling(provider, temperature, seed) self.llm: ChatModel @@ -632,6 +634,7 @@ def _int(v): t["prompt_tokens"] += _int(usage.prompt_tokens) t["completion_tokens"] += _int(usage.completion_tokens) t["total_tokens"] += _int(usage.total_tokens) + self._resolved_models.update(n for n in usage.resolved_model.split(",") if n) cost = usage.cost_usd if isinstance(cost, (int, float)): t["cost_usd"] += float(cost) @@ -649,6 +652,9 @@ def _build_evaluation_output(self, evaluations: List[Dict[str, Any]]) -> Dict[st } with self._usage_lock: totals = dict(self._usage_totals) + resolved = sorted(self._resolved_models) + if resolved: + output["judge_resolved_models"] = resolved # cost_usd is null (not 0.0) when no judge call had known pricing, so a # missing price reads distinct from a genuinely free judge. output["judge_metrics"] = { diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index 22da74e3f..bfca86be8 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -810,7 +810,7 @@ def _cache_extra(self, stream: bool, max_tokens: Optional[int]) -> Dict[str, Any # reasoning-starved completion is a different answer). temperature/seed # belong in it too, only when set: a pinned run must not be served a # free-sampling reply, and entries cached before they existed still hit. - return {"stream": stream, "max_tokens": max_tokens, + return {"stream": stream, "max_tokens": max_tokens, **llm_cli.cache_extra(self.provider), **_sampling_kwargs(temperature=self.temperature, seed=self.seed)} def _labels(self) -> Tuple[str, str]: diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 4ff92d6b0..1915dd40f 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -41,6 +41,7 @@ from __future__ import annotations +import functools import json import os from dataclasses import dataclass @@ -191,16 +192,17 @@ def _claude_parse(done: process.Completed, model: str, provider: str) -> CliResu completion = int(u.get("output_tokens") or 0) model_usage = payload.get("modelUsage") or {} list_cost = payload.get("total_cost_usd") + resolved = _main_model(model_usage, model) usage = UsageRecord( prompt_tokens=prompt, completion_tokens=completion, total_tokens=prompt + completion, cache_read_tokens=cache_read, cache_write_tokens=cache_write, cost_usd=None, latency_sec=done.elapsed_sec, model=model, provider=provider, - n_calls=1, cost_known_calls=0, + n_calls=1, cost_known_calls=0, resolved_model=resolved or "", ) return CliResult( text=str(payload.get("result") or ""), usage=usage, - model_id=_main_model(model_usage, model), + model_id=resolved, list_cost_usd=float(list_cost) if isinstance(list_cost, (int, float)) else None, finish_reason="length" if payload.get("stop_reason") == "max_tokens" else "stop", ) @@ -313,6 +315,33 @@ def _env(cli: CliBackend) -> process.Env: return process.Env.only([k for k in os.environ if _passed_on(k, cli)]) +def cache_extra(provider: str) -> Dict[str, str]: + """What the response-cache key takes from a CLI backend: its version, and nothing otherwise. + + An alias resolves inside the CLI, so a new release can answer ``sonnet`` with another + model, and its replies must not be replayed for the new one. The resolved id is known + only after the call, so the version stands in for it; a move the CLI does not ship + with is not seen. + """ + return {"cli_version": cli_version(provider)} if backend(provider) is not None else {} + + +def cli_version(provider: str) -> str: + """The CLI's version line, or ``""`` when it cannot be read. Read once per process.""" + return _version_line(provider) + + +@functools.lru_cache(maxsize=None) +def _version_line(provider: str) -> str: + cli = BACKENDS[provider] + try: + done = process.run([cli.binary, "--version"], timeout=30, env=_env(cli)) + except OSError: + return "" + lines = done.stdout.strip().splitlines() + return lines[0] if lines else "" + + def call(provider: str, *, model: str, prompt: str, system: Optional[str] = None, timeout: float = DEFAULT_TIMEOUT_SEC, runner: Optional[process.Runner] = None) -> CliResult: """One completion through *provider*'s CLI; the prompt goes on stdin (no ``ARG_MAX``). diff --git a/dokimi/dokimi/metrics/usage.py b/dokimi/dokimi/metrics/usage.py index 594cc1172..e1d9e744f 100644 --- a/dokimi/dokimi/metrics/usage.py +++ b/dokimi/dokimi/metrics/usage.py @@ -44,6 +44,9 @@ class UsageRecord: cost_known_calls: int = 0 # served from the response cache: cost_usd is the original call's, not this process's bill replayed: bool = False + # The model the provider resolved the request to, when it says (a CLI alias names a + # version). A sum of rows that resolved differently lists each, comma-separated. + resolved_model: str = "" def __add__(self, other: "UsageRecord | int") -> "UsageRecord": """Sum two records: tokens/cost/latency/n_calls add; first ttft wins. @@ -78,6 +81,7 @@ def __add__(self, other: "UsageRecord | int") -> "UsageRecord": n_calls=self.n_calls + other.n_calls, cost_known_calls=_known_cost_calls(self) + _known_cost_calls(other), replayed=self.replayed and other.replayed, + resolved_model=_joined_models(self.resolved_model, other.resolved_model), ) __radd__ = __add__ # enables sum([...]) — 0 + record is handled in __add__ @@ -123,6 +127,8 @@ def as_metrics_dict( "cost_known_calls": _known_cost_calls(self), "role": self.role, } + if self.resolved_model: + m["resolved_model"] = self.resolved_model if answer_chars is not None: m["raw_answer_chars"] = answer_chars m["final_answer_chars"] = final_chars if final_chars is not None else answer_chars @@ -170,6 +176,7 @@ def to_record( n_calls=(int(obj["n_calls"]) if obj.get("n_calls") is not None else 1), cost_known_calls=int(obj.get("cost_known_calls", 0) or 0), + resolved_model=str(obj.get("resolved_model") or ""), ) raise TypeError(f"Cannot coerce {type(obj).__name__} into UsageRecord") @@ -216,6 +223,12 @@ def _add_optional_costs(a: Optional[float], b: Optional[float]) -> Optional[floa return round((a or 0.0) + (b or 0.0), 6) +def _joined_models(a: str, b: str) -> str: + """The distinct resolved model names of two rows, sorted and comma-separated.""" + names = {n for n in (*a.split(","), *b.split(",")) if n} + return ",".join(sorted(names)) + + def _known_cost_calls(rec: UsageRecord) -> int: """Number of LLM calls whose dollar cost is represented in ``cost_usd``.""" if rec.cost_known_calls: diff --git a/dokimi/tests/test_judge_sampling.py b/dokimi/tests/test_judge_sampling.py index 197ede47b..710627487 100644 --- a/dokimi/tests/test_judge_sampling.py +++ b/dokimi/tests/test_judge_sampling.py @@ -18,6 +18,16 @@ def test_judge_hands_sampling_params_to_its_client_and_output(): assert j._build_evaluation_output([])["judge_temperature"] == 0.0 +def test_the_judge_record_names_the_resolved_models_it_was_judged_by(): + from dokimi.metrics.usage import UsageRecord + + j = Judge(model="sonnet", provider="claude-cli") + assert "judge_resolved_models" not in j._build_evaluation_output([]) + j._accumulate_usage(UsageRecord(resolved_model="claude-sonnet-5-5")) + j._accumulate_usage(UsageRecord(resolved_model="claude-sonnet-5-5")) + assert j._build_evaluation_output([])["judge_resolved_models"] == ["claude-sonnet-5-5"] + + def test_judge_defaults_leave_sampling_unset(): j = Judge(model="m", provider="openai") assert j.temperature is None diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index e487ab4fa..2817163c1 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -15,6 +15,7 @@ import pytest from dokimi import llm, llm_cli, process +from dokimi.metrics.usage import UsageRecord from dokimi.process import FakeRunner, reply @@ -47,6 +48,8 @@ def runner(monkeypatch, tmp_path): fake = FakeRunner(lambda call: fake.answers.pop(0)) fake.answers = [] # type: ignore[attr-defined] monkeypatch.setattr(llm_cli.process, "run", fake) + # The version is part of the cache key; a test of the key sets its own. + monkeypatch.setattr(llm_cli, "cli_version", lambda provider: "2.1.303 (Claude Code)") return fake @@ -268,7 +271,7 @@ def test_benchmark_preflight_checks_the_login_not_a_key(monkeypatch): errors, _ = _provider_checks(["claude-cli"]) assert any("not installed" in e for e in errors) monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY") errors, _ = _provider_checks(["openai"]) assert any("missing OPENAI_API_KEY" in e for e in errors) @@ -523,3 +526,52 @@ def test_the_session_turns_flag_defaults_to_none(): args = parse_args(["arms", "run", "--product", "p", "--questions", "q.json", "--arms", "baseline", "--provider", "claude-cli", "--model", "sonnet"]) assert args.session_turns is None + + +def test_the_resolved_model_is_on_the_usage_row_and_in_its_metrics(runner): + runner.answers.append(reply(_ok())) + usage = llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[1] + assert usage.resolved_model == "claude-sonnet-5-5" + assert usage.as_metrics_dict()["resolved_model"] == "claude-sonnet-5-5" + assert "resolved_model" not in UsageRecord().as_metrics_dict() + + +def test_a_sum_of_usage_rows_keeps_every_resolved_model_it_mixes(): + sonnet = UsageRecord(resolved_model="claude-sonnet-5-5") + assert (sonnet + UsageRecord()).resolved_model == "claude-sonnet-5-5" + assert (sonnet + UsageRecord(resolved_model="claude-sonnet-5-5")).resolved_model == "claude-sonnet-5-5" + mixed = sonnet + UsageRecord(resolved_model="claude-opus-5-0") + assert mixed.resolved_model == "claude-opus-5-0,claude-sonnet-5-5" + + +def test_a_cached_cli_reply_replays_the_resolved_model(runner, monkeypatch, tmp_path): + monkeypatch.setenv("DOKIMI_RESPONSE_CACHE", str(tmp_path / "responses")) + runner.answers.append(reply(_ok())) + first = llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[1] + second = llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[1] + assert len(runner.calls) == 1 # the second reply came from the cache + assert second.replayed and second.resolved_model == first.resolved_model == "claude-sonnet-5-5" + + +def test_a_new_cli_version_misses_the_response_cache(runner, monkeypatch, tmp_path): + # An alias resolves inside the CLI: a new release can move "sonnet" to another model, + # so its answers must not be replayed for the new one. + monkeypatch.setenv("DOKIMI_RESPONSE_CACHE", str(tmp_path / "responses")) + runner.answers.extend([reply(_ok("OLD")), reply(_ok("NEW"))]) + monkeypatch.setattr(llm_cli, "cli_version", lambda provider: "2.1.303 (Claude Code)") + assert llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[0] == "OLD" + monkeypatch.setattr(llm_cli, "cli_version", lambda provider: "2.1.400 (Claude Code)") + assert llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[0] == "NEW" + assert len(runner.calls) == 2 + + +def test_the_cli_version_is_its_first_line_and_is_read_once(monkeypatch): + llm_cli._version_line.cache_clear() + fake = FakeRunner(lambda call: reply("2.1.303 (Claude Code)\nmore")) + monkeypatch.setattr(llm_cli.process, "run", fake) + try: + assert llm_cli.cli_version("claude-cli") == "2.1.303 (Claude Code)" + assert llm_cli.cli_version("claude-cli") == "2.1.303 (Claude Code)" + assert len(fake.calls) == 1 + finally: + llm_cli._version_line.cache_clear() From 1688652d1a2663b6e67d6eedf46c5e6d0df3e9cc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:03:51 -0700 Subject: [PATCH 15/42] test(llm): drop the raising=False on the cli_version patches, now the attribute exists Review round 4 (nit). The flag was only needed before the attribute existed. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_llm_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 2817163c1..d5a84bea2 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -271,7 +271,7 @@ def test_benchmark_preflight_checks_the_login_not_a_key(monkeypatch): errors, _ = _provider_checks(["claude-cli"]) assert any("not installed" in e for e in errors) monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) - monkeypatch.delenv("OPENAI_API_KEY") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) errors, _ = _provider_checks(["openai"]) assert any("missing OPENAI_API_KEY" in e for e in errors) From a3c2923eccc0025908a346c567e501d13be11ceb Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 20:49:03 -0700 Subject: [PATCH 16/42] test(llm): pin the benchmark preflight exit status on the CLI login The login checks were covered through _provider_checks only, so a preflight that dropped an error, skipped the judge's login, or lost the unknown-login warning still passed. The new test runs cmd_benchmark_preflight end to end with a fake claude: signed out exits 1, an unanswered login prints the warning and passes, a signed-in judge passes, and a signed-out judge with an API answerer exits 1. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_llm_cli.py | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index d5a84bea2..328a4bf12 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -575,3 +575,44 @@ def test_the_cli_version_is_its_first_line_and_is_read_once(monkeypatch): assert len(fake.calls) == 1 finally: llm_cli._version_line.cache_clear() + + +@pytest.mark.parametrize( + "answer, status, exits, printed", + [ + ("claude-cli", {"loggedIn": False}, True, "is not signed in"), + ("claude-cli", "no answer", False, "could not check the login"), + ("claude-cli", {"loggedIn": True}, False, "Preflight OK"), + ("openai", {"loggedIn": False}, True, "is not signed in"), + ], + ids=["signed-out-is-an-error", "no-answer-is-a-warning", "signed-in-passes", "judge-login-is-checked"], +) +def test_benchmark_preflight_exit_status_follows_the_cli_login(monkeypatch, tmp_path, capsys, + answer, status, exits, printed): + from dokimi.cli import build_parser + from dokimi.commands import benchmark + from dokimi.registry import ProductEntry + + class Registry: + def get(self, key): + return ProductEntry(key="onetbb", name="oneTBB", description="TBB", doc_sources=["local:docs"]) + + monkeypatch.setenv("OPENAI_API_KEY", "test") + monkeypatch.setattr(benchmark, "_load_registry", lambda args: Registry()) + monkeypatch.setattr("dokimi.mcp.factory.create_doc_source_client", lambda doc_source: object()) + monkeypatch.setattr(process, "run", _auth_runner(status)) + args = build_parser().parse_args([ + "benchmark", "preflight", "--library", "onetbb", "--doc-source", "local:docs", + "--provider", answer, "--model", "sonnet" if answer == "claude-cli" else "gpt-4o", + "--judge-provider", "claude-cli", "--judge-model", "opus", + "--output-dir", str(tmp_path / "out"), + ]) + if exits: + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + else: + args.func(args) + captured = capsys.readouterr() + assert printed in captured.out + captured.err + assert ("Preflight OK" in captured.out) is (not exits) From 090122c05db97e114f47de2d672d85e5dfed4746 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 22:33:22 -0700 Subject: [PATCH 17/42] fix(llm): refuse a CLI matrix before its first cell; close review test gaps - arms matrix-run refuses a claude-cli cell with an agentic arm before any cell is answered, as the sampling check already does (_refuse_cli_cell). - drop the usage-limit entry from the retryable words: _claude_error refuses that text before the tables are read, so the entry was dead. - tests: answers generate joins the parse-time model refusal; the novelty default judge is pinned for full-id answerers (claude-opus-5-1) as well as bare aliases. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/arms.py | 10 +++++++++ dokimi/dokimi/llm_cli.py | 2 +- dokimi/tests/test_llm_cli.py | 39 +++++++++++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index e424aa9d0..2898e2b6b 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -126,6 +126,7 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: cell_args = argparse.Namespace(**vars(args)) _apply_matrix_cell(cell_args, cell) _check_sampling_support(cell_args) + _refuse_cli_cell(cell_args, specs) except (ValueError, FileNotFoundError) as exc: print(f"Error loading matrix run inputs: {exc}", file=sys.stderr) sys.exit(1) @@ -562,6 +563,15 @@ def _apply_matrix_cell(args: argparse.Namespace, matrix_cell) -> None: args.plugins = ",".join(matrix_cell.plugin_specs) +def _refuse_cli_cell(args: argparse.Namespace, specs: list[str]) -> None: + """A matrix cell on a CLI provider with an agentic arm is refused before the first cell is answered.""" + from dokimi import llm_cli + from dokimi.treatments import create_treatments + + if llm_cli.backend(args.provider) is not None: + refuse_cli_arms(args, create_treatments(specs, top_k=args.top_k, rerank_threshold=args.rerank_threshold)) + + def _check_sampling_support(args: argparse.Namespace) -> None: """Reject sampling flags the answer provider would refuse on every answer.""" from dokimi.llm import unsupported_sampling_params diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 1915dd40f..852b70462 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -75,7 +75,7 @@ #: ``api_error_status`` values worth another attempt. _RETRYABLE_STATUS = frozenset({408, 429, 500, 502, 503, 504, 529}) -_RETRYABLE_WORDS = ("rate limit", "rate_limit", "overloaded", "usage limit", "timed out") +_RETRYABLE_WORDS = ("rate limit", "rate_limit", "overloaded", "timed out") DEFAULT_TIMEOUT_SEC = 600.0 diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 328a4bf12..402eb6303 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -385,11 +385,12 @@ def test_the_independence_warning_fires_for_an_alias_against_its_full_id(capsys) assert not _warn_judge_independence("claude-cli", "sonnet", "claude-cli", "claude-opus-5-5") -def test_the_novelty_default_judge_on_the_cli_is_another_family(): +@pytest.mark.parametrize("answerer", ["sonnet", "opus", "claude-opus-5-1", "claude-sonnet-5-5"]) +def test_the_novelty_default_judge_on_the_cli_is_another_family(answerer): from dokimi.novelty.client import default_judge - judge, provider = default_judge("sonnet", "claude-cli") - assert provider == "claude-cli" and not llm_cli.same_model("claude-cli", judge, "claude-cli", "sonnet") + judge, provider = default_judge(answerer, "claude-cli") + assert provider == "claude-cli" and not llm_cli.same_model("claude-cli", judge, "claude-cli", answerer) def test_a_novelty_judge_of_the_answerers_family_counts_as_the_answerer(): @@ -436,6 +437,7 @@ def test_every_command_with_a_hidden_or_default_model_refuses_the_cli_without_a_ from dokimi.cli import parse_args cases = [ + ["answers", "generate", "--product", "p", "--questions", "q.json", "--provider", "claude-cli"], ["questions", "generate", "--product", "p", "--personas", "p.json", "--provider", "claude-cli"], ["questions", "analyze", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], ["questions", "panel-review", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], @@ -500,6 +502,37 @@ def __init__(self, *args, **kwargs): arms._build_arms_output(args, ["baseline", "agent"], []) +def test_a_matrix_with_a_cli_agentic_cell_is_refused_before_the_first_cell_is_answered(monkeypatch, tmp_path, capsys): + from dokimi.commands import arms + from dokimi.eval import arm_runner + from dokimi.eval import cells as eval_cells + + class RunnerMustNotStart: + def __init__(self, *args, **kwargs): + raise AssertionError("a cell was answered before the CLI refusal") + + cells = [ + argparse.Namespace(name="api", model="gpt-6.1-sol", provider="openai", harness="arms-runner", + plugin_specs=[]), + argparse.Namespace(name="cli", model="sonnet", provider="claude-cli", harness="arms-runner", + plugin_specs=[]), + ] + monkeypatch.setattr(arm_runner, "ArmRunner", RunnerMustNotStart) + monkeypatch.setattr(eval_cells, "load_matrix_cells", lambda path: cells) + monkeypatch.setattr(arms, "_load_questions", lambda path: []) + monkeypatch.setattr(arms, "_validate_supported_matrix_harnesses", lambda cells: None) + monkeypatch.setattr(arms, "_validate_unique_cell_filenames", lambda cells: None) + args = argparse.Namespace(arms="baseline,agent", questions="q.json", matrix_cells="m.json", + product="p", out_dir=str(tmp_path), top_k=3, rerank_threshold=0.3, + temperature=None, seed=None, judge=False, session_turns=None, + plugins="", harness="arms-runner", matrix_cell=None, judge_provider=None, + judge_model=None, max_iterations=5, context7_id=None) + with pytest.raises(SystemExit): + arms.cmd_arms_matrix_run(args) + assert "agentic arm" in capsys.readouterr().err + assert not tmp_path.joinpath("cells").exists() + + def test_an_alias_and_a_bedrock_or_prefixed_full_id_are_one_model_for_the_identity_checks(): assert llm_cli.same_model("claude-cli", "sonnet", "anthropic", "us.anthropic.claude-sonnet-5-5-v1:0") assert llm_cli.same_model("claude-cli", "sonnet", "litellm", "anthropic/claude-sonnet-5-5") From c8f0ca9fec9874e5478af6d19a6803c0a6497998 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 28 Oct 2026 22:36:25 -0700 Subject: [PATCH 18/42] fix(novelty): refuse an API judge of a CLI alias answerer's family up front default_judge compared bare names, so --model sonnet on claude-cli with --judge-provider anthropic picked claude-sonnet-5-5 as judge. same_client refused it only after the paid extraction; compare by family instead, so the run stops in role selection, as it does for a full id on main. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/novelty/client.py | 6 ++---- dokimi/tests/test_novelty_cli.py | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index 765c6fedd..41dce22c6 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -282,7 +282,7 @@ def default_judge(model: str, provider: str | None = None, Raises: ValueError: ``provider`` runs no default judge other than ``model``. """ - from dokimi.llm_cli import backend + from dokimi.llm_cli import backend, same_model bare = model.rsplit("/", 1)[-1].rsplit("anthropic.", 1)[-1] if provider is None and answerer_provider is not None and backend(answerer_provider) is not None: @@ -290,8 +290,6 @@ def default_judge(model: str, provider: str | None = None, # Hard-coded for claude-cli; a second CLI backend (codex-cli) needs its own alias list here. if provider is not None and provider == "claude-cli": # The CLI's aliases: the first one not of the answerer's family, so the judge is never the answerer. - from dokimi.llm_cli import same_model - for alias in ("opus", "sonnet", "haiku"): if not same_model("claude-cli", alias, "claude-cli", bare): return alias, provider @@ -299,7 +297,7 @@ def default_judge(model: str, provider: str | None = None, "pass --judge-model as well") candidates = [(m, p) for m, p in ((DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER), (DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER)) - if m != bare] + if m != bare and not same_model(answerer_provider or "", bare, p, m)] if provider is None: return candidates[0] for m, p in candidates: diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index 62b7f1f5f..87fec6730 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -72,6 +72,9 @@ def test_a_judge_provider_alone_takes_the_generator_when_the_judge_is_measured() ("--model", f"us.anthropic.{DEFAULT_JUDGE_MODEL}", "--provider", "amazon-bedrock", "--judge-provider", "amazon-bedrock"), ("--model", DEFAULT_JUDGE_MODEL, "--provider", "anthropic", "--judge-provider", "anthropic"), + # A CLI alias is its family's full id: refused here, not after the paid extraction. + ("--model", "sonnet", "--provider", "claude-cli", "--judge-provider", "anthropic"), + ("--model", "sonnet", "--provider", "claude-cli", "--judge-provider", "amazon-bedrock"), # The default answerer is the generator, so openai has no default judge left either. ("--judge-provider", "openai"), ]) From dbe7e2838f5e1da522c4c4de7becd3fe4a004456 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 28 Oct 2026 22:36:33 -0700 Subject: [PATCH 19/42] test(novelty): pin the CLI default judge reached through the command The only test called default_judge with an explicit claude-cli provider, so dropping the CLI-answerer inheritance or the same-family skip survived the suite. Parse novelty analyze with no judge flags instead. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_novelty_cli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index 87fec6730..dd366d02f 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -84,6 +84,17 @@ def test_a_judge_provider_with_no_default_judge_there_exits(argv): novelty_cmd._clients(args) +@pytest.mark.parametrize("model", ["sonnet", "opus", "haiku", "claude-opus-5-5"]) +def test_a_cli_answerer_with_no_judge_flags_is_judged_on_the_cli_by_another_family(model): + from dokimi.llm_cli import same_model + + args = _parse("novelty", "analyze", "--artifact", "skill:x", "--model", model, + "--provider", "claude-cli") + _extractor, _answerer, judge = novelty_cmd._clients(args) + assert judge.provider == "claude-cli" + assert not same_model("claude-cli", judge.model, "claude-cli", model) + + def test_a_named_judge_without_a_provider_runs_on_the_answer_provider(): args = _parse("novelty", "analyze", "--artifact", "skill:x", "--provider", "openrouter", "--judge-model", "j") From 52a1f383efde65ef9437c680d04dadc320db2b33 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 28 Oct 2026 22:37:54 -0700 Subject: [PATCH 20/42] fix(arms): record judge_resolved_models in the arms judge record arms run and matrix-run build the judge record in ArmRunner, which copied the judge's usage totals but not the models a CLI alias resolved to, so only eval score wrote judge_resolved_models as the changelog says. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/eval/arm_runner.py | 11 ++++++----- dokimi/tests/test_judge_sampling.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index 4b6045055..c262ebd06 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -742,11 +742,12 @@ def build_output( # cost_summary so a full run's spend is auditable from one artifact). if judge is not None and hasattr(judge, "_usage_totals"): with judge._usage_lock: - totals = dict(judge._usage_totals) - totals["cost_usd"] = ( - round(totals["cost_usd"], 6) if totals["cost_known_calls"] else None - ) - out["judge_metrics"] = totals + t = dict(judge._usage_totals) + resolved = sorted(getattr(judge, "_resolved_models", ())) + t["cost_usd"] = round(t["cost_usd"], 6) if t["cost_known_calls"] else None + out["judge_metrics"] = t + if resolved: # the model a CLI alias judge ran as, as eval score records it + out["judge_resolved_models"] = resolved return out def _summarize( diff --git a/dokimi/tests/test_judge_sampling.py b/dokimi/tests/test_judge_sampling.py index 710627487..682f4317b 100644 --- a/dokimi/tests/test_judge_sampling.py +++ b/dokimi/tests/test_judge_sampling.py @@ -28,6 +28,19 @@ def test_the_judge_record_names_the_resolved_models_it_was_judged_by(): assert j._build_evaluation_output([])["judge_resolved_models"] == ["claude-sonnet-5-5"] +def test_the_arms_judge_record_names_the_resolved_models_too(): + # arms run and matrix-run build the judge record in ArmRunner, not in Judge. + from dokimi.eval.arm_runner import ArmRunner + from dokimi.metrics.usage import UsageRecord + from dokimi.treatments import BaselineTreatment + + runner, j = ArmRunner([BaselineTreatment()]), Judge(model="opus", provider="claude-cli") + assert "judge_resolved_models" not in runner.build_output("oneTBB", [], [], judge=j) + j._accumulate_usage(UsageRecord(resolved_model="claude-opus-5-5")) + out = runner.build_output("oneTBB", [], [], judge=j) + assert out["judge_resolved_models"] == ["claude-opus-5-5"] + + def test_judge_defaults_leave_sampling_unset(): j = Judge(model="m", provider="openai") assert j.temperature is None From 42558758754d0beac50f9776499f9123b7d75700 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 28 Oct 2026 22:39:08 -0700 Subject: [PATCH 21/42] fix(arms): refuse a matrix cell pairing a CLI provider with a non-Claude model The parse-time gate reads --model only; a matrix cell sets the provider and model after it, so claude-cli/gpt-6.1-sol in a cell sent `claude --model gpt-6.1-sol` for every answer. Check the pair where the cell is applied. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/arms.py | 7 +++++++ dokimi/tests/test_llm_cli.py | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 2898e2b6b..32cf7b109 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -556,6 +556,13 @@ def _apply_named_matrix_cell(args: argparse.Namespace) -> None: def _apply_matrix_cell(args: argparse.Namespace, matrix_cell) -> None: + from dokimi import llm_cli + + if llm_cli.backend(matrix_cell.provider) and not llm_cli.accepts_model(matrix_cell.provider, + matrix_cell.model): + # The parse-time model gate reads --model only; a cell sets the pair after it. + raise ValueError(f"matrix cell {matrix_cell.name!r}: provider {matrix_cell.provider} takes a " + f"Claude model (sonnet, haiku, opus or a claude-* id), not {matrix_cell.model!r}") args.matrix_cell = matrix_cell.name args.model = matrix_cell.model args.provider = matrix_cell.provider diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 402eb6303..749744e3e 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -533,6 +533,41 @@ def __init__(self, *args, **kwargs): assert not tmp_path.joinpath("cells").exists() +def _matrix_run(monkeypatch, tmp_path, cli_model, arms_spec, capsys): + """matrix-run on an API cell then a CLI cell, where no cell may run: the error it exits with.""" + from dokimi.cli import build_parser + from dokimi.commands import arms + + cells = tmp_path / "cells.json" + cells.write_text(json.dumps({"matrix": {"cells": [ + {"id": "api", "model": "gpt-6.1-sol", "provider": "openai", "harness": "arms-runner"}, + {"id": "cli", "model": cli_model, "provider": "claude-cli", "harness": "arms-runner"}, + ]}}), encoding="utf-8") + questions = tmp_path / "q.json" + questions.write_text(json.dumps([{"id": "q1", "question": "Q?"}]), encoding="utf-8") + def cell_must_not_run(*a, **kw): + raise AssertionError("a matrix cell ran before the CLI cell was refused") + + monkeypatch.setattr(arms, "_build_arms_output", cell_must_not_run) + args = build_parser().parse_args( + ["arms", "matrix-run", "--product", "p", "--questions", str(questions), "--arms", arms_spec, + "--matrix-cells", str(cells), "--out-dir", str(tmp_path / "out")]) + with pytest.raises(SystemExit): + args.func(args) + return capsys.readouterr().err + + +def test_a_matrix_cell_pairing_a_cli_provider_with_a_non_claude_model_is_refused( + monkeypatch, tmp_path, capsys): + from dokimi.commands import arms + + assert "takes a Claude model" in _matrix_run(monkeypatch, tmp_path, "gpt-6.1-sol", "baseline", capsys) + # arms run --matrix-cells/--matrix-cell applies the cell the same way + named = argparse.Namespace(matrix_cells=str(tmp_path / "cells.json"), matrix_cell="cli") + with pytest.raises(ValueError, match="takes a Claude model"): + arms._apply_named_matrix_cell(named) + + def test_an_alias_and_a_bedrock_or_prefixed_full_id_are_one_model_for_the_identity_checks(): assert llm_cli.same_model("claude-cli", "sonnet", "anthropic", "us.anthropic.claude-sonnet-5-5-v1:0") assert llm_cli.same_model("claude-cli", "sonnet", "litellm", "anthropic/claude-sonnet-5-5") From 3d6bc5c05cfd4d84cfef98b5a967cee25e55b4a7 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 11:48:21 +0000 Subject: [PATCH 22/42] Test that a bad request or auth failure is not retried Signed-off-by: Anton Volkov --- dokimi/tests/test_llm_cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 749744e3e..20f578a72 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -129,6 +129,14 @@ def test_each_retryable_word_is_retried_without_a_status(runner, word): assert info.value.retryable is True +@pytest.mark.parametrize("status", [None, 400, 401, 404, 501]) +def test_a_bad_request_or_auth_failure_is_not_retried(runner, status): + runner.answers.append(reply(_err("invalid request error", status), returncode=1)) + with pytest.raises(llm_cli.CliCallError) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False + + @pytest.mark.parametrize("status", [None, 429]) def test_a_subscription_usage_limit_is_not_retried(runner, status): # It resets after hours: three more CLI starts would only delay the same failure. From 5ebba4fa5d49b3b2372bfd8a3da323e154aaa492 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 11:48:32 +0000 Subject: [PATCH 23/42] Recognise the model family in Vertex ids with an @ version Signed-off-by: Anton Volkov --- dokimi/dokimi/llm_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 852b70462..1bc0bb80b 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -44,6 +44,7 @@ import functools import json import os +import re from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Dict, Optional, Tuple @@ -257,7 +258,7 @@ def _family(model: str) -> Optional[str]: if name in CLAUDE_CLI.aliases: return name if name.startswith("claude-"): - for part in name.split("-"): + for part in re.split(r"[-@]", name): if part in CLAUDE_CLI.aliases: return part return None From 925efd8863d1fe14437236fdc12778597a40ed0e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Thu, 29 Oct 2026 11:48:58 +0000 Subject: [PATCH 24/42] Describe the full environment drop in the subscription backends doc Signed-off-by: Anton Volkov --- dokimi/docs/subscription-backends.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 521af5d34..43024f526 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -81,11 +81,13 @@ carries about 37k tokens of agent prompt, tool schemas and `CLAUDE.md`. - `--bare` is never used: it reads only `ANTHROPIC_API_KEY`, so a subscription login fails with "Authentication error". -- The child gets the caller's environment minus the variables a Claude Code - session sets for its children (`CLAUDECODE`, `CLAUDE_CODE_SESSION_ID`, …). If - dokimi itself runs inside Claude Code and they were passed on, the child would - join the parent's session. `ANTHROPIC_API_KEY` is dropped too: set, it would - win over the login and the run would bill the key. +- The child gets the caller's environment minus every variable starting with + `ANTHROPIC_`, `CLAUDE_` or `CLAUDECODE`, except `CLAUDE_CONFIG_DIR` and + `CLAUDE_CODE_OAUTH_TOKEN`. That drops the session variables, an API key or + token that would win over the login, and a base URL, model alias or + Bedrock/Vertex switch, so the CLI runs on its own login. If dokimi itself + runs inside Claude Code and the session variables were passed on, the child + would join the parent's session. - The CLI keeps its own config directory, because that is where the login is stored. User settings and `CLAUDE.md` stay out because of `--setting-sources ""`. From 481c61dd33ab08437cc82069adb9e14e53faba57 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:41:50 -0700 Subject: [PATCH 25/42] fix(benchmark): probe a CLI provider once in the preflight _credential_error and _login_warning each called llm_cli.detect, so a CLI provider ran `--version` and `auth status` twice, and a flaky `auth status` could give two answers in one preflight. _provider_checks now detects once per provider and derives the error and the warning from that status. Thread: PRRT_kwDOVBg7g86xlz5y Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/benchmark.py | 26 ++++++++++++++++++-------- dokimi/tests/test_llm_cli.py | 10 ++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index bddb78fa6..dcc44c234 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -83,14 +83,17 @@ def _credential_env_var(provider: str) -> str: }.get(provider, "OPENAI_API_KEY") -def _credential_error(provider: str) -> str | None: - """Why *provider* cannot run here, or ``None``: a missing key, or a CLI not signed in.""" +def _credential_error(provider: str, status=None) -> str | None: + """Why *provider* cannot run here, or ``None``: a missing key, or a CLI not signed in. + + *status* is the provider's ``llm_cli.detect`` result when the caller already has it. + """ from dokimi import llm_cli if llm_cli.backend(provider) is None: key = _credential_env_var(provider) return None if os.environ.get(key) else f"missing {key} for provider {provider}" - status = llm_cli.detect(provider) + status = status or llm_cli.detect(provider) if not status.installed: return f"{provider}: '{status.binary}' is not installed" if status.authenticated is False: @@ -99,20 +102,27 @@ def _credential_error(provider: str) -> str | None: def _provider_checks(providers: list[str]) -> tuple[list[str], list[str]]: - """The errors (a missing key, a CLI not installed or signed in) and warnings for *providers*.""" + """The errors (a missing key, a CLI not installed or signed in) and warnings for *providers*. + + A CLI provider is probed once: ``detect`` runs ``--version`` and ``auth status``, and two + probes of a flaky ``auth status`` could give an error-free pass and a warning in one preflight. + """ + from dokimi import llm_cli + names = sorted(set(providers)) - errors = [e for e in map(_credential_error, names) if e] - warnings = [w for w in map(_login_warning, names) if w] + statuses = {n: llm_cli.detect(n) for n in names if llm_cli.backend(n) is not None} + errors = [e for e in (_credential_error(n, statuses.get(n)) for n in names) if e] + warnings = [w for w in (_login_warning(n, statuses.get(n)) for n in names) if w] return errors, warnings -def _login_warning(provider: str) -> str | None: +def _login_warning(provider: str, status=None) -> str | None: """A warning when a CLI provider's login could not be checked (no answer from ``auth status``).""" from dokimi import llm_cli if llm_cli.backend(provider) is None: return None - status = llm_cli.detect(provider) + status = status or llm_cli.detect(provider) if status.installed and status.authenticated is None: return f"{provider}: could not check the login ('{status.binary} auth status' gave no answer)" return None diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 20f578a72..8769e646d 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -284,6 +284,16 @@ def test_benchmark_preflight_checks_the_login_not_a_key(monkeypatch): assert any("missing OPENAI_API_KEY" in e for e in errors) +def test_the_preflight_probes_a_cli_provider_once_for_its_error_and_its_warning(monkeypatch): + from dokimi.commands.benchmark import _provider_checks + + fake = _auth_runner("no answer") + monkeypatch.setattr(process, "run", fake) + errors, warnings = _provider_checks(["claude-cli", "claude-cli"]) + assert errors == [] and len(warnings) == 1 + assert [c.argv[-1] for c in fake.calls] == ["--version", "status"] # one detect, not one per check + + def test_a_judge_panel_refuses_sampling_on_the_cli_with_explicit_judges(): from dokimi.eval.panel import JudgeConfig, JudgePanel From 2876e3bde2b59196a42e4d3c7708fcf2039a6010 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:42:17 -0700 Subject: [PATCH 26/42] fix(utils): get_llm imports llm_cli from its own module It took llm_cli through `from dokimi.llm import ...`, which works only while llm.py happens to import the module. Thread: PRRT_kwDOVBg7g86xlz6D Signed-off-by: Nikolay Petrov --- dokimi/dokimi/utils.py | 3 ++- dokimi/tests/test_llm_cli.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index 6f9a2b379..9e637e2e5 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -161,7 +161,8 @@ def get_llm(provider: str, model: str, api_key: str | None = None, ``max_tokens=2000``, Bedrock reads only its bearer token) and its cache keys, until the judge, generator and validator move to ``LLM``. """ - from dokimi.llm import LLM, _GetLLMCompat, _Invoker, llm_cli + from dokimi import llm_cli + from dokimi.llm import LLM, _GetLLMCompat, _Invoker if llm_cli.backend(provider) is not None: # a logged-in CLI: no route, no key return _Invoker(LLM(model, provider, temperature=temperature, seed=seed)) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 8769e646d..79eb9c491 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -231,6 +231,19 @@ def test_judge_and_get_llm_reach_the_cli_backend(runner): assert resp.content == "generated" and resp.usage.provider == "claude-cli" +def test_get_llm_imports_llm_cli_from_its_own_module(): + import ast + import inspect + + from dokimi.utils import get_llm + + tree = ast.parse(inspect.getsource(get_llm)) + borrowed = [alias.name for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module == "dokimi.llm" + for alias in node.names if alias.name == "llm_cli"] + assert borrowed == [] # it works only while llm.py happens to import the module + + def _auth_runner(status: dict | None, version: str = "2.1.304 (Claude Code)"): def handle(call): if call.argv[-1] == "--version": From 75991a4e4efce7b10bccd6be71c0ea23118a871d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:43:00 -0700 Subject: [PATCH 27/42] fix(judge): report a CLI judge's sampling refusal as an error line, exit 2 check_sampling raised a plain ValueError that cli.main did not catch, so `eval score` and `eval panel-score` with --judge-provider claude-cli and a temperature printed a traceback. It now raises SamplingError (a ValueError, so the existing catches still work), which cli.main reports with exit 2. The message no longer names --seed, which the panel does not have. Thread: PRRT_kwDOVBg7g86xlz59 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/cli.py | 2 +- dokimi/dokimi/llm_cli.py | 9 +++++++-- dokimi/tests/test_llm_cli.py | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index 8b90c0d83..887ee65cf 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -120,7 +120,7 @@ def main() -> None: try: args.func(args) - except (EmptyQuestionError, InputError) as exc: # a bad input record, not a bug: no traceback + except (EmptyQuestionError, InputError, llm_cli.SamplingError) as exc: # bad input, not a bug: no traceback print(f"Error: {exc}", file=sys.stderr) # 2, as the commands' other input errors: 1 is the verdict of `eval judge-validity` # and of the --strict gates, and a malformed file must not read as a failed gate. diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 1bc0bb80b..34261e288 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -113,6 +113,10 @@ class AuthStatus: hint: str = "" +class SamplingError(ValueError): + """A temperature or seed given to a CLI provider, which takes neither; ``cli.main`` reports it (exit 2).""" + + @dataclass(frozen=True) class CliBackend: """One CLI: how to call it, read its reply, and ask it whether it is signed in.""" @@ -284,9 +288,10 @@ def same_model(provider_a: str, model_a: str, provider_b: str, model_b: str) -> def check_sampling(provider: str, temperature: Optional[float], seed: Optional[int]) -> None: - """Raise ``ValueError`` before a run when a CLI *provider* is given a temperature or seed.""" + """Raise :class:`SamplingError` before a run when a CLI *provider* is given a temperature or seed.""" if provider in BACKENDS and (temperature is not None or seed is not None): - raise ValueError(f"{provider} takes no temperature or seed: drop --judge-temperature/--seed") + raise SamplingError(f"{provider} takes no temperature or seed: leave out the judge temperature " + "and seed options") def work_dir(name: str) -> Path: diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 79eb9c491..91f12846b 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -307,6 +307,23 @@ def test_the_preflight_probes_a_cli_provider_once_for_its_error_and_its_warning( assert [c.argv[-1] for c in fake.calls] == ["--version", "status"] # one detect, not one per check +@pytest.mark.parametrize("argv", [ + ["eval", "score", "--judge-provider", "claude-cli", "--judge-temperature", "0"], + ["eval", "panel-score", "--provider", "claude-cli", "--judge-temperature", "0"], +]) +def test_a_cli_judge_given_a_temperature_is_an_error_line_and_exit_2(argv, tmp_path, monkeypatch, capsys): + from dokimi import cli + + answers = tmp_path / "answers.json" + answers.write_text("[]", encoding="utf-8") + monkeypatch.setattr("sys.argv", ["dokimi", *argv, "--product", "p", "--answers", str(answers)]) + with pytest.raises(SystemExit) as exc: + cli.main() + err = capsys.readouterr().err + assert exc.value.code == 2 and err.startswith("Error: claude-cli takes no temperature") + assert "Traceback" not in err + + def test_a_judge_panel_refuses_sampling_on_the_cli_with_explicit_judges(): from dokimi.eval.panel import JudgeConfig, JudgePanel From 96ca858917f542a06d1070f08e7833b3c8cb974b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:43:32 -0700 Subject: [PATCH 28/42] docs(reviewing): list claude-cli among the providers The guide gave the valid --provider values as an exhaustive set and left out claude-cli. A test now checks it names every answer provider. Thread: PRR_kwDOVBg7g88AAAABU43URA.a Signed-off-by: Nikolay Petrov --- dokimi/docs/reviewing.md | 4 +++- dokimi/tests/test_llm_cli.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/reviewing.md b/dokimi/docs/reviewing.md index 8a0f8982b..0fdddc833 100644 --- a/dokimi/docs/reviewing.md +++ b/dokimi/docs/reviewing.md @@ -20,7 +20,9 @@ pip install -e . # installs the `dokimi` command The `novelty` and `arms` steps take `--model` and `--provider`; `arms run` also needs `--judge-provider` (it does not fall back to `--provider` for the judge); `tasks run` and `repo` take `--model` only. The provider is one of `openai`, -`anthropic`, `amazon-bedrock`, `google-vertex`, `openrouter` or `openai-codex`; +`anthropic`, `amazon-bedrock`, `google-vertex`, `openrouter`, `openai-codex` or +`claude-cli` (a logged-in Claude Code CLI, no key; see +[subscription backends](subscription-backends.md)); `arms run --judge-provider` takes `google`, an alias of `google-vertex`, but not `openai-codex`. Credentials are read from the environment: `OPENAI_API_KEY`, diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 91f12846b..faa28ad7b 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -244,6 +244,13 @@ def test_get_llm_imports_llm_cli_from_its_own_module(): assert borrowed == [] # it works only while llm.py happens to import the module +def test_the_reviewing_guide_lists_every_answer_provider(): + from pathlib import Path + + text = (Path(__file__).resolve().parents[1] / "docs" / "reviewing.md").read_text(encoding="utf-8") + assert [p for p in llm.ANSWER_PROVIDERS if f"`{p}`" not in text] == [] + + def _auth_runner(status: dict | None, version: str = "2.1.304 (Claude Code)"): def handle(call): if call.argv[-1] == "--version": From b03bd76156b105d1442f96b4e5e037d39b55226a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 11:51:18 -0700 Subject: [PATCH 29/42] Address review: four reviews whose findings earlier commits fixed Thread: PRR_kwDOVBg7g88AAAABUbOhiQ: already fixed in 35c5219b and 9147dfa4 Thread: PRR_kwDOVBg7g88AAAABUgBkVw: already fixed in 9147dfa4, 4937a078 and 9ad3fe17 Thread: PRR_kwDOVBg7g88AAAABUp_h4A: already fixed in 35c5219b, a3c2923e and 9147dfa4 Thread: PRR_kwDOVBg7g88AAAABUqQ6ag: already fixed in 9147dfa4, 090122c0 and 4937a078 Signed-off-by: Nikolay Petrov From 7ccbb93edded81ef432f0eceb689362bc9ad7994 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:53:18 -0700 Subject: [PATCH 30/42] fix(backends): an installed CLI with no version output is 'unknown', not 'not installed' Review of intel/dokimi#255 (backends.py:26): detect() sets installed=True with an empty version when --version prints nothing or times out, and the version column then read 'not installed' beside 'yes'/'no'. The column now keys off installed. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/backends.py | 3 ++- dokimi/tests/test_llm_cli.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/backends.py b/dokimi/dokimi/commands/backends.py index 9a71495ea..46a96b64c 100644 --- a/dokimi/dokimi/commands/backends.py +++ b/dokimi/dokimi/commands/backends.py @@ -22,8 +22,9 @@ def cmd_backends(args: argparse.Namespace) -> None: print("|---|---|---|---|---|---|") for r in rows: signed = {True: "yes", False: "no", None: "unknown"}[r.authenticated] if r.installed else "-" + version = r.version or ("unknown" if r.installed else "not installed") models = ", ".join(llm_cli.BACKENDS[r.backend].aliases) - print(f"| {r.backend} | {r.binary} | {r.version or 'not installed'} | {signed} " + print(f"| {r.backend} | {r.binary} | {version} | {signed} " f"| {r.method or '-'} | {models} |") for r in rows: if r.hint: diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index faa28ad7b..a849c6e6f 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -286,6 +286,19 @@ def test_backends_command_prints_one_row_per_backend(monkeypatch, capsys): assert row["backend"] == "claude-cli" and row["authenticated"] is True +def test_an_installed_cli_that_prints_no_version_is_unknown_not_not_installed(monkeypatch, capsys): + from dokimi.commands.backends import cmd_backends + + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True}, version="")) + cmd_backends(argparse.Namespace(json=False)) + (row,) = [line for line in capsys.readouterr().out.splitlines() if line.startswith("| claude-cli |")] + assert row.split("|")[3:5] == [" unknown ", " yes "] + monkeypatch.setattr(process, "run", _auth_runner(None)) + cmd_backends(argparse.Namespace(json=False)) + (gone,) = [line for line in capsys.readouterr().out.splitlines() if line.startswith("| claude-cli |")] + assert gone.split("|")[3] == " not installed " + + def test_benchmark_preflight_checks_the_login_not_a_key(monkeypatch): from dokimi.commands.benchmark import _provider_checks From 509867cc92df53963a5a75f8a2fade52eb1c8f8d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:06 -0700 Subject: [PATCH 31/42] fix(llm_cli): a failure with no JSON reply is retried on its transient words Review of intel/dokimi#255 (llm_cli.py:190): a non-zero exit with no JSON on stdout was always non-retryable, so 'API Error: 503 ... overloaded' or 'rate limit' on stderr failed the run at once. The same words are now checked on that text, as they are on a JSON result. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/llm_cli.py | 7 +++++-- dokimi/tests/test_llm_cli.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 34261e288..58fd557f5 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -186,8 +186,11 @@ def _claude_parse(done: process.Completed, model: str, provider: str) -> CliResu except ValueError: payload = None if not isinstance(payload, dict): - raise CliCallError(f"claude-cli: exit {done.returncode}, no JSON reply: " - f"{(done.stderr or done.stdout).strip()[:300]}", retryable=False) + text = (done.stderr or done.stdout).strip() + # No JSON, so no status: the same transient words as a JSON result are checked on the text. + retryable = any(w in text.lower() for w in _RETRYABLE_WORDS) + raise CliCallError(f"claude-cli: exit {done.returncode}, no JSON reply: {text[:300]}", + retryable=retryable) if payload.get("is_error") or done.returncode != 0: raise _claude_error(payload, done) u = payload.get("usage") or {} diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index a849c6e6f..eb8a33588 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -102,6 +102,7 @@ def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): (reply(_err("Overloaded", 529), returncode=1), True, "Overloaded"), (reply("", timed_out=True), True, "no reply"), (reply("segfault", stderr="boom", returncode=139), False, "no JSON reply"), + (reply("", stderr="API Error: 503 Service Unavailable, overloaded", returncode=1), True, "no JSON reply"), ]) def test_a_failed_call_says_whether_another_attempt_can_help(runner, answer, retryable, words): runner.answers.append(answer) From e643423c13e68660417de1df5d5ef98c90ea1dcc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:28 -0700 Subject: [PATCH 32/42] fix(llm_cli): pass CLAUDE_CODE_GIT_BASH_PATH to the CLI Review of intel/dokimi#255 (llm_cli.py:71): the ANTHROPIC_/CLAUDE_ prefix drop also removed CLAUDE_CODE_GIT_BASH_PATH, which Claude Code on Windows reads to find Git Bash. There, every call failed with 'no JSON reply'. It changes neither the billing nor the session, so it is kept with the login variables. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/llm_cli.py | 7 ++++--- dokimi/tests/test_llm_cli.py | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 58fd557f5..325fe0846 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -66,9 +66,10 @@ #: provider switch such as CLAUDE_CODE_USE_BEDROCK) cannot change who is billed or which #: session the child joins. CLAUDE_ENV_DROP_PREFIXES = ("ANTHROPIC_", "CLAUDE_", "CLAUDECODE") -#: Kept despite a prefix: where the account's login lives, and the token ``claude setup-token`` -#: makes for a headless login. -CLAUDE_ENV_KEEP = frozenset({"CLAUDE_CONFIG_DIR", "CLAUDE_CODE_OAUTH_TOKEN"}) +#: Kept despite a prefix: where the account's login lives, the token ``claude setup-token`` +#: makes for a headless login, and where Claude Code on Windows finds Git Bash (without it +#: ``claude -p`` cannot start there). +CLAUDE_ENV_KEEP = frozenset({"CLAUDE_CONFIG_DIR", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CODE_GIT_BASH_PATH"}) #: The flags that strip the CLI to a bare completion; see the module docstring. CLAUDE_ISOLATION = ("--tools", "", "--setting-sources", "", "--strict-mcp-config", diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index eb8a33588..0b70d68a1 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -411,6 +411,14 @@ def test_the_child_env_drops_billing_and_session_variables_by_prefix(runner, mon assert env["CLAUDE_CONFIG_DIR"] == "/accounts/work" +def test_the_git_bash_path_a_windows_claude_needs_is_passed_on(runner, monkeypatch): + bash = r"C:\Program Files\Git\bin\bash.exe" + monkeypatch.setenv("CLAUDE_CODE_GIT_BASH_PATH", bash) + runner.answers.append(reply(_ok())) + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert runner.calls[0].env["CLAUDE_CODE_GIT_BASH_PATH"] == bash + + def test_the_work_dir_is_under_the_users_cache_not_the_shared_temp_dir(monkeypatch, tmp_path): import tempfile From 2881692db4947dc63ef1113d0e0b6911160f4a4b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:54:47 -0700 Subject: [PATCH 33/42] fix(llm_cli): accept the 1M-context form (sonnet[1m]) for claude-cli Review of intel/dokimi#255 (llm_cli.py:256): claude --model takes the [1m] context-window suffix, but accepts_model refused it, so a matrix cell failed with a usage error. A trailing [...] is now stripped in accepts_model, _family and same_model, so sonnet[1m] is the sonnet family for the self-judge check as well. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/llm_cli.py | 19 +++++++++++++++---- dokimi/tests/test_llm_cli.py | 11 +++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 325fe0846..ffb853fa0 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -254,15 +254,26 @@ def backend(provider: str) -> Optional[CliBackend]: return BACKENDS.get(provider) +#: The 1M-context suffix ``claude --model`` takes after an alias or a full id: ``sonnet[1m]``. +_CONTEXT_SUFFIX = re.compile(r"\[[^\]]*\]$") + + +def _without_context(model: str) -> str: + """*model* with a trailing ``[...]`` (the context-window form) removed, and blanks stripped.""" + return _CONTEXT_SUFFIX.sub("", model.strip()) + + def accepts_model(provider: str, model: str) -> bool: - """Whether *provider*'s CLI takes *model*: one of its aliases, or a full id it names.""" + """Whether *provider*'s CLI takes *model*: one of its aliases, or a full id it names (either may + carry a ``[1m]``-style context suffix).""" cli = BACKENDS[provider] - return model in cli.aliases or model.startswith(cli.model_prefixes) + bare = _without_context(model) + return bare in cli.aliases or bare.startswith(cli.model_prefixes) def _family(model: str) -> Optional[str]: """The alias a Claude model name belongs to: ``sonnet``, or of ``claude-sonnet-5-5``.""" - name = model.strip().lower().rsplit("/", 1)[-1].rsplit("anthropic.", 1)[-1] + name = _without_context(model.lower().rsplit("/", 1)[-1].rsplit("anthropic.", 1)[-1]) if name in CLAUDE_CLI.aliases: return name if name.startswith("claude-"): @@ -282,7 +293,7 @@ def same_model(provider_a: str, model_a: str, provider_b: str, model_b: str) -> """ if provider_a not in BACKENDS and provider_b not in BACKENDS: return False - a, b = model_a.strip().lower(), model_b.strip().lower() + a, b = _without_context(model_a.lower()), _without_context(model_b.lower()) if a == b: return True if a not in CLAUDE_CLI.aliases and b not in CLAUDE_CLI.aliases: diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 0b70d68a1..f2c612d35 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -454,6 +454,17 @@ def test_an_alias_and_its_full_id_are_one_model_for_the_identity_checks(): assert not same("openai", "gpt-6.1-sol", "claude-cli", "sonnet") +def test_the_one_million_context_form_is_a_claude_model_on_the_cli(): + # claude --model takes sonnet[1m]; it is the same model as the alias, with a larger window. + assert llm_cli.accepts_model("claude-cli", "sonnet[1m]") + assert llm_cli.accepts_model("claude-cli", "opus[1m]") + assert llm_cli.accepts_model("claude-cli", "claude-sonnet-5-5[1m]") + assert not llm_cli.accepts_model("claude-cli", "gpt-6.1-sol[1m]") + assert llm_cli._family("sonnet[1m]") == "sonnet" + assert llm_cli.same_model("claude-cli", "sonnet[1m]", "claude-cli", "claude-sonnet-5-5") + assert not llm_cli.same_model("claude-cli", "sonnet[1m]", "claude-cli", "claude-opus-5-5") + + def test_the_independence_warning_fires_for_an_alias_against_its_full_id(capsys): from dokimi.commands.evaluate import _warn_judge_independence From d436854cb17985d1240525d7a06544cab31bd9c6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:55:16 -0700 Subject: [PATCH 34/42] fix(llm_cli): stop git at the work dir's parent so a repo above it is not read Review of intel/dokimi#255 (llm_cli.py:305): the work dir is under the user's cache dir, which can sit inside a git repository (a home directory under git). The CLI then put that repository's branch and status into every prompt. The child now gets GIT_CEILING_DIRECTORIES set to the work dir's parent, so git does not search above it. The test runs git with the call's own env and cwd. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/llm_cli.py | 20 ++++++++++++++++---- dokimi/tests/test_llm_cli.py | 17 +++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index ffb853fa0..36d19ecfd 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -332,8 +332,19 @@ def _passed_on(name: str, cli: CliBackend) -> bool: return name not in cli.env_drop and not name.startswith(cli.env_drop_prefixes) -def _env(cli: CliBackend) -> process.Env: - return process.Env.only([k for k in os.environ if _passed_on(k, cli)]) +def _env(cli: CliBackend, extra: Optional[Dict[str, str]] = None) -> process.Env: + return process.Env.only([k for k in os.environ if _passed_on(k, cli)], set=extra) + + +def _call_env(cli: CliBackend, cwd: Path) -> process.Env: + """The env of a call: the parent's, minus the drops, and git stopped at the work dir's parent. + + The CLI reads the git status of its working directory into the prompt. The work dir sits + under the user's cache dir, which can lie inside a repository (a home directory under git), + so git must not search above that cache directory for a repository. + """ + ceiling = [str(cwd.parent)] + [p for p in os.environ.get("GIT_CEILING_DIRECTORIES", "").split(os.pathsep) if p] + return _env(cli, {"GIT_CEILING_DIRECTORIES": os.pathsep.join(ceiling)}) def cache_extra(provider: str) -> Dict[str, str]: @@ -372,9 +383,10 @@ def call(provider: str, *, model: str, prompt: str, system: Optional[str] = None KeyError: *provider* is not a CLI backend. """ cli = BACKENDS[provider] + cwd = work_dir(cli.name) try: - done = (runner or process.run)(cli.build_argv(model, system), timeout=timeout, env=_env(cli), - cwd=work_dir(cli.name), input=prompt, capture_limit=None) + done = (runner or process.run)(cli.build_argv(model, system), timeout=timeout, env=_call_env(cli, cwd), + cwd=cwd, input=prompt, capture_limit=None) except FileNotFoundError as exc: raise CliCallError(f"{provider}: '{cli.binary}' is not installed or not on PATH ({exc})", retryable=False) from exc diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index f2c612d35..c35c98ba0 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -419,6 +419,23 @@ def test_the_git_bash_path_a_windows_claude_needs_is_passed_on(runner, monkeypat assert runner.calls[0].env["CLAUDE_CODE_GIT_BASH_PATH"] == bash +def test_git_does_not_read_a_repository_above_the_work_dir(runner, monkeypatch, tmp_path): + # The cache dir sits inside a git repository (a home directory under git): the CLI must not + # read that repository's branch and status into its prompt. Probed with the call's own env. + import subprocess + + home = tmp_path / "home" + subprocess.run(["git", "init", "-q", str(home)], check=True) + monkeypatch.setenv("XDG_CACHE_HOME", str(home / "cache")) + runner.answers.append(reply(_ok())) + llm_cli.call("claude-cli", model="sonnet", prompt="p") + call = runner.calls[0] + env = {k: v for k, v in call.env.items() if k not in ("GIT_DIR", "GIT_WORK_TREE")} + probe = subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], cwd=call.cwd, env=env, + capture_output=True, text=True) + assert probe.returncode != 0, probe.stdout + + def test_the_work_dir_is_under_the_users_cache_not_the_shared_temp_dir(monkeypatch, tmp_path): import tempfile From efd401f0c317bb8d64c573ee0599a7d0469b6cb5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:56:47 -0700 Subject: [PATCH 35/42] test(llm_cli): pin the empty reply shapes the parser branches on Review of intel/dokimi#255 (test_llm_cli.py:410): every success payload in the suite had a non-empty result, usage and modelUsage, so the empty branches had no test. A reply with modelUsage {}, a null usage and a null result, and one with no modelUsage key, now must give no model id, no tokens and no text. Each of those fails when its guard is removed. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_llm_cli.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index c35c98ba0..5b5d84ca3 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -455,6 +455,22 @@ def test_a_reply_names_the_main_model_the_cli_resolved(runner): assert out.model_id == "claude-sonnet-5-5" +def test_an_empty_reply_gives_no_model_no_tokens_and_no_text(runner): + # The parser's empty branches: no modelUsage entries, a null usage and a null result. + runner.answers.append(reply(_ok(modelUsage={}, usage=None, result=None))) + out = llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert out.model_id is None and out.text == "" + assert (out.usage.prompt_tokens, out.usage.completion_tokens, out.usage.resolved_model) == (0, 0, "") + + +def test_a_reply_without_a_model_usage_key_names_no_model(runner): + payload = json.loads(_ok()) + del payload["modelUsage"] + runner.answers.append(reply(json.dumps(payload))) + out = llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert out.model_id is None and out.usage.resolved_model == "" + + def test_the_llm_reply_carries_the_resolved_model_id(runner): runner.answers.append(reply(_ok())) request = llm.Request(model="sonnet", provider="claude-cli", litellm_model="sonnet", From 720da725fd9471d589f169408a6e44daa0da67cc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 17:55:33 -0700 Subject: [PATCH 36/42] docs(subscription-backends): name tools as the command without a CLI provider Review of intel/dokimi#255 (subscription-backends.md:16): 'wherever a command takes --provider' is too broad, since tools preference takes only the API providers (it needs tool calling). The sentence names that exception, and a test pins it against tools.PROVIDERS. Signed-off-by: Nikolay Petrov --- dokimi/docs/subscription-backends.md | 2 +- dokimi/tests/test_llm_cli.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 43024f526..995e7bda1 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -13,7 +13,7 @@ SPDX-License-Identifier: Apache-2.0 The Q&A track (answers, judging, question generation, personas, novelty) can call a model through a coding-agent CLI the machine is already signed in to, instead of an API key. Today one backend exists, `claude-cli` (Claude Code's `claude -p`). -Pass it wherever a command takes `--provider` or `--judge-provider`. +Pass it wherever a command takes `--provider` or `--judge-provider`, except `tools`, which needs tool calling. ## Check what this machine can use diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 5b5d84ca3..51c87e58a 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -245,6 +245,16 @@ def test_get_llm_imports_llm_cli_from_its_own_module(): assert borrowed == [] # it works only while llm.py happens to import the module +def test_the_backend_doc_names_tools_as_the_command_that_refuses_a_cli_provider(): + from pathlib import Path + + from dokimi.commands import tools + + text = (Path(__file__).resolve().parents[1] / "docs" / "subscription-backends.md").read_text(encoding="utf-8") + assert "claude-cli" not in tools.PROVIDERS + assert "`--provider` or `--judge-provider`, except `tools`" in text + + def test_the_reviewing_guide_lists_every_answer_provider(): from pathlib import Path From 988089e3ecf73a2e5e28616ed30e78b12f2b3ade Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 29 Oct 2026 19:32:22 -0700 Subject: [PATCH 37/42] docs(ci): keep the comment blocks this PR edits to two lines The comment-block gate (scripts/check_comment_blocks.py, from #298) fails a PR that adds or edits a line of a block longer than two lines. After the merge of intel/main it flagged three blocks: the CLAUDE_ENV_DROP_PREFIXES and CLAUDE_ENV_KEEP comments in llm_cli.py, and the golden_manifest bounds comment in spec.v1.yaml, which this PR edits for the new doc count. Each block is cut to two lines that keep the reason. The bounds comment drops its "8 markdown files live outside docs/" backstory; the max_docs value says the count. Red: check_comment_blocks.py --base intel/main --head HEAD exits 1 with three errors. Green: it exits 0 after this commit. Signed-off-by: Nikolay Petrov --- dokimi/benchmarks/spec.v1.yaml | 8 ++------ dokimi/dokimi/llm_cli.py | 10 ++++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index f47f15b71..fbaf738fb 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -72,12 +72,8 @@ future: enabled: false min_score: 0.85 -# Bounds on the document set the static track scores. They are a sanity check -# on the include/exclude globs, not a target: min_docs catches a glob that -# matched almost nothing, max_docs catches one that escaped into the tree (8 -# markdown files live outside docs/). Measured selection at the time of -# writing: 36 files. Adding a doc past the ceiling is meant to be a deliberate -# edit here, so raise max_docs in the same commit that adds the doc. +# Sanity bounds on the static track's doc set, not targets: min_docs catches a glob +# that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 max_docs: 36 diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py index 36d19ecfd..f1eedbf66 100644 --- a/dokimi/dokimi/llm_cli.py +++ b/dokimi/dokimi/llm_cli.py @@ -62,13 +62,11 @@ # Set, they win over the CLI's login: the run would bill the key or token, not the subscription. "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", }) -#: Dropped by prefix, so a variable not yet listed (an endpoint, a model alias override, a -#: provider switch such as CLAUDE_CODE_USE_BEDROCK) cannot change who is billed or which -#: session the child joins. +#: Dropped by prefix, so a variable not yet listed (an endpoint, a model alias, a provider switch +#: such as CLAUDE_CODE_USE_BEDROCK) cannot change who is billed or which session the child joins. CLAUDE_ENV_DROP_PREFIXES = ("ANTHROPIC_", "CLAUDE_", "CLAUDECODE") -#: Kept despite a prefix: where the account's login lives, the token ``claude setup-token`` -#: makes for a headless login, and where Claude Code on Windows finds Git Bash (without it -#: ``claude -p`` cannot start there). +#: Kept despite a prefix: the login's location, the ``claude setup-token`` token, and Git Bash +#: for Claude Code on Windows (without it ``claude -p`` cannot start there). CLAUDE_ENV_KEEP = frozenset({"CLAUDE_CONFIG_DIR", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CODE_GIT_BASH_PATH"}) #: The flags that strip the CLI to a bare completion; see the module docstring. From 772801c51854f2677f81ce31befb821b3a8a4fc1 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:05:33 +0000 Subject: [PATCH 38/42] docs(llm): list every kept env variable and the git ceiling Signed-off-by: Anton Volkov --- .../20261026_150000_napetrov_claude-cli-backend.md | 5 +++-- dokimi/docs/subscription-backends.md | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md index f0c78654e..8869daf28 100644 --- a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md +++ b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md @@ -28,7 +28,8 @@ - Independence checks treat an alias and its full id as one model (`opus` and `claude-opus-…`). The novelty default judge on `claude-cli` is another Claude family than the answerer. - The CLI child process drops `ANTHROPIC_*`, `CLAUDE_*` and `CLAUDECODE` variables from the - environment (except `CLAUDE_CONFIG_DIR` and `CLAUDE_CODE_OAUTH_TOKEN`), so a billing or - session setting in the caller does not reroute the call. A Bedrock or Vertex route set + environment (except `CLAUDE_CONFIG_DIR`, `CLAUDE_CODE_OAUTH_TOKEN` and + `CLAUDE_CODE_GIT_BASH_PATH` (Windows)), so a billing or + session setting in the caller does not reroute the call. Git is stopped at the work dir's parent. A Bedrock or Vertex route set in that environment is dropped too: the CLI then uses its own login, and `backends` reports that login, not the route. diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 995e7bda1..63e6f1883 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -82,12 +82,14 @@ carries about 37k tokens of agent prompt, tool schemas and `CLAUDE.md`. - `--bare` is never used: it reads only `ANTHROPIC_API_KEY`, so a subscription login fails with "Authentication error". - The child gets the caller's environment minus every variable starting with - `ANTHROPIC_`, `CLAUDE_` or `CLAUDECODE`, except `CLAUDE_CONFIG_DIR` and - `CLAUDE_CODE_OAUTH_TOKEN`. That drops the session variables, an API key or + `ANTHROPIC_`, `CLAUDE_` or `CLAUDECODE`, except `CLAUDE_CONFIG_DIR`, + `CLAUDE_CODE_OAUTH_TOKEN` and `CLAUDE_CODE_GIT_BASH_PATH` (Windows). That + drops the session variables, an API key or token that would win over the login, and a base URL, model alias or Bedrock/Vertex switch, so the CLI runs on its own login. If dokimi itself runs inside Claude Code and the session variables were passed on, the child - would join the parent's session. + would join the parent's session. Git is stopped at the work dir's parent + (`GIT_CEILING_DIRECTORIES`), so it finds no repository above it. - The CLI keeps its own config directory, because that is where the login is stored. User settings and `CLAUDE.md` stay out because of `--setting-sources ""`. From 9134bca0fe6986c6c1ae27348e659d456ef75568 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:05:44 +0000 Subject: [PATCH 39/42] docs(llm): say the exclude-dynamic flag is not passed Signed-off-by: Anton Volkov --- dokimi/docs/subscription-backends.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 63e6f1883..1e99d6402 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -59,8 +59,9 @@ What this means for results: `anthropic:claude-sonnet-5-5` get different prompts (the CLI adds its own text) and different sampling. Compare arms within one backend. - **The CLI states its environment, and nothing turns this off.** It does so - with no tools, with its own system prompt replaced, and with - `--exclude-dynamic-system-prompt-sections`. Tested and not working: + with no tools and with its own system prompt replaced; + `--exclude-dynamic-system-prompt-sections` was tried and does not remove it. + Tested and not working: `CLAUDE_CODE_HIDE_CWD`, `CLAUDE_CODE_DISABLE_GIT_INSTRUCTIONS` (Claude Code 2.1.304). dokimi runs every call in one fixed empty directory (`$XDG_CACHE_HOME/dokimi/cli/claude-cli`, default From cbefbe38865841bec56812f7d3647098cabae14f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:05:55 +0000 Subject: [PATCH 40/42] docs(llm): note the system prompt is bound by the argument limit Signed-off-by: Anton Volkov --- dokimi/docs/subscription-backends.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 1e99d6402..7336d6002 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -77,7 +77,7 @@ What this means for results: Each call is `claude -p --input-format text --output-format json --model --tools "" --setting-sources "" --strict-mcp-config --no-session-persistence --system-prompt `. The prompt goes on stdin, so its size is not bounded by -the argument limit. Without these flags, a plain `claude -p` in a repository +the argument limit; the system prompt is an argument and is. Without these flags, a plain `claude -p` in a repository carries about 37k tokens of agent prompt, tool schemas and `CLAUDE.md`. - `--bare` is never used: it reads only `ANTHROPIC_API_KEY`, so a subscription From 1b6d131602ab8c27110ddc72a0e97e51de808457 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:06:06 +0000 Subject: [PATCH 41/42] docs(llm): list the errors that fail on the first attempt Signed-off-by: Anton Volkov --- dokimi/docs/subscription-backends.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md index 7336d6002..0ba42ef49 100644 --- a/dokimi/docs/subscription-backends.md +++ b/dokimi/docs/subscription-backends.md @@ -98,8 +98,8 @@ carries about 37k tokens of agent prompt, tool schemas and `CLAUDE.md`. ## Errors and retries Rate limits, overload, 5xx and a timeout are retried with the same backoff as -an API provider. An unknown model, a missing login or a missing binary fail on -the first attempt. +an API provider. An unknown model, a missing login, a missing binary, an +unusable work directory and a subscription usage limit fail on the first attempt. ## Terms of use From 3aed8c8bc73b8d57827181cc8bb7b57b764f7835 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Fri, 30 Oct 2026 10:06:27 +0000 Subject: [PATCH 42/42] test(llm): pin that the caller's git ceiling entries are kept Signed-off-by: Anton Volkov --- dokimi/tests/test_llm_cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index 51c87e58a..cbd0b64bb 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -432,14 +432,18 @@ def test_the_git_bash_path_a_windows_claude_needs_is_passed_on(runner, monkeypat def test_git_does_not_read_a_repository_above_the_work_dir(runner, monkeypatch, tmp_path): # The cache dir sits inside a git repository (a home directory under git): the CLI must not # read that repository's branch and status into its prompt. Probed with the call's own env. + import os import subprocess home = tmp_path / "home" subprocess.run(["git", "init", "-q", str(home)], check=True) monkeypatch.setenv("XDG_CACHE_HOME", str(home / "cache")) + monkeypatch.setenv("GIT_CEILING_DIRECTORIES", str(tmp_path / "users-own")) runner.answers.append(reply(_ok())) llm_cli.call("claude-cli", model="sonnet", prompt="p") call = runner.calls[0] + # The caller's own ceiling entry is kept next to the one for the work dir's parent. + assert call.env["GIT_CEILING_DIRECTORIES"].split(os.pathsep) == [str(call.cwd.parent), str(tmp_path / "users-own")] env = {k: v for k, v in call.env.items() if k not in ("GIT_DIR", "GIT_WORK_TREE")} probe = subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], cwd=call.cwd, env=env, capture_output=True, text=True)