diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index cde41b078..fbaf738fb 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -72,15 +72,11 @@ future: enabled: false min_score: 0.85 -# Bounds on the document set the static track scores. They are a sanity check -# on the include/exclude globs, not a target: min_docs catches a glob that -# matched almost nothing, max_docs catches one that escaped into the tree (8 -# markdown files live outside docs/). Measured selection at the time of -# writing: 35 files. Adding a doc past the ceiling is meant to be a deliberate -# edit here, so raise max_docs in the same commit that adds the doc. +# Sanity bounds on the static track's doc set, not targets: min_docs catches a glob +# that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 - max_docs: 35 + max_docs: 36 include: - docs/**/*.md exclude: diff --git a/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md new file mode 100644 index 000000000..8869daf28 --- /dev/null +++ b/dokimi/changelog.d/20261026_150000_napetrov_claude-cli-backend.md @@ -0,0 +1,35 @@ +### Added + +- **`--provider claude-cli` / `--judge-provider claude-cli`**: answers, judging, + question generation, personas and novelty can run through a logged-in Claude + Code CLI (`claude -p`) instead of an API key. Each call runs with no tools, no + user or project settings and no MCP servers, in one fixed empty directory, with + the prompt on stdin. A requested `temperature` or `seed` is refused, since + the CLI cannot honour them. `cost_usd` is `None`, because the CLI's figure is + list price. Rate limits, overload and timeouts are retried; an unknown model or + a missing login is not. See `docs/subscription-backends.md`. +- **`backends`**: lists the CLI backends with install, version and login status; + calls no model. `benchmark preflight` checks a CLI backend's login instead + of an API key. + +### Changed + +- The `--provider` choices of every command come from one tuple, + `dokimi.llm.ANSWER_PROVIDERS`. `tools` keeps API providers only, since it needs + tool calling. +- A CLI provider refuses an agentic arm (`mcp_agent`, `skill_agent`, `skill_catalog`) and + `--session-turns` in `arms run` and `arms matrix-run`, before any answer is paid for: the CLI + runs one prompt with no tools and no conversation. +- A CLI reply records the model the CLI resolved its alias to: `resolved_model` on each usage + row (answers, judge calls), `judge_resolved_models` in the judge record. The response-cache + key takes the CLI version, so a new release that moves `sonnet` does not replay the old + model's replies. The resolved id is known only after a call, so a move the CLI does not + ship with is not caught. +- Independence checks treat an alias and its full id as one model (`opus` and + `claude-opus-…`). The novelty default judge on `claude-cli` is another Claude family than the answerer. +- The CLI child process drops `ANTHROPIC_*`, `CLAUDE_*` and `CLAUDECODE` variables from the + environment (except `CLAUDE_CONFIG_DIR`, `CLAUDE_CODE_OAUTH_TOKEN` and + `CLAUDE_CODE_GIT_BASH_PATH` (Windows)), so a billing or + session setting in the caller does not reroute the call. Git is stopped at the work dir's parent. A Bedrock or Vertex route set + in that environment is dropped too: the CLI then uses its own login, and `backends` reports + that login, not the route. diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index c336eecb0..68bf82147 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -313,6 +313,8 @@ code. verifier pass rate against a baseline. - [answer-formats.md](answer-formats.md) — run Inspect AI, OpenAI evals, skill-creator and promptfoo answer-only sets on the answers track +- [subscription-backends.md](subscription-backends.md) — run the Q&A track through a + logged-in coding-agent CLI (`--provider claude-cli`) instead of an API key - [context-window.md](context-window.md) — read a run's context window from its Claude Code trace with `cli.py context trace`: start-up load, growth, compactions, tool-result weight. No LLM call. diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 6dd46dc4f..4bc5e7617 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -411,6 +411,10 @@ Guide: [run-consistency.md](run-consistency.md). adapter seam: `LiteLLMAdapter` in production, `FakeAdapter` in tests (the `fake_llm` fixture). `ask_parsed` is the redraw-until-it-parses loop. `llm_call`, `chat_completion` and `utils.get_llm` delegate to `LLM`. + `dokimi/llm_cli.py` serves the CLI providers (`claude-cli`): `LiteLLMAdapter` + hands their requests to it, and it runs the logged-in CLI through `process.run` + ([subscription-backends.md](subscription-backends.md)). `dokimi/llm_json.py` + holds the JSON extraction helpers `llm.py` re-exports. - `dokimi/caching.py` — the two caches that hang off `llm.py`: provider prompt caching (the stable part of a prompt goes first, so the provider can reuse it) and an opt-in response cache for byte-identical single-shot requests diff --git a/dokimi/docs/reviewing.md b/dokimi/docs/reviewing.md index b40065add..98c8e2292 100644 --- a/dokimi/docs/reviewing.md +++ b/dokimi/docs/reviewing.md @@ -20,7 +20,9 @@ pip install -e . # installs the `dokimi` command The `novelty` and `arms` steps take `--model` and `--provider`; `arms run` also needs `--judge-provider` (it does not fall back to `--provider` for the judge); `tasks run` and `repo` take `--model` only. The provider is one of `openai`, -`anthropic`, `amazon-bedrock`, `google-vertex`, `openrouter` or `openai-codex`; +`anthropic`, `amazon-bedrock`, `google-vertex`, `openrouter`, `openai-codex` or +`claude-cli` (a logged-in Claude Code CLI, no key; see +[subscription backends](subscription-backends.md)); `arms run --judge-provider` takes `google`, an alias of `google-vertex`, but not `openai-codex`. Credentials are read from the environment: `OPENAI_API_KEY`, diff --git a/dokimi/docs/subscription-backends.md b/dokimi/docs/subscription-backends.md new file mode 100644 index 000000000..0ba42ef49 --- /dev/null +++ b/dokimi/docs/subscription-backends.md @@ -0,0 +1,107 @@ +--- +depends_on: + - dokimi/dokimi/llm_cli.py + - dokimi/dokimi/commands/backends.py +--- + + +# Subscription backends: a logged-in CLI as the LLM provider + +The Q&A track (answers, judging, question generation, personas, novelty) can call +a model through a coding-agent CLI the machine is already signed in to, instead +of an API key. Today one backend exists, `claude-cli` (Claude Code's `claude -p`). +Pass it wherever a command takes `--provider` or `--judge-provider`, except `tools`, which needs tool calling. + +## Check what this machine can use + +```bash +python cli.py backends # one row per backend: installed, version, signed in, how +python cli.py backends --json +``` + +This calls no model. A backend that is not signed in prints how to fix it. On a +machine without a browser, `claude setup-token` prints a token; export it as +`CLAUDE_CODE_OAUTH_TOKEN`. `benchmark preflight` checks the login the same way, +instead of looking for an API key. + +## Run + +```bash +python cli.py answers generate --product oneTBB --questions questions/oneTBB.json \ + --output answers/oneTBB.json --provider claude-cli --model haiku +python cli.py eval score --product oneTBB --answers answers/oneTBB.json \ + --output eval/oneTBB.json --judge-provider claude-cli --judge-model sonnet +``` + +`--model` takes what `claude --model` takes: an alias (`haiku`, `sonnet`, +`opus`) or a full id (`claude-sonnet-5-5`). The default judge model works on +`claude-cli` as it does on `anthropic`. When the answerer and the judge are the +same model, the judge grades its own answers. + +## How it differs from an API provider + +| | API provider | `claude-cli` | +|---|---|---| +| Credential | `*_API_KEY` | the CLI's own login | +| `--temperature`, `--seed` | sent when set | refused before the run (`unsupported_sampling_params`) | +| `max_tokens` | output ceiling | ignored: the CLI has no flag for it | +| Hidden prompt | none | fixed CLI preamble (≈0.5k–1.3k tokens, by CLI version), plus working directory, git status, platform and date | +| Tools | `chat(tools=...)` | refused: one prompt, one reply | +| `cost_usd` | litellm price | `None`: the CLI's `total_cost_usd` is list price, not what a subscription is charged | +| Latency | model only | plus ≈2–3 s CLI start-up per call | + +What this means for results: + +- **Don't pool scores across transports.** `claude-cli:sonnet` and + `anthropic:claude-sonnet-5-5` get different prompts (the CLI adds its own text) + and different sampling. Compare arms within one backend. +- **The CLI states its environment, and nothing turns this off.** It does so + with no tools and with its own system prompt replaced; + `--exclude-dynamic-system-prompt-sections` was tried and does not remove it. + Tested and not working: + `CLAUDE_CODE_HIDE_CWD`, `CLAUDE_CODE_DISABLE_GIT_INSTRUCTIONS` (Claude Code + 2.1.304). dokimi runs every call in one fixed empty directory + (`$XDG_CACHE_HOME/dokimi/cli/claude-cli`, default + `~/.cache/dokimi/cli/claude-cli`), so the stated environment is the same in the + with-docs and the without-docs arm, and the response cache still hits. A + without-docs answer may still mention it ("the working directory isn't a git + repository"). +- **Repeats differ.** With no temperature or seed, the same prompt gives a + different answer each time; measure variance with repeats, not a pinned seed. + +## Isolation + +Each call is `claude -p --input-format text --output-format json --model +--tools "" --setting-sources "" --strict-mcp-config --no-session-persistence +--system-prompt `. The prompt goes on stdin, so its size is not bounded by +the argument limit; the system prompt is an argument and is. Without these flags, a plain `claude -p` in a repository +carries about 37k tokens of agent prompt, tool schemas and `CLAUDE.md`. + +- `--bare` is never used: it reads only `ANTHROPIC_API_KEY`, so a subscription + login fails with "Authentication error". +- The child gets the caller's environment minus every variable starting with + `ANTHROPIC_`, `CLAUDE_` or `CLAUDECODE`, except `CLAUDE_CONFIG_DIR`, + `CLAUDE_CODE_OAUTH_TOKEN` and `CLAUDE_CODE_GIT_BASH_PATH` (Windows). That + drops the session variables, an API key or + token that would win over the login, and a base URL, model alias or + Bedrock/Vertex switch, so the CLI runs on its own login. If dokimi itself + runs inside Claude Code and the session variables were passed on, the child + would join the parent's session. Git is stopped at the work dir's parent + (`GIT_CEILING_DIRECTORIES`), so it finds no repository above it. +- The CLI keeps its own config directory, because that is where the login is + stored. User settings and `CLAUDE.md` stay out because of + `--setting-sources ""`. + +## Errors and retries + +Rate limits, overload, 5xx and a timeout are retried with the same backoff as +an API provider. An unknown model, a missing login, a missing binary, an +unusable work directory and a subscription usage limit fail on the first attempt. + +## Terms of use + +A consumer subscription is for personal use. Keep CI and shared benchmark +runs on API keys. diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index 32ee6e64c..887ee65cf 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -14,6 +14,7 @@ from dokimi.commands import ( answers, arms, + backends, baseline, benchmark, catalog, @@ -35,7 +36,7 @@ tasks, tools, ) -from dokimi import __version__, process +from dokimi import __version__, llm_cli, process from dokimi.console import tolerate_console_encoding from dokimi.defaults import JUDGE_MODEL_FOR_PROVIDER, default_judge_model @@ -81,6 +82,7 @@ def positive_int(value: str) -> int: repo.register(sub, positive_int) # repo (repositories as subjects) tools.register(sub, positive_int) # tools (tool-choice probes) context.register(sub, positive_int) # context (context window of a run) + backends.register(sub, positive_int) # backends (CLI providers) return p @@ -99,6 +101,11 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.error(f"--judge-provider {args.judge_provider} has no default judge model; " f"pass --judge-model") args.judge_model = model + provider, model = getattr(args, "provider", None), getattr(args, "model", None) + if provider in llm_cli.BACKENDS and model is not None and not llm_cli.accepts_model(provider, model): + # The default answer model is not a Claude model: left out, every call would fail. + parser.error(f"--provider {provider} takes a Claude model (sonnet, haiku, opus or a claude-* id), " + f"not {model!r}: pass --model") return args @@ -113,7 +120,7 @@ def main() -> None: try: args.func(args) - except (EmptyQuestionError, InputError) as exc: # a bad input record, not a bug: no traceback + except (EmptyQuestionError, InputError, llm_cli.SamplingError) as exc: # bad input, not a bug: no traceback print(f"Error: {exc}", file=sys.stderr) # 2, as the commands' other input errors: 1 is the verdict of `eval judge-validity` # and of the --strict gates, and a malformed file must not read as a failed gate. diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py index 4b08abe6a..f93828616 100644 --- a/dokimi/dokimi/commands/answers.py +++ b/dokimi/dokimi/commands/answers.py @@ -9,6 +9,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands.evaluate import _require_ragas, _run_ragas_eval from dokimi.defaults import ( DEFAULT_ANSWER_MODEL, @@ -158,7 +159,7 @@ def register(sub, positive_int) -> None: "Sets output to results/{product}_{run_id}/answers/{product}.json") gen_a_p.add_argument("--output", default=None, help="Output file (default: answers/{product}.json)") gen_a_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM model for answering") - gen_a_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + gen_a_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) gen_a_p.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, help="Max tokens to retrieve per question") gen_a_p.add_argument("--top-k", type=int, default=None, help=f"Number of docs to retrieve before reranking (default from config/products.yaml retrieval.top_k or {DEFAULT_RETRIEVAL_TOP_K})") gen_a_p.add_argument("--rerank-threshold", type=float, default=0.3, help="Min relevance score (0-1) to keep docs") diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 0198feb0d..a7934d166 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -11,6 +11,8 @@ import sys from pathlib import Path +from dokimi.commands.arms_refusals import refuse_cli_arms, refuse_cli_cell, refuse_cli_cell_model +from dokimi.llm import ANSWER_PROVIDERS from dokimi.utils import question_id from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -123,6 +125,7 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: cell_args = argparse.Namespace(**vars(args)) _apply_matrix_cell(cell_args, cell) _check_sampling_support(cell_args) + refuse_cli_cell(cell_args, specs) except (ValueError, OSError, RecursionError) as exc: print(f"Error loading matrix run inputs: {exc}", file=sys.stderr) sys.exit(1) @@ -233,7 +236,7 @@ def register(sub, positive_int) -> None: ) arms_run_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM for answering") arms_run_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, - choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + choices=ANSWER_PROVIDERS) arms_run_p.add_argument("--harness", default="arms-runner", help="Execution harness label stamped into output (default: arms-runner)") arms_run_p.add_argument("--plugins", default="", @@ -379,7 +382,7 @@ def _add_common_run_args(parser, positive_int, *, matrix_required: bool = False) ) parser.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM for answering") parser.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, - choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + choices=ANSWER_PROVIDERS) parser.add_argument("--harness", default="arms-runner", help="Execution harness label stamped into output (default: arms-runner)") parser.add_argument("--plugins", default="", @@ -552,6 +555,7 @@ def _apply_named_matrix_cell(args: argparse.Namespace) -> None: def _apply_matrix_cell(args: argparse.Namespace, matrix_cell) -> None: + refuse_cli_cell_model(matrix_cell) args.matrix_cell = matrix_cell.name args.model = matrix_cell.model args.provider = matrix_cell.provider @@ -625,6 +629,7 @@ def _build_arms_output(args: argparse.Namespace, specs: list[str], questions: li plugins = create_plugins([s.strip() for s in args.plugins.split(",") if s.strip()]) validate_plugins_for_harness(plugins, args.harness) treatments = wrap_treatments(treatments, plugins) + refuse_cli_arms(args, treatments) plugin_set = plugin_set_metadata(plugins) print(f"Arms: {', '.join(t.name for t in treatments)}") if args.matrix_cell: diff --git a/dokimi/dokimi/commands/arms_refusals.py b/dokimi/dokimi/commands/arms_refusals.py new file mode 100644 index 000000000..af8b503c3 --- /dev/null +++ b/dokimi/dokimi/commands/arms_refusals.py @@ -0,0 +1,62 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Refusals for the arms commands on a CLI provider, raised before any answer is paid for.""" + +from __future__ import annotations + +import argparse + + +def _is_agentic_arm(treatment) -> bool: + """True when the arm offers tools, so it runs the agentic loop (see ``AgentConfig.tools``). + + The tools are added in ``prepare()``, which needs a question and may retrieve, so the + arm types that offer tools are named here instead of probed per question. + """ + from dokimi.plugins import PluginWrappedTreatment + from dokimi.treatments.arms import ( + CombinedTreatment, + MCPAgentTreatment, + SkillAgentTreatment, + SkillCatalogTreatment, + ) + + inner = treatment.inner if isinstance(treatment, PluginWrappedTreatment) else treatment + if isinstance(inner, CombinedTreatment): + return any(_is_agentic_arm(sub) for sub in inner.treatments) + return isinstance(inner, (MCPAgentTreatment, SkillAgentTreatment, SkillCatalogTreatment)) + + +def refuse_cli_arms(args: argparse.Namespace, treatments: list) -> None: + """Refused before any answer: a CLI provider runs one prompt, with no tools and no conversation.""" + from dokimi import llm_cli + + if llm_cli.backend(args.provider) is None: + return + agentic = [t.name for t in treatments if _is_agentic_arm(t)] + if agentic: + raise ValueError(f"--provider {args.provider} runs one prompt with no tools, so it cannot run " + f"the agentic arm(s) {', '.join(agentic)}: use an API provider for them") + if getattr(args, "session_turns", None) is not None: + raise ValueError(f"--session-turns needs a conversation, which --provider {args.provider} " + "does not have: use an API provider for a session") + + +def refuse_cli_cell_model(matrix_cell) -> None: + """A matrix cell sets provider and model after the parse-time model gate has read only --model.""" + from dokimi import llm_cli + + if llm_cli.backend(matrix_cell.provider) and not llm_cli.accepts_model(matrix_cell.provider, + matrix_cell.model): + raise ValueError(f"matrix cell {matrix_cell.name!r}: provider {matrix_cell.provider} takes a " + f"Claude model (sonnet, haiku, opus or a claude-* id), not {matrix_cell.model!r}") + + +def refuse_cli_cell(args: argparse.Namespace, specs: list[str]) -> None: + """A matrix cell on a CLI provider with an agentic arm is refused before the first cell is answered.""" + from dokimi import llm_cli + from dokimi.treatments import create_treatments + + if llm_cli.backend(args.provider) is not None: + refuse_cli_arms(args, create_treatments(specs, top_k=args.top_k, rerank_threshold=args.rerank_threshold)) diff --git a/dokimi/dokimi/commands/backends.py b/dokimi/dokimi/commands/backends.py new file mode 100644 index 000000000..46a96b64c --- /dev/null +++ b/dokimi/dokimi/commands/backends.py @@ -0,0 +1,42 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""backends command: which coding-agent CLIs can serve as LLM providers here.""" + +from __future__ import annotations + +import argparse +import dataclasses +import json + + +def cmd_backends(args: argparse.Namespace) -> None: + """One row per CLI backend: installed, version, signed in and how. Calls no model.""" + from dokimi import llm_cli + + rows = [llm_cli.detect(name) for name in llm_cli.BACKENDS] + if args.json: + print(json.dumps([dataclasses.asdict(r) for r in rows], indent=2)) + return + print("| provider | binary | version | signed in | method | models |") + print("|---|---|---|---|---|---|") + for r in rows: + signed = {True: "yes", False: "no", None: "unknown"}[r.authenticated] if r.installed else "-" + version = r.version or ("unknown" if r.installed else "not installed") + models = ", ".join(llm_cli.BACKENDS[r.backend].aliases) + print(f"| {r.backend} | {r.binary} | {version} | {signed} " + f"| {r.method or '-'} | {models} |") + for r in rows: + if r.hint: + print(f"{r.backend}: {r.hint}") + + +def register(sub, positive_int) -> None: + """Add the `backends` command.""" + p = sub.add_parser( + "backends", + help="Coding-agent CLIs usable as --provider (claude-cli): installed, version, signed " + "in. No model call.", + ) + p.add_argument("--json", action="store_true", help="Print the rows as JSON") + p.set_defaults(func=cmd_backends) diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index 2c50c724e..dcc44c234 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, List, cast +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands.evaluate import _warn_judge_independence from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -82,6 +83,51 @@ def _credential_env_var(provider: str) -> str: }.get(provider, "OPENAI_API_KEY") +def _credential_error(provider: str, status=None) -> str | None: + """Why *provider* cannot run here, or ``None``: a missing key, or a CLI not signed in. + + *status* is the provider's ``llm_cli.detect`` result when the caller already has it. + """ + from dokimi import llm_cli + + if llm_cli.backend(provider) is None: + key = _credential_env_var(provider) + return None if os.environ.get(key) else f"missing {key} for provider {provider}" + status = status or llm_cli.detect(provider) + if not status.installed: + return f"{provider}: '{status.binary}' is not installed" + if status.authenticated is False: + return f"{provider}: '{status.binary}' is not signed in ({status.hint})" + return None + + +def _provider_checks(providers: list[str]) -> tuple[list[str], list[str]]: + """The errors (a missing key, a CLI not installed or signed in) and warnings for *providers*. + + A CLI provider is probed once: ``detect`` runs ``--version`` and ``auth status``, and two + probes of a flaky ``auth status`` could give an error-free pass and a warning in one preflight. + """ + from dokimi import llm_cli + + names = sorted(set(providers)) + statuses = {n: llm_cli.detect(n) for n in names if llm_cli.backend(n) is not None} + errors = [e for e in (_credential_error(n, statuses.get(n)) for n in names) if e] + warnings = [w for w in (_login_warning(n, statuses.get(n)) for n in names) if w] + return errors, warnings + + +def _login_warning(provider: str, status=None) -> str | None: + """A warning when a CLI provider's login could not be checked (no answer from ``auth status``).""" + from dokimi import llm_cli + + if llm_cli.backend(provider) is None: + return None + status = status or llm_cli.detect(provider) + if status.installed and status.authenticated is None: + return f"{provider}: could not check the login ('{status.binary} auth status' gave no answer)" + return None + + def _resolved_benchmark_plan(entry, args: argparse.Namespace) -> dict: doc_source = getattr(args, "doc_source", None) or (entry.doc_sources[0] if entry.doc_sources else "context7") output_dir = getattr(args, "output_dir", None) or f"results/{entry.key}{DEFAULT_RESULTS_SUFFIX}" @@ -138,13 +184,7 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None: plan = _resolved_benchmark_plan(entry, args) _print_plan(plan) - errors: list[str] = [] - warnings: list[str] = [] - - for provider in {args.provider, args.judge_provider}: - key = _credential_env_var(provider) - if not os.environ.get(key): - errors.append(f"missing {key} for provider {provider}") + errors, warnings = _provider_checks([args.provider, args.judge_provider]) try: Path(plan["output_dir"]).mkdir(parents=True, exist_ok=True) @@ -364,7 +404,7 @@ def register(sub, positive_int) -> None: bench_run_p.add_argument("--output-dir", default=None, dest="output_dir", help="Output directory (default: results/{target}_final)") bench_run_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL) - bench_run_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + bench_run_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) bench_run_p.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, dest="judge_model", help=JUDGE_MODEL_HELP) bench_run_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider", @@ -400,7 +440,7 @@ def register(sub, positive_int) -> None: bench_pre_p.add_argument("--doc-source", default=None, dest="doc_source", help="Override doc/context source") bench_pre_p.add_argument("--output-dir", default=None, dest="output_dir", help="Output directory (default: results/{target}_final)") bench_pre_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL) - bench_pre_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + bench_pre_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) bench_pre_p.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, dest="judge_model", help=JUDGE_MODEL_HELP) bench_pre_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider", choices=JUDGE_PROVIDERS) @@ -422,7 +462,7 @@ def register(sub, positive_int) -> None: help="Run for all libraries in registry") bench_batch_p.add_argument("--output-dir", default="results", dest="output_dir") bench_batch_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL) - bench_batch_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + bench_batch_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) bench_batch_p.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, dest="judge_model", help=JUDGE_MODEL_HELP) bench_batch_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider", diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 9d3602f31..0652e0101 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -11,6 +11,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.defaults import ( DEFAULT_CONCURRENCY, DEFAULT_JUDGE_MODEL, @@ -31,7 +32,10 @@ def _warn_judge_independence( context: str = "run", ) -> bool: """Emit a non-fatal warning when answer and judge use the same model/provider.""" - same = normalize_model_ref(answer_provider, answer_model) == normalize_model_ref(judge_provider, judge_model) + from dokimi.llm_cli import same_model + + same = (normalize_model_ref(answer_provider, answer_model) == normalize_model_ref(judge_provider, judge_model) + or same_model(answer_provider, answer_model, judge_provider, judge_model)) if same: print( "⚠ Evaluator independence warning " @@ -571,7 +575,7 @@ def register(sub, positive_int) -> None: panel_p.add_argument("--product", required=True, help="Product name (e.g., oneTBB)") panel_p.add_argument("--output", default=None, help="Output file (default: eval/{product}_panel.json)") panel_p.add_argument("--model", default=DEFAULT_JUDGE_MODEL, help="Default LLM model for all judges") - panel_p.add_argument("--provider", default=DEFAULT_JUDGE_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + panel_p.add_argument("--provider", default=DEFAULT_JUDGE_PROVIDER, choices=ANSWER_PROVIDERS) panel_p.add_argument("--roles", default=None, help="Comma-separated judge roles (default: technical_expert,developer_advocate,doc_reviewer)") panel_p.add_argument( diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index c22fdf419..61a73f426 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -28,12 +28,11 @@ DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, ) +from dokimi.llm import ANSWER_PROVIDERS as PROVIDERS if TYPE_CHECKING: from dokimi.novelty.tiers import RunSettings -PROVIDERS = ["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"] - def cmd_novelty_analyze(args: argparse.Namespace) -> None: """Run the full pipeline over one artifact and write the novelty artifact + views.""" @@ -1007,8 +1006,8 @@ def _role_models(args: argparse.Namespace) -> dict[str, tuple[str, str]]: if args.judge_model: judge_model, judge_provider = args.judge_model, args.judge_provider or args.provider else: - try: - judge_model, judge_provider = default_judge(args.model, args.judge_provider) + try: # a CLI answerer with no judge provider judges on the same CLI, another family + judge_model, judge_provider = default_judge(args.model, args.judge_provider, args.provider) except ValueError as exc: raise SystemExit(str(exc)) from None extract_model = args.extract_model or judge_model diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py index f31b9bbac..b037d6854 100644 --- a/dokimi/dokimi/commands/orchestrate.py +++ b/dokimi/dokimi/commands/orchestrate.py @@ -9,6 +9,8 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS +from dokimi.llm_cli import check_sampling from dokimi.commands.evaluate import _warn_judge_independence from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -27,6 +29,11 @@ def cmd_evaluate(args: argparse.Namespace) -> None: if not getattr(args, "repo", None) and not getattr(args, "description", None): print("Error: either --repo or --description must be provided.", file=sys.stderr) sys.exit(1) + try: # before any persona, question or answer is paid for; the judge would refuse later + check_sampling(args.judge_provider, getattr(args, "judge_temperature", None), None) + except ValueError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) print(f"Starting full evaluation pipeline for {args.product}") if args.repo: @@ -117,7 +124,7 @@ def register(sub, positive_int) -> None: "is looked up, for example a checkout of the evaluation-content " "repository: --questions-root /path/to/content-repo/questions") eval_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM model for generation/answering") - eval_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + eval_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) eval_p.add_argument("--judge-model", default=JUDGE_MODEL_FOR_PROVIDER, help=JUDGE_MODEL_HELP) eval_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS) eval_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", diff --git a/dokimi/dokimi/commands/personas.py b/dokimi/dokimi/commands/personas.py index 73248afdd..ef30a23b4 100644 --- a/dokimi/dokimi/commands/personas.py +++ b/dokimi/dokimi/commands/personas.py @@ -10,6 +10,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands._output import exit_on_unwritable from dokimi.defaults import DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER from dokimi.personas.analyzer import PersonaAnalyzer @@ -133,7 +134,7 @@ def register(sub, positive_int) -> None: discover_p.add_argument("--output", default=None, help="Output file (default: personas/{product}.json)") discover_p.add_argument("--count", type=int, default=5, help="Target number of personas (5-8)") discover_p.add_argument("--model", default=DEFAULT_GENERATOR_MODEL, help="LLM model for generation") - discover_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + discover_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=ANSWER_PROVIDERS) discover_p.add_argument("--github-token", default=None, help="GitHub token (or set GITHUB_TOKEN env)") discover_p.add_argument("--save-analysis", action="store_true", help="Save intermediate analysis JSON") discover_p.set_defaults(func=cmd_personas_discover) diff --git a/dokimi/dokimi/commands/questions.py b/dokimi/dokimi/commands/questions.py index 89cc2d795..5cf3980f7 100644 --- a/dokimi/dokimi/commands/questions.py +++ b/dokimi/dokimi/commands/questions.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, Dict +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands._output import exit_on_unwritable from dokimi.defaults import DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER @@ -389,7 +390,7 @@ def register(sub, positive_int) -> None: "(sentence-transformers, no API key), or auto " "(openai when a key is set, else local)") gen_q_p.add_argument("--model", default=DEFAULT_GENERATOR_MODEL, help="LLM model for generation") - gen_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + gen_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=ANSWER_PROVIDERS) gen_q_p.add_argument("--doc-source", default="context7", help="Documentation source for topic extraction: 'context7', 'local:', 'url:'") gen_q_p.add_argument("--context7-id", default=None, dest="context7_id", @@ -402,7 +403,7 @@ def register(sub, positive_int) -> None: analyze_q_p.add_argument("--product", required=True, help="Product name (e.g., oneTBB)") analyze_q_p.add_argument("--output", default=None, help="Output report JSON (default: reports/{product}_question_quality.json)") analyze_q_p.add_argument("--model", default=DEFAULT_GENERATOR_MODEL, help="LLM model for classification") - analyze_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + analyze_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=ANSWER_PROVIDERS) analyze_q_p.add_argument("--concurrency", type=int, default=5, help="Parallel classification requests") analyze_q_p.set_defaults(func=cmd_questions_analyze) @@ -433,7 +434,7 @@ def register(sub, positive_int) -> None: panel_q_p.add_argument("--output", default=None, help="Output report JSON (default: reports/{product}_question_panel.json)") panel_q_p.add_argument("--model", default=DEFAULT_GENERATOR_MODEL) - panel_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + panel_q_p.add_argument("--provider", default=DEFAULT_GENERATOR_PROVIDER, choices=ANSWER_PROVIDERS) panel_q_p.add_argument("--reviewers", default=None, help="Comma-separated reviewers (default: domain_expert,user_advocate,qa_engineer)") panel_q_p.add_argument("--concurrency", type=positive_int, default=6) diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index e60b662ae..43ab120da 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -10,6 +10,7 @@ import sys from pathlib import Path +from dokimi.llm import ANSWER_PROVIDERS from dokimi.commands._output import exit_on_unwritable from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.defaults import ( @@ -209,7 +210,7 @@ def register(sub, positive_int) -> None: run_p.add_argument("--out-md", default=None, dest="out_md") run_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help=argparse.SUPPRESS) run_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, - choices=["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"]) + choices=ANSWER_PROVIDERS) run_p.add_argument("--harness", default="arms-runner", help=argparse.SUPPRESS) run_p.add_argument("--context7-id", default=None, dest="context7_id") run_p.add_argument("--judge", action="store_true") diff --git a/dokimi/dokimi/commands/tools.py b/dokimi/dokimi/commands/tools.py index 5908fab7a..0b14c7ac7 100644 --- a/dokimi/dokimi/commands/tools.py +++ b/dokimi/dokimi/commands/tools.py @@ -15,9 +15,10 @@ import time from pathlib import Path +from dokimi.llm import PROVIDERS as API_PROVIDERS from dokimi.defaults import DEFAULT_ANSWER_PROVIDER -PROVIDERS = ["openai", "anthropic", "amazon-bedrock", "google-vertex", "openrouter", "openai-codex"] +PROVIDERS = list(API_PROVIDERS) # tool calling: no CLI backend class CutBeforeToolCall(RuntimeError): diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py index 71056525c..10edc9e20 100644 --- a/dokimi/dokimi/defaults.py +++ b/dokimi/dokimi/defaults.py @@ -67,7 +67,8 @@ def default_judge_model(provider: str) -> str | None: Pairing a provider with another provider's model (``openai/claude-sonnet-5-5``) fails at the first call, so a command with no default here asks for ``--judge-model``. """ - if provider == DEFAULT_JUDGE_PROVIDER or (provider == "amazon-bedrock" + # claude-cli takes Anthropic's model ids, as the anthropic provider does. + if provider in (DEFAULT_JUDGE_PROVIDER, "claude-cli") or (provider == "amazon-bedrock" and DEFAULT_JUDGE_MODEL in BEDROCK_MODEL_IDS): return DEFAULT_JUDGE_MODEL if provider == DEFAULT_GENERATOR_PROVIDER: @@ -84,7 +85,7 @@ class _JudgeModelForProvider(str): #: ``dokimi.cli.parse_args`` reads it as ``DEFAULT_JUDGE_MODEL``. JUDGE_MODEL_FOR_PROVIDER = _JudgeModelForProvider(DEFAULT_JUDGE_MODEL) JUDGE_MODEL_HELP = (f"Judge model (default: the default judge for --judge-provider: " - f"{DEFAULT_JUDGE_MODEL} on {DEFAULT_JUDGE_PROVIDER} and amazon-bedrock, " + f"{DEFAULT_JUDGE_MODEL} on {DEFAULT_JUDGE_PROVIDER}, amazon-bedrock and claude-cli, " f"{DEFAULT_GENERATOR_MODEL} on {DEFAULT_GENERATOR_PROVIDER}; other providers " f"need this flag)") diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py index 1dbdc0612..5e2dca750 100644 --- a/dokimi/dokimi/eval/answerer.py +++ b/dokimi/dokimi/eval/answerer.py @@ -16,6 +16,7 @@ from dokimi.utils import question_id from dokimi.artifacts import save_artifact from dokimi.llm import ChatModel +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS from dokimi.llm import llm_call_with_usage, ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER from dokimi.metrics.usage import UsageRecord, to_record @@ -101,12 +102,12 @@ def __init__( elif provider == "anthropic": self.llm = ChatAnthropic(model=model, api_key=api_key) elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai"): + "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): self.llm = None # llm_call_with_usage handles these natively else: raise ValueError( f"Unsupported provider: '{provider}'. " - "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'." + "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) self.top_k = top_k self.debug_retrieval = debug_retrieval diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index 4b6045055..c262ebd06 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -742,11 +742,12 @@ def build_output( # cost_summary so a full run's spend is auditable from one artifact). if judge is not None and hasattr(judge, "_usage_totals"): with judge._usage_lock: - totals = dict(judge._usage_totals) - totals["cost_usd"] = ( - round(totals["cost_usd"], 6) if totals["cost_known_calls"] else None - ) - out["judge_metrics"] = totals + t = dict(judge._usage_totals) + resolved = sorted(getattr(judge, "_resolved_models", ())) + t["cost_usd"] = round(t["cost_usd"], 6) if t["cost_known_calls"] else None + out["judge_metrics"] = t + if resolved: # the model a CLI alias judge ran as, as eval score records it + out["judge_resolved_models"] = resolved return out def _summarize( diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py index f07c9cc4d..2f035df55 100644 --- a/dokimi/dokimi/eval/judge.py +++ b/dokimi/dokimi/eval/judge.py @@ -18,6 +18,7 @@ from dokimi import caching from dokimi.artifacts import save_artifact from dokimi.llm import ChatModel, ask_parsed +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS, check_sampling from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.eval.diagnoser import diagnose from dokimi.eval.question_text import require_question_texts @@ -161,7 +162,8 @@ MAX_GROUND_TRUTH_CHARS = 4000 #: Providers ``Judge.__init__`` accepts; CLI ``--judge-provider`` choices read this. -JUDGE_PROVIDERS = ("openai", "anthropic", "google", "google-vertex", "openrouter", "amazon-bedrock") +JUDGE_PROVIDERS = ("openai", "anthropic", "google", "google-vertex", "openrouter", "amazon-bedrock", + *CLI_BACKENDS) def _clip_ground_truth(ground_truth: Optional[str], limit: int = MAX_GROUND_TRUTH_CHARS) -> str: @@ -241,7 +243,10 @@ def __init__( "prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "redrawn_judgements": 0, } + # The model names a CLI judge's aliases resolved to, for the judge record. + self._resolved_models: set = set() + check_sampling(provider, temperature, seed) self.llm: ChatModel if provider == "openai": self.llm = ChatOpenAI(model=model, api_key=api_key, @@ -250,7 +255,7 @@ def __init__( self.llm = ChatAnthropic(model=model, api_key=api_key, temperature=temperature, seed=seed) elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai"): + "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): from dokimi.utils import get_llm import os fallback_key = os.environ.get("OPENROUTER_API_KEY") if provider == "openrouter" else None @@ -259,7 +264,7 @@ def __init__( else: raise ValueError( f"Unsupported provider: '{provider}'. " - "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'." + "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) logger.info(f"Judge initialized: {provider}/{model}") @@ -629,6 +634,7 @@ def _int(v): t["prompt_tokens"] += _int(usage.prompt_tokens) t["completion_tokens"] += _int(usage.completion_tokens) t["total_tokens"] += _int(usage.total_tokens) + self._resolved_models.update(n for n in usage.resolved_model.split(",") if n) cost = usage.cost_usd if isinstance(cost, (int, float)): t["cost_usd"] += float(cost) @@ -646,6 +652,9 @@ def _build_evaluation_output(self, evaluations: List[Dict[str, Any]]) -> Dict[st } with self._usage_lock: totals = dict(self._usage_totals) + resolved = sorted(self._resolved_models) + if resolved: + output["judge_resolved_models"] = resolved # cost_usd is null (not 0.0) when no judge call had known pricing, so a # missing price reads distinct from a genuinely free judge. output["judge_metrics"] = { diff --git a/dokimi/dokimi/eval/panel.py b/dokimi/dokimi/eval/panel.py index 4aa0b0ed4..d107c3882 100644 --- a/dokimi/dokimi/eval/panel.py +++ b/dokimi/dokimi/eval/panel.py @@ -27,6 +27,7 @@ from dokimi import caching from dokimi.llm import ask_parsed, llm_call +from dokimi.llm_cli import check_sampling from dokimi.defaults import DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER from dokimi.eval.question_text import require_question_texts # The panel had no numeric-claims criterion (see `eval/rubric.py`) until it was @@ -203,6 +204,8 @@ def __init__( temperature=temperature, seed=seed) for role in DEFAULT_PANEL ] + for j in self.judges: + check_sampling(j.provider, j.temperature, j.seed) self.concurrency = concurrency @property diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index b5afaca10..29a92292c 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -28,7 +28,9 @@ :data:`PROVIDER_ALIASES`. :func:`resolve_model` is this module's model-name map and :func:`api_key_env` its key-variable map. A provider outside the table is passed to litellm as its own route prefix (``azure/``), reading - ``OPENAI_API_KEY``. An explicit ``api_key`` wins over the environment; either + ``OPENAI_API_KEY``. A provider in :data:`dokimi.llm_cli.BACKENDS` (``claude-cli``) + is a logged-in coding-agent CLI, not an API: no key, no tools, no sampling + control (:mod:`dokimi.llm_cli`). An explicit ``api_key`` wins over the environment; either may be ``file:``, and an unreadable file raises ``ValueError``. - **Bedrock bare names.** A name in :data:`dokimi.defaults.BEDROCK_MODEL_IDS` runs under its inference profile (:func:`dokimi.defaults.bedrock_profile`); @@ -64,7 +66,6 @@ from __future__ import annotations import contextlib -import json import logging import os import re @@ -73,8 +74,10 @@ from dataclasses import dataclass, field, replace from typing import Any, Callable, Dict, Iterator, List, Optional, Protocol, Tuple, TypeVar, cast -from dokimi import caching +from dokimi import caching, llm_cli from dokimi.defaults import bedrock_profile +from dokimi.llm_json import extract_json_array as extract_json_array +from dokimi.llm_json import extract_json_object as extract_json_object from dokimi.metrics.usage import UsageRecord logger = logging.getLogger(__name__) @@ -93,6 +96,9 @@ "openrouter": "OPENROUTER_API_KEY", } +#: ``--provider`` choices: the API providers, then the CLI backends. +ANSWER_PROVIDERS: Tuple[str, ...] = (*PROVIDERS, *llm_cli.BACKENDS) + #: Other spellings of a canonical provider. PROVIDER_ALIASES: Dict[str, str] = { "google": "google-vertex", @@ -112,12 +118,12 @@ def api_key_env(provider: str) -> str: def resolve_api_key(provider: str, api_key: Optional[str] = None) -> Optional[str]: """*api_key*, else the provider's environment variable; a ``file:`` value is read. - Returns ``None`` when there is no key. + Returns ``None`` when there is no key; a CLI provider takes its login, never a key. Raises: ValueError: a ``file:`` key is unreadable. """ - key = api_key or os.environ.get(api_key_env(provider), "") + key = "" if llm_cli.backend(provider) else (api_key or os.environ.get(api_key_env(provider), "")) if key.startswith("file:"): path = os.path.expanduser(key[len("file:"):]) try: @@ -199,6 +205,9 @@ def unsupported_sampling_params( requested = list(_sampling_kwargs(temperature=temperature, seed=seed)) if not requested: return [] + cli = llm_cli.backend(_canonical(provider)) + if cli is not None: + return [n for n in requested if n not in cli.supports] try: from litellm import get_supported_openai_params @@ -264,6 +273,9 @@ def is_retryable(exc: BaseException) -> bool: if its message happens to contain a transient-sounding word, so the non-retryable set is consulted first. """ + flag = getattr(exc, "retryable", None) # set by an error that knows (llm_cli.CliCallError) + if isinstance(flag, bool): + return flag name = type(exc).__name__.lower() if name in _NON_RETRYABLE_EXC_NAMES: return False @@ -441,6 +453,8 @@ class Reply: finish_reason: Optional[str] = None tool_calls: Tuple[ToolCall, ...] = () response: Any = field(default=None, compare=False, repr=False) + #: The model the provider resolved to, when it says (a CLI names the version behind an alias). + model_id: Optional[str] = None @property def truncated(self) -> bool: @@ -516,10 +530,28 @@ def _stream(completion: Any, request: Request) -> Reply: return Reply(text="".join(parts), usage=usage, finish_reason=finish_reason) +def _cli_reply(request: Request) -> Reply: + """A request for a CLI provider (:mod:`dokimi.llm_cli`): one prompt, an optional system.""" + turns = [m for m in request.messages if m.get("role") != "system"] + if request.tools or len(turns) != 1 or request.temperature is not None or request.seed is not None: + raise llm_cli.CliCallError(f"{request.provider} takes one prompt: no tools, no conversation, " + "no temperature or seed", retryable=False) + system = [prompt_text(replace(request, messages=[m])) + for m in request.messages if m.get("role") == "system"] + out = llm_cli.call(request.provider, model=request.model, prompt=prompt_text(request), + system="\n\n".join(system) or None) + return Reply(text=out.text, usage=out.usage, finish_reason=out.finish_reason, model_id=out.model_id) + + class LiteLLMAdapter: - """The production adapter: one ``litellm.completion`` call, priced by litellm.""" + """The production adapter: one ``litellm.completion`` call, priced by litellm. + + A CLI provider (:data:`dokimi.llm_cli.BACKENDS`) goes to its CLI instead. + """ def __call__(self, request: Request) -> Reply: + if llm_cli.backend(request.provider) is not None: + return _cli_reply(request) from litellm import completion if request.stream: @@ -779,7 +811,7 @@ def _cache_extra(self, stream: bool, max_tokens: Optional[int]) -> Dict[str, Any # reasoning-starved completion is a different answer). temperature/seed # belong in it too, only when set: a pinned run must not be served a # free-sampling reply, and entries cached before they existed still hit. - return {"stream": stream, "max_tokens": max_tokens, + return {"stream": stream, "max_tokens": max_tokens, **llm_cli.cache_extra(self.provider), **_sampling_kwargs(temperature=self.temperature, seed=self.seed)} def _labels(self) -> Tuple[str, str]: @@ -1059,60 +1091,3 @@ def chat_completion( llm = LLM(model, provider, api_key, max_tokens=max_tokens, temperature=temperature, seed=seed, max_retries=max_retries, retry_delay=retry_delay) return llm.chat(messages, tools=tools, tool_choice=tool_choice).response - - -# ── Robust JSON extraction ───────────────────────────────────────────────────── - - -def extract_json_object(text: str) -> dict: - """Extract a JSON object from LLM output, handling markdown fences and leading text.""" - text = (text or "").strip() - if not text: - raise ValueError("LLM returned empty response") - # Direct parse - try: - return json.loads(text) - except Exception: - pass - # Markdown fence - fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) - if fence: - try: - return json.loads(fence.group(1).strip()) - except Exception: - pass - # First { ... } block - m = re.search(r"\{[\s\S]*\}", text) - if m: - return json.loads(m.group(0)) - raise ValueError(f"No JSON object found in LLM response: {text[:200]!r}") - - -def extract_json_array(text: str) -> list: - """Extract a JSON array from LLM output, handling markdown fences and leading text.""" - text = (text or "").strip() - if not text: - raise ValueError("LLM returned empty response") - # Direct parse - try: - result = json.loads(text) - if isinstance(result, list): - return result - except Exception: - pass - # Markdown fence - fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) - if fence: - try: - result = json.loads(fence.group(1).strip()) - if isinstance(result, list): - return result - except Exception: - pass - # First [ ... ] block - m = re.search(r"\[[\s\S]*\]", text) - if m: - result = json.loads(m.group(0)) - if isinstance(result, list): - return result - raise ValueError(f"No JSON array found in LLM response: {text[:200]!r}") diff --git a/dokimi/dokimi/llm_cli.py b/dokimi/dokimi/llm_cli.py new file mode 100644 index 000000000..f1eedbf66 --- /dev/null +++ b/dokimi/dokimi/llm_cli.py @@ -0,0 +1,412 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Coding-agent CLIs as LLM providers: a completion through a locally logged-in CLI. + +A provider in :data:`BACKENDS` (``claude-cli``) runs one prompt through the CLI the +user is already signed in to (a Claude subscription, ``claude setup-token``, or +whatever ``claude auth status`` reports) instead of an API key. :mod:`dokimi.llm` +routes a :class:`~dokimi.llm.Request` for such a provider here, so retry, the +response cache and usage totals work as for any provider. + +What differs from an API provider, and what a caller can rely on: + +- **No sampling control.** The CLI takes no temperature, seed or output ceiling. + :data:`CliBackend.supports` lists none of them, so + :func:`dokimi.llm.unsupported_sampling_params` names a requested ``temperature`` + or ``seed`` and a run refuses before it starts. ``max_tokens`` is dropped: every + caller sets one, and refusing it would refuse every run. +- **Hidden context.** Even with no tools, no settings and its own system prompt, the + CLI adds a fixed preamble (about 0.5k-1.3k tokens, by version) and states the + working directory, git status, platform and date. The child runs in one fixed + empty directory per backend (:func:`work_dir`), so that text is the same for + every call of a run, with and without docs. Do not pool its scores with the same + model through an API provider. +- **Isolation.** ``--tools "" --setting-sources "" --strict-mcp-config + --no-session-persistence``: no tools, no user or project settings (so no + ``CLAUDE.md``), no MCP servers, no session written. Never ``--bare``: it reads only + ``ANTHROPIC_API_KEY``, so a subscription login fails with "Authentication error". + The parent's environment is passed on except :data:`CLAUDE_ENV_DROP`; without + that, a child started inside a Claude Code session joins the parent's session. +- **Usage.** Tokens and the resolved model id come from the CLI's JSON. + ``cost_usd`` is ``None``: ``total_cost_usd`` is a list-price estimate, not what a + subscription is charged; it is kept as :attr:`CliResult.list_cost_usd`. +- **Errors.** A failed call raises :class:`CliCallError` with ``retryable`` set: + rate limits, overload, 5xx and timeouts are retried; an unknown model, a missing + login or a missing binary are not. + +:func:`detect` reports whether a backend's binary is installed and signed in +without calling a model (``cli.py backends``). +""" + +from __future__ import annotations + +import functools +import json +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Dict, Optional, Tuple + +from dokimi import process +from dokimi.metrics.usage import UsageRecord + +#: Variables a Claude Code session sets for its own children, plus the API key. Dropped, so +#: a call made from inside a session is a session of its own and runs on the CLI's login. +CLAUDE_ENV_DROP = frozenset({ + "CLAUDECODE", "CLAUDE_CODE_SESSION_ID", "CLAUDE_CODE_CHILD_SESSION", + "CLAUDE_CODE_ENTRYPOINT", "CLAUDE_CODE_SSE_PORT", + "CLAUDE_CODE_MESSAGING_SOCKET", "CLAUDE_CODE_MESSAGING_TOKEN", "CLAUDE_PID", + "CLAUDE_EFFORT", "CLAUDE_CODE_SESSION_ATTENDED", + # Set, they win over the CLI's login: the run would bill the key or token, not the subscription. + "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", +}) +#: Dropped by prefix, so a variable not yet listed (an endpoint, a model alias, a provider switch +#: such as CLAUDE_CODE_USE_BEDROCK) cannot change who is billed or which session the child joins. +CLAUDE_ENV_DROP_PREFIXES = ("ANTHROPIC_", "CLAUDE_", "CLAUDECODE") +#: Kept despite a prefix: the login's location, the ``claude setup-token`` token, and Git Bash +#: for Claude Code on Windows (without it ``claude -p`` cannot start there). +CLAUDE_ENV_KEEP = frozenset({"CLAUDE_CONFIG_DIR", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CODE_GIT_BASH_PATH"}) + +#: The flags that strip the CLI to a bare completion; see the module docstring. +CLAUDE_ISOLATION = ("--tools", "", "--setting-sources", "", "--strict-mcp-config", + "--no-session-persistence") + +#: ``api_error_status`` values worth another attempt. +_RETRYABLE_STATUS = frozenset({408, 429, 500, 502, 503, 504, 529}) +_RETRYABLE_WORDS = ("rate limit", "rate_limit", "overloaded", "timed out") + +DEFAULT_TIMEOUT_SEC = 600.0 + + +class CliCallError(RuntimeError): + """A CLI call that produced no reply; ``retryable`` is what :func:`dokimi.llm.is_retryable` reads.""" + + def __init__(self, message: str, *, retryable: bool): + super().__init__(message) + self.retryable = retryable + + +@dataclass(frozen=True) +class CliResult: + """One CLI reply: the text, its usage, and the model id the CLI resolved.""" + + text: str + usage: UsageRecord + model_id: Optional[str] = None + list_cost_usd: Optional[float] = None + finish_reason: Optional[str] = None + + +@dataclass(frozen=True) +class AuthStatus: + """What :func:`detect` found. ``authenticated`` is ``None`` when the CLI cannot tell.""" + + backend: str + binary: str + installed: bool + version: str = "" + authenticated: Optional[bool] = None + method: str = "" + hint: str = "" + + +class SamplingError(ValueError): + """A temperature or seed given to a CLI provider, which takes neither; ``cli.main`` reports it (exit 2).""" + + +@dataclass(frozen=True) +class CliBackend: + """One CLI: how to call it, read its reply, and ask it whether it is signed in.""" + + name: str + binary: str + build_argv: Callable[[str, Optional[str]], list] # (model, system) -> argv; prompt goes on stdin + parse: Callable[[process.Completed, str, str], CliResult] # (completed, model, provider) + auth_argv: Tuple[str, ...] + parse_auth: Callable[[process.Completed], Tuple[Optional[bool], str]] # -> (authenticated, method) + env_drop: frozenset = frozenset() + env_drop_prefixes: Tuple[str, ...] = () + env_keep: frozenset = frozenset() + supports: frozenset = frozenset() # of "temperature", "seed", "max_tokens" + login_hint: str = "" + aliases: Tuple[str, ...] = () + model_prefixes: Tuple[str, ...] = () # full model ids the CLI takes, by prefix + + +# ── claude-cli ──────────────────────────────────────────────────────────────── + + +def _claude_argv(model: str, system: Optional[str]) -> list: + argv = ["claude", "-p", "--input-format", "text", "--output-format", "json", + "--model", model, *CLAUDE_ISOLATION] + # Without --system-prompt the CLI sends its own agent prompt (~37k tokens with tools). + return argv + ["--system-prompt", system or "You are a helpful assistant."] + + +def _claude_error(payload: Dict[str, Any], done: process.Completed) -> CliCallError: + message = str(payload.get("result") or done.stderr.strip() or f"exit {done.returncode}")[:500] + status = payload.get("api_error_status") + lowered = message.lower() + if any(w in lowered for w in ("authentication", "not logged in", "/login")): + return CliCallError(f"claude-cli: {message} (sign in: claude setup-token, or claude /login)", + retryable=False) + if "usage limit" in lowered: # a subscription cap: it resets in hours, a retry cannot help + return CliCallError(f"claude-cli: {message}", retryable=False) + retryable = status in _RETRYABLE_STATUS or any(w in lowered for w in _RETRYABLE_WORDS) + return CliCallError(f"claude-cli: {message}", retryable=retryable) + + +def _main_model(model_usage: Dict[str, Any], requested: str) -> Optional[str]: + """The model the CLI answered with: the requested family if listed, else the one with most output. + + ``modelUsage`` also lists the helper models a call used (a title or a classifier), so + its first key is not always the answering model. + """ + names = [n for n in model_usage if isinstance(n, str)] + if not names: + return None + family = _family(requested) + for name in names: + if family is not None and _family(name) == family: + return name + def output(name: str) -> int: + stats = model_usage.get(name) + return int(stats.get("outputTokens") or 0) if isinstance(stats, dict) else 0 + return max(names, key=output) + + +def _claude_parse(done: process.Completed, model: str, provider: str) -> CliResult: + if done.timed_out: + raise CliCallError(f"claude-cli: no reply in {done.elapsed_sec:.0f}s", retryable=True) + try: + payload = json.loads(done.stdout) + except ValueError: + payload = None + if not isinstance(payload, dict): + text = (done.stderr or done.stdout).strip() + # No JSON, so no status: the same transient words as a JSON result are checked on the text. + retryable = any(w in text.lower() for w in _RETRYABLE_WORDS) + raise CliCallError(f"claude-cli: exit {done.returncode}, no JSON reply: {text[:300]}", + retryable=retryable) + if payload.get("is_error") or done.returncode != 0: + raise _claude_error(payload, done) + u = payload.get("usage") or {} + cache_write = int(u.get("cache_creation_input_tokens") or 0) + cache_read = int(u.get("cache_read_input_tokens") or 0) + prompt = int(u.get("input_tokens") or 0) + cache_write + cache_read + completion = int(u.get("output_tokens") or 0) + model_usage = payload.get("modelUsage") or {} + list_cost = payload.get("total_cost_usd") + resolved = _main_model(model_usage, model) + usage = UsageRecord( + prompt_tokens=prompt, completion_tokens=completion, total_tokens=prompt + completion, + cache_read_tokens=cache_read, cache_write_tokens=cache_write, + cost_usd=None, latency_sec=done.elapsed_sec, model=model, provider=provider, + n_calls=1, cost_known_calls=0, resolved_model=resolved or "", + ) + return CliResult( + text=str(payload.get("result") or ""), + usage=usage, + model_id=resolved, + list_cost_usd=float(list_cost) if isinstance(list_cost, (int, float)) else None, + finish_reason="length" if payload.get("stop_reason") == "max_tokens" else "stop", + ) + + +def _claude_auth(done: process.Completed) -> Tuple[Optional[bool], str]: + try: + status = json.loads(done.stdout) + except ValueError: + return None, "" + if not isinstance(status, dict): + return None, "" + method = str(status.get("authMethod") or "") + if status.get("apiProvider"): + method = f"{method} ({status['apiProvider']})" if method else str(status["apiProvider"]) + return bool(status.get("loggedIn")), method + + +CLAUDE_CLI = CliBackend( + name="claude-cli", + binary="claude", + build_argv=_claude_argv, + parse=_claude_parse, + auth_argv=("claude", "auth", "status"), + parse_auth=_claude_auth, + env_drop=CLAUDE_ENV_DROP, + env_drop_prefixes=CLAUDE_ENV_DROP_PREFIXES, + env_keep=CLAUDE_ENV_KEEP, + login_hint="claude setup-token (headless) or claude, then /login", + aliases=("haiku", "sonnet", "opus"), + model_prefixes=("claude-",), +) + +#: CLI backends by provider name. +BACKENDS: Dict[str, CliBackend] = {CLAUDE_CLI.name: CLAUDE_CLI} + + +def backend(provider: str) -> Optional[CliBackend]: + """The CLI backend for *provider*, or ``None`` for an API provider.""" + return BACKENDS.get(provider) + + +#: The 1M-context suffix ``claude --model`` takes after an alias or a full id: ``sonnet[1m]``. +_CONTEXT_SUFFIX = re.compile(r"\[[^\]]*\]$") + + +def _without_context(model: str) -> str: + """*model* with a trailing ``[...]`` (the context-window form) removed, and blanks stripped.""" + return _CONTEXT_SUFFIX.sub("", model.strip()) + + +def accepts_model(provider: str, model: str) -> bool: + """Whether *provider*'s CLI takes *model*: one of its aliases, or a full id it names (either may + carry a ``[1m]``-style context suffix).""" + cli = BACKENDS[provider] + bare = _without_context(model) + return bare in cli.aliases or bare.startswith(cli.model_prefixes) + + +def _family(model: str) -> Optional[str]: + """The alias a Claude model name belongs to: ``sonnet``, or of ``claude-sonnet-5-5``.""" + name = _without_context(model.lower().rsplit("/", 1)[-1].rsplit("anthropic.", 1)[-1]) + if name in CLAUDE_CLI.aliases: + return name + if name.startswith("claude-"): + for part in re.split(r"[-@]", name): + if part in CLAUDE_CLI.aliases: + return part + return None + + +def same_model(provider_a: str, model_a: str, provider_b: str, model_b: str) -> bool: + """Whether two model refs name one model, for the independence and self-judge checks. + + An alias the CLI resolves (``sonnet``) is the same model as a full id of its family + (``claude-sonnet-5-5``): which version the alias is today is not known here, so the + check errs toward "same", and a warning is the cost of being wrong. Refs that involve + no CLI provider are left to the caller's own comparison (this returns ``False``). + """ + if provider_a not in BACKENDS and provider_b not in BACKENDS: + return False + a, b = _without_context(model_a.lower()), _without_context(model_b.lower()) + if a == b: + return True + if a not in CLAUDE_CLI.aliases and b not in CLAUDE_CLI.aliases: + return False # two full ids that differ are two models + fa, fb = _family(a), _family(b) + return fa is not None and fa == fb + + +def check_sampling(provider: str, temperature: Optional[float], seed: Optional[int]) -> None: + """Raise :class:`SamplingError` before a run when a CLI *provider* is given a temperature or seed.""" + if provider in BACKENDS and (temperature is not None or seed is not None): + raise SamplingError(f"{provider} takes no temperature or seed: leave out the judge temperature " + "and seed options") + + +def work_dir(name: str) -> Path: + """The fixed empty working directory a backend's calls run in. + + Fixed, not fresh per call: the CLI states its working directory in the prompt, and + a random path would make every request, and its response-cache key, differ. + Under the user's cache dir, not the shared temp dir: another account there could + pre-create it, and the CLI reads the git status of its working directory into the prompt. + """ + base = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") + path = base / "dokimi" / "cli" / name + try: + path.mkdir(parents=True, exist_ok=True, mode=0o700) + except OSError as exc: + raise CliCallError(f"{name}: cannot create the work dir {path} ({exc})", retryable=False) from exc + return path + + +def _passed_on(name: str, cli: CliBackend) -> bool: + if name in cli.env_keep: + return True + return name not in cli.env_drop and not name.startswith(cli.env_drop_prefixes) + + +def _env(cli: CliBackend, extra: Optional[Dict[str, str]] = None) -> process.Env: + return process.Env.only([k for k in os.environ if _passed_on(k, cli)], set=extra) + + +def _call_env(cli: CliBackend, cwd: Path) -> process.Env: + """The env of a call: the parent's, minus the drops, and git stopped at the work dir's parent. + + The CLI reads the git status of its working directory into the prompt. The work dir sits + under the user's cache dir, which can lie inside a repository (a home directory under git), + so git must not search above that cache directory for a repository. + """ + ceiling = [str(cwd.parent)] + [p for p in os.environ.get("GIT_CEILING_DIRECTORIES", "").split(os.pathsep) if p] + return _env(cli, {"GIT_CEILING_DIRECTORIES": os.pathsep.join(ceiling)}) + + +def cache_extra(provider: str) -> Dict[str, str]: + """What the response-cache key takes from a CLI backend: its version, and nothing otherwise. + + An alias resolves inside the CLI, so a new release can answer ``sonnet`` with another + model, and its replies must not be replayed for the new one. The resolved id is known + only after the call, so the version stands in for it; a move the CLI does not ship + with is not seen. + """ + return {"cli_version": cli_version(provider)} if backend(provider) is not None else {} + + +def cli_version(provider: str) -> str: + """The CLI's version line, or ``""`` when it cannot be read. Read once per process.""" + return _version_line(provider) + + +@functools.lru_cache(maxsize=None) +def _version_line(provider: str) -> str: + cli = BACKENDS[provider] + try: + done = process.run([cli.binary, "--version"], timeout=30, env=_env(cli)) + except OSError: + return "" + lines = done.stdout.strip().splitlines() + return lines[0] if lines else "" + + +def call(provider: str, *, model: str, prompt: str, system: Optional[str] = None, + timeout: float = DEFAULT_TIMEOUT_SEC, runner: Optional[process.Runner] = None) -> CliResult: + """One completion through *provider*'s CLI; the prompt goes on stdin (no ``ARG_MAX``). + + Raises: + CliCallError: the CLI is missing, failed, timed out or returned no reply. + KeyError: *provider* is not a CLI backend. + """ + cli = BACKENDS[provider] + cwd = work_dir(cli.name) + try: + done = (runner or process.run)(cli.build_argv(model, system), timeout=timeout, env=_call_env(cli, cwd), + cwd=cwd, input=prompt, capture_limit=None) + except FileNotFoundError as exc: + raise CliCallError(f"{provider}: '{cli.binary}' is not installed or not on PATH ({exc})", + retryable=False) from exc + return cli.parse(done, model, provider) + + +def detect(provider: str, runner: Optional[process.Runner] = None) -> AuthStatus: + """Whether *provider*'s binary is installed and signed in. Calls no model.""" + cli = BACKENDS[provider] + runner = runner or process.run + env = _env(cli) + try: + version = runner([cli.binary, "--version"], timeout=30, env=env) + except OSError: + return AuthStatus(provider, cli.binary, installed=False, hint=f"install '{cli.binary}'") + try: + auth = runner(list(cli.auth_argv), timeout=30, env=env) + authenticated, method = cli.parse_auth(auth) + except OSError: + authenticated, method = None, "" + return AuthStatus( + provider, cli.binary, installed=True, version=version.stdout.strip().splitlines()[0] + if version.stdout.strip() else "", authenticated=authenticated, method=method, + hint="" if authenticated else cli.login_hint, + ) diff --git a/dokimi/dokimi/llm_json.py b/dokimi/dokimi/llm_json.py new file mode 100644 index 000000000..397b013b5 --- /dev/null +++ b/dokimi/dokimi/llm_json.py @@ -0,0 +1,63 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Robust JSON extraction from LLM output; re-exported by :mod:`dokimi.llm`.""" + +from __future__ import annotations + +import json +import re + + +def extract_json_object(text: str) -> dict: + """Extract a JSON object from LLM output, handling markdown fences and leading text.""" + text = (text or "").strip() + if not text: + raise ValueError("LLM returned empty response") + # Direct parse + try: + return json.loads(text) + except Exception: + pass + # Markdown fence + fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + if fence: + try: + return json.loads(fence.group(1).strip()) + except Exception: + pass + # First { ... } block + m = re.search(r"\{[\s\S]*\}", text) + if m: + return json.loads(m.group(0)) + raise ValueError(f"No JSON object found in LLM response: {text[:200]!r}") + + +def extract_json_array(text: str) -> list: + """Extract a JSON array from LLM output, handling markdown fences and leading text.""" + text = (text or "").strip() + if not text: + raise ValueError("LLM returned empty response") + # Direct parse + try: + result = json.loads(text) + if isinstance(result, list): + return result + except Exception: + pass + # Markdown fence + fence = re.search(r"```(?:json)?\s*([\s\S]*?)```", text) + if fence: + try: + result = json.loads(fence.group(1).strip()) + if isinstance(result, list): + return result + except Exception: + pass + # First [ ... ] block + m = re.search(r"\[[\s\S]*\]", text) + if m: + result = json.loads(m.group(0)) + if isinstance(result, list): + return result + raise ValueError(f"No JSON array found in LLM response: {text[:200]!r}") diff --git a/dokimi/dokimi/metrics/usage.py b/dokimi/dokimi/metrics/usage.py index 594cc1172..e1d9e744f 100644 --- a/dokimi/dokimi/metrics/usage.py +++ b/dokimi/dokimi/metrics/usage.py @@ -44,6 +44,9 @@ class UsageRecord: cost_known_calls: int = 0 # served from the response cache: cost_usd is the original call's, not this process's bill replayed: bool = False + # The model the provider resolved the request to, when it says (a CLI alias names a + # version). A sum of rows that resolved differently lists each, comma-separated. + resolved_model: str = "" def __add__(self, other: "UsageRecord | int") -> "UsageRecord": """Sum two records: tokens/cost/latency/n_calls add; first ttft wins. @@ -78,6 +81,7 @@ def __add__(self, other: "UsageRecord | int") -> "UsageRecord": n_calls=self.n_calls + other.n_calls, cost_known_calls=_known_cost_calls(self) + _known_cost_calls(other), replayed=self.replayed and other.replayed, + resolved_model=_joined_models(self.resolved_model, other.resolved_model), ) __radd__ = __add__ # enables sum([...]) — 0 + record is handled in __add__ @@ -123,6 +127,8 @@ def as_metrics_dict( "cost_known_calls": _known_cost_calls(self), "role": self.role, } + if self.resolved_model: + m["resolved_model"] = self.resolved_model if answer_chars is not None: m["raw_answer_chars"] = answer_chars m["final_answer_chars"] = final_chars if final_chars is not None else answer_chars @@ -170,6 +176,7 @@ def to_record( n_calls=(int(obj["n_calls"]) if obj.get("n_calls") is not None else 1), cost_known_calls=int(obj.get("cost_known_calls", 0) or 0), + resolved_model=str(obj.get("resolved_model") or ""), ) raise TypeError(f"Cannot coerce {type(obj).__name__} into UsageRecord") @@ -216,6 +223,12 @@ def _add_optional_costs(a: Optional[float], b: Optional[float]) -> Optional[floa return round((a or 0.0) + (b or 0.0), 6) +def _joined_models(a: str, b: str) -> str: + """The distinct resolved model names of two rows, sorted and comma-separated.""" + names = {n for n in (*a.split(","), *b.split(",")) if n} + return ",".join(sorted(names)) + + def _known_cost_calls(rec: UsageRecord) -> int: """Number of LLM calls whose dollar cost is represented in ``cost_usd``.""" if rec.cost_known_calls: diff --git a/dokimi/dokimi/novelty/analyze.py b/dokimi/dokimi/novelty/analyze.py index aa5afab29..bfa4659d6 100644 --- a/dokimi/dokimi/novelty/analyze.py +++ b/dokimi/dokimi/novelty/analyze.py @@ -239,7 +239,9 @@ def analyze( concurrency=config.concurrency, allow_self_judge=config.allow_self_judge, placebo_text=config.placebo_text, seed=config.seed, ) - if config.allow_self_judge and judge.model == answerer.model: + from dokimi.novelty.client import same_client + + if config.allow_self_judge and same_client(judge, answerer): warnings.append( "judge and answerer are the same model (--allow-self-judge): scores are inflated" ) diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index 386467ee4..41dce22c6 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -260,7 +260,15 @@ def roll_up_cost(clients: list[LLMClient]) -> dict[str, Any]: } -def default_judge(model: str, provider: str | None = None) -> tuple[str, str]: +def same_client(a: LLMClient, b: LLMClient) -> bool: + """Whether two clients run one model: the same name, or a CLI alias and a full id of its family.""" + from dokimi.llm_cli import same_model + + return a.model == b.model or same_model(a.provider, a.model, b.provider, b.model) + + +def default_judge(model: str, provider: str | None = None, + answerer_provider: str | None = None) -> tuple[str, str]: """A (model, provider) judge that is not the answerer, when the caller did not name one. The default judge, unless it is the model under measurement (by bare name, so a @@ -269,14 +277,27 @@ def default_judge(model: str, provider: str | None = None) -> tuple[str, str]: only a default that runs there is taken: its own provider, or Bedrock for a name in ``BEDROCK_MODEL_IDS``. Pairing the provider with a model of another provider would send e.g. ``bedrock/gpt-6.1-sol``, so with no such default this raises instead. + A CLI ``answerer_provider`` with no ``provider`` judges on the same CLI. Raises: ValueError: ``provider`` runs no default judge other than ``model``. """ + from dokimi.llm_cli import backend, same_model + bare = model.rsplit("/", 1)[-1].rsplit("anthropic.", 1)[-1] + if provider is None and answerer_provider is not None and backend(answerer_provider) is not None: + provider = answerer_provider + # Hard-coded for claude-cli; a second CLI backend (codex-cli) needs its own alias list here. + if provider is not None and provider == "claude-cli": + # The CLI's aliases: the first one not of the answerer's family, so the judge is never the answerer. + for alias in ("opus", "sonnet", "haiku"): + if not same_model("claude-cli", alias, "claude-cli", bare): + return alias, provider + raise ValueError(f"judge provider {provider}: no judge of another family than {model}; " + "pass --judge-model as well") candidates = [(m, p) for m, p in ((DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER), (DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER)) - if m != bare] + if m != bare and not same_model(answerer_provider or "", bare, p, m)] if provider is None: return candidates[0] for m, p in candidates: diff --git a/dokimi/dokimi/novelty/measure.py b/dokimi/dokimi/novelty/measure.py index 2143fcb58..678a8e250 100644 --- a/dokimi/dokimi/novelty/measure.py +++ b/dokimi/dokimi/novelty/measure.py @@ -255,7 +255,9 @@ def run_measurement( irrelevant context actively harms them. seed: Seed for the ``shuffled`` control, so the control is reproducible. """ - if judge.model == answerer.model and not allow_self_judge: + from dokimi.novelty.client import same_client + + if same_client(judge, answerer) and not allow_self_judge: raise SelfJudgeError( f"judge model {judge.model!r} is the answering model; grading a response with " "the model that produced it inflates the score. Pass a different --judge-model, " diff --git a/dokimi/dokimi/personas/generator.py b/dokimi/dokimi/personas/generator.py index d7f00f9d1..2936cd3c1 100644 --- a/dokimi/dokimi/personas/generator.py +++ b/dokimi/dokimi/personas/generator.py @@ -9,6 +9,7 @@ import json from dokimi.llm import ChatModel +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.defaults import DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER @@ -105,13 +106,13 @@ def __init__( elif provider == "anthropic": self.llm = ChatAnthropic(model=model, api_key=api_key) elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai"): + "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): from dokimi.utils import get_llm self.llm = get_llm(provider, model, api_key) else: raise ValueError( f"Unsupported provider: '{provider}'. " - "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'." + "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) logger.info(f"Initialized PersonaGenerator with {provider}/{model}") diff --git a/dokimi/dokimi/questions/llm_gen.py b/dokimi/dokimi/questions/llm_gen.py index 9cfcb6680..2ca5a397b 100644 --- a/dokimi/dokimi/questions/llm_gen.py +++ b/dokimi/dokimi/questions/llm_gen.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) from dokimi.llm import ChatModel +from dokimi.llm_cli import BACKENDS as CLI_BACKENDS from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.defaults import DEFAULT_GENERATOR_MODEL, DEFAULT_GENERATOR_PROVIDER @@ -98,7 +99,7 @@ def __init__( elif provider == "anthropic": self.llm = ChatAnthropic(model=model, api_key=api_key) elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai"): + "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): from dokimi.utils import get_llm import os fallback_key = os.environ.get("OPENROUTER_API_KEY") if provider == "openrouter" else None @@ -106,7 +107,7 @@ def __init__( else: raise ValueError( f"Unsupported provider: '{provider}'. " - "Use 'openai', 'anthropic', 'google', 'openrouter', or 'amazon-bedrock'." + "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) logger.info(f"QuestionGenerator initialized: {provider}/{model}") diff --git a/dokimi/dokimi/utils.py b/dokimi/dokimi/utils.py index cda2ff68c..9e637e2e5 100644 --- a/dokimi/dokimi/utils.py +++ b/dokimi/dokimi/utils.py @@ -161,8 +161,11 @@ def get_llm(provider: str, model: str, api_key: str | None = None, ``max_tokens=2000``, Bedrock reads only its bearer token) and its cache keys, until the judge, generator and validator move to ``LLM``. """ - from dokimi.llm import _GetLLMCompat, _Invoker + from dokimi import llm_cli + from dokimi.llm import LLM, _GetLLMCompat, _Invoker + if llm_cli.backend(provider) is not None: # a logged-in CLI: no route, no key + return _Invoker(LLM(model, provider, temperature=temperature, seed=seed)) api_key = _read_key(api_key) if api_key else api_key max_tokens: int | None = None key: str | None diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index f22f10047..168ec6061 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -117,6 +117,7 @@ nav: - tool-preference.md - run-consistency.md - coding-harnesses.md + - subscription-backends.md - answer-formats.md - context-window.md - Content novelty: diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 7ad83253b..002c2f2c9 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,9 +1,9 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 1192, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --repeats refused under --all-vs-all; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 1191, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --repeats refused under --all-vs-all; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/llm.py": {"lines": 1118, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, + "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1207, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1176, "rationale": "pre-existing when the gate was introduced"}, "tests/test_benchflow_harness.py": {"lines": 1716, "rationale": "pre-existing when the gate was introduced"}, @@ -19,7 +19,7 @@ "dokimi/commands/arms.py::_build_arms_output": {"complexity": 13}, "dokimi/commands/arms.py::_validate_expected_skills": {"complexity": 11}, "dokimi/commands/benchmark.py::cmd_benchmark_batch": {"complexity": 11}, - "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 21}, + "dokimi/commands/benchmark.py::cmd_benchmark_preflight": {"complexity": 19}, "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 15}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 30, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index 92c46ae7d..0e8949a3e 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -69,7 +69,8 @@ # process chokepoints, console output. ("foundation", ( "dokimi", "dokimi._version", "dokimi.artifacts", "dokimi.caching", "dokimi.config", - "dokimi.console", "dokimi.container_policy", "dokimi.defaults", "dokimi.intents", "dokimi.llm", "dokimi.llm_pricing", + "dokimi.console", "dokimi.container_policy", "dokimi.defaults", "dokimi.intents", + "dokimi.llm", "dokimi.llm_cli", "dokimi.llm_json", "dokimi.llm_pricing", "dokimi.process", "dokimi.records", "dokimi.registry", "dokimi.utils", # Token/cost record the LLM chokepoint fills; it lives under metrics/ but # depends on nothing and is read by every LLM caller. diff --git a/dokimi/tests/test_judge_sampling.py b/dokimi/tests/test_judge_sampling.py index 197ede47b..682f4317b 100644 --- a/dokimi/tests/test_judge_sampling.py +++ b/dokimi/tests/test_judge_sampling.py @@ -18,6 +18,29 @@ def test_judge_hands_sampling_params_to_its_client_and_output(): assert j._build_evaluation_output([])["judge_temperature"] == 0.0 +def test_the_judge_record_names_the_resolved_models_it_was_judged_by(): + from dokimi.metrics.usage import UsageRecord + + j = Judge(model="sonnet", provider="claude-cli") + assert "judge_resolved_models" not in j._build_evaluation_output([]) + j._accumulate_usage(UsageRecord(resolved_model="claude-sonnet-5-5")) + j._accumulate_usage(UsageRecord(resolved_model="claude-sonnet-5-5")) + assert j._build_evaluation_output([])["judge_resolved_models"] == ["claude-sonnet-5-5"] + + +def test_the_arms_judge_record_names_the_resolved_models_too(): + # arms run and matrix-run build the judge record in ArmRunner, not in Judge. + from dokimi.eval.arm_runner import ArmRunner + from dokimi.metrics.usage import UsageRecord + from dokimi.treatments import BaselineTreatment + + runner, j = ArmRunner([BaselineTreatment()]), Judge(model="opus", provider="claude-cli") + assert "judge_resolved_models" not in runner.build_output("oneTBB", [], [], judge=j) + j._accumulate_usage(UsageRecord(resolved_model="claude-opus-5-5")) + out = runner.build_output("oneTBB", [], [], judge=j) + assert out["judge_resolved_models"] == ["claude-opus-5-5"] + + def test_judge_defaults_leave_sampling_unset(): j = Judge(model="m", provider="openai") assert j.temperature is None diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py new file mode 100644 index 000000000..cbd0b64bb --- /dev/null +++ b/dokimi/tests/test_llm_cli.py @@ -0,0 +1,821 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for dokimi.llm_cli: a logged-in coding-agent CLI as an LLM provider. + +The payloads are trimmed from real ``claude -p --output-format json`` replies +(Claude Code 2.1.3xx): a success, an unknown model, a failed login. +""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from dokimi import llm, llm_cli, process +from dokimi.metrics.usage import UsageRecord +from dokimi.process import FakeRunner, reply + + +def _ok(text: str = "PONG", **over) -> str: + payload = { + "type": "result", "is_error": False, "result": text, "stop_reason": "end_turn", + "total_cost_usd": 0.005394, + "usage": {"input_tokens": 2, "cache_creation_input_tokens": 1340, + "cache_read_input_tokens": 10, "output_tokens": 3}, + "modelUsage": {"claude-sonnet-5-5": {"costUSD": 0.005394, "costBasis": "list"}}, + } + payload.update(over) + return json.dumps(payload) + + +def _err(result: str, status) -> str: + return json.dumps({"type": "result", "is_error": True, "result": result, + "api_error_status": status, "usage": {}, "modelUsage": {}}) + + +UNKNOWN_MODEL = _err("There's an issue with the selected model (claude-nope-1). " + "It may not exist or you may not have access to it.", 404) +NO_LOGIN = _err("Authentication error · This may be a temporary network issue, please try again", None) + + +@pytest.fixture +def runner(monkeypatch, tmp_path): + """Every CLI call answers from ``runner.answers`` (a list, consumed in order).""" + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + fake = FakeRunner(lambda call: fake.answers.pop(0)) + fake.answers = [] # type: ignore[attr-defined] + monkeypatch.setattr(llm_cli.process, "run", fake) + # The version is part of the cache key; a test of the key sets its own. + monkeypatch.setattr(llm_cli, "cli_version", lambda provider: "2.1.303 (Claude Code)") + return fake + + +def test_a_reply_reads_text_tokens_and_model_and_leaves_cost_unknown(runner): + runner.answers.append(reply(_ok())) + out = llm_cli.call("claude-cli", model="sonnet", prompt="ping", system="Be terse.") + assert out.text == "PONG" + assert out.model_id == "claude-sonnet-5-5" + u = out.usage + assert (u.prompt_tokens, u.completion_tokens, u.total_tokens) == (1352, 3, 1355) + assert (u.cache_write_tokens, u.cache_read_tokens) == (1340, 10) + # A subscription is not billed per call: the CLI's figure is list price, kept apart. + assert u.cost_usd is None and u.cost_known_calls == 0 + assert out.list_cost_usd == pytest.approx(0.005394) + assert (u.model, u.provider) == ("sonnet", "claude-cli") + + +def test_the_call_is_isolated_and_the_prompt_goes_on_stdin(runner, monkeypatch): + monkeypatch.setenv("CLAUDECODE", "1") + for name in llm_cli.CLAUDE_ENV_DROP: + monkeypatch.setenv(name, "parent") + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-would-bill") + monkeypatch.setenv("DOKIMI_TEST_KEEP", "kept") + runner.answers.extend([reply(_ok()), reply(_ok())]) + llm_cli.call("claude-cli", model="haiku", prompt="x" * 300_000, system="Grade it.") + llm_cli.call("claude-cli", model="haiku", prompt="again") + first, second = runner.calls + argv = list(first.argv) + assert argv[:2] == ["claude", "-p"] and "--bare" not in argv + for flag in ("--strict-mcp-config", "--no-session-persistence"): + assert flag in argv + assert argv[argv.index("--output-format") + 1] == "json" + assert argv[argv.index("--tools") + 1] == "" and argv[argv.index("--setting-sources") + 1] == "" + assert argv[argv.index("--system-prompt") + 1] == "Grade it." + assert first.input == b"x" * 300_000 and "x" * 100 not in " ".join(argv) + assert not llm_cli.CLAUDE_ENV_DROP & set(first.env) + assert {"CLAUDE_CODE_ENTRYPOINT", "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_MESSAGING_TOKEN", "CLAUDE_PID"} <= llm_cli.CLAUDE_ENV_DROP + assert first.env["DOKIMI_TEST_KEEP"] == "kept" + # One fixed, empty directory: the cwd the CLI states is the same in every call. + assert first.cwd == second.cwd and list(first.cwd.iterdir()) == [] + # No system prompt is still a system prompt: never the CLI's own agent prompt. + assert "--system-prompt" in second.argv + + +@pytest.mark.parametrize("answer, retryable, words", [ + (reply(UNKNOWN_MODEL, returncode=1), False, "selected model"), + (reply(NO_LOGIN, returncode=1), False, "setup-token"), + (reply(_err("Rate limit reached", 429), returncode=1), True, "Rate limit"), + (reply(_err("Overloaded", 529), returncode=1), True, "Overloaded"), + (reply("", timed_out=True), True, "no reply"), + (reply("segfault", stderr="boom", returncode=139), False, "no JSON reply"), + (reply("", stderr="API Error: 503 Service Unavailable, overloaded", returncode=1), True, "no JSON reply"), +]) +def test_a_failed_call_says_whether_another_attempt_can_help(runner, answer, retryable, words): + runner.answers.append(answer) + with pytest.raises(llm_cli.CliCallError, match=words) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is retryable + assert llm.is_retryable(info.value) is retryable + + +# The status and word lists are written out here, not read from llm_cli: dropping a +# member from the module must fail a test, not drop the test with it. +@pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504, 529]) +def test_each_retryable_status_is_retried_whatever_the_message(runner, status): + runner.answers.append(reply(_err("something went wrong", status), returncode=1)) + with pytest.raises(llm_cli.CliCallError) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is True + + +@pytest.mark.parametrize("word", ["rate limit", "rate_limit", "overloaded", "timed out"]) +def test_each_retryable_word_is_retried_without_a_status(runner, word): + runner.answers.append(reply(_err(f"API said: {word}", None), returncode=1)) + with pytest.raises(llm_cli.CliCallError) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is True + + +@pytest.mark.parametrize("status", [None, 400, 401, 404, 501]) +def test_a_bad_request_or_auth_failure_is_not_retried(runner, status): + runner.answers.append(reply(_err("invalid request error", status), returncode=1)) + with pytest.raises(llm_cli.CliCallError) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False + + +@pytest.mark.parametrize("status", [None, 429]) +def test_a_subscription_usage_limit_is_not_retried(runner, status): + # It resets after hours: three more CLI starts would only delay the same failure. + runner.answers.append(reply(_err("Claude AI usage limit reached|1761660000", status), returncode=1)) + with pytest.raises(llm_cli.CliCallError, match="usage limit") as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False + + +@pytest.mark.parametrize("answer", [ + reply(_ok("bad", is_error=True), returncode=0), # is_error alone, exit 0 + reply(_ok("boom"), returncode=1), # exit 1 alone, is_error false +]) +def test_is_error_and_the_exit_code_each_mark_a_failure(runner, answer): + runner.answers.append(answer) + with pytest.raises(llm_cli.CliCallError) as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False + + +def test_an_unusable_work_dir_is_a_permanent_cli_error(runner, monkeypatch, tmp_path): + blocker = tmp_path / "a-file" + blocker.write_text("not a directory", encoding="utf-8") + monkeypatch.setenv("XDG_CACHE_HOME", str(blocker)) + with pytest.raises(llm_cli.CliCallError, match="work dir") as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False and runner.calls == [] + + +def test_a_cli_provider_never_reads_an_api_key_file(runner, monkeypatch, tmp_path): + monkeypatch.setenv("OPENAI_API_KEY", f"file:{tmp_path / 'missing'}") + assert llm.resolve_api_key("claude-cli") is None + runner.answers.append(reply(_ok("4"))) + assert llm.LLM("sonnet", provider="claude-cli", max_tokens=50).complete("2+2?").text == "4" + + +def test_a_missing_binary_is_a_permanent_error(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + + def missing(call): + raise FileNotFoundError("claude") + monkeypatch.setattr(llm_cli.process, "run", FakeRunner(missing)) + with pytest.raises(llm_cli.CliCallError, match="not installed") as info: + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert info.value.retryable is False + + +def test_llm_routes_a_cli_provider_retries_it_and_meters_it(runner): + runner.answers.extend([reply(_err("Rate limit reached", 429), returncode=1), reply(_ok("4"))]) + model = llm.LLM("sonnet", provider="claude-cli", max_tokens=50, retry_delay=0) + out = model.complete("2+2?", system="Answer with a number.") + assert out.text == "4" and len(runner.calls) == 2 + assert runner.calls[1].input == b"2+2?" + assert model.metrics().completion_tokens == 3 and model.metrics().cost_usd is None + + +def test_tools_or_a_conversation_are_refused_before_any_call(runner): + model = llm.LLM("sonnet", provider="claude-cli") + with pytest.raises(llm_cli.CliCallError, match="one prompt"): + model.chat([{"role": "user", "content": "hi"}], + tools=[{"type": "function", "function": {"name": "f"}}]) + with pytest.raises(llm_cli.CliCallError, match="one prompt"): + model.chat([{"role": "user", "content": "a"}, {"role": "assistant", "content": "b"}, + {"role": "user", "content": "c"}]) + from dokimi.eval.judge import Judge + from dokimi.eval.panel import JudgePanel + for sampling in ({"temperature": 0.0}, {"seed": 7}): # --judge-temperature / --seed: before the run + with pytest.raises(ValueError, match="temperature or seed"): + Judge(model="sonnet", provider="claude-cli", **sampling) + with pytest.raises(ValueError, match="temperature or seed"): # eval panel-score + JudgePanel(default_provider="claude-cli", **sampling) + JudgePanel(default_provider="claude-cli") + with pytest.raises(llm_cli.CliCallError, match="temperature"): + llm.LLM("sonnet", provider="claude-cli", seed=1).complete("x") + assert runner.calls == [] + + +def test_sampling_the_cli_cannot_honour_is_named(): + assert llm.unsupported_sampling_params("sonnet", "claude-cli", temperature=0.0, seed=7) == [ + "temperature", "seed"] + assert llm.unsupported_sampling_params("sonnet", "claude-cli") == [] + + +def test_judge_and_get_llm_reach_the_cli_backend(runner): + from dokimi.eval.judge import JUDGE_PROVIDERS, Judge + from dokimi.utils import get_llm + + assert "claude-cli" in JUDGE_PROVIDERS and "claude-cli" in llm.ANSWER_PROVIDERS + runner.answers.extend([reply(_ok("judged")), reply(_ok("generated"))]) + assert Judge(model="sonnet", provider="claude-cli").llm.invoke("grade").content == "judged" + resp = get_llm("claude-cli", "haiku").invoke("write") + assert resp.content == "generated" and resp.usage.provider == "claude-cli" + + +def test_get_llm_imports_llm_cli_from_its_own_module(): + import ast + import inspect + + from dokimi.utils import get_llm + + tree = ast.parse(inspect.getsource(get_llm)) + borrowed = [alias.name for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module == "dokimi.llm" + for alias in node.names if alias.name == "llm_cli"] + assert borrowed == [] # it works only while llm.py happens to import the module + + +def test_the_backend_doc_names_tools_as_the_command_that_refuses_a_cli_provider(): + from pathlib import Path + + from dokimi.commands import tools + + text = (Path(__file__).resolve().parents[1] / "docs" / "subscription-backends.md").read_text(encoding="utf-8") + assert "claude-cli" not in tools.PROVIDERS + assert "`--provider` or `--judge-provider`, except `tools`" in text + + +def test_the_reviewing_guide_lists_every_answer_provider(): + from pathlib import Path + + text = (Path(__file__).resolve().parents[1] / "docs" / "reviewing.md").read_text(encoding="utf-8") + assert [p for p in llm.ANSWER_PROVIDERS if f"`{p}`" not in text] == [] + + +def _auth_runner(status: dict | None, version: str = "2.1.304 (Claude Code)"): + def handle(call): + if call.argv[-1] == "--version": + if status is None: + raise FileNotFoundError("claude") + return reply(version + "\n") + return reply(json.dumps(status)) + return FakeRunner(handle) + + +def test_detect_reports_install_login_and_method_without_a_model_call(): + fake = _auth_runner({"loggedIn": True, "authMethod": "oauth_token"}) + got = llm_cli.detect("claude-cli", runner=fake) + assert (got.installed, got.authenticated, got.method, got.version) == ( + True, True, "oauth_token", "2.1.304 (Claude Code)") + assert got.hint == "" and [c.argv[1:] for c in fake.calls] == [("--version",), ("auth", "status")] + out = llm_cli.detect("claude-cli", runner=_auth_runner({"loggedIn": False})) + assert out.authenticated is False and "setup-token" in out.hint + gone = llm_cli.detect("claude-cli", runner=_auth_runner(None)) + assert gone.installed is False and gone.authenticated is None + + +def test_backends_command_prints_one_row_per_backend(monkeypatch, capsys): + from dokimi.commands.backends import cmd_backends + + monkeypatch.setattr(process, "run", _auth_runner( + {"loggedIn": True, "authMethod": "third_party", "apiProvider": "bedrock"})) + cmd_backends(argparse.Namespace(json=False)) + text = capsys.readouterr().out + assert "| claude-cli | claude | 2.1.304 (Claude Code) | yes | third_party (bedrock) |" in text + cmd_backends(argparse.Namespace(json=True)) + (row,) = json.loads(capsys.readouterr().out) + assert row["backend"] == "claude-cli" and row["authenticated"] is True + + +def test_an_installed_cli_that_prints_no_version_is_unknown_not_not_installed(monkeypatch, capsys): + from dokimi.commands.backends import cmd_backends + + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True}, version="")) + cmd_backends(argparse.Namespace(json=False)) + (row,) = [line for line in capsys.readouterr().out.splitlines() if line.startswith("| claude-cli |")] + assert row.split("|")[3:5] == [" unknown ", " yes "] + monkeypatch.setattr(process, "run", _auth_runner(None)) + cmd_backends(argparse.Namespace(json=False)) + (gone,) = [line for line in capsys.readouterr().out.splitlines() if line.startswith("| claude-cli |")] + assert gone.split("|")[3] == " not installed " + + +def test_benchmark_preflight_checks_the_login_not_a_key(monkeypatch): + from dokimi.commands.benchmark import _provider_checks + + monkeypatch.setenv("ANTHROPIC_API_KEY", "") # no key anywhere: the CLI login is what counts + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": False})) + errors, _ = _provider_checks(["claude-cli"]) + assert any("not signed in" in e for e in errors) + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) + assert _provider_checks(["claude-cli"]) == ([], []) + monkeypatch.setattr(process, "run", _auth_runner(None)) + errors, _ = _provider_checks(["claude-cli"]) + assert any("not installed" in e for e in errors) + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + errors, _ = _provider_checks(["openai"]) + assert any("missing OPENAI_API_KEY" in e for e in errors) + + +def test_the_preflight_probes_a_cli_provider_once_for_its_error_and_its_warning(monkeypatch): + from dokimi.commands.benchmark import _provider_checks + + fake = _auth_runner("no answer") + monkeypatch.setattr(process, "run", fake) + errors, warnings = _provider_checks(["claude-cli", "claude-cli"]) + assert errors == [] and len(warnings) == 1 + assert [c.argv[-1] for c in fake.calls] == ["--version", "status"] # one detect, not one per check + + +@pytest.mark.parametrize("argv", [ + ["eval", "score", "--judge-provider", "claude-cli", "--judge-temperature", "0"], + ["eval", "panel-score", "--provider", "claude-cli", "--judge-temperature", "0"], +]) +def test_a_cli_judge_given_a_temperature_is_an_error_line_and_exit_2(argv, tmp_path, monkeypatch, capsys): + from dokimi import cli + + answers = tmp_path / "answers.json" + answers.write_text("[]", encoding="utf-8") + monkeypatch.setattr("sys.argv", ["dokimi", *argv, "--product", "p", "--answers", str(answers)]) + with pytest.raises(SystemExit) as exc: + cli.main() + err = capsys.readouterr().err + assert exc.value.code == 2 and err.startswith("Error: claude-cli takes no temperature") + assert "Traceback" not in err + + +def test_a_judge_panel_refuses_sampling_on_the_cli_with_explicit_judges(): + from dokimi.eval.panel import JudgeConfig, JudgePanel + + with pytest.raises(ValueError, match="takes no temperature"): + JudgePanel(judges=[JudgeConfig(role="a", model="sonnet", provider="claude-cli", temperature=0.0)]) + + +def test_an_unanswered_login_check_is_a_warning_not_a_pass(monkeypatch): + from dokimi.commands.benchmark import _login_warning, _provider_checks + + monkeypatch.setattr(process, "run", _auth_runner("no answer")) + assert _provider_checks(["claude-cli"]) == ([], [ + "claude-cli: could not check the login ('claude auth status' gave no answer)"]) + assert "could not check the login" in _login_warning("claude-cli") + monkeypatch.setattr(process, "run", _auth_runner({"loggedIn": True})) + assert _login_warning("claude-cli") is None + + +def test_the_default_judge_runs_on_the_cli(): + from dokimi.defaults import DEFAULT_JUDGE_MODEL, default_judge_model + + assert default_judge_model("claude-cli") == DEFAULT_JUDGE_MODEL + + +def test_a_cli_provider_needs_a_claude_model_on_the_command_line(): + from dokimi.cli import parse_args + + base = ["answers", "generate", "--product", "p", "--questions", "q.json", "--provider", "claude-cli"] + with pytest.raises(SystemExit): + parse_args(base) # the default answer model is not a Claude model + assert parse_args([*base, "--model", "haiku"]).model == "haiku" + assert parse_args([*base, "--model", "claude-sonnet-5-5"]).model == "claude-sonnet-5-5" + + +def test_answerer_and_generators_accept_the_cli_provider(runner): + from dokimi.eval.answerer import Answerer + from dokimi.personas.generator import PersonaGenerator + from dokimi.questions.llm_gen import QuestionGenerator + + Answerer(model="haiku", provider="claude-cli") + runner.answers.extend([reply(_ok("persona")), reply(_ok("question"))]) + assert PersonaGenerator(model="haiku", provider="claude-cli").llm.invoke("p").content == "persona" + assert QuestionGenerator(model="haiku", provider="claude-cli").llm.invoke("q").content == "question" + + +def test_the_child_env_drops_billing_and_session_variables_by_prefix(runner, monkeypatch): + # Named in review: an endpoint, a model alias override and a provider switch each + # change who is billed; a messaging socket joins the parent's session. + for name, value in { + "ANTHROPIC_BASE_URL": "https://gateway.example", "ANTHROPIC_DEFAULT_SONNET_MODEL": "other", + "ANTHROPIC_CUSTOM_HEADERS": "x: y", "CLAUDE_CODE_USE_BEDROCK": "1", + "CLAUDE_CODE_MESSAGING_SOCKET": "messaging.sock", "CLAUDE_CODE_SSE_PORT": "1234", + "CLAUDE_CODE_OAUTH_TOKEN": "headless-token", "CLAUDE_CONFIG_DIR": "/accounts/work", + }.items(): + monkeypatch.setenv(name, value) + runner.answers.append(reply(_ok())) + llm_cli.call("claude-cli", model="sonnet", prompt="p") + env = runner.calls[0].env + for name in ("ANTHROPIC_BASE_URL", "ANTHROPIC_DEFAULT_SONNET_MODEL", "ANTHROPIC_CUSTOM_HEADERS", + "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_MESSAGING_SOCKET", "CLAUDE_CODE_SSE_PORT"): + assert name not in env, name + # The login is kept: the headless token and the account's config dir. + assert env["CLAUDE_CODE_OAUTH_TOKEN"] == "headless-token" + assert env["CLAUDE_CONFIG_DIR"] == "/accounts/work" + + +def test_the_git_bash_path_a_windows_claude_needs_is_passed_on(runner, monkeypatch): + bash = r"C:\Program Files\Git\bin\bash.exe" + monkeypatch.setenv("CLAUDE_CODE_GIT_BASH_PATH", bash) + runner.answers.append(reply(_ok())) + llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert runner.calls[0].env["CLAUDE_CODE_GIT_BASH_PATH"] == bash + + +def test_git_does_not_read_a_repository_above_the_work_dir(runner, monkeypatch, tmp_path): + # The cache dir sits inside a git repository (a home directory under git): the CLI must not + # read that repository's branch and status into its prompt. Probed with the call's own env. + import os + import subprocess + + home = tmp_path / "home" + subprocess.run(["git", "init", "-q", str(home)], check=True) + monkeypatch.setenv("XDG_CACHE_HOME", str(home / "cache")) + monkeypatch.setenv("GIT_CEILING_DIRECTORIES", str(tmp_path / "users-own")) + runner.answers.append(reply(_ok())) + llm_cli.call("claude-cli", model="sonnet", prompt="p") + call = runner.calls[0] + # The caller's own ceiling entry is kept next to the one for the work dir's parent. + assert call.env["GIT_CEILING_DIRECTORIES"].split(os.pathsep) == [str(call.cwd.parent), str(tmp_path / "users-own")] + env = {k: v for k, v in call.env.items() if k not in ("GIT_DIR", "GIT_WORK_TREE")} + probe = subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], cwd=call.cwd, env=env, + capture_output=True, text=True) + assert probe.returncode != 0, probe.stdout + + +def test_the_work_dir_is_under_the_users_cache_not_the_shared_temp_dir(monkeypatch, tmp_path): + import tempfile + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path / "shared-tmp")) + path = llm_cli.work_dir("claude-cli") + assert path == tmp_path / "cache" / "dokimi" / "cli" / "claude-cli" and path.is_dir() + assert not (tmp_path / "shared-tmp").exists() + + +def test_a_reply_names_the_main_model_the_cli_resolved(runner): + # modelUsage lists a helper model too; the main one has the output tokens. + payload = json.loads(_ok(modelUsage={"claude-haiku-5-5": {"outputTokens": 9}, + "claude-sonnet-5-5": {"outputTokens": 3}})) + runner.answers.append(reply(json.dumps(payload))) + out = llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert out.model_id == "claude-sonnet-5-5" + + +def test_an_empty_reply_gives_no_model_no_tokens_and_no_text(runner): + # The parser's empty branches: no modelUsage entries, a null usage and a null result. + runner.answers.append(reply(_ok(modelUsage={}, usage=None, result=None))) + out = llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert out.model_id is None and out.text == "" + assert (out.usage.prompt_tokens, out.usage.completion_tokens, out.usage.resolved_model) == (0, 0, "") + + +def test_a_reply_without_a_model_usage_key_names_no_model(runner): + payload = json.loads(_ok()) + del payload["modelUsage"] + runner.answers.append(reply(json.dumps(payload))) + out = llm_cli.call("claude-cli", model="sonnet", prompt="p") + assert out.model_id is None and out.usage.resolved_model == "" + + +def test_the_llm_reply_carries_the_resolved_model_id(runner): + runner.answers.append(reply(_ok())) + request = llm.Request(model="sonnet", provider="claude-cli", litellm_model="sonnet", + messages=[{"role": "user", "content": "ping"}], max_tokens=50) + assert llm._cli_reply(request).model_id == "claude-sonnet-5-5" + + +def test_an_alias_and_its_full_id_are_one_model_for_the_identity_checks(): + same = llm_cli.same_model + assert same("claude-cli", "sonnet", "claude-cli", "claude-sonnet-5-5") + assert same("anthropic", "claude-sonnet-5-5", "claude-cli", "sonnet") + assert not same("claude-cli", "sonnet", "claude-cli", "claude-opus-5-5") + assert not same("claude-cli", "claude-sonnet-5-5", "claude-cli", "claude-sonnet-4-5") + assert not same("openai", "gpt-6.1-sol", "claude-cli", "sonnet") + + +def test_the_one_million_context_form_is_a_claude_model_on_the_cli(): + # claude --model takes sonnet[1m]; it is the same model as the alias, with a larger window. + assert llm_cli.accepts_model("claude-cli", "sonnet[1m]") + assert llm_cli.accepts_model("claude-cli", "opus[1m]") + assert llm_cli.accepts_model("claude-cli", "claude-sonnet-5-5[1m]") + assert not llm_cli.accepts_model("claude-cli", "gpt-6.1-sol[1m]") + assert llm_cli._family("sonnet[1m]") == "sonnet" + assert llm_cli.same_model("claude-cli", "sonnet[1m]", "claude-cli", "claude-sonnet-5-5") + assert not llm_cli.same_model("claude-cli", "sonnet[1m]", "claude-cli", "claude-opus-5-5") + + +def test_the_independence_warning_fires_for_an_alias_against_its_full_id(capsys): + from dokimi.commands.evaluate import _warn_judge_independence + + assert _warn_judge_independence("claude-cli", "sonnet", "claude-cli", "claude-sonnet-5-5") + assert "independence warning" in capsys.readouterr().err + assert not _warn_judge_independence("claude-cli", "sonnet", "claude-cli", "claude-opus-5-5") + + +@pytest.mark.parametrize("answerer", ["sonnet", "opus", "claude-opus-5-1", "claude-sonnet-5-5"]) +def test_the_novelty_default_judge_on_the_cli_is_another_family(answerer): + from dokimi.novelty.client import default_judge + + judge, provider = default_judge(answerer, "claude-cli") + assert provider == "claude-cli" and not llm_cli.same_model("claude-cli", judge, "claude-cli", answerer) + + +def test_a_novelty_judge_of_the_answerers_family_counts_as_the_answerer(): + # run_measurement refuses a judge that same_client matches with the answerer, unless allowed. + from dokimi.novelty.client import LLMClient, same_client + + judge = LLMClient(model="claude-sonnet-5-5", provider="anthropic") + answerer = LLMClient(model="sonnet", provider="claude-cli") + assert same_client(judge, answerer) and same_client(answerer, judge) + assert not same_client(LLMClient(model="opus", provider="claude-cli"), answerer) + + +def test_an_agentic_arm_or_a_session_is_refused_on_the_cli_before_any_answer(): + from dokimi.commands import arms_refusals + from dokimi.treatments import create_treatments + + plain = create_treatments(["baseline", "docs"], top_k=3, rerank_threshold=0.3) + agentic = create_treatments(["baseline", "agent"], top_k=3, rerank_threshold=0.3) + cli = argparse.Namespace(provider="claude-cli", session_turns=None) + arms_refusals.refuse_cli_arms(cli, plain) + with pytest.raises(ValueError, match="agentic arm"): + arms_refusals.refuse_cli_arms(cli, agentic) + with pytest.raises(ValueError, match="--session-turns"): + arms_refusals.refuse_cli_arms(argparse.Namespace(provider="claude-cli", session_turns=3), plain) + arms_refusals.refuse_cli_arms(argparse.Namespace(provider="openai", session_turns=3), agentic) + + +def test_a_cli_judge_given_a_temperature_stops_the_full_pipeline_before_any_work(monkeypatch, capsys): + from dokimi.commands import orchestrate + from dokimi import orchestrator + + def pipeline_must_not_start(*args, **kwargs): + raise AssertionError("the pipeline started before the sampling refusal") + + monkeypatch.setattr(orchestrator, "EvaluationPipeline", pipeline_must_not_start) + args = argparse.Namespace(repo="owner/repo", description=None, product="demo", + judge_provider="claude-cli", judge_temperature=0.0) + with pytest.raises(SystemExit): + orchestrate.cmd_evaluate(args) + assert "takes no temperature" in capsys.readouterr().err + + +def test_every_command_with_a_hidden_or_default_model_refuses_the_cli_without_a_claude_model(capsys): + from dokimi.cli import parse_args + + cases = [ + ["answers", "generate", "--product", "p", "--questions", "q.json", "--provider", "claude-cli"], + ["questions", "generate", "--product", "p", "--personas", "p.json", "--provider", "claude-cli"], + ["questions", "analyze", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], + ["questions", "panel-review", "--questions", "q.json", "--product", "p", "--provider", "claude-cli"], + ["personas", "discover", "--product", "p", "--provider", "claude-cli"], + ["subjects", "run", "subject.toml", "--provider", "claude-cli"], + ] + for argv in cases: + with pytest.raises(SystemExit): + parse_args(argv) + # the refusal is the model gate, not an unrelated argparse error + assert "takes a Claude model" in capsys.readouterr().err, argv + assert parse_args(["subjects", "run", "subject.toml", "--provider", "claude-cli", + "--model", "haiku"]).model == "haiku" + + +def test_every_agentic_arm_kind_is_refused_on_the_cli_including_wrapped_and_combined(demo_skill): + from dokimi.skills import load_skill + + from dokimi.commands import arms_refusals + from dokimi.plugins import PluginWrappedTreatment + from dokimi.treatments.arms import ( + BaselineTreatment, + CombinedTreatment, + MCPAgentTreatment, + SkillAgentTreatment, + SkillCatalogTreatment, + ) + + cli = argparse.Namespace(provider="claude-cli", session_turns=None) + skill = load_skill(demo_skill) + base = BaselineTreatment() + agentic = { + "mcp_agent": MCPAgentTreatment(mcp_client=None), + "skill_agent": SkillAgentTreatment(skill), + "skill_catalog": SkillCatalogTreatment([skill]), + "combined_with_agent": CombinedTreatment([base, SkillAgentTreatment(skill)]), + "plugin_wrapped_agent": PluginWrappedTreatment(SkillAgentTreatment(skill), []), + } + for arm in agentic.values(): + with pytest.raises(ValueError, match="agentic arm"): + arms_refusals.refuse_cli_arms(cli, [arm]) + arms_refusals.refuse_cli_arms( + cli, [base, CombinedTreatment([base, base]), PluginWrappedTreatment(base, [])]) + + +def test_the_arms_entry_point_refuses_a_cli_agentic_arm_before_any_answer(monkeypatch): + from dokimi.commands import arms + from dokimi.eval import arm_runner + + class RunnerMustNotStart: + def __init__(self, *args, **kwargs): + raise AssertionError("the arm runner started before the CLI refusal") + + monkeypatch.setattr(arm_runner, "ArmRunner", RunnerMustNotStart) + monkeypatch.setattr(arms, "_resolve_doc_library_id", lambda *a, **k: None) + monkeypatch.setattr(arms, "_validate_expected_skills", lambda *a, **k: None) + args = argparse.Namespace(provider="claude-cli", model="sonnet", judge=False, session_turns=None, + top_k=3, rerank_threshold=0.3, product="p", context7_id=None, + plugins="", harness="arms-runner", matrix_cell=None, judge_provider=None, + judge_model=None, max_iterations=5) + with pytest.raises(ValueError, match="agentic arm"): + arms._build_arms_output(args, ["baseline", "agent"], []) + + +def test_a_matrix_with_a_cli_agentic_cell_is_refused_before_the_first_cell_is_answered(monkeypatch, tmp_path, capsys): + from dokimi.commands import arms + from dokimi.eval import arm_runner + from dokimi.eval import cells as eval_cells + + class RunnerMustNotStart: + def __init__(self, *args, **kwargs): + raise AssertionError("a cell was answered before the CLI refusal") + + cells = [ + argparse.Namespace(name="api", model="gpt-6.1-sol", provider="openai", harness="arms-runner", + plugin_specs=[]), + argparse.Namespace(name="cli", model="sonnet", provider="claude-cli", harness="arms-runner", + plugin_specs=[]), + ] + monkeypatch.setattr(arm_runner, "ArmRunner", RunnerMustNotStart) + monkeypatch.setattr(eval_cells, "load_matrix_cells", lambda path: cells) + monkeypatch.setattr(arms, "_load_questions", lambda path: []) + monkeypatch.setattr(arms, "_validate_supported_matrix_harnesses", lambda cells: None) + monkeypatch.setattr(arms, "_validate_unique_cell_filenames", lambda cells: None) + args = argparse.Namespace(arms="baseline,agent", questions="q.json", matrix_cells="m.json", + product="p", out_dir=str(tmp_path), top_k=3, rerank_threshold=0.3, + temperature=None, seed=None, judge=False, session_turns=None, + plugins="", harness="arms-runner", matrix_cell=None, judge_provider=None, + judge_model=None, max_iterations=5, context7_id=None) + with pytest.raises(SystemExit): + arms.cmd_arms_matrix_run(args) + assert "agentic arm" in capsys.readouterr().err + assert not tmp_path.joinpath("cells").exists() + + +def _matrix_run(monkeypatch, tmp_path, cli_model, arms_spec, capsys): + """matrix-run on an API cell then a CLI cell, where no cell may run: the error it exits with.""" + from dokimi.cli import build_parser + from dokimi.commands import arms + + cells = tmp_path / "cells.json" + cells.write_text(json.dumps({"matrix": {"cells": [ + {"id": "api", "model": "gpt-6.1-sol", "provider": "openai", "harness": "arms-runner"}, + {"id": "cli", "model": cli_model, "provider": "claude-cli", "harness": "arms-runner"}, + ]}}), encoding="utf-8") + questions = tmp_path / "q.json" + questions.write_text(json.dumps([{"id": "q1", "question": "Q?"}]), encoding="utf-8") + def cell_must_not_run(*a, **kw): + raise AssertionError("a matrix cell ran before the CLI cell was refused") + + monkeypatch.setattr(arms, "_build_arms_output", cell_must_not_run) + args = build_parser().parse_args( + ["arms", "matrix-run", "--product", "p", "--questions", str(questions), "--arms", arms_spec, + "--matrix-cells", str(cells), "--out-dir", str(tmp_path / "out")]) + with pytest.raises(SystemExit): + args.func(args) + return capsys.readouterr().err + + +def test_a_matrix_cell_pairing_a_cli_provider_with_a_non_claude_model_is_refused( + monkeypatch, tmp_path, capsys): + from dokimi.commands import arms + + assert "takes a Claude model" in _matrix_run(monkeypatch, tmp_path, "gpt-6.1-sol", "baseline", capsys) + # arms run --matrix-cells/--matrix-cell applies the cell the same way + named = argparse.Namespace(matrix_cells=str(tmp_path / "cells.json"), matrix_cell="cli") + with pytest.raises(ValueError, match="takes a Claude model"): + arms._apply_named_matrix_cell(named) + + +def test_an_alias_and_a_bedrock_or_prefixed_full_id_are_one_model_for_the_identity_checks(): + assert llm_cli.same_model("claude-cli", "sonnet", "anthropic", "us.anthropic.claude-sonnet-5-5-v1:0") + assert llm_cli.same_model("claude-cli", "sonnet", "litellm", "anthropic/claude-sonnet-5-5") + assert not llm_cli.same_model("claude-cli", "sonnet", "anthropic", "us.anthropic.claude-opus-5-1-v1:0") + + +def test_the_main_model_falls_back_to_most_output_when_the_requested_family_is_unknown(): + usage = {"helper-a": {"outputTokens": 5}, "helper-b": {"outputTokens": 500}} + assert llm_cli._main_model(usage, "claude-fable-5-1") == "helper-b" + + +def test_the_login_check_runs_with_the_same_reduced_env_as_the_call(monkeypatch): + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://gateway.example") + fake = _auth_runner({"loggedIn": True, "authMethod": "oauth_token"}) + llm_cli.detect("claude-cli", runner=fake) + assert fake.calls + for call in fake.calls: + assert "ANTHROPIC_BASE_URL" not in call.env + + +def test_the_session_turns_flag_defaults_to_none(): + from dokimi.cli import parse_args + + args = parse_args(["arms", "run", "--product", "p", "--questions", "q.json", "--arms", "baseline", + "--provider", "claude-cli", "--model", "sonnet"]) + assert args.session_turns is None + + +def test_the_resolved_model_is_on_the_usage_row_and_in_its_metrics(runner): + runner.answers.append(reply(_ok())) + usage = llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[1] + assert usage.resolved_model == "claude-sonnet-5-5" + assert usage.as_metrics_dict()["resolved_model"] == "claude-sonnet-5-5" + assert "resolved_model" not in UsageRecord().as_metrics_dict() + + +def test_a_sum_of_usage_rows_keeps_every_resolved_model_it_mixes(): + sonnet = UsageRecord(resolved_model="claude-sonnet-5-5") + assert (sonnet + UsageRecord()).resolved_model == "claude-sonnet-5-5" + assert (sonnet + UsageRecord(resolved_model="claude-sonnet-5-5")).resolved_model == "claude-sonnet-5-5" + mixed = sonnet + UsageRecord(resolved_model="claude-opus-5-0") + assert mixed.resolved_model == "claude-opus-5-0,claude-sonnet-5-5" + + +def test_a_cached_cli_reply_replays_the_resolved_model(runner, monkeypatch, tmp_path): + monkeypatch.setenv("DOKIMI_RESPONSE_CACHE", str(tmp_path / "responses")) + runner.answers.append(reply(_ok())) + first = llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[1] + second = llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[1] + assert len(runner.calls) == 1 # the second reply came from the cache + assert second.replayed and second.resolved_model == first.resolved_model == "claude-sonnet-5-5" + + +def test_a_new_cli_version_misses_the_response_cache(runner, monkeypatch, tmp_path): + # An alias resolves inside the CLI: a new release can move "sonnet" to another model, + # so its answers must not be replayed for the new one. + monkeypatch.setenv("DOKIMI_RESPONSE_CACHE", str(tmp_path / "responses")) + runner.answers.extend([reply(_ok("OLD")), reply(_ok("NEW"))]) + monkeypatch.setattr(llm_cli, "cli_version", lambda provider: "2.1.303 (Claude Code)") + assert llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[0] == "OLD" + monkeypatch.setattr(llm_cli, "cli_version", lambda provider: "2.1.400 (Claude Code)") + assert llm.llm_call_with_usage("ping", model="sonnet", provider="claude-cli")[0] == "NEW" + assert len(runner.calls) == 2 + + +def test_the_cli_version_is_its_first_line_and_is_read_once(monkeypatch): + llm_cli._version_line.cache_clear() + fake = FakeRunner(lambda call: reply("2.1.303 (Claude Code)\nmore")) + monkeypatch.setattr(llm_cli.process, "run", fake) + try: + assert llm_cli.cli_version("claude-cli") == "2.1.303 (Claude Code)" + assert llm_cli.cli_version("claude-cli") == "2.1.303 (Claude Code)" + assert len(fake.calls) == 1 + finally: + llm_cli._version_line.cache_clear() + + +@pytest.mark.parametrize( + "answer, status, exits, printed", + [ + ("claude-cli", {"loggedIn": False}, True, "is not signed in"), + ("claude-cli", "no answer", False, "could not check the login"), + ("claude-cli", {"loggedIn": True}, False, "Preflight OK"), + ("openai", {"loggedIn": False}, True, "is not signed in"), + ], + ids=["signed-out-is-an-error", "no-answer-is-a-warning", "signed-in-passes", "judge-login-is-checked"], +) +def test_benchmark_preflight_exit_status_follows_the_cli_login(monkeypatch, tmp_path, capsys, + answer, status, exits, printed): + from dokimi.cli import build_parser + from dokimi.commands import benchmark + from dokimi.registry import ProductEntry + + class Registry: + def get(self, key): + return ProductEntry(key="onetbb", name="oneTBB", description="TBB", doc_sources=["local:docs"]) + + monkeypatch.setenv("OPENAI_API_KEY", "test") + monkeypatch.setattr(benchmark, "_load_registry", lambda args: Registry()) + monkeypatch.setattr("dokimi.mcp.factory.create_doc_source_client", lambda doc_source: object()) + monkeypatch.setattr(process, "run", _auth_runner(status)) + args = build_parser().parse_args([ + "benchmark", "preflight", "--library", "onetbb", "--doc-source", "local:docs", + "--provider", answer, "--model", "sonnet" if answer == "claude-cli" else "gpt-4o", + "--judge-provider", "claude-cli", "--judge-model", "opus", + "--output-dir", str(tmp_path / "out"), + ]) + if exits: + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + else: + args.func(args) + captured = capsys.readouterr() + assert printed in captured.out + captured.err + assert ("Preflight OK" in captured.out) is (not exits) diff --git a/dokimi/tests/test_novelty_cli.py b/dokimi/tests/test_novelty_cli.py index 9db901a78..5fb37bece 100644 --- a/dokimi/tests/test_novelty_cli.py +++ b/dokimi/tests/test_novelty_cli.py @@ -80,6 +80,9 @@ def test_a_judge_provider_alone_takes_the_generator_when_the_judge_is_measured() ("--model", f"us.anthropic.{DEFAULT_JUDGE_MODEL}", "--provider", "amazon-bedrock", "--judge-provider", "amazon-bedrock"), ("--model", DEFAULT_JUDGE_MODEL, "--provider", "anthropic", "--judge-provider", "anthropic"), + # A CLI alias is its family's full id: refused here, not after the paid extraction. + ("--model", "sonnet", "--provider", "claude-cli", "--judge-provider", "anthropic"), + ("--model", "sonnet", "--provider", "claude-cli", "--judge-provider", "amazon-bedrock"), # The default answerer is the generator, so openai has no default judge left either. ("--judge-provider", "openai"), ]) @@ -89,6 +92,17 @@ def test_a_judge_provider_with_no_default_judge_there_exits(argv): novelty_cmd._clients(args) +@pytest.mark.parametrize("model", ["sonnet", "opus", "haiku", "claude-opus-5-5"]) +def test_a_cli_answerer_with_no_judge_flags_is_judged_on_the_cli_by_another_family(model): + from dokimi.llm_cli import same_model + + args = _parse("novelty", "analyze", "--artifact", "skill:x", "--model", model, + "--provider", "claude-cli") + _extractor, _answerer, judge = novelty_cmd._clients(args) + assert judge.provider == "claude-cli" + assert not same_model("claude-cli", judge.model, "claude-cli", model) + + def test_a_named_judge_without_a_provider_runs_on_the_answer_provider(): args = _parse("novelty", "analyze", "--artifact", "skill:x", "--provider", "openrouter", "--judge-model", "j")