From fe6ef4a0c402ddebea26db25796d36d412791ee1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Wed, 28 Oct 2026 22:21:29 -0700 Subject: [PATCH 01/51] fix(cli): input and usage refusals exit 2, not 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exit-code table (CLI API v2 §2.6) gives 2 to a missing, unreadable or malformed input, a flag combination a command refuses, an output file that cannot be written, and a benchmark run or preflight with no provider credential. Exit 1 stays for a verdict or a gate that ran and failed. Refusals raised as SystemExit() now go through usage_exit, which prints the message to stderr and exits 2. Messages are unchanged. Output-write failures (exit_on_unwritable, write_text_or_exit) exit 2. Benchmark preflight exits 2 on a missing credential and keeps 1 for its question-set checks. Verdict, gate, infra and tool-output sites are not changed; see exit_audit for the open questions. Signed-off-by: Nikolay Petrov --- ...20000_napetrov_fix-cli-exit-codes-audit.md | 3 + dokimi/dokimi/commands/_output.py | 6 +- dokimi/dokimi/commands/answers.py | 15 +-- dokimi/dokimi/commands/arms.py | 9 +- dokimi/dokimi/commands/baseline.py | 11 +- dokimi/dokimi/commands/benchmark.py | 14 ++- dokimi/dokimi/commands/catalog.py | 41 +++---- dokimi/dokimi/commands/catalog_probe.py | 3 +- dokimi/dokimi/commands/dashboard.py | 5 +- dokimi/dokimi/commands/evaluate.py | 11 +- dokimi/dokimi/commands/explore.py | 15 ++- dokimi/dokimi/commands/inputs.py | 5 +- dokimi/dokimi/commands/library.py | 9 +- dokimi/dokimi/commands/novelty.py | 70 ++++++------ dokimi/dokimi/commands/novelty_view_inputs.py | 3 +- dokimi/dokimi/commands/orchestrate.py | 5 +- dokimi/dokimi/commands/personas.py | 19 ++-- dokimi/dokimi/commands/questions.py | 29 ++--- dokimi/dokimi/commands/repo.py | 21 ++-- dokimi/dokimi/commands/repo_run.py | 5 +- dokimi/dokimi/commands/report.py | 27 ++--- dokimi/dokimi/commands/run.py | 3 +- dokimi/dokimi/commands/subjects.py | 23 ++-- dokimi/dokimi/commands/tasks.py | 25 +++-- dokimi/dokimi/commands/tools.py | 11 +- dokimi/dokimi/exit_codes.py | 27 +++++ dokimi/dokimi/novelty/artifact.py | 3 +- dokimi/dokimi/novelty/atoms.py | 3 +- dokimi/dokimi/report/model_compare.py | 37 +++--- dokimi/dokimi/utils.py | 3 +- dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_answers_command.py | 20 ++-- .../tests/test_baseline_unreadable_input.py | 18 +-- .../test_benchmark_command_question_set.py | 5 +- dokimi/tests/test_dashboard_cli.py | 4 +- dokimi/tests/test_eval_output_write_errors.py | 2 +- dokimi/tests/test_evaluate_cli_inputs.py | 8 +- dokimi/tests/test_exit_codes_input_errors.py | 105 ++++++++++++++++++ dokimi/tests/test_import_architecture.py | 2 +- dokimi/tests/test_library_cli_inputs.py | 2 +- dokimi/tests/test_matrix_rollup.py | 6 +- dokimi/tests/test_novelty_settings_parity.py | 16 ++- dokimi/tests/test_novelty_view_inputs.py | 2 +- dokimi/tests/test_personas_approve.py | 8 +- dokimi/tests/test_personas_discover_output.py | 12 +- dokimi/tests/test_questions_artifact.py | 12 +- .../tests/test_questions_generate_config.py | 4 +- .../tests/test_questions_generate_inputs.py | 20 ++-- dokimi/tests/test_questions_input_errors.py | 4 +- .../tests/test_questions_output_unwritable.py | 8 +- dokimi/tests/test_ragas_runtime.py | 4 +- dokimi/tests/test_record_readers.py | 4 +- dokimi/tests/test_repo_budget.py | 4 +- dokimi/tests/test_repo_compare.py | 2 +- dokimi/tests/test_repo_inspect_cli.py | 4 +- dokimi/tests/test_repo_manifest.py | 4 +- dokimi/tests/test_repo_reach.py | 4 +- dokimi/tests/test_report_cli_inputs.py | 18 +-- dokimi/tests/test_report_eval_input_errors.py | 8 +- dokimi/tests/test_runner_run.py | 4 +- dokimi/tests/test_skill_loader.py | 4 +- dokimi/tests/test_skill_selection.py | 6 +- dokimi/tests/test_subjects.py | 2 +- dokimi/tests/test_subjects_run_input.py | 26 ++--- .../tests/test_tools_preference_items_out.py | 8 +- 65 files changed, 495 insertions(+), 328 deletions(-) create mode 100644 dokimi/changelog.d/20261028_120000_napetrov_fix-cli-exit-codes-audit.md create mode 100644 dokimi/dokimi/exit_codes.py create mode 100644 dokimi/tests/test_exit_codes_input_errors.py diff --git a/dokimi/changelog.d/20261028_120000_napetrov_fix-cli-exit-codes-audit.md b/dokimi/changelog.d/20261028_120000_napetrov_fix-cli-exit-codes-audit.md new file mode 100644 index 000000000..453254fc1 --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_napetrov_fix-cli-exit-codes-audit.md @@ -0,0 +1,3 @@ +### Changed + +- **Input and usage refusals exit 2, not 1** — the exit-code table (CLI API v2 §2.6) gives 2 to a missing, unreadable, malformed or unusable input, a flag combination a command refuses, an output file that cannot be written, and a benchmark run or preflight with no provider credential. Exit 1 is left for a verdict or a gate that ran and failed. The messages are unchanged. Scripts that branch on `$? -eq 1` for these cases must also accept 2. Affected: `personas` (approve, discover), `questions`, `eval` (score, ragas, grounding, validity, panel-score), `orchestrate`, `report` (eval, generate, model-compare), `baseline`, `novelty`, `catalog` (and `catalog probe`), `answers`, `arms`, `explore`, `tasks`, `subjects`, `repo` (and `repo run`), `library`, `tools`, `dashboard`, `run`, and `benchmark` (run, preflight, batch). A preflight that fails only on a missing credential exits 2; one that fails on its question-set checks still exits 1. diff --git a/dokimi/dokimi/commands/_output.py b/dokimi/dokimi/commands/_output.py index c2c6eed40..d25f04a83 100644 --- a/dokimi/dokimi/commands/_output.py +++ b/dokimi/dokimi/commands/_output.py @@ -9,12 +9,14 @@ import sys from pathlib import Path +from dokimi.exit_codes import EXIT_USAGE + @contextlib.contextmanager def exit_on_unwritable(path: Path): - """Exit 1 naming ``path`` when an output write inside the block fails.""" + """Exit 2 naming ``path`` when an output write inside the block fails (exit-code table, §2.6).""" try: yield except OSError as exc: print(f"Error: cannot write {path}: {exc}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py index 4b08abe6a..8a998ee5f 100644 --- a/dokimi/dokimi/commands/answers.py +++ b/dokimi/dokimi/commands/answers.py @@ -19,6 +19,7 @@ DEFAULT_MAX_TOKENS_PER_QUESTION, DEFAULT_RETRIEVAL_TOP_K, ) +from dokimi.exit_codes import EXIT_USAGE def _load_runtime_config() -> dict: @@ -33,18 +34,18 @@ def _load_runtime_config() -> dict: loaded = yaml.safe_load(config_path.read_text(encoding="utf-8")) except yaml.YAMLError as exc: print(f"Error: invalid YAML in {config_path}: {exc}", file=sys.stderr) - raise SystemExit(1) from exc + raise SystemExit(EXIT_USAGE) from exc except (OSError, ValueError) as exc: # ValueError covers bad UTF-8 print(f"Error: cannot read {config_path}: {exc}", file=sys.stderr) - raise SystemExit(1) from exc + raise SystemExit(EXIT_USAGE) from exc if loaded is not None and not isinstance(loaded, dict): print(f"Error: expected a mapping in {config_path}", file=sys.stderr) - raise SystemExit(1) + raise SystemExit(EXIT_USAGE) config = loaded or {} for key in ("products", "retrieval"): if config.get(key) is not None and not isinstance(config[key], dict): print(f"Error: '{key}' in {config_path} must be a mapping", file=sys.stderr) - raise SystemExit(1) + raise SystemExit(EXIT_USAGE) return config @@ -55,10 +56,10 @@ def _load_questions(path: str) -> list: return load_questions(path).questions except ValueError as exc: # ArtifactValidationError: names the record and the fix print(f"Error: {exc}", file=sys.stderr) - raise SystemExit(1) from exc + raise SystemExit(EXIT_USAGE) from exc except OSError as exc: print(f"Error: cannot read {path}: {exc.strerror or exc}", file=sys.stderr) - raise SystemExit(1) from exc + raise SystemExit(EXIT_USAGE) from exc def _output_path(args: argparse.Namespace) -> Path: @@ -104,7 +105,7 @@ def cmd_answers_generate(args: argparse.Namespace) -> None: ) except ValueError as exc: print(f"Error: {exc}", file=sys.stderr) - raise SystemExit(1) from exc + raise SystemExit(EXIT_USAGE) from exc print(f"Generating answers for {args.product} (library_id={library_id})") print(f"Using model: {args.provider}/{args.model}") diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 5696708f8..a027d3960 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -52,6 +52,7 @@ from dokimi.eval.question_text import EmptyQuestionError +from dokimi.exit_codes import EXIT_USAGE def forwarded_run_config(args: argparse.Namespace, **overrides) -> dict: """Copy the shared run-configuration fields off *args*. @@ -90,7 +91,7 @@ def cmd_arms_run(args: argparse.Namespace) -> None: raise # a bad input record, not an arm setup error: cli.main reports it (exit 2) except (ValueError, FileNotFoundError) as exc: print(f"Error building arms/plugins: {exc}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) out_json = Path(args.out_json) if args.out_json else Path(f"results/arms/{args.product}.json") out_md = Path(args.out_md) if args.out_md else Path(f"results/arms/{args.product}.md") @@ -126,7 +127,7 @@ def cmd_arms_matrix_run(args: argparse.Namespace) -> None: _check_sampling_support(cell_args) except (ValueError, FileNotFoundError) as exc: print(f"Error loading matrix run inputs: {exc}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) out_dir = Path(args.out_dir) if args.out_dir else Path(f"results/arms/{args.product}-matrix") out_dir.mkdir(parents=True, exist_ok=True) @@ -191,7 +192,7 @@ def cmd_arms_plugin_delta(args: argparse.Namespace) -> None: output = compare_plugin_runs(baseline, plugin) except (PluginDeltaError, ValueError, FileNotFoundError) as exc: print(f"Error computing plugin delta: {exc}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) out_json = Path(args.out_json) if args.out_json else Path("results/arms/plugin_delta.json") save_artifact("plugin_delta", output, out_json) @@ -622,7 +623,7 @@ def _build_arms_output(args: argparse.Namespace, specs: list[str], questions: li library_id = _resolve_doc_library_id(treatments, args.product, args.context7_id) except ValueError as exc: print(f"Error: {exc}", file=sys.stderr) - raise SystemExit(1) from exc + raise SystemExit(EXIT_USAGE) from exc plugins = create_plugins([s.strip() for s in args.plugins.split(",") if s.strip()]) validate_plugins_for_harness(plugins, args.harness) treatments = wrap_treatments(treatments, plugins) diff --git a/dokimi/dokimi/commands/baseline.py b/dokimi/dokimi/commands/baseline.py index 20769c53d..bdcbd3f95 100644 --- a/dokimi/dokimi/commands/baseline.py +++ b/dokimi/dokimi/commands/baseline.py @@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional, Tuple from dokimi.utils import normalize_model_ref +from dokimi.exit_codes import EXIT_USAGE BASELINES_DIR = Path("baselines/eval") @@ -37,7 +38,7 @@ def _read_json_object(path: Path, code: int) -> Dict[str, Any]: return data -def _baseline_manifest(code: int = 1) -> Dict[str, Any]: +def _baseline_manifest(code: int = EXIT_USAGE) -> Dict[str, Any]: """Load or create the baseline manifest; a corrupt one exits ``code``.""" manifest_path = BASELINES_DIR / "manifest.json" if not manifest_path.exists(): @@ -140,7 +141,7 @@ def cmd_baseline_save(args: argparse.Namespace) -> None: eval_path = Path(args.from_eval) manifest = _baseline_manifest() # first: a corrupt manifest fails fast and leaves no orphan file - eval_data = _read_json_object(eval_path, 1) + eval_data = _read_json_object(eval_path, EXIT_USAGE) product = args.product or eval_data.get("product", "unknown") timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%f") name = args.name or f"{product}-{timestamp}" @@ -151,13 +152,13 @@ def cmd_baseline_save(args: argparse.Namespace) -> None: summary = _compute_summary(eval_data) except RecordError as exc: print(f"❌ {eval_path}: {exc}", file=sys.stderr) - raise SystemExit(1) from exc + raise SystemExit(EXIT_USAGE) from exc BASELINES_DIR.mkdir(parents=True, exist_ok=True) dest = BASELINES_DIR / f"{name}.json" if dest.exists(): print(f"❌ Baseline already exists: {dest} (choose a different --name)", file=sys.stderr) - raise SystemExit(1) + raise SystemExit(EXIT_USAGE) dest.write_text(json.dumps(eval_data, indent=2), encoding="utf-8") manifest["baselines"].append({ @@ -205,7 +206,7 @@ def _pick_entry(args: argparse.Namespace, baselines: list, eval_data: dict) -> d entry = next((b for b in baselines if b["name"] == args.baseline), None) if not entry: print(f"❌ Baseline '{args.baseline}' not found.") - raise SystemExit(1) + raise SystemExit(EXIT_USAGE) else: product = args.product or eval_data.get("product") candidates = [b for b in baselines if b.get("product") == product] if product else baselines diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index 2c50c724e..c67d7647f 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -25,6 +25,7 @@ JUDGE_MODEL_HELP, ) from dokimi.commands.library import _load_registry +from dokimi.exit_codes import EXIT_USAGE def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None) -> dict: @@ -133,18 +134,20 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None: entry = registry.get(args.library) except KeyError as exc: print(f"Error: {exc}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) plan = _resolved_benchmark_plan(entry, args) _print_plan(plan) errors: list[str] = [] warnings: list[str] = [] + credential_errors: list[str] = [] for provider in {args.provider, args.judge_provider}: key = _credential_env_var(provider) if not os.environ.get(key): - errors.append(f"missing {key} for provider {provider}") + credential_errors.append(f"missing {key} for provider {provider}") + errors.extend(credential_errors) try: Path(plan["output_dir"]).mkdir(parents=True, exist_ok=True) @@ -207,7 +210,8 @@ def cmd_benchmark_preflight(args: argparse.Namespace) -> None: print("Errors:", file=sys.stderr) for item in errors: print(f" ✗ {item}", file=sys.stderr) - sys.exit(1) + # No credentials is a usage refusal (2); the other preflight errors are a failed gate (1). + sys.exit(EXIT_USAGE if credential_errors else 1) print("✅ Preflight OK") @@ -219,7 +223,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: entry = registry.get(args.library) except KeyError as exc: print(f"Error: {exc}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) output_dir = args.output_dir or f"results/{entry.key}{DEFAULT_RESULTS_SUFFIX}" n_runs = getattr(args, "multi_run", 1) @@ -231,7 +235,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: _, _, problem = _check_questions_from(questions_from, entry.key) if problem: print(f"Error: {problem}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) if n_runs == 1: _run_single_library( diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index f81c5a7bd..f1b89225c 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -16,6 +16,7 @@ import json import sys from pathlib import Path +from dokimi.exit_codes import usage_exit def _skill_row(skill, review) -> dict: @@ -62,7 +63,7 @@ def _scan_skills(args: argparse.Namespace, root: Path, report: dict, as_json: bo try: skills = read_skill_manifest(root) except ValueError as exc: - raise SystemExit(f"skills.yaml: {exc}") from exc + usage_exit(f"skills.yaml: {exc}") reviews = review_skills(skills) counts = {"ok": 0, "warn": 0, "error": 0} failed = False @@ -102,7 +103,7 @@ def _scan_tasks(args: argparse.Namespace, root: Path, report: dict, as_json: boo try: tasks = read_task_manifest(root, tasks_subdir=args.tasks_subdir) except ValueError as exc: - raise SystemExit(str(exc)) from exc + usage_exit(str(exc)) if not as_json: print(f"# tasks ({len(tasks)} declared under {root})") failed = False @@ -121,7 +122,7 @@ def _scan_tasks(args: argparse.Namespace, root: Path, report: dict, as_json: boo def cmd_catalog_scan(args: argparse.Namespace) -> None: root = Path(args.root) if not root.is_dir(): - raise SystemExit(f"--root is not a directory: {root}") + usage_exit(f"--root is not a directory: {root}") as_json = args.format == "json" report: dict = {"root": str(root), "skills": [], "tasks": []} @@ -154,9 +155,9 @@ def _write_trigger_set(args: argparse.Namespace, queries, catalog_root: Path) -> else ts.to_skill_creator(queries, args.target)) Path(args.out).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") except ArtifactValidationError as exc: # the path was writable; the payload failed the schema - raise SystemExit(f"Error: questions set rejected: {exc}") from exc + usage_exit(f"Error: questions set rejected: {exc}") except (OSError, ValueError) as exc: # unwritable --out: a directory, a file as parent, no permission, empty path (pathlib ValueError) - raise SystemExit(f"Error: cannot write --out {args.out}: {exc}") from exc + usage_exit(f"Error: cannot write --out {args.out}: {exc}") def _discover_catalog_skills(catalog_root: Path) -> list: @@ -165,7 +166,7 @@ def _discover_catalog_skills(catalog_root: Path) -> list: try: return discover_skills(catalog_root) except (OSError, ValueError, RecursionError) as exc: # unreadable, non-UTF-8, bad frontmatter or nesting too deep - raise SystemExit(f"--catalog {catalog_root}: {exc}") from exc + usage_exit(f"--catalog {catalog_root}: {exc}") def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: @@ -175,25 +176,25 @@ def cmd_catalog_trigger_set(args: argparse.Namespace) -> None: catalog_root = Path(args.catalog) if not catalog_root.is_dir(): - raise SystemExit(f"--catalog is not a directory: {catalog_root}") + usage_exit(f"--catalog is not a directory: {catalog_root}") catalog = _discover_catalog_skills(catalog_root) # Checked before any --rewrite call, so a bad invocation costs no LLM calls. if args.format == "skill-creator": if not args.target: - raise SystemExit("--format skill-creator needs --target ") + usage_exit("--format skill-creator needs --target ") if args.target not in {s.name for s in catalog}: - raise SystemExit(f"--target {args.target!r} is not a skill in --catalog " - f"(have: {', '.join(sorted(s.name for s in catalog))})") + usage_exit(f"--target {args.target!r} is not a skill in --catalog " + f"(have: {', '.join(sorted(s.name for s in catalog))})") sources = _question_sources(args.questions or []) sources += _trigger_set_sources(args.trigger_set or [], catalog) if not sources: - raise SystemExit("give --questions PATH=LABEL or --trigger-set PATH[=TARGET]") + usage_exit("give --questions PATH=LABEL or --trigger-set PATH[=TARGET]") specs = list(args.questions or []) + list(args.trigger_set or []) rewrite = None if args.rewrite: if not args.model: - raise SystemExit("--rewrite needs --model") + usage_exit("--rewrite needs --model") from dokimi.llm import llm_call def rewrite(text: str) -> str: @@ -202,7 +203,7 @@ def rewrite(text: str) -> str: try: queries = ts.build_trigger_set(sources, catalog, rewrite=rewrite) except ValueError as exc: - raise SystemExit(str(exc)) from exc + usage_exit(str(exc)) # An @field rule that resolves for no question in its source is a typo in the # path, not a set of negatives; refuse it rather than write "nothing loads". start = 0 @@ -241,7 +242,7 @@ def cmd_catalog_formats(args: argparse.Namespace) -> None: root = Path(args.root) if not root.is_dir(): - raise SystemExit(f"--root is not a directory: {root}") + usage_exit(f"--root is not a directory: {root}") packages = formats.detect_skill_packages(root) summary = formats.summarize(packages) if args.json: @@ -266,11 +267,11 @@ def _question_sources(specs: list) -> list: for spec in specs: path, _, rule = spec.rpartition("=") # labels never contain '=', paths may if not path or not rule: - raise SystemExit(f"--questions needs PATH=LABEL (a skill name, 'none' or '@field'): {spec}") + usage_exit(f"--questions needs PATH=LABEL (a skill name, 'none' or '@field'): {spec}") data = read_json_input(path, flag="--questions") questions = data.get("questions", data) if isinstance(data, dict) else data if not isinstance(questions, list) or not all(isinstance(q, dict) for q in questions): - raise SystemExit(f"{path}: expected a list of question objects") + usage_exit(f"{path}: expected a list of question objects") sources.append((questions, rule)) return sources @@ -295,16 +296,16 @@ def _trigger_set_sources(specs: list, catalog) -> list: try: fmt, queries = ts.read_trigger_set(data, target=target) except ValueError as exc: - raise SystemExit(f"{path}: {exc}") from exc + usage_exit(f"{path}: {exc}") if fmt == ts.FORMAT_SKILL_CREATOR: if target not in names: - raise SystemExit(f"{spec}: skill-creator target {target!r} is not a skill in " - f"--catalog (have: {', '.join(sorted(names))})") + usage_exit(f"{spec}: skill-creator target {target!r} is not a skill in " + f"--catalog (have: {', '.join(sorted(names))})") if len(names) > 1: print(f"note: {path}: skill-creator negatives are read as 'no skill loads'; in a " f"catalog of {len(names)} a rival load counts as a false trigger", file=sys.stderr) elif target: - raise SystemExit(f"{spec}: =TARGET applies to skill-creator sets only, this is {fmt}") + usage_exit(f"{spec}: =TARGET applies to skill-creator sets only, this is {fmt}") print(f"read {path} as {fmt} ({len(queries)} queries)", file=sys.stderr) rows = ts.to_questions(queries)["questions"] for row, q in zip(rows, queries, strict=True): diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index abecfcdda..4cfb97a28 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -13,6 +13,7 @@ import threading from concurrent.futures import ThreadPoolExecutor from pathlib import Path +from dokimi.exit_codes import usage_exit def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: @@ -109,7 +110,7 @@ def _labelled_queries(args: argparse.Namespace, names: set) -> list: data = json.loads(Path(args.trigger_set).read_text(encoding="utf-8")) _fmt, queries = ts.read_trigger_set(data, target=args.target) except (OSError, ValueError) as exc: - raise SystemExit(f"--trigger-set: {exc}") from exc + usage_exit(f"--trigger-set: {exc}") queries = [q for q in queries if q.labelled][: args.limit or None] if not queries: # else nothing runs, and an empty summary reads as a probe that ran raise InputError(f"--trigger-set has no labelled queries: {args.trigger_set}" diff --git a/dokimi/dokimi/commands/dashboard.py b/dokimi/dokimi/commands/dashboard.py index f0087e70a..af64a5fec 100644 --- a/dokimi/dokimi/commands/dashboard.py +++ b/dokimi/dokimi/commands/dashboard.py @@ -8,6 +8,7 @@ import argparse import sys from pathlib import Path +from dokimi.exit_codes import EXIT_USAGE def cmd_dashboard_generate(args: argparse.Namespace) -> None: @@ -24,7 +25,7 @@ def cmd_dashboard_generate(args: argparse.Namespace) -> None: if not data.products: print("⚠ No evaluation results found. Run benchmark first.", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) print(f"Found {len(data.products)} product(s): {', '.join(p.product for p in data.products)}") @@ -40,7 +41,7 @@ def cmd_dashboard_generate(args: argparse.Namespace) -> None: print(f"✅ JSON data: {json_path}") except OSError as exc: print(f"Error: cannot write the dashboard to {output_dir}: {exc}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) # Print quick summary to stdout print(f"\n{'─'*50}") diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 9d3602f31..1889d4c7a 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -21,6 +21,7 @@ from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs from dokimi.utils import normalize_model_ref, write_text_or_exit +from dokimi.exit_codes import EXIT_USAGE, usage_exit def _warn_judge_independence( @@ -53,7 +54,7 @@ def _load_answers(path: str): sys.exit(2) -def _write_json_or_exit(path: Path, payload: dict, code: int = 1) -> None: +def _write_json_or_exit(path: Path, payload: dict, code: int = EXIT_USAGE) -> None: """Write ``payload`` as JSON to ``path``, or exit ``code`` naming ``path`` when it cannot be written.""" write_text_or_exit(path, json.dumps(payload, indent=2), code) @@ -61,7 +62,7 @@ def _write_json_or_exit(path: Path, payload: dict, code: int = 1) -> None: def _require_ragas() -> None: """Exit before any model call when the ragas extra is not installed.""" if importlib.util.find_spec("ragas") is None: - sys.exit("RAGAS meta-evaluation needs the ragas extra: pip install 'dokimi[ragas]'") + usage_exit("RAGAS meta-evaluation needs the ragas extra: pip install 'dokimi[ragas]'") def _run_ragas_eval(answers, output_path, args) -> None: @@ -110,11 +111,11 @@ def cmd_eval_panel_score(args: argparse.Namespace) -> None: f"Error: --judge-configs entry '{entry}' must be 'role:model:provider'.", file=sys.stderr, ) - sys.exit(1) + sys.exit(EXIT_USAGE) role, model, provider = parts if role not in JUDGE_ROLES: print(f"Error: unknown role '{role}'. Valid: {list(JUDGE_ROLES)}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) judges.append(JudgeConfig(role=role.strip(), model=model.strip(), provider=provider.strip(), temperature=args.judge_temperature)) roles = [j.role for j in judges] @@ -123,7 +124,7 @@ def cmd_eval_panel_score(args: argparse.Namespace) -> None: unknown = [r for r in roles if r not in JUDGE_ROLES] if unknown: print(f"Error: unknown roles: {unknown}. Valid: {list(JUDGE_ROLES)}", file=sys.stderr) - sys.exit(1) + sys.exit(EXIT_USAGE) judges = [JudgeConfig(role=r, model=args.model, provider=args.provider, temperature=args.judge_temperature) for r in roles] diff --git a/dokimi/dokimi/commands/explore.py b/dokimi/dokimi/commands/explore.py index 64900fdc9..d4dff01de 100644 --- a/dokimi/dokimi/commands/explore.py +++ b/dokimi/dokimi/commands/explore.py @@ -7,6 +7,7 @@ import argparse from pathlib import Path +from dokimi.exit_codes import usage_exit _TASKS_ROOT_HELP = ( "Directory of exploration task YAML files (default: data/explore_tasks of a " @@ -27,7 +28,7 @@ def cmd_explore_list(args: argparse.Namespace) -> None: try: tasks = discover_explore_tasks(root, default_repo_root=default_repo_root) except FileNotFoundError as exc: - raise SystemExit(str(exc)) from exc + usage_exit(str(exc)) for task in tasks: n_refs = len(task.references) print(f"{task.id}\t{task.product}\t{n_refs} ref(s)\t{task.query[:60]}") @@ -55,7 +56,7 @@ def _build_arms(specs: list[str], output_root: Path): elif spec.startswith("command:"): template = spec[len("command:"):] if not template: - raise SystemExit("'command:' arm requires a template") + usage_exit("'command:' arm requires a template") def _factory(task, _template=template, _spec=spec): return CommandExplorer( @@ -66,9 +67,7 @@ def _factory(task, _template=template, _spec=spec): arms[spec] = _factory else: - raise SystemExit( - f"Unknown explore arm: '{spec}'. Valid: oracle, empty, command: