ci(mutation): mutmut pilot over the stats, gates and cache key

.github/.licenserc.yaml

@@ -45,6 +45,7 @@ header:
     - 'dokimi/requirements.lock.txt'
     - 'dokimi/requirements-docs.lock.txt'
     - 'dokimi/requirements-ci.lock.txt'
+    - 'dokimi/requirements-mutation.lock.txt'
     # Read by agents verbatim: a header would become part of the instructions,
     # persona prompt or skill being measured.
     - 'AGENTS.md'

.github/renovate.json5

@@ -51,7 +51,7 @@
   // and updates the SHA and the tag together.
   "pre-commit": { enabled: true },
   "pip-compile": {
-    managerFilePatterns: ["/(^|/)requirements(-ci)?\\.lock\\.txt$/"],
+    managerFilePatterns: ["/(^|/)requirements(-ci|-mutation)?\\.lock\\.txt$/"],
   },
   // Inside a range: bump the lock only. Outside it: move the range too.
   rangeStrategy: "update-lockfile",

.github/workflows/mutation.yml

@@ -0,0 +1,96 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: mutation
+
+# Mutation pilot: mutmut over the measurement core (statistics, gate decisions, cache key), where a surviving mutant
+# is a wrong number that ships silently. Weekly and informational: the score goes to the step summary, never a failure.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '17 5 * * 1'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+defaults:
+  run:
+    working-directory: dokimi
+
+jobs:
+  mutmut:
+    name: mutmut (pilot modules)
+    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+        with:
+          python-version: '3.13'
+      # The runtime lock for the tests, and mutmut's own lock (requirements-mutation.txt),
+      # which no other workflow installs.
+      - name: Install deps
+        run: pip install --require-hashes -r requirements.lock.txt -r requirements-mutation.lock.txt
+      - name: Run mutmut
+        # Crashes are reported below, not failed on. The step timeout sits inside the job's 90 minutes, so the summary
+        # step still runs; a job-level timeout would fail the whole job instead.
+        timeout-minutes: 75
+        continue-on-error: true
+        run: mutmut run --max-children "$(nproc)"
+      - name: Score and survivors
+        if: always()
+        run: |
+          mutmut results --all true > mutmut-results.txt || true
+          python - <<'EOF' >> "$GITHUB_STEP_SUMMARY"
+          import collections
+
+          counts = collections.defaultdict(collections.Counter)
+          survivors = collections.defaultdict(list)
+          with open("mutmut-results.txt", encoding="utf-8") as fh:
+              for line in fh:
+                  name, sep, status = line.strip().rpartition(": ")
+                  if not sep:
+                      continue
+                  # <module>.x_<function>__mutmut_<n>, or x<sep><class><sep><method>
+                  module = name.rsplit(".", 1)[0]
+                  counts[module][status] += 1
+                  if status == "survived":
+                      survivors[module].append(name)
+
+          print("## Mutation pilot\n")
+          if not counts:
+              print("No results: see the `Run mutmut` step log.")
+          else:
+              print("| Module | Killed | Survived | Other | Score |")
+              print("|---|---:|---:|---:|---:|")
+              for module in sorted(counts):
+                  c = counts[module]
+                  killed, survived = c["killed"], c["survived"]
+                  other = sum(c.values()) - killed - survived
+                  scored = killed + survived
+                  score = f"{100 * killed / scored:.1f}%" if scored else "n/a"
+                  print(f"| `{module}` | {killed} | {survived} | {other} | {score} |")
+              unchecked = sum(c["not checked"] for c in counts.values())
+              unscored = sorted(m for m, c in counts.items() if not c["killed"] + c["survived"])
+              if unchecked or unscored:
+                  parts = []
+                  if unchecked:
+                      parts.append(f"{unchecked} mutants were not checked (a step timeout or a crash cut the run short)")
+                  if unscored:
+                      parts.append(f"nothing was scored in {', '.join(f'`{m}`' for m in unscored)}")
+                  print("\n> **Incomplete run: do not read the scores as final.** "
+                        + "; ".join(parts) + ". See the `Run mutmut` step log.")
+              print("\nOther: no tests, timeout, suspicious or not checked.")
+              print("`mutmut show <name>` prints a mutant's diff.\n")
+              for module in sorted(survivors):
+                  print(f"<details><summary>{module}: {len(survivors[module])} survivors</summary>\n")
+                  print("```")
+                  print("\n".join(survivors[module]))
+                  print("```\n</details>\n")
+          EOF

.github/workflows/pip-audit.yml

@@ -48,19 +48,13 @@ jobs:
       # pip-audit comes from the hash-locked tools lock (requirements-ci.txt pins it).
       - name: Install tools
         run: pip install --require-hashes -r requirements-ci.lock.txt
-      # The locks are the full closure (requirements-docs.lock.txt for the
-      # documentation site, requirements-ci.lock.txt for the tools), so --no-deps --disable-pip audits them as written
-      # without resolving or installing anything. Each --ignore-vuln is
-      # an advisory SECURITY.md documents and dokimi/osv-scanner.toml ignores;
-      # add or drop them together, in all three.
-      # pip-audit skips a pin whose marker is false on this 3.13 runner (the
-      # pins for older interpreters, for one), so markers are stripped, a pin
-      # the locks share is kept once, a name pinned twice goes to a second file,
-      # and the trailing `\` of a hashed pin is dropped (its --hash lines are skipped).
+      # Full-closure locks, so --no-deps --disable-pip audits them as written; markers are stripped (3.13 skips false ones).
+      # Each --ignore-vuln is in SECURITY.md and osv-scanner.toml: add or drop it in all three, together.
       - name: pip-audit
         run: |
           sed -e 's/ *\\$//' -e 's/ *;.*//' \
-              requirements.lock.txt requirements-docs.lock.txt requirements-ci.lock.txt | sort -u \
+              requirements.lock.txt requirements-docs.lock.txt requirements-ci.lock.txt \
+              requirements-mutation.lock.txt | sort -u \
             | awk -F'==' '/^[A-Za-z0-9]/ { print > (ENVIRON["RUNNER_TEMP"] "/pins" seen[$1]++ ".txt") }'
           for f in "$RUNNER_TEMP"/pins*.txt; do
             pip-audit -r "$f" --no-deps --disable-pip --progress-spinner off \

.gitignore

@@ -13,6 +13,9 @@ coverage.json
 /dokimi/outcomes-result.json
 /dokimi/journeys-result.json
 /dokimi/journeys-result.md
+# mutmut's working copy and results (`mutmut run`, the mutation workflow)
+/dokimi/mutants/
+/dokimi/mutmut-results.txt
 
 # Local environments and credentials
 .venv/

dokimi/CONTRIBUTING.md

@@ -154,19 +154,36 @@ pre-commit run --all-files                     # the whole tree, as CI's pre-com
 mypy and pytest are not hooks: they need the full environment and pytest takes
 minutes. `make ci-local` runs both; `make hooks` runs mypy before a push.
 
+**Mutation pilot.** mutmut (pinned in `requirements-mutation.txt`, locked apart from the tools lock) mutates the
+measurement core: the statistics in `dokimi/eval/stats.py` and
+`binary_stats.py`, the score, regression, critical-band and trust gates in
+`dokimi/gate/` (not `novelty_gate.py`) and the response-cache key in
+`dokimi/caching.py` (the list is `only_mutate` under `[tool.mutmut]` in
+`pyproject.toml`). A mutant that survives is a change to that code, such as
+a flipped comparison, an off-by-one or a dropped term, that no test notices.
+The `mutation` workflow runs it weekly and on demand and writes the score per
+module and the survivors to the job summary; it never fails a build. From
+`dokimi/`: `mutmut run --max-children 8`, then `mutmut results` and
+`mutmut show <name>` for a survivor's diff. A test that kills a survivor
+asserts the value the mutant changes. Some survivors are equivalent (the mutant
+computes the same result, e.g. a `<` that differs only at a value no input
+reaches); leave those alone. `mutants/` is scratch and git-ignored.
+
 Dependencies install from `requirements.lock.txt`; the docs build's from
 `requirements-docs.lock.txt` on top of it, and the tools CI, `make dev`, the
 release job and the eval image use (`requirements-ci.txt`) from
 `requirements-ci.lock.txt`. That lock also pins the build backend, setuptools:
 package-smoke and the release job build with `python -m build --no-isolation`,
 so the published sdist and wheel are not built by a backend resolved fresh from
-PyPI. When you change a range in
+PyPI. The mutation workflow adds mutmut's closure
+(`requirements-mutation.txt`) from `requirements-mutation.lock.txt`, which
+nothing else installs, and pip-audit audits with the others. When you change a range in
 `requirements*.txt`, regenerate the lock with the command in its header: run it
 from `dokimi/`, and add `--upgrade` to pick up new versions (without it, uv
 keeps the current pins). The command's `--exclude-newer=P7D` skips releases
 less than 7 days old; if a security fix is newer than that, its vulnerability PR
 fails to resolve, so regenerate the lock by hand without the flag. The docs lock
-and the tools lock (`requirements-ci.lock.txt`) are constrained to the main one,
+and the tools locks (`requirements-ci.lock.txt`, `requirements-mutation.lock.txt`) are constrained to the main one,
 so regenerate them (command in each header) after the main lock changes too;
 `tests/test_dependency_locks.py` fails while a pin they share differs.
 

dokimi/README.md

@@ -202,7 +202,7 @@ The default static benchmark is configured in `benchmarks/spec.v1.yaml` and curr
 | `readability` | Estimates text readability with a Flesch-Kincaid-style score. |
 | `example_pass_rate` | Executes supported fenced code examples in isolation. |
 
-Weights are normalized across enabled metrics. The benchmark also supports soft gates, hard gates, critical bands, and regression thresholds. Hard gates and critical bands are enforced only when `--strict` is passed. With `--strict`, a spec without a `golden_manifest` also exits 1 when no document was measured. A spec with one, as `spec.v1.yaml` has, exits 2 with or without `--strict` when fewer documents than its `min_docs` match, none included. An empty or absolute `include` pattern exits 2 as well. Exit 1 is not only the `--strict` verdict: a document that cannot be read, or an `--out-json` or `--out-md` path that cannot be written, also exits 1, with an `Error:` or `❌` line on stderr saying which.
+Weights are normalized across enabled metrics. The benchmark also supports soft gates, hard gates, critical bands, and regression thresholds. Hard gates and critical bands are enforced only when `--strict` is passed. With `--strict`, a spec without a `golden_manifest` also exits 1 when no document was measured. A spec with one, as `spec.v1.yaml` has, exits 2 with or without `--strict` when fewer documents than its `min_docs` match, none included. An empty or absolute `include` pattern exits 2 as well. With `--strict`, a critical band the gate cannot read (an unknown condition, or a `None` metric) exits 2 too. Exit 1 is not only the `--strict` verdict: a document that cannot be read, or an `--out-json` or `--out-md` path that cannot be written, also exits 1, with an `Error:` or `❌` line on stderr saying which.
 
 ## LLM and treatment evaluation workflow
 

dokimi/SECURITY.md

@@ -45,7 +45,8 @@ The project is pre-1.0; only the latest `main` is supported. Fixes land on
 ## Known advisories in dependencies
 
 `pip-audit` against the locks (`requirements.lock.txt`,
-`requirements-docs.lock.txt` and `requirements-ci.lock.txt`) reports these advisories with no
+`requirements-docs.lock.txt`, `requirements-ci.lock.txt` and
+`requirements-mutation.lock.txt`) reports these advisories with no
 fix the locks can take:
 
 - **filelock 3.32.7 — thirteen advisories** (fixed in 4.2.1, GHSA-j8f7-rjxc-mr56

dokimi/changelog.d/20261021_160000_napetrov_mutation-pilot.md

@@ -0,0 +1,15 @@
+### Added
+
+- **Mutation pilot over the measurement core** — `.github/workflows/mutation.yml`
+  runs mutmut 3.8.0 (hash-pinned in its own `requirements-mutation.lock.txt`, installed by that workflow alone) weekly and on demand over
+  `eval/stats.py`, `eval/binary_stats.py`, the score, regression, critical-band and
+  trust gates and `caching.py`, and writes the score per module and the surviving
+  mutants to the job summary. It never fails a build. New tests kill the survivors
+  that were real gaps (inclusive thresholds, the Wilcoxon veto, per-condition metric
+  keys, the bootstrap tail index, the cache key's role and sampling parameters), which
+  takes the pilot from 75.3% to 92.0% of mutants killed. Configuration:
+  `[tool.mutmut]` in `pyproject.toml`; usage: CONTRIBUTING.md.
+
+### Changed
+
+- **`run --strict` reads the critical bands strictly** — a NaN metric now violates its band; it passed silently before. A band the gate cannot read (an unknown condition, or a `None` metric) exits 2 and names the spec, instead of ending in a traceback.

dokimi/dokimi/commands/run.py

@@ -73,7 +73,11 @@ def cmd_run(args: argparse.Namespace) -> None:
         from dokimi.gate.critical_bands import check_critical_bands
 
         hard_gate = check_gate(result["summary"], spec, "hard")
-        bands = check_critical_bands(result["summary"], spec)
+        try:
+            bands = check_critical_bands(result["summary"], spec)
+        except ValueError as exc:  # a band the gate cannot read: a bad spec, named, not a traceback
+            print(f"Error: {spec_path}: {exc}", file=sys.stderr)
+            sys.exit(2)
 
         # Exit 1 if hard gate fails
         if hard_gate.enabled and not hard_gate.passed:

dokimi/dokimi/gate/critical_bands.py

@@ -5,6 +5,7 @@
 
 from __future__ import annotations
 
+import math
 from dataclasses import dataclass
 
 
@@ -36,6 +37,19 @@ def has_violations(self) -> bool:
         return any(v.violated for v in self.violations)
 
 
+def _metric(summary: dict, key: str, condition: str) -> float:
+    """The summary value a condition reads. A missing key reads as 0.0; None is an error."""
+    raw = summary.get(key, 0.0)
+    if raw is None:
+        raise ValueError(f"critical_bands {condition}: summary metric {key!r} is None, not a number")
+    return float(raw)
+
+
+def _below(actual: float, threshold: float) -> bool:
+    """Strictly below the threshold. NaN is not a number the gate can pass, so it violates."""
+    return math.isnan(actual) or actual < threshold
+
+
 def check_critical_bands(summary: dict, spec: dict) -> CriticalBandsResult:
     """Check if summary violates any critical bands.
 
@@ -62,20 +76,20 @@ def check_critical_bands(summary: dict, spec: dict) -> CriticalBandsResult:
         threshold = float(band.get("value", 0.0))
 
         if condition == "score_below":
-            actual = float(summary.get("score", 0.0))
-            violated = actual < threshold
+            actual = _metric(summary, "score", "score_below")
+            violated = _below(actual, threshold)
             violations.append(BandViolation(condition, threshold, actual, violated))
         elif condition == "coverage_below":
-            actual = float(summary.get("coverage", 0.0))
-            violated = actual < threshold
+            actual = _metric(summary, "coverage", "coverage_below")
+            violated = _below(actual, threshold)
             violations.append(BandViolation(condition, threshold, actual, violated))
         elif condition == "freshness_below":
-            actual = float(summary.get("freshness_lite", 0.0))
-            violated = actual < threshold
+            actual = _metric(summary, "freshness_lite", "freshness_below")
+            violated = _below(actual, threshold)
             violations.append(BandViolation(condition, threshold, actual, violated))
         elif condition == "readability_below":
-            actual = float(summary.get("readability", 0.0))
-            violated = actual < threshold
+            actual = _metric(summary, "readability", "readability_below")
+            violated = _below(actual, threshold)
             violations.append(BandViolation(condition, threshold, actual, violated))
 
     return CriticalBandsResult(enabled=True, violations=violations)

dokimi/pyproject.toml

@@ -275,3 +275,36 @@ filterwarnings = [
 # Test order is shuffled and `random` reseeded per test (pytest-randomly), so a
 # test that leans on another's leftovers fails. The header prints the seed;
 # `--randomly-seed=<seed>` replays that order, `-p no:randomly` keeps file order.
+
+# Mutation pilot (mutmut 3.x, run by mutation.yml): only_mutate is the statistics, gate decisions and cache key.
+# novelty_gate.py is left out (one 440-line function, copied per mutant). source_paths is the whole package.
+[tool.mutmut]
+source_paths = ["dokimi"]
+only_mutate = [
+  "dokimi/eval/stats.py",
+  "dokimi/eval/binary_stats.py",
+  "dokimi/gate/critical_bands.py",
+  "dokimi/gate/regression.py",
+  "dokimi/gate/score_gate.py",
+  "dokimi/gate/trust_gate.py",
+  "dokimi/caching.py",
+]
+pytest_add_cli_args = ["-p", "no:randomly", "-p", "no:cacheprovider"]
+pytest_add_cli_args_test_selection = [
+  "tests/test_stats.py",
+  "tests/test_stats_properties.py",
+  "tests/test_binary_stats.py",
+  "tests/test_novelty_stats_point_biserial.py",
+  "tests/test_pr26_properties.py",
+  "tests/test_skill_lift_summary.py",
+  "tests/test_model_compare.py",
+  "tests/test_repo_compare.py",
+  "tests/test_critical_bands.py",
+  "tests/test_regression.py",
+  "tests/test_score_gate.py",
+  "tests/test_gate_call_sites.py",
+  "tests/test_trust_gate.py",
+  "tests/test_caching.py",
+  "tests/test_judge_response_cache.py",
+  "tests/test_arm_runner.py",
+]