diff --git a/.gitattributes b/.gitattributes index f37e6548..eb23256e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -20,3 +20,10 @@ tests/fixtures/skill_packages/** -text # Gate releases and signed documents must survive checkout byte-for-byte; # `-text`, as above, so the catalogs stay reviewable in a diff. design/83f3-gate/** -text + +# Recorded llama.cpp HTTP responses (design/84): replayed byte-for-byte as +# server-sent events, whose framing is a bare LF. A CRLF translation on Windows +# breaks every replay — measured on the workstation, 2026-10-02: the gate +# selftest went to 3 failed / 9 errors. `-text`, so they stay diffable. +tests/fixtures/llamacpp/** -text + diff --git a/CLAUDE.md b/CLAUDE.md index 79bd5afa..97dbede1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -657,7 +657,10 @@ because a block looks small. Standing rules that support the above: one worktree per concurrent agent, never `git add -A`, never `pip install -e .` while another tree is live. Push uses a -non-default SSH key (`GIT_SSH_COMMAND="ssh -i ~/.ssh/yonce"`). Rig-facing +non-default SSH key (`GIT_SSH_COMMAND="ssh -i ~/.ssh/yonce"`) to `origin` — on +the machine that has that key. Elsewhere, check `git remote -v` and ask which +remote a block's branches go to (design/84's go to the operator's fork, `ian`, +with the default key). Rig-facing commands must be PowerShell/cmd-safe. Rig facts belong in gate docs, design notes, and rig profiles — never in `microclaw/`. diff --git a/README.md b/README.md index eadbfbf6..d03cb013 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ User (natural language) → Agent → Tools → Safety Guard → Microscope Cont | [Using Microclaw](docs/using-microclaw.md) | What a session is, what leaves the machine, skills, hooks, compiling a session to a standalone script | | [Safety configuration](docs/safety-config.md) | The `safety_config.yaml` schema, every gate, and what a section's absence means | | [The browser GUI](docs/browser-gui.md) | The chat interface, the desktop shortcut, API keys, remote access | +| [Using a local model](docs/local-llm-model.md) | Serving a model with llama.cpp, the settings Microclaw needs, what to expect, troubleshooting, models tested | | [CLI reference](docs/cli-reference.md) | Flags and subcommands, the history viewer, where files live | | [Publishing a skill package](docs/publishing-skill-packages.md) | Ship your own skill or analysis worker from your own repository: manifest, locks, worker protocol, signing | | [Developing Microclaw](docs/development.md) | Source install, the test suite, building a safety profile by hand | diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 5c8e1801..2cf4ab9b 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -26,7 +26,7 @@ installed distributions and review notices for the versions they distribute. | keyring | optional `serve` | MIT | | pytest | test only | MIT | | pytest-mock | test only | MIT | -| HTTPX | test only | BSD 3-Clause | +| HTTPX | runtime | BSD 3-Clause | | setuptools | build only | MIT | Transitive dependencies are not enumerated because they vary by platform and diff --git a/design/70-carried-forward-register.md b/design/70-carried-forward-register.md index 2af3711b..74fd45b9 100644 --- a/design/70-carried-forward-register.md +++ b/design/70-carried-forward-register.md @@ -121,6 +121,8 @@ Rows added since the triage: | `R157` | `design/88` block 88b's demo gate and spike rerun, scored from artifacts 2026-10-18 | | `R158` | `design/88` block 88b's Windows CI run, 2026-10-18 (not 88b's code) | | `R159` | `design/89` block 89a's demo run, 2026-10-19 — the M2 limbs it could not run | +| `R160`–`R164` | `design/91`, the local OpenAI-compatible backend, 2026-10-02 to 2026-10-14 | +| `R165` | PR #62's demo-machine run (Qwen3.5-4B, local llama.cpp), scored from artifacts 2026-10-27 | ## The work queue @@ -199,7 +201,7 @@ Sorted by ease, then by importance. `→` names an existing block; do the block, | `R79` | [Saved knowledge does not separate measurement from inference](#r79) | HIGH | LARGE | | | `R02` | [Agent offers to split an acquisition the backend already splits](#r02) | MEDIUM | SMALL | | | `R16` | [get_device_property_info errors on a guessed property name without naming the real ones](#r16) | MEDIUM | SMALL | | -| `R17` | [An aborted turn's error never reaches the transcript](#r17) | MEDIUM | SMALL | | +| `R17` | [An aborted turn's error never reaches the transcript](#r17) | HIGH | SMALL | | | `R21` | [Nothing reports the bounds microclaw is enforcing](#r21) | MEDIUM | SMALL | | | `R24` | [Four of five connect sites' actionable refusal never reaches the user](#r24) | MEDIUM | SMALL | | | `R25` | [An exported script writes its dataset beside the script, silently](#r25) | MEDIUM | SMALL | | @@ -235,6 +237,11 @@ Sorted by ease, then by importance. `→` names an existing block; do the block, | `R97` | [The runtime deadline includes Acquisition() construction, and nothing records how much](#r97) | LOW | SMALL | | | ~~`R155`~~ | [Ctrl-C during a pycro-manager teardown wait can make a live waiter look finished](#r155) | MEDIUM | SMALL | **90a** | | ~~`R158`~~ | [84a's window test reads the final display once, without a wait](#r158) | LOW | SMALL | **90a** | +| `R160` | [`estimate_tokens` counts a thumbnail's base64 bytes as text](#r160) | MEDIUM | SMALL | | +| `R161` | [One `read_hook_log` result can take a third of a local model's context](#r161) | HIGH | SMALL | | +| `R162` | [`test_built_wheel_contains_the_source_tree_skill_catalog` fails intermittently](#r162) | LOW | SMALL | | +| `R164` | [No route to combine saved channels into a colour composite](#r164) | MEDIUM | MEDIUM | | +| `R165` | [A local reply with no visible text ends the turn silently and is stored empty](#r165) | LOW | SMALL | | **Demo machine** @@ -283,6 +290,7 @@ Sorted by ease, then by importance. `→` names an existing block; do the block, |---|---|---|---|---| | `R06` | [probe_hint emitted for a lock with no usable status property](#r06) | MEDIUM | SMALL | | | `R45` | [Block 13 G3 - transmitted-light SNR refusal unmeasured on any rig](#r45) | LOW | SMALL | → Block 13 G3 | +| `R163` | [At `reasoning_effort=low` the local model reported a dataset it never acquired](#r163) | HIGH | SMALL | | **Blocked on a Nikon Ti — no reachable machine** @@ -826,7 +834,8 @@ error path and this merge is already carrying unverified change. **Failed agent turns appear in the browser stream but leave no durable error record in the transcript.** - **Status** — OPEN - `microclaw/webserve.py:989-994` still emits exceptions only to SSE, while `microclaw/conversation.py:77-112` still reloads every JSONL object without role validation. -- **Importance** — MEDIUM - This loses useful failure evidence, but the browser displays the error and the operator can record it manually. +- **Importance** — HIGH (raised from MEDIUM, 2026-10-27). The MEDIUM rested on "the browser displays the error and the operator can record it manually", and that premise is false: `serve.html` shows a turn-ending `error` event only as a `toast`, which clears itself. +- **Seen again** — PR #62 demo run, 2026-10-27 (Qwen3.5-9B, local llama.cpp): `serve` logged `seq 5 error: Local server http://127.0.0.1:8081 context limit is 65536 tokens; request uses 70881 tokens.` The operator saw "nothing at all, it just ran", and the history holds the tool result and no reply. The error was lost from the screen as well as from the transcript. A fix should make a turn-ending error persist on the page (as design/91 91d did for its compaction notices) as well as record it. - **Where** — LOCAL - A mocked agent exception and audit/reload assertions can settle the behavior without hardware. - **Block** — NONE - No later design block claims or closes this transcript gap. - **Effort** — SMALL @@ -3403,3 +3412,80 @@ the model" as the actual blast radius. - **Importance** — MEDIUM. Recommended before 89b, not required: 89b's own M2 gate compares cadence with feedback on and off, so this would only find a problem earlier and more cheaply (operator decision, 2026-10-19). - **Effort** — SMALL - **Provenance** — `design/89` 89a, 2026-10-19. + + +### R160 — `estimate_tokens` counts a thumbnail's base64 bytes as text + +**One 512 px PNG thumbnail of a noisy field estimates at 121,748 tokens — above the Anthropic path's 120k high-water on its own.** + +- **What happened** — measured 2026-10-02 (design/91, coordinator): `make_thumbnail` of a 2048² uint16 noise frame is 287,144 base64 bytes, and `estimate_tokens` over one `tool_result` holding it returns 121,748; the same call on a smooth ramp is 6,040 bytes and 2,636. `estimate_tokens` divides the JSON of the whole history by 2.36, image data included. Real camera frames are noise-dominated, so their thumbnails are the large case. +- **Why it matters** — a single `snap_and_analyze` that keeps its thumbnail can push `ConversationStore` past high-water and fold every older turn into a checkpoint on the next round, rewriting the cached prefix (design/82's cost lever) for an image the API bills at a few hundred tokens. Not observed in a recorded session yet: no artifact has been checked for a thumbnail-triggered compaction. +- **What a fix would look like** — estimate an image block at its billed size (a per-image constant from the API's image-token rule) rather than its byte length; check an archived design/82 history for a compaction that followed a thumbnail. design/91 91b fixes the local text-only path only, where the image is never sent. +- **Provenance** — design/91 91b planning, 2026-10-02. + +### R161 — One `read_hook_log` result can take a third of a local model's context + +**On the local llama.cpp server, `read_hook_log(limit=200)` returned 140,232 characters (~59k tokens): 104 s to process, and once more than the remaining context could hold.** + +- **What happened** — design/91 gate 91 round 3, 2026-10-06, `n_ctx` 131,072: record 16 processed 59,262 new tokens in 104.0 s for that one tool result; a later turn asking for the same read made the next request too large and the server refused it (context exceeded, the turn rolled back). design/82 F2/D3 bounded the read by entry count (`limit`), which the model may raise to 200. +- **Why it matters** — on a local model the cost of a large result is prefill time on every cold path and a hard context limit, not just money; a bound in entries is not a bound in tokens. +- **Seen again, at the default `limit=50`** — PR #62 demo run, 2026-10-27, Qwen3.5-9B Q4_K_M at `n_ctx` 65,536 (`fixed=35579, high=17669, low=13251`). Turn 3 read 50 of 200 entries (~16k tokens) and declined compaction (`estimate=24085 threshold=17669`); its final request fit with 50 tokens spare (57,294 + 8,192). Turn 4 read the same 50 entries again, and the next request was 70,881 tokens; llama.cpp refused it, and the operator saw nothing (`R17`). At this context one 50-entry read is as large as the whole history budget. +- **Why compaction did not help** — `ConversationStore.model_messages` never folds the two most recent complete turns (`MIN_RECENT_COMPLETE_TURNS = 2`, on `main` since `dacab18`, 2026-07-29, written for the Anthropic path), and when that floor makes the target impossible it deliberately "let[s] the API report its real context limit". Turn 3's read sat inside the floor, so only ~5k tokens were foldable. design/91's server-fit valve respects the floor. +- **What a fix would look like** — (1) when the calibrated request will not fit the server, let the server-fit valve fold into the recent-turn floor: the context size is a hard limit and the floor is a preference (estimated, not measured: folding turn 3 brings the refused request to roughly 55–60k, which fits; replay the 2026-10-27 history through `model_messages` to confirm); (2) if even a full fold cannot fit (the turn in flight alone is too large), refuse before sending, with a message that persists; (3) bound tool-result size in characters relative to the session's remaining budget (the store knows it), stated in the result as design/82 D3 requires, or have `read_hook_log` cap its returned characters and point at `rank_hook_log`. +- **Where** — LOCAL (replay the recorded history), then the demo machine at `-c 65536`, which reproduces it. +- **Block** — NONE; taken as a follow-up after PR #62 merges (operator, 2026-10-27). +- **Importance** — HIGH (raised from MEDIUM, 2026-10-27): seen twice, on two machines and two context sizes, each time losing the turn. +- **Effort** — SMALL +- **Provenance** — design/91 gate 91 round 3, scored from artifacts 2026-10-06; extended from the PR #62 demo run, 2026-10-27. + +### R162 — `test_built_wheel_contains_the_source_tree_skill_catalog` fails intermittently + +**Failed in 2 of the full-suite runs during design/91 and passed alone each time (38 s).** + +- **What happened** — design/91, 2026-10-02: once at the 91a baseline (before any 91 code, during a network drop) and once on 91b's tip; never reproduced in isolation. +- **Why it matters** — an intermittent in the full suite is read as noise, which is how a real failure in the same test would be missed. +- **What a fix would look like** — capture its failure output on the next occurrence (the runs here did not keep the traceback) and decide whether it builds a wheel over the network. +- **Where** — LOCAL. +- **Block** — NONE. +- **Importance** — LOW +- **Effort** — SMALL +- **Provenance** — design/91 coordinator full-suite runs, 2026-10-02. + +### R163 — At `reasoning_effort=low` the local model reported a dataset it never acquired + +**Told the operator "Saved to disk: `red_1`, `green_1`, `blue_1` … These are the real, verified acquisitions" when only red and green had been acquired in that turn.** + +- **What happened** — design/91, real-microscope session `20261014_141904_322634` (Qwen 3.8 27B, `MICROCLAW_LLM_REASONING_EFFORT=low`), 2026-10-14: the acquisitions record holds two acquisitions for the turn (18:34:34, 18:34:51 UTC); the model switched the turret to DAPI and wrote a composite script referencing a `blue_1` that does not exist. Not seen in the `xhigh` or `medium` sessions of the same script — one session each, so not evidence either way. +- **Why it matters** — it breaks the system prompt's central reporting rule (every claim from a tool result) in the most consequential place: what is on disk. A local model at low effort is the setting an operator chooses for speed. +- **What a fix would look like** — first measure: two or three sessions per effort level, scoring every "saved"/"acquired" claim in the replies against the acquisitions record (scriptable from the session files). If it recurs, a tool-side check is stronger than prompt wording: e.g. a turn-end summary of what was actually acquired, from the acquisitions record, that the model must relay. +- **Where** — WORKSTATION + RIG (the operator's Zeiss). +- **Block** — NONE. +- **Importance** — HIGH +- **Effort** — SMALL to measure +- **Provenance** — design/91 reasoning-effort sessions, scored from artifacts 2026-10-14. + +### R164 — No route to combine saved channels into a colour composite + +**Asked to "combine into a 3 colour image and save", the model had no tool for it at any reasoning level.** + +- **What happened** — design/91 real-microscope sessions, 2026-10-14: `low` wrote an unrun composite script and later gave Fiji steps; the request is ordinary for a three-channel fluorescence image. +- **Why it matters** — a routine deliverable with no route invites exactly the hand-written, unexecuted script the system prompt warns about. +- **What a fix would look like** — an offline adapter on the existing `run_analysis_on_saved_dataset` route (analysis lives in hooks/adapters, CLAUDE.md) that renders a composite from a channel axis or from named single-channel datasets, written as an artifact `open_artifact` can show. Simpler once the rig's `Channel` group gives one multi-channel dataset. +- **Where** — LOCAL, then the rig. +- **Block** — NONE. +- **Importance** — MEDIUM +- **Effort** — MEDIUM +- **Provenance** — design/91 reasoning-effort sessions, 2026-10-14. + +### R165 — A local reply with no visible text ends the turn silently and is stored empty + +**Qwen3.5-4B called a tool, got its result, then produced only reasoning; the operator saw no answer and the history kept `{"role": "assistant", "content": []}`.** + +- **What happened** — PR #62 demo run, 2026-10-27, `unsloth/Qwen3.5-4B-GGUF:Q8_0` at `n_ctx` 131,072: 3 of the 4 turns that reached the model ended this way, e.g. 99 output tokens against 372 reasoning characters and no `delta.content`. Turn 2's mean intensity and SNR were never shown. Qwen3.5-9B on the same machine did not do it (5 of 5 final replies had text). `_stream_one_round`'s local branch builds `blocks = []` and returns `end_turn`. +- **Why it matters** — the user is left with nothing, not even a notice; the answer may have been inside the reasoning, which is deliberately not shown or stored. Unverified: an empty assistant message in the history may be refused by the Anthropic API if the session switches model. +- **What a fix would look like** — when a local reply ends with no text and no tool call, show a notice ("the model returned no visible reply") and do not store the empty message; consider one retry. +- **Where** — LOCAL (a recorded stream with reasoning only). +- **Block** — NONE. +- **Importance** — LOW: model-dependent, and it did not stop the session being usable (operator, 2026-10-27). +- **Effort** — SMALL +- **Provenance** — PR #62 demo run, scored from artifacts 2026-10-27. diff --git a/design/82-block82c-r63-probe.py b/design/82-block82c-r63-probe.py index 81fdb2cc..7b07733f 100644 --- a/design/82-block82c-r63-probe.py +++ b/design/82-block82c-r63-probe.py @@ -12,10 +12,14 @@ turns, just as in the cost reconstruction. Usage turn IDs must follow those same boundaries. Missing/misaligned records cannot pin replay partitions. * The real ConversationStore.model_messages(history[:i]) reproduces the context - under the tree and window being tested. Usage compaction counts are checked - on every call; disagreement is reported and partitions remain replay estimates, - not a claim about the historical context. A usage log from a tree with a - different estimate_tokens pins WHEN compaction fired, not WHERE it cut. + under the tree and window being tested. This is always a replay under the + current tree's compaction policy. Since 84c, D9 commits only when a fold clears + high-water; replaying a pre-84c history can therefore produce a different + partition even when its usage log has matching per-call compaction counts. + Usage compaction counts are checked on every call; disagreement is reported + and partitions remain replay estimates, not a claim about the historical + context. Matching counts pin WHEN compaction fired, not WHERE it cut. A usage + log from a tree with a different estimate_tokens pins WHEN, not WHERE. Compare current/recorded estimated tokens only while both are uncompacted: near 1 supports estimator equivalence and replay partition trust; away from 1 indicates a different estimator and a replay context the model never saw. @@ -138,12 +142,16 @@ def score(messages, usage=None, *, high_water=DEFAULT_CONTEXT_HIGH_WATER_TOKENS, return dict(calls=calls, turns=turn, events=events, mentions=mentions, usage_records=len(usage) if usage is not None else None, usage_events=usage_events, disagreements=disagreements, - estimate_ratios=estimate_ratios) + estimate_ratios=estimate_ratios, + replay_policy='current ConversationStore (D9 high-water gate)') def report(result): print(f"{result['calls']} assistant calls, {result['turns']} turns, " f"{len(result['events'])} compactions; usage records: {result['usage_records']}") + print('Replay policy: current ConversationStore (D9 high-water gate). ' + 'A pre-84c session may have different checkpoint partitions; replay ' + 'partitions are not historical proof.') if result['usage_records'] is None: print('Replay estimates only: usage-pinned timestamps and turn IDs unavailable.') else: diff --git a/design/91-block91e-check.md b/design/91-block91e-check.md new file mode 100644 index 00000000..9472776c --- /dev/null +++ b/design/91-block91e-check.md @@ -0,0 +1,81 @@ +# Block 91e check — change the reasoning level within a session + +**Machine: the llama.cpp workstation**, Micro-Manager running (demo config or the +real rig — either works; this check makes no exposures beyond what you ask +for). **Windows PowerShell 5.1** and **Chrome**. About **10 minutes**. + +What this checks: the browser's reasoning-level control changes the level for +the session, shows what the change costs, and every model call records the level +it ran at. + +## 0 — the model server + +Running as before (`… -c 131072 … --api-key testkey`). From a second window: + +```powershell +Invoke-RestMethod http://127.0.0.1:8080/health +``` + +## 1 — get the branch + +```powershell +cd C:\code\microclaw +git fetch origin +git checkout design91/91e-effort +git pull +git merge-base --is-ancestor e611762 HEAD +if ($LASTEXITCODE -eq 0) { "PIN OK" } else { "STOP - wrong tree" } +``` + +## 2 — start Microclaw with no level set + +```powershell +cd C:\code\microclaw +$env:MICROCLAW_LLM_BASE_URL = 'http://127.0.0.1:8080' +$env:MICROCLAW_LLM_REASONING_EFFORT = '' +.venv\Scripts\python.exe -m microclaw serve +``` + +Open `http://127.0.0.1:8000` in Chrome and enter the key `testkey` from the key +chip as before. + +**(E1)** Beside the model button there is a reasoning control. What does it +show? (Expected: **Server default**.) + +## 3 — three short turns, switching between them + +Send each, waiting for the reply: + +> What is the stage position? + +Now set the control to **Medium**. **(E2)** Copy the notice that appears (it +stays until you click its ✕). + +> What is the stage position? + +Set the control back to **Server default**, then: + +> What is the stage position? + +**(E3)** Did the replies after a switch feel slower than the first? Roughly how +many seconds each? + +Press **Ctrl+C** in the `serve` window. + +## 4 — read the record + +```powershell +cd C:\code\microclaw +.venv\Scripts\python.exe design\91-block91e-check.py --session . +``` + +Expect three or more calls, the `effort` column reading `server default`, then +`medium`, then `server default`, and a last line +`levels in order: server default -> medium -> server default`. The `processed` +column shows how much each call made the server re-process; a level the server +has not seen this run shows a large number on its first call. + +## 5 — send + +The output of step 4, your answers to **E1–E3**, and the session's +`*_microclaw_usage.jsonl` (newest in `C:\code\microclaw`). diff --git a/design/91-block91e-check.py b/design/91-block91e-check.py new file mode 100644 index 00000000..4e72997f --- /dev/null +++ b/design/91-block91e-check.py @@ -0,0 +1,51 @@ +"""Block 91e check: one line per model call of the newest session in a folder. + + .venv\\Scripts\\python.exe design\\91-block91e-check.py --session . + +Prints, per call, the reasoning level recorded for it, the tokens the server +processed and reused, and the hidden-reasoning length, then the distinct levels +seen. A session that switched levels shows each switch as a change in the +`effort` column, and the call after a switch to a level not used before as a +large `processed` count. Reading only: it never contacts the server. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--session", default=".", help="folder holding *_microclaw_usage.jsonl") + args = parser.parse_args() + target = Path(args.session) + files = sorted(target.glob("*_microclaw_usage.jsonl")) if target.is_dir() else [target] + if not files or not files[-1].exists(): + print(f"NOT EXERCISED - no *_microclaw_usage.jsonl in {target}") + return 1 + path = files[-1] + records = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() + if line.strip()] + print(f"{path.name}: {len(records)} model calls") + if records and "reasoning_effort" not in records[0]: + print("NOT EXERCISED - these records carry no reasoning_effort field: " + "the session ran on a tree without block 91e") + return 1 + print(f"{'call':>4} {'effort':<16} {'processed':>9} {'reused':>7} {'reasoning':>9}") + for index, record in enumerate(records): + effort = record.get("reasoning_effort") or "server default" + print(f"{index:>4} {effort:<16} {record.get('prompt_n') or 0:>9} " + f"{record.get('cache_n') or 0:>7} {record.get('reasoning_characters') or 0:>9}") + seen = [] + for record in records: + effort = record.get("reasoning_effort") or "server default" + if not seen or seen[-1] != effort: + seen.append(effort) + print("levels in order:", " -> ".join(seen)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/design/91-gate-selftest.py b/design/91-gate-selftest.py new file mode 100644 index 00000000..709290c1 --- /dev/null +++ b/design/91-gate-selftest.py @@ -0,0 +1,247 @@ +"""Selftest for design/91-gate.py: every graded limb discriminates. + + uv run python -m pytest -q design\\91-gate-selftest.py + +A session is produced by the real `run_agent_iter`, the real +`ConversationStore` and the real `AuditLog`, against llama.cpp responses +recorded from a real server (`tests/fixtures/llamacpp/`) replayed through +`httpx.MockTransport`, so the product's own request building and SSE parsing +run. The usage sink adds what `webserve`'s does (estimate, compaction count, +turn id). Each graded limb is then run on a mutant that breaks exactly that +limb, and must not pass. The thumbnail is a real `make_thumbnail` of a noise +frame, because a smooth image encodes to almost nothing and would hide the +defect the limb exists for (design/70 R145). +""" +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import httpx +import numpy as np +import pytest + +from microclaw import agent +from microclaw.conversation import AuditLog, ConversationStore +from microclaw.image_analysis import make_thumbnail + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests" / "fixtures" / "llamacpp" +BASE = "http://gate91-recorded" +KEY = "testkey" + +spec = importlib.util.spec_from_file_location("gate91", Path(__file__).with_name("91-gate.py")) +gate = importlib.util.module_from_spec(spec) +spec.loader.exec_module(gate) + + +def replay(name: str) -> httpx.Response: + head, body = (FIXTURES / f"{name}.http").read_bytes().split(b"\n\n", 1) + status, headers = head.split(b"\n", 1) + return httpx.Response(int(status.split()[1]), headers=json.loads(headers), content=body) + + +def install_server(monkeypatch) -> None: + chats = iter(["stream_tool_calls", "stream_text_reply"] * 50) + + def handler(request: httpx.Request) -> httpx.Response: + # The real server with --api-key answers only /health without the key + # (measured 2026-10-02: /props, /v1/models, /tokenize, /apply-template + # all 401), so the fake refuses the same way. + if request.headers.get("authorization") != f"Bearer {KEY}": + return replay("error_bad_key") + path = request.url.path + if path.endswith("/props"): + return replay("get_props") + if path.endswith("/v1/models"): + return replay("get_v1_models") + if path.endswith("/apply-template"): + return replay("post_apply_template") + if path.endswith("/tokenize"): + return replay("post_tokenize") + return replay(next(chats)) + + monkeypatch.setattr(agent, "_local_http", httpx.Client(transport=httpx.MockTransport(handler))) + monkeypatch.setattr(agent, "_local_vision", {}) + monkeypatch.setattr(agent, "_local_models", None) + monkeypatch.setenv("MICROCLAW_LLM_BASE_URL", BASE) + monkeypatch.setenv("MICROCLAW_LLM_API_KEY", KEY) + + +def noise_thumbnail_result() -> list[dict]: + frame = np.random.default_rng(0).integers(100, 4000, (2048, 2048), dtype=np.uint16) + return [ + {"type": "text", "text": json.dumps({"z_um": 12.5})}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": make_thumbnail(frame, max_size=512)}}, + ] + + +# The recorded tool-call stream asks for set_exposure and get_z_position. Both +# are real registered names, so the scorer's registry limb sees real names; +# their implementations here are fakes, and get_z_position returns a thumbnail. +REGISTRY = { + "set_exposure": lambda ctrl, guard, ms: {"exposure_ms": ms}, + "get_z_position": lambda ctrl, guard: noise_thumbnail_result(), +} + + +def drive(tmp_path: Path, *, turns: int, high: int, low: int, text_only: bool) -> Path: + stem = tmp_path / "20261002_120000_000000" + history_audit = AuditLog(str(stem) + "_microclaw_history.jsonl") + usage_audit = AuditLog(str(stem) + "_microclaw_usage.jsonl") + store = ConversationStore(history_audit, high_water_tokens=high, + low_water_tokens=low, text_only=text_only) + history: list[dict] = [] + for turn in range(turns): + def sink(record, turn=turn): + usage_audit.append({**record, "estimated_tokens": store.last_estimated_tokens, + "compaction_count": store.compaction_count, + "turn_id": f"turn-{turn}"}) + for _ in agent.run_agent_iter( + f"turn {turn}", None, None, history, model="recorded", + context_provider=store.model_messages, on_message=store.append, + tool_schemas=[], tool_registry=REGISTRY, usage_sink=sink, + ): + pass + return stem + + +def verdicts(capsys, stem: Path, base_url: str | None = BASE, + api_key: str | None = KEY) -> dict[str, str]: + usage, history = gate.find_session(stem) + gate.score(usage, history, base_url, api_key) + out = capsys.readouterr().out + rows = {} + for line in out.splitlines(): + for verdict in ("NOT EXERCISED", "PASS", "FAIL", "REPORT"): + if line.startswith(verdict): + rows[line[len(verdict):].strip().split(" - ")[0]] = verdict + break + return rows + + +def rewrite(stem: Path, suffix: str, edit) -> None: + path = Path(str(stem) + suffix) + rows = gate.read_jsonl(path) + edit(rows) + path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") + + +LOCAL = "every model call went through the local backend" +ONE = "one usage record per model reply" +NAMES = "every tool call names a registered tool" +FIT = "every request fit the server's context" +D9 = "no compaction was committed above high-water" +COMPACTED = "the session compacted at least once" +PREFIX = "each later round reused the previous round's prompt" +THUMB = "a thumbnail did not inflate the history estimate" +GRADED = (LOCAL, ONE, NAMES, FIT, D9, COMPACTED, PREFIX, THUMB) + + +@pytest.fixture +def good(tmp_path, monkeypatch): + install_server(monkeypatch) + # high 2500 / low 1500, 10 turns: each turn adds ~430 estimated tokens and + # a fold must clear high-water to commit (design/91 D9), so the checkpoint + # plus the two retained turns has to fit; measured, this commits twice + # (estimates 1502, 1858). At 1200 / 600 no fold ever clears high-water. + return drive(tmp_path, turns=10, high=2500, low=1500, text_only=True) + + +def test_the_real_session_passes_every_graded_limb(good, capsys): + rows = verdicts(capsys, good) + assert {name: rows.get(name) for name in GRADED} == {name: "PASS" for name in GRADED} + + +def test_anthropic_shaped_records_fail_the_backend_limb(good, capsys): + rewrite(good, "_microclaw_usage.jsonl", + lambda rows: [r.pop("prompt_ms") for r in rows]) + assert verdicts(capsys, good)[LOCAL] == "FAIL" + + +def test_a_missing_usage_record_fails_the_count_limb(good, capsys): + rewrite(good, "_microclaw_usage.jsonl", lambda rows: rows.pop()) + assert verdicts(capsys, good)[ONE] == "FAIL" + + +def test_an_unregistered_tool_name_fails_the_registry_limb(good, capsys): + def rename(rows): + for message in rows: + for block in message.get("content") or []: + if isinstance(block, dict) and block.get("type") == "tool_use": + block["name"] = "set_exposur" + return + rewrite(good, "_microclaw_history.jsonl", rename) + assert verdicts(capsys, good)[NAMES] == "FAIL" + + +def test_a_prompt_with_no_room_for_the_reply_fails_the_fit_limb(good, capsys): + # The recorded server reports n_ctx 131072; 125,000 + the 8,192 reply + # reserve does not fit. + rewrite(good, "_microclaw_usage.jsonl", + lambda rows: rows[0].update(input_tokens=125_000)) + assert verdicts(capsys, good)[FIT] == "FAIL" + + +def test_a_compaction_committed_above_high_water_fails_the_d9_limb(good, capsys): + # Gate 91 round 1's shape: the compacting call's estimate is still above the + # high-water the product derives from the server (84,920 there; here the + # recorded server's tokenize fixture makes it larger, so use n_ctx itself). + def above(rows): + for i in range(1, len(rows)): + if rows[i]["compaction_count"] > rows[i - 1]["compaction_count"]: + rows[i]["estimated_tokens"] = 131_072 + return + rewrite(good, "_microclaw_usage.jsonl", above) + assert verdicts(capsys, good)[D9] == "FAIL" + + +def test_no_base_url_is_not_exercised_not_passed(good, capsys): + rows = verdicts(capsys, good, base_url=None) + assert (rows[FIT], rows[D9]) == ("NOT EXERCISED", "NOT EXERCISED") + + +def test_no_key_for_a_keyed_server_is_not_exercised_not_passed(good, capsys): + # Without the key the server answers 401 and the product falls back to a + # guessed budget; the limb must not grade the session against that guess. + rows = verdicts(capsys, good, api_key=None) + assert (rows[FIT], rows[D9]) == ("NOT EXERCISED", "NOT EXERCISED") + + +def test_a_session_that_never_compacts_is_not_exercised(tmp_path, monkeypatch, capsys): + install_server(monkeypatch) + stem = drive(tmp_path, turns=3, high=10_000_000, low=5_000_000, text_only=True) + assert verdicts(capsys, stem)[COMPACTED] == "NOT EXERCISED" + + +def test_a_lost_prefix_fails_the_reuse_limb(good, capsys): + def cold(rows): + for i in range(1, len(rows)): + if rows[i]["turn_id"] == rows[i - 1]["turn_id"] \ + and rows[i]["compaction_count"] == rows[i - 1]["compaction_count"]: + rows[i]["cache_n"] = 0 + return + rewrite(good, "_microclaw_usage.jsonl", cold) + assert verdicts(capsys, good)[PREFIX] == "FAIL" + + +def test_a_thumbnail_counted_as_text_fails_the_image_limb(tmp_path, monkeypatch, capsys): + # The mutant is the product defect itself: the Anthropic-path estimate, + # which counts base64. Huge watermarks so no compaction hides the call. + install_server(monkeypatch) + stem = drive(tmp_path, turns=2, high=10_000_000, low=5_000_000, text_only=False) + assert verdicts(capsys, stem)[THUMB] == "FAIL" + + +def test_the_same_session_with_the_fix_passes_the_image_limb(tmp_path, monkeypatch, capsys): + install_server(monkeypatch) + stem = drive(tmp_path, turns=2, high=10_000_000, low=5_000_000, text_only=True) + assert verdicts(capsys, stem)[THUMB] == "PASS" + + +def test_progress_reads_the_newest_session(good, capsys): + usage, _ = gate.find_session(good.parent) + assert gate.progress(usage) == 0 + assert "compactions" in capsys.readouterr().out diff --git a/design/91-gate.md b/design/91-gate.md new file mode 100644 index 00000000..d98b56c4 --- /dev/null +++ b/design/91-gate.md @@ -0,0 +1,231 @@ +# Gate 91, round 3 — the same session on block 91d + +**Machine: the llama.cpp workstation** (`IIC-LLS-Wkstn`), with Micro-Manager's +stock demo configuration — no microscope. **Windows PowerShell 5.1** and +**Chrome**. Microclaw and the model run on the same machine, so the timings are +a local deployment's. + +What this checks: every model call goes to the local server (91a), the +conversation budget comes from the server and compaction happens before the +server's context fills (91b), the server's prompt cache is reused between +rounds, a thumbnail does not count as text, and the browser works with a +keyless or keyed local server. The scorer reads the session's own history and +usage files; steps 6 and 7 are the parts only you can judge. + +**Round 3 (block 91d).** Round 1 paid three slow rebuilds (366 s); round 2, +on 91c, paid two (197 s), the first landing 671 tokens under the threshold and +buying one round. 91d commits a compaction only when it reaches the low-water +mark, unless the conversation would otherwise not fit the server — judged with +the estimate calibrated against the server's own token counts. Replaying both +earlier sessions' real histories through 91d gives **no** compaction for round +1's session and **one**, landing well under low-water, for round 2's; every +request still fits. Same session, same server settings; steps 1–3 are safe to +re-run. Leave `MICROCLAW_LLM_REASONING_EFFORT` unset for this round, so it +compares with rounds 1 and 2. + +About **30–45 minutes**, most of it waiting for the model. Nothing here costs +money or exposes a sample. + +## 0 — the model server + +In its own window, leave this running for the whole gate (the command you +already use, unchanged): + +```powershell +llama-server.exe -m Qwen3.8-27B-UD-Q4_K_XL.gguf --host 0.0.0.0 --port 8080 --jinja -c 131072 -np 1 -fa on --cache-type-k q8_0 --cache-type-v q8_0 --api-key testkey +``` + +Check it from a second PowerShell window: + +```powershell +Invoke-RestMethod http://127.0.0.1:8080/health +``` + +Expect `status` `ok`. + +## 1 — get the branch + +```powershell +if (-not (Test-Path C:\code\microclaw)) { git clone https://github.com/iandobbie/microclaw.git C:\code\microclaw } +cd C:\code\microclaw +git fetch origin +git checkout design91/91d-calibrate +git pull +git merge-base --is-ancestor cde64fe HEAD +if ($LASTEXITCODE -eq 0) { "PIN OK - the implementation is in this tree" } else { "STOP - wrong tree, do not run the gate" } +``` + +Expect `PIN OK`. + +If this clone existed before `988abea`, its recorded fixtures were checked out +with Windows line endings and every replay fails; `git pull` does not rewrite +them. Refresh them once (harmless on a fresh clone): + +```powershell +Remove-Item -Recurse -Force tests\fixtures\llamacpp +git checkout -- tests/fixtures/llamacpp +``` + +## 2 — the environment + +This is the README's source install. It needs the network once. + +```powershell +cd C:\code\microclaw +uv venv --python 3.12 +uv pip install --python .venv\Scripts\python.exe -e ".[serve,test]" +.venv\Scripts\python.exe -c "import microclaw, sys; print(microclaw.__file__)" +``` + +The last line must print a path under `C:\code\microclaw\microclaw\`. A +`SyntaxWarning` about `'\P'` in `mmpycorex\install.py` is harmless. + +Check the scorer before spending an hour on a session: + +```powershell +.venv\Scripts\python.exe -m pytest -q design\91-gate-selftest.py +``` + +Expect `13 passed`. Each graded limb is run there on a real session built from +recordings of this server, and on a mutant that breaks exactly that limb. If it +fails, stop and send the output. + +## 3 — safety bounds + +There is no microscope, so copy the reviewed demo bounds rather than run setup. +Round 1 copied them already; this copies only if none is there and never +overwrites one: + +```powershell +New-Item -ItemType Directory -Force "$env:APPDATA\microclaw" | Out-Null +if (Test-Path "$env:APPDATA\microclaw\safety_config.yaml") { "safety_config.yaml present - leaving it" } else { Copy-Item tests\fixtures\50a-demo-safety_config.yaml "$env:APPDATA\microclaw\safety_config.yaml" } +``` + +## 4 — Micro-Manager + +Start Micro-Manager with the stock **`MMConfig_demo.cfg`**, then tick +**Tools → Options → Run pycro-manager server on port 4827**. Check: + +```powershell +.venv\Scripts\python.exe -m microclaw check-bridge +``` + +Expect it to report the bridge answering and exit without an error. + +## 5 — start Microclaw on the local model + +Leave this window open and **unredirected** for the whole session: `serve` +prints nothing under redirection until it exits. + +```powershell +cd C:\code\microclaw +$env:MICROCLAW_LLM_BASE_URL = 'http://127.0.0.1:8080' +.venv\Scripts\python.exe -m microclaw serve +``` + +No key is set yet, and this server requires one, so expect +`[microclaw] Local context budget: n_ctx=32768 (default; …), …, high=4096, +low=3072 (provisional 4096-token history floor); probe failures: /props HTTP +401, …` and `No local LLM API key set; none is needed unless the server was +started with --api-key.` That is correct: the budget is re-read when you enter +the key in step 6. A browser opens at `http://127.0.0.1:8000`; if it +is not Chrome, copy that address into Chrome. Close the other browser's tab. + +`$env:` here sets the variable for this window only; nothing is written to the +machine's settings. + +## 6 — drive the session + +**Before entering any key**, note: **(O1)** is the **Send** button enabled, and +is there no "No … key" banner? (A key chip labelled `key` should be visible.) + +**Turn 0** — send this with no key set. The server requires `testkey`, so this +turn must fail: + +> Call get_system_state once and tell me the stage position. + +**(O2)** Copy the error text it shows. It should say the local server rejected +its API key and name `MICROCLAW_LLM_API_KEY`. + +Now click the **key** chip, enter `testkey`, leave **remember** unticked, and +save. **(O3)** What did the confirmation toast say? The `serve` window should +now print `[microclaw] Local context budget: n_ctx=131072, fixed=33864, +high=84920, low=63690` — **(O3b)** copy that line. + +**Turn 1:** + +> Call get_system_state once and tell me the stage XY and Z position and the current channel. + +**Turn 2:** + +> Snap one image with snap_and_analyze with return_thumbnail set to true, and tell me its mean intensity, snr and whether focus_metric_valid is true. + +**Turn 3:** + +> Run a 200-frame timelapse on the demo camera at 5 ms exposure with the snr_observer hook, saving into a folder called block91-frames, and tell me the hook log path when it finishes. + +**Turns 4 onward** — send this, unchanged, until the meter below shows +`compactions 1` or more, then send it **once more**: + +> Read the hook log from the timelapse you just ran and tell me whether the SNR drifted over the run. + +The meter: in the second PowerShell window, after each reply, + +```powershell +cd C:\code\microclaw +.venv\Scripts\python.exe design\91-gate.py --session . --progress +``` + +Each read adds roughly 19,000 tokens of history; compaction should come within +about five reads. **(O4)** The two compaction notices now stay up until you +click them, so copy each one's text before dismissing it; the `serve` window +also prints each with its numbers. +- **"Conversation history is above the compaction threshold; … being sent as + is."** — expected once, around the large hook-log read; the replies after it + should **not** be slow. +- **"Compacted context: A → B estimated tokens; rebuilding N calibrated tokens, + about S s. …"** — expected **at most once**, and B should be well below the + low-water mark (63,690). If it appears, note S and roughly how long it was + from **Send** to the first text. + +Say how many slow turns this round had (round 1: three, round 2: two). + +**(O6)** After you entered the key, what does the model button at the top of the +page say? It should name the Qwen model, not a Claude id. + +**Last turn:** + +> Before we finish, tell me the illumination state. + +**(O5)** Did it call `get_system_state` for this (visible in the transcript), +and does its answer say only what that tool returned — including `unknown` +where the tool said unknown? + +Then press **Ctrl+C** in the `serve` window. + +## 7 — score it + +The server must still be running: the budget limb reads the server's `n_ctx` +and asks the product's own `make_conversation_store` what this server's budget +is, with the key, because the server answers nothing but `/health` without it. + +```powershell +cd C:\code\microclaw +New-Item -ItemType Directory -Force block91d-evidence | Out-Null +.venv\Scripts\python.exe design\91-gate.py --session . --base-url http://127.0.0.1:8080 --api-key testkey > block91d-evidence\gate.log 2>&1 +Get-Content block91d-evidence\gate.log +Copy-Item (Get-ChildItem *_microclaw_history.jsonl | Sort-Object LastWriteTime | Select-Object -Last 1) block91d-evidence\ +Copy-Item (Get-ChildItem *_microclaw_usage.jsonl | Sort-Object LastWriteTime | Select-Object -Last 1) block91d-evidence\ +Copy-Item (Get-ChildItem *_microclaw_confirmations.jsonl | Sort-Object LastWriteTime | Select-Object -Last 1) block91d-evidence\ -ErrorAction SilentlyContinue +``` + +The last line of `gate.log` reads `/ graded limbs passed`. **NOT +EXERCISED is never a pass.** REPORT lines are measurements and are not graded. + +## 8 — send + +Send the `block91d-evidence` folder, your answers to **O1–O6**, and a copy-paste +of the `serve` window's output (select all in that window). The datasets in +`block91-frames` can stay where they are. + +Afterwards, to remove the demo bounds: `Remove-Item "$env:APPDATA\microclaw\safety_config.yaml"`. diff --git a/design/91-gate.py b/design/91-gate.py new file mode 100644 index 00000000..17404b5e --- /dev/null +++ b/design/91-gate.py @@ -0,0 +1,320 @@ +"""Gate 91 — score one driven session against a local llama.cpp server. + +design/91 puts a local OpenAI-compatible model behind the same agent loop +(91a) and sizes the conversation budget from the server (91b). This scores the +artifacts of one session driven on the machine that runs the server. + + .venv\\Scripts\\python.exe design\\91-gate.py --session . --base-url http://127.0.0.1:8080 --api-key testkey + .venv\\Scripts\\python.exe design\\91-gate.py --session . --progress + +`--progress` prints one line about the newest session and exits 0; it is the +meter the runbook watches between turns. Without it every limb runs. + +Every limb is independent: one FAIL never hides the limbs after it. A limb that +could not run its mechanism reports NOT EXERCISED, which is never a pass, and +the script exits nonzero on either. REPORT lines are measurements, not grades: +design/79's lesson is that naming the winner turns attribution into +label-reading, so timing is printed with its derivation and not judged. + +The budget limb asks the product, not this file, what the budget is: it calls +`agent.make_conversation_store` against the same server, so the scorer cannot +drift from the code it scores. That needs the server up while scoring, and its +key: llama.cpp with `--api-key` answers nothing but `/health` without one. +""" +from __future__ import annotations + +import argparse +import json +import os +import statistics +import sys +from pathlib import Path + +LOCAL_FIELDS = ("prompt_n", "cache_n", "prompt_ms", "predicted_n", "predicted_ms", + "reasoning_characters") + + +class Score: + """One line per limb, no cascade, and NOT EXERCISED is never a pass.""" + + def __init__(self) -> None: + self.rows: list[tuple[str, str, str]] = [] + + def _add(self, verdict: str, name: str, detail: str) -> None: + self.rows.append((verdict, name, detail)) + print(f"{verdict:<14} {name} - {detail}", flush=True) + + def check(self, name: str, ok: bool, detail: str) -> None: + self._add("PASS" if ok else "FAIL", name, detail) + + def skip(self, name: str, detail: str) -> None: + self._add("NOT EXERCISED", name, detail) + + def report(self, name: str, detail: str) -> None: + self._add("REPORT", name, detail) + + def exit_code(self) -> int: + bad = [r for r in self.rows if r[0] in ("FAIL", "NOT EXERCISED")] + passed = sum(1 for r in self.rows if r[0] == "PASS") + graded = sum(1 for r in self.rows if r[0] != "REPORT") + print(f"\n{passed}/{graded} graded limbs passed" + f"{'' if not bad else ' - ' + ', '.join(f'{v}: {n}' for v, n, _ in bad)}") + return 1 if bad else 0 + + +def find_session(target: Path) -> tuple[Path | None, Path | None]: + """Accept a directory (newest session in it) or a stem; return (usage, history).""" + if target.is_dir(): + usage = sorted(target.glob("*_microclaw_usage.jsonl")) + if not usage: + return None, None + stem = str(usage[-1]).replace("_microclaw_usage.jsonl", "") + else: + stem = str(target) + for suffix in ("_microclaw_usage.jsonl", "_microclaw_history.jsonl"): + stem = stem.replace(suffix, "") + usage_path = Path(stem + "_microclaw_usage.jsonl") + history_path = Path(stem + "_microclaw_history.jsonl") + return (usage_path if usage_path.exists() else None, + history_path if history_path.exists() else None) + + +def read_jsonl(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() + if line.strip()] + + +def blocks(message: dict) -> list[dict]: + content = message.get("content") + return [b for b in content if isinstance(b, dict)] if isinstance(content, list) else [] + + +def progress(usage_path: Path | None) -> int: + if usage_path is None: + print("no session yet (no *_microclaw_usage.jsonl here)") + return 0 + records = read_jsonl(usage_path) + if not records: + print(f"{usage_path.name}: no model calls yet") + return 0 + last = records[-1] + print(f"{usage_path.name}: {len(records)} calls, estimated history " + f"{last.get('estimated_tokens')} tokens, compactions {last.get('compaction_count')}, " + f"last prompt {last.get('input_tokens')} tokens " + f"({last.get('prompt_n')} processed, {last.get('cache_n')} cached, " + f"{(last.get('prompt_ms') or 0) / 1000:.1f} s)") + return 0 + + +def server_budget(base_url: str, api_key: str | None) -> tuple[int, int, int] | str: + """(high, low, n_ctx) as the product computes them for this server, or a reason. + + The scorer reads `n_ctx` itself first. If it cannot (a 401 without the key, + an unreachable server), the product would be computing a fallback budget, + and grading the session against a fallback is grading nothing. + """ + import httpx + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + try: + from microclaw import agent + response = agent._get_local_http().get(base_url.rstrip("/") + "/props", + headers=headers, timeout=30) + response.raise_for_status() + n_ctx = response.json()["default_generation_settings"]["n_ctx"] + except (httpx.HTTPError, KeyError, ValueError) as exc: + return f"the scorer could not read the server's n_ctx: {type(exc).__name__}: {exc}" + os.environ["MICROCLAW_LLM_BASE_URL"] = base_url + if api_key: + os.environ["MICROCLAW_LLM_API_KEY"] = api_key + try: + from microclaw.conversation import AuditLog + store = agent.make_conversation_store(AuditLog("gate91-unused", enabled=False)) + except Exception as exc: # the limb reports it; it must not stop the others + return f"make_conversation_store failed: {type(exc).__name__}: {exc}" + return store.high_water_tokens, store.low_water_tokens, n_ctx + + +def score(usage_path: Path | None, history_path: Path | None, base_url: str | None, + api_key: str | None = None) -> int: + s = Score() + if usage_path is None or history_path is None: + s.skip("the session wrote its history and usage files", + f"usage={usage_path}, history={history_path}") + return s.exit_code() + records = read_jsonl(usage_path) + history = read_jsonl(history_path) + s.check("the session wrote its history and usage files", bool(records), + f"{usage_path.name}: {len(records)} model calls; " + f"{history_path.name}: {len(history)} messages") + if not records: + return s.exit_code() + + # --- 91a: the calls went to the local server ----------------------------- + # prompt_ms and cache_n exist only on the local path (llama.cpp `timings`); + # an Anthropic call cannot produce them, so this is the backend's signature. + models = sorted({str(r.get("model")) for r in records}) + local = [r for r in records if all(r.get(f) is not None for f in LOCAL_FIELDS)] + s.check("every model call went through the local backend", + len(local) == len(records) and not any(m.startswith("claude") for m in models), + f"{len(local)}/{len(records)} records carry llama.cpp timings; model(s) {models}") + + assistant = [m for m in history if m.get("role") == "assistant"] + s.check("one usage record per model reply", len(assistant) == len(records), + f"{len(records)} usage records vs {len(assistant)} assistant messages") + + from microclaw.tools import TOOL_REGISTRY + calls = [b for m in assistant for b in blocks(m) if b.get("type") == "tool_use"] + unknown = sorted({b.get("name") for b in calls if b.get("name") not in TOOL_REGISTRY}) + unparsed = [b for b in calls if "_microclaw_parse_error" in (b.get("input") or {})] + if not calls: + s.skip("every tool call names a registered tool", "the session made no tool calls") + else: + s.check("every tool call names a registered tool", not unknown, + f"{len(calls)} calls, {len(set(b.get('name') for b in calls))} distinct" + + (f"; unknown: {unknown}" if unknown else "")) + s.report("tool calls whose arguments did not parse", + f"{len(unparsed)} of {len(calls)}" + + (f": {[b.get('name') for b in unparsed]}" if unparsed else "")) + + # --- 91b: the budget comes from the server ------------------------------- + peak = max(r.get("input_tokens") or 0 for r in records) + compactions = [i for i in range(1, len(records)) + if (records[i].get("compaction_count") or 0) + > (records[i - 1].get("compaction_count") or 0)] + # Three questions, graded separately (gate 91 round 1, F2: one limb that + # skipped compaction records passed a checkpoint committed above high-water). + FIT = "every request fit the server's context" + D9 = "no compaction was committed above high-water" + if base_url is None: + for name in (FIT, D9): + s.skip(name, "no --base-url given, so the server's budget was not read") + else: + budget = server_budget(base_url, api_key) + if isinstance(budget, str): + for name in (FIT, D9): + s.skip(name, budget) + else: + high, low, n_ctx = budget + from microclaw.agent import MAX_OUTPUT_TOKENS + # The real limit: the server refuses a prompt that leaves no room for + # the reply. Everything else is the store's policy. + too_big = [(i, r.get("input_tokens")) for i, r in enumerate(records) + if (r.get("input_tokens") or 0) + MAX_OUTPUT_TOKENS > n_ctx] + s.check(FIT, not too_big, + f"server n_ctx {n_ctx}; peak prompt {peak} + {MAX_OUTPUT_TOKENS} reply " + f"reserve = {peak + MAX_OUTPUT_TOKENS}" + + (f"; over at (record, prompt) {too_big}" if too_big else "")) + # D9 (design/91 91c): a fold that cannot reach high-water is not + # committed. The record of the compacting call carries the estimate + # of the view it sent, so a committed compaction above high is visible. + bad = [(i, records[i].get("estimated_tokens")) for i in compactions + if (records[i].get("estimated_tokens") or 0) > high] + if not compactions: + s.skip(D9, "the session never compacted, so no commit could be judged") + else: + s.check(D9, not bad, + f"high {high}, low {low}; estimate after each compaction " + f"{[records[i].get('estimated_tokens') for i in compactions]}" + + (f"; committed above high at (record, estimate) {bad}" if bad else "")) + # D9 deliberately sends an over-threshold view when no fold would help; + # that is policy, so it is reported, not graded. + above = [i for i, r in enumerate(records) + if (r.get("estimated_tokens") or 0) > high and i not in compactions] + s.report("requests sent above high-water without compacting", + f"{len(above)}" + (f" at record(s) {above}" if above else "")) + + if not compactions: + s.skip("the session compacted at least once", + f"compaction_count stayed {records[-1].get('compaction_count')}; peak prompt " + f"{peak} tokens - drive more hook-log reads (runbook step 4)") + else: + s.check("the session compacted at least once", True, + f"{len(compactions)} compaction(s), at record(s) {compactions}") + for i in compactions: + r = records[i] + s.report(f"the round after compaction {records[i].get('compaction_count')}", + f"record {i}: prompt {r.get('input_tokens')} tokens, " + f"{r.get('prompt_n')} processed in {(r.get('prompt_ms') or 0) / 1000:.1f} s, " + f"{r.get('cache_n')} reused; estimate after compaction " + f"{r.get('estimated_tokens')}") + + # --- D7: the prefix is reused between rounds ----------------------------- + # Within a turn, round k+1's prompt is round k's prompt plus the reply and the + # tool results, so llama.cpp should reuse nearly all of round k. A compaction + # rewrites the history part, so those rounds are reported above, not graded. + pairs = [(records[i - 1], records[i]) for i in range(1, len(records)) + if records[i].get("turn_id") == records[i - 1].get("turn_id") + and i not in compactions] + if not pairs: + s.skip("each later round reused the previous round's prompt", + "no turn had two rounds without a compaction between them") + else: + # 64 tokens of slack: the assistant generation prompt and the turn's + # closing markers are re-rendered (measured 4-56 on 2026-10-02). + missed = [(a.get("input_tokens"), b.get("cache_n")) for a, b in pairs + if (b.get("cache_n") or 0) < (a.get("input_tokens") or 0) - 64] + s.check("each later round reused the previous round's prompt", not missed, + f"{len(pairs) - len(missed)}/{len(pairs)} rounds reused all but <= 64 tokens" + + (f"; misses (previous prompt, reused): {missed[:5]}" if missed else "")) + + # --- 91a D4: an image the text-only server never sees --------------------- + # The model call that follows an image result is the one whose estimate would + # carry the thumbnail. Records align with assistant messages in order, so + # count assistant messages to find it. Only that call is examined: a hook-log + # read legitimately adds ~19k tokens, so a session-wide "largest rise" would + # fail every real run. + after_image = [] + replies_so_far = 0 + for message in history: + if message.get("role") == "assistant": + replies_so_far += 1 + elif any(b.get("type") == "tool_result" and isinstance(b.get("content"), list) + and any(isinstance(p, dict) and p.get("type") == "image" for p in b["content"]) + for b in blocks(message)): + after_image.append(replies_so_far) + rises = [(i, (records[i].get("estimated_tokens") or 0) + - (records[i - 1].get("estimated_tokens") or 0)) + for i in after_image if 0 < i < len(records) and i not in compactions] + if not rises: + s.skip("a thumbnail did not inflate the history estimate", + "no tool result carried an image followed by a model call without a " + "compaction - runbook turn 2 asks for one") + else: + # A thumbnail counted as text adds 20k-120k tokens (design/70 R145); the + # snap's own JSON payload is about a thousand. + worst = max(rise for _, rise in rises) + s.check("a thumbnail did not inflate the history estimate", worst < 5000, + f"{len(rises)} call(s) after an image result; estimate rose by " + f"{[rise for _, rise in rises]} tokens") + + # --- timing, reported ----------------------------------------------------- + prompt_s = [(r.get("prompt_ms") or 0) / 1000 for r in records] + gen = [r["predicted_n"] / (r["predicted_ms"] / 1000) for r in records + if r.get("predicted_n") and r.get("predicted_ms")] + reasoning = [r.get("reasoning_characters") or 0 for r in records] + s.report("prompt processing per call", + f"median {statistics.median(prompt_s):.2f} s, max {max(prompt_s):.1f} s, " + f"first call {prompt_s[0]:.1f} s; peak prompt {peak} tokens") + if gen: + s.report("generation rate", f"median {statistics.median(gen):.1f} tokens/s over " + f"{len(gen)} calls") + s.report("reasoning text per call (not shown, not stored)", + f"median {statistics.median(reasoning):.0f} chars, max {max(reasoning)}") + return s.exit_code() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--session", default=".", help="folder (newest session) or a session stem") + parser.add_argument("--base-url", default=None, help="the llama.cpp server, for the budget limb") + parser.add_argument("--api-key", default=None, help="the server's --api-key, if it has one") + parser.add_argument("--progress", action="store_true", help="print one meter line and exit") + args = parser.parse_args() + usage_path, history_path = find_session(Path(args.session)) + if args.progress: + return progress(usage_path) + return score(usage_path, history_path, args.base_url, args.api_key) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/design/91-local-openai-compatible-backend.md b/design/91-local-openai-compatible-backend.md new file mode 100644 index 00000000..28bf61eb --- /dev/null +++ b/design/91-local-openai-compatible-backend.md @@ -0,0 +1,501 @@ +# A local model behind the same loop + +Status: **PROPOSED**, 2026-10-01. Nothing implemented. The evidence below is a +three-request spike, not a session. + +## Problem + +Microclaw can only think through the Anthropic API. An operator wants to run it +against a local Qwen served by llama.cpp on another machine — no per-call bill +(design/82: $102 over five sessions), and no dependence on an external service. +Microclaw must not anchor on that machine, that model or that server: the +target is "any OpenAI-compatible `/v1/chat/completions` endpoint". + +The seam is one function. `agent._stream_one_round` is the only SDK call site; +it returns a `Message` whose `content` blocks (`text`, `tool_use`) and +`stop_reason` drive `run_agent_iter`. Everything downstream — the history +JSONL, `ConversationStore`, `export_session_script`, the viewer — reads the +Anthropic-shaped history, not the SDK. + +## Evidence (spike, 2026-10-01) + +Server: llama.cpp `b11260-e904318a2`, `Qwen3.8-27B-UD-Q4_K_XL.gguf`, +`-c 65536 -np 1 --jinja`, `n_ctx_train 262144`, `modalities.vision: false`, +chat template supports tools and ``. Request: `SYSTEM_PROMPT` alone (no +knowledge base, no rig interview) plus all 83 `TOOLS` translated to +`{"type":"function","function":{name, description, parameters}}`, non-streamed. +Main at `9e9271a`. No tool was executed; round 3 answered with a **synthetic** +payload that said so in a `note` field. + +| request | prompt tokens | new / cached | prefill | wall | result | +|---|---|---|---|---|---| +| cold "stage position?" | 33,592 | 33,592 / 0 | 37.3 s | 44.0 s | `get_xy_position` + `get_z_position` | +| warm "50 ms, DAPI" | 33,600 | 26 / 33,574 | 0.17 s | 4.6 s | `set_exposure {"ms":50}` + `get_available_channels` | +| tool result → reply | 33,688 | 114 / 33,574 | 0.27 s | 11.8 s | `end_turn` prose | + +Prefill ~900 tok/s, generation ~30 tok/s. Every tool name called exists and +every argument matched its schema. The same three requests on the older +`128c478` tree (81 tools, 32,499 tokens) gave the same shape. + +**One behavioural observation, n=2.** On `128c478` the reply relayed the +payload's "synthetic" note; on `9e9271a` it reported the synthetic numbers as +"Current stage position" with no caveat. That is the *say only what a tool told +you* rule, and it is what the gate scores — not a finding about the model. + +**At `-c 131072 -fa on --cache-type-k q8_0 --cache-type-v q8_0`** (operator +restart, 2026-10-01; ~3 GB VRAM still free, KV size not printed): the same +three calls with the same arguments, cold prefill 37.9 s, warm 0.19 / 0.29 s, +generation **26.4 tok/s against 30.0** — 12 % slower, not attributed between +the quantised KV, flash attention and the larger context. The reply relayed the +synthetic note again, so that rule held in **2 of 3** runs. + +What the spike did **not** measure: streaming, a session near the context +limit, compaction, the knowledge-base and skill-discovery blocks, images, and +any prompt-level rule over a real session. + +## Decisions + +**D1 — One backend, translated at the seam.** The canonical history stays +Anthropic-shaped. A second branch of `_stream_one_round` translates the request +out and the reply back, and returns a real `anthropic.types.Message` built with +`Message.model_validate` — so `run_agent_iter` (which reads `block.type`, +`block.id`, `response.stop_reason` by attribute), the history JSONL and the +exporter cannot tell which backend produced it. Checked: a translated +`tool_use` reply validates and round-trips `id`, `stop_reason` and +`cache_read_input_tokens`. No provider registry, no abstract base class: two +branches of one function. + +**D2 — Translation.** + +| Anthropic (history) | chat completions (wire) | +|---|---| +| `system` blocks | one `system` message, blocks joined; `cache_control` dropped | +| `tools[i]` | `{"type":"function","function":{name, description, parameters: input_schema}}`; `cache_control` dropped | +| assistant `text` + `tool_use` blocks | one assistant message: `content` + `tool_calls[{id, function:{name, arguments: json}}]` | +| user `tool_result` blocks | one `tool` message per block, `tool_call_id`, text content; `is_error` prefixed into the text | +| `image` block in a tool result | text-only model: replaced by `{"image_omitted": "this model does not accept images"}` — never dropped silently. Vision model: a following `user` message with `image_url` data URIs | +| `finish_reason` `stop` / `tool_calls` / `length` | `stop_reason` `end_turn` / `tool_use` / `max_tokens` | +| `delta.content` | `text_delta` event | +| `delta.reasoning_content` | not streamed as prose, not stored in history (the template strips it on replay); its length goes to `usage_sink` | +| `usage.prompt_tokens`, `prompt_tokens_details.cached_tokens` | `input_tokens`, `cache_read_input_tokens` in the existing `usage_sink` record | + +Tool-call arguments arrive as JSON *text*; an unparsable one becomes a +`tool_result` error naming the parse failure, so the model sees and corrects it. + +**D3 — Selection is one setting.** `--llm-base-url` / `MICROCLAW_LLM_BASE_URL` +set means the local backend; unset means Anthropic, unchanged. The model id +defaults to the server's single `/v1/models` entry, so the operator names +nothing else; the existing model picker lists that endpoint's models. The key, +when the server has one, goes through `credentials.py` under its own name — +never the Anthropic one — and the startup checks (`webserve.py`'s "No Anthropic +API key", `__main__`) accept a configured base URL instead. The URL is +operator configuration, never a constant in `microclaw/`. + +**D4 — Vision is read, not assumed.** If the server answers llama.cpp's +`/props`, `modalities.vision` decides. Otherwise text-only, the safe direction: +an omitted-image note is honest, an image sent to a model that cannot read it +is a 400 or a hallucination. + +**D5 — Errors map onto the existing two.** Connection refused/reset, 503 and +429 → the transient retry path, message naming the base URL. 404 on the model +→ `_BadModel`. A 400 whose body reports the context exceeded → an error that +states the server's `n_ctx` and the request's token count, not a generic +failure. `TRANSIENT_API_ERROR_MESSAGE` stops saying "Anthropic" on this path. + +**D6 — The context budget comes from the server.** `ConversationStore`'s +120k/90k watermarks are Anthropic-sized; at `n_ctx 65536` a session would +overflow long before it compacted. Read `n_ctx` (llama.cpp `/props` or +`/v1/models` `meta.n_ctx`; otherwise a stated default) and set +`high = n_ctx − fixed_prefix − max_output`, low at 75 % of high. At the spike's +spike's numbers: at `n_ctx 65536` that is ≈ 23.7k tokens of history; at the +operator's chosen `131072`, **≈ 89.3k**, low water ≈ 67k. + +A larger window is not free, and the cost moves rather than vanishing: +compaction becomes rarer, but the round after it re-prefills the whole +low-water prompt — ≈ 100k tokens, **≈ 110 s at the measured 900 tok/s**, and +prefill rate falls with length, so likely more. 91b must measure that round +rather than this estimate, and the operator should see it coming: a progress +event saying the context is being rebuilt, not a silent two-minute pause. + +**D7 — Timing contract.** Prefix reuse is what makes this usable: 37 s cold +against 0.2 s warm per round. Anything that rewrites the prompt's front pays the +cold cost again — compaction, a changed knowledge base or skill-discovery block, +another client on a one-slot server. The local branch must keep the prefix +byte-stable between rounds (same system string, same tool order, same +serialisation of earlier turns), and `usage_sink` records `prompt_n`, +`cache_n` and `prompt_ms` so a cold hit is visible per round rather than +inferred. Not attributed yet: how often a real session goes cold. + +**D8 — Export is untouched.** `export_session_script` reads recorded tool calls +and results; the history shape does not change, so no emitter changes. A test +asserts a session driven through the local branch exports the same script as the +same calls driven through the Anthropic one. + +## Blocks + +**91a — the backend.** D1–D5 and D8. Streaming chat-completions client over +`httpx` (already a direct dependency; no new package), translation +both ways, selection, credentials, startup checks, `usage_sink` mapping. +Tests: a fake server written from llama.cpp's actual SSE shape (`tool_calls` +deltas split across chunks, `reasoning_content` deltas, the final `timings` +object), not from our caller. Watched-fail evidence on each translation row. + +**91b — the budget and the prefix.** D6 and D7: watermarks from `n_ctx`, a +prefix-stability test (two consecutive rounds serialise an identical leading +byte range), compaction measured as a cold hit. + +**Gate 91 — one driven session against a real llama.cpp server**, from a +machine that can reach it. Score from the history and usage JSONL: per-round +`prompt_ms` and `cache_n` (how many cold rounds, and which events caused each), +whether compaction fired before overflow, and the reporting rules (numbers only +from tool results; `unknown` relayed as `unknown`; illumination state at +hand-off). Image tools exercised once to see the omitted-image note reach the +model. + +## Open questions for the operator + +- **Q1 — context size. Settled, 2026-10-01:** `-c 131072` with q8_0 KV and + flash attention, ~3 GB VRAM spare. Not pushed to 262,144: a cold round after + compaction would then cost minutes (D6). +- **Q2 — gate machine. Settled, 2026-10-01:** the llama.cpp workstation + itself, with Micro-Manager's demo config and Microclaw installed beside the + model (operator to install). No network path between the gate's Microclaw and + its server, so the timings are a local deployment's, not a tunnel's. The + earlier gate machines are at another site and are not assumed to reach it. + +## Run ledger + +Notebook drafted on `Qwen-local-model` from `main` `9e9271a`. + +| block | branch | start | state | +|---|---|---|---| +| 91a | `design91/91a-backend` | `55ccc95` (fixtures) on `4b2e9cf` | **merged** to fork `main` `24a28b3`, 2026-10-06 | +| 91b | `design91/91b-budget` (on 91a) | `3b0f1c7` | **merged** `24a28b3`, 2026-10-06 (gate 91 round 1) | +| 91c | `design91/91c-compaction` (on 91b) | `3e778b5` | **merged** `24a28b3`, 2026-10-06 (gate 91 round 2: 9/9; F3 → 91d) | +| 91d | `design91/91d-calibrate` | `4859ce2` (fork `main` + httpx fix) | **merged** to fork `main`, 2026-10-06 (gate 91 round 3: 9/9) | +| 91e | `design91/91e-effort` | fork `main` `4a01af9` | **merged** to fork `main`, 2026-10-22 (check passed) | + +**91a review, 2026-10-02.** Baseline at `55ccc95`: 5512 passed, 108 skipped, +1 failed (`test_built_wheel_contains_the_source_tree_skill_catalog`, passes +alone; ran during a network drop). Three runner turns: `f6917b3` (incomplete +by its own report), `ec16e85` (round 1: eleven defects F1–F11, among them the +compaction checkpoint silently dropped and `--llm-base-url` lost before +`serve`), `64065e6` (round 2: G1–G7; turn killed by the coordinator's 2-hour +background limit during its final check, committed as found, then reviewed). +Coordinator full suite on `64065e6`: **5532 passed, 108 skipped**. Mutation +check, each D2 row and error mapping broken in turn: 11/14 caught on +`ec16e85`, **19/19** on `64065e6`. Live against the workstation through +`_stream_one_round` with `_system_blocks()` (33,889 tokens): 124 streamed +deltas, first at 8.1 s, `end_turn`; second round `get_z_position {}` with +`cache_n 33853`, `prompt_ms 174`. Known nits left: one-line `if`s and a +redundant `kind` branch in `webserve.py`'s key route. + +**Runner lesson.** A revision turn outlived Claude Code's 2-hour background cap +when the network dropped mid-turn. Launch long runner turns under `nohup` +(CLAUDE.md step 2) rather than as session background tasks. + +**91b review, 2026-10-02.** Four runner turns: `246a72f` (budget from the +server), `1dcb5b1` (H1: fixed prompt counted through `/apply-template` + +`/tokenize` = **33,864**, equal to the server's own `prompt_tokens`; the +untemplated count was 30,118), `befe1a7` (K1: the browser locked Send without a +key on a keyless local server), `0ecbbcc` (L1–L4: with `--api-key` every +llama.cpp endpoint but `/health` 401s, the store was built before the key was +loaded, and failed probes were cached — the budget clamped to **2**). +Coordinator fixes: `55af3bb` (a stored key was no longer registered with the +history audit's redaction after that reorder), `2780191` (`keyRequired` +declared with the page state; an absent `key_required` reads as required), +`557200f` (a stale round-1 test). Live on the workstation: budget 84,920 / +63,690 with the key; without it a stated provisional 4,096 / 3,072 and every +401 named, re-read exactly when the key is entered. Prefix reuse measured over +a 5-round tool loop: each round reused all but 4–56 tokens of the previous. +Full suite on `a8c4f81`: **5547 passed, 108 skipped, 1 failed** — +`test_built_wheel_contains_the_source_tree_skill_catalog`, which passes alone +and failed the same way at 91a's baseline (2 of 6 full runs this session). + +**D6 correction.** After compaction the system+tools prefix stays cached, so +the slow round re-processes the history only: ≈ low-water (63,690) at +~900 tok/s, about 70 s, not the whole prompt. Gate 91 measures it. + +**For later, not a block yet.** The model's chat template renders "Reasoning +effort is set to xhigh" by default; a lower per-request effort may cut reply +latency substantially. Measure against gate 91's baseline before changing it. + +## Gate 91, round 1 (workstation, 2026-10-05) — scored from artifacts + +Session `20261005_153316_502746`: 14 model calls, all local; 8 tool calls, all +registered, none unparsed; the no-key turn gave the 401 error naming +`MICROCLAW_LLM_API_KEY` (O2) as a toast, Send stayed enabled with no banner +(O1); entering the key re-read the budget exactly (O3b: 33,864 / 84,920 / +63,690); the illumination answer relayed `unknown` from `get_system_state` +(O5). Seven same-turn rounds reused all but ≤ 64 tokens of the previous prompt. +The thumbnail moved the estimate by 519 tokens. Peak prompt + output reserve +120,205 of 131,072. + +**F1 — a compaction that cannot get under high-water is committed anyway, and +repeats.** One `read_hook_log(limit=200)` returned 140,218 characters (~59k +tokens). `ConversationStore` could fold only the small early turns (the +two-recent-turns floor), committed a checkpoint at **87,067 > 84,920**, and +compacted again on each of the next two calls (86,773, then 67,021). Each +commit rewrites the history head, so each paid a full history re-prefill: +**134.0 s, 133.3 s, 98.7 s — 366 of the session's 443 s of prompt +processing**, the second for ~300 tokens of saving. The operator saw a +"rebuilding" toast and minutes of silence, with nothing saying it had not +helped (O4: "several minutes"). + +**F2 — the gate's budget limb could not fail on F1.** It excluded compaction +records, so it passed with a post-compaction estimate above high-water. + +**Measured, not graded.** Prefill slows with position: 826 tok/s at 34k, 671 at +54k, 583 at 112k. Generation was 464 s of the session (more than prompt +processing), with 28,209 characters of hidden reasoning. The history estimate +ran 1.12–1.63× the server's own history count (safe direction). + +## 91c — compaction only when it helps, and say so + +**D9 — commit a compaction only if it reaches high-water or below.** Otherwise +send the current view unchanged: the prefix (and on the Anthropic path the +cache) survives, and the real limit is the server's, which the request still +fits (F1's peak: 120,205 of 131,072). Both backends; the Anthropic path gains +the same saving on cache rewrites (design/82). + +**D10 — the operator sees what a compaction does and costs.** The `compaction` +event carries the estimate before and after, and an expected rebuild time from +this session's own measured prefill rate (not a constant); a declined +compaction emits an information event saying the history is above the +threshold and recent turns cannot be folded yet. Information, never a +confirmation (CLAUDE.md). + +**D11 — the context window is operator configuration, measured, not coded.** +91b already reads `n_ctx`. A larger `-c` makes compaction rarer but each rebuild +longer, and slower per token (above): at `-c 262144` a low-water near 160k would +re-prefill ~125k tokens at well under 583 tok/s — several minutes per rebuild. +Gate 91c measures one larger window rather than assuming it helps. + +**Gate 91c.** Re-run gate 91 on the workstation with the fixed scorer (F2: grade +prompt + reserve ≤ `n_ctx` and report every request sent above high-water; a +mutant for each), expecting one compaction-cost round where round 1 had three. + +**91c review, 2026-10-05.** Three runner turns: `fa61636` (D9/D10; three +existing tests weakened until they no longer tested their purpose — 91a's F1 +checkpoint-on-the-wire test among them), `ce3d2fc` (N1–N4: purposes restored at +watermarks where D9 commits; the gate shape pinned — old store 1, 2, 3 commits; +rebuild estimate text-only; declined notice once per stretch), `4c99b81` (six +more fixtures outside the targeted files resized; the design/82 R63 probe now +says it replays under the current policy, so a pre-91c session's partition is +not historical proof). Coordinator: `f26df3e` (gate scorer F2 — fit and D9 +graded separately; re-scored on round 1's real artifacts it fails D9 at records +10 and 11), `c650837` (round-2 runbook). Replay of round 1's real history +through the final store at the gate's watermarks: old 3 commits at calls 10, 11, +12; new: declined once at 10, sent as is at 11, **one** commit at 12. Full suite +on `4c99b81`: **5551 passed, 108 skipped**. + +## Gate 91, round 2 (workstation, 2026-10-06) — scored from artifacts + +Session `20261006_090604_910375`, 9/9 graded limbs. Two compactions where round +1 had three, both committed under high-water; two over-threshold requests sent as +is, each reusing the cache (68,538 and 109,934 tokens). Rebuilds 125.3 s + 72.1 s += 197 s against round 1's 366 s. Peak prompt + reserve 121,586 of 131,072. The +two sessions differ (Qwen read the log with `limit: 150` this time), so the +comparison is qualitative; the per-record mechanism is not. + +**F3 — a fold that clears high-water by a hair buys nothing.** Compaction 1 left +84,249 (671 under 84,920); the next call was over again and compaction 2 +followed two calls later. Replaying the real history with "commit only at +low-water" gives one compaction (at call 14) and saves the 125 s rebuild, at the +cost of sending call 13 about 5k tokens larger — which needs a calibrated +"would not fit" valve, because the estimator runs ~1.2× high for Qwen and in raw +estimate units that valve fires at call 12 and reproduces today's behaviour. + +**F4 — the rebuild prediction.** 150 s and 111.5 s predicted against 125.3 s +and 72.1 s measured, because the estimate counts ~1.2× the server's tokens; and +the operator's 210 s to first text includes the model's hidden reasoning after +the rebuild (5,097 characters on that turn), which the toast does not mention. + +**F5 — the toasts vanish before they can be read, and `serve` logs only +`compaction`.** **F6 — generation now dominates:** 728 s against 382 s of prompt +processing; one round wrote 8,656 reasoning characters before a tool call. +**F7 — the model button says `claude-opus-4-8` on the local path** until the +server answers `/v1/models`, which a keyed server does only after the key. + +## 91d — compaction that buys headroom, numbers that are true + +**D12 (F3) — on the local path, commit a fold only when it reaches low-water, +unless the current view would not fit the server.** Otherwise send as is (D9's +decline). The safety valve compares the *calibrated* view (D13) with what the +server can take — `n_ctx − fixed − MAX_OUTPUT_TOKENS − margin`, in server +tokens — and, when the view would not fit, commits the deepest fold at or below +high-water as 91c does. The Anthropic path keeps 91c's D9 unchanged: it has no +server-reported limit to calibrate against. + +**D13 (F3, F4) — calibrate the estimate against the server's own count.** Each +local response reports `prompt_tokens`; with the fixed prompt known (91b), the +server's history tokens over the store's estimate of the history that was sent +is a measured ratio (round 1: 1.12–1.63). The valve (D12) and the rebuild +prediction use the calibrated value; before any response, the raw estimate, +which errs high (the safe direction for both). + +**D14 (F4, F5) — the operator can read what happened.** The compaction and +declined notices stay up long enough to read (until dismissed or the next turn, +not a few seconds), say that the predicted time covers the rebuild only and the +model then thinks before it replies, and `serve` prints each with its numbers. + +**D15 (F7) — the model button names the model that is answering.** On the local +path it shows the server's model id, refreshed when a key is entered; before the +server has answered it says so, and it never shows a Claude id. + +**D16 (F6) — reasoning effort: an optional setting, default unchanged** +(operator decision, 2026-10-06). `MICROCLAW_LLM_REASONING_EFFORT`, unset by +default, is forwarded unchanged as `chat_template_kwargs.reasoning_effort`, and +the fixed-prompt count renders with it. Measured on Microclaw's real prompt and +tools, 2 runs per cell: median generation per call 16.5 s (`xhigh`), 6.0 s +(`medium`), 9.1 s (`low`); simple turns fell from 7–14 s to ~2 s, the 3×3 +bead-count planning turn did not reliably fall (80 s `xhigh`, 85–93 s +`medium`, 20 s and 92 s `low`; 217 s once at `xhigh`). No quality loss visible +at that sample size, which cannot show equivalence. The operator tries it out +of band. + +**91d review, 2026-10-06.** Two runner turns: `0b55c88` (D13–D15 right; D12 +not — the local path returned before trying any fold whenever the calibrated +view fitted, so a low-water fold never committed and no notice fired; +`resolve_model` sent `"model": null`), `9372a89` (P1–P5: the D12 sequence, +the ceiling stated, the gate-shape test watched failing on 91c's store, a +neutral `"default"` model string, and D16). Coordinator: `cde64fe` (an empty +effort means unset). Replays of both real gate sessions through `9372a89`'s +store, calibrated from each session's own usage: round 1's history **0** +compactions (one decline), round 2's **1**, landing at 55,222; largest request +~126.6k of 131,072 with the reply reserve. Live: budget unchanged with the +setting unset; with `medium` the probe counted 33,826 fixed and the request was +33,831. Full suite on `cde64fe`: **5559 passed, 108 skipped**. + +**Found on the way, fixed on `main` first** (`54e1055`): `httpx` was declared +only in the `test` extra while `agent.py` imports it; `anthropic` now depends on +`httpx2`, so a fresh `.[serve]` install could not import `microclaw.agent`. The +gates installed `.[serve,test]` and never saw it. + +## Gate 91, round 3 (workstation, 2026-10-06) — scored from artifacts + +Session `20261006_135341_257860`, 9/9 graded limbs, reasoning effort unset. +**One** compaction (95,316 → 6,925 estimated), its rebuild **8.6 s** (5,823 +tokens processed, 33,366 reused; predicted 12.7 s) — against 366 s in round 1 +and 197 s in round 2. Five requests went out above high-water without +compacting (records 16–20), each reusing the cache (0.3–7.2 s); the largest +calibrated view was 83,624 against the 84,920 ceiling, so the valve correctly +held. Peak prompt + reserve 125,680 of 131,072. O6: the model button named the +Qwen model. Operator: "no appreciable delay" on the compaction round. + +Not compaction: record 16's 104 s was the server processing one 140,232- +character `read_hook_log(limit=200)` result (59,262 new tokens), and a later turn +asking for the same read overflowed the context; the server refused with its own +numbers and the turn rolled back (`R160`). Model time was 200 s of prompt +processing against **938 s of generation** — one round wrote 11,781 reasoning +characters in 187 s — which is what the D16 setting is for. + +Fixed before merge (`75fe21b`): the rebuild count is whole tokens (the toast +had shown `6,106.033`); persistent notices close from their own button so their +text can be selected and copied; `serve` prints an error event's message, not +only `error`. + +## Reasoning effort on a real microscope (operator, 2026-10-14) — scored from artifacts + +Three sessions on a real Zeiss rig (HXP 120 V fluorescence via the HXP shutter, +reflector turret DsRed / GFP / DAPI), one per level, the same six-prompt script: +`xhigh` `20261014_132137_063462`, `medium` `20261014_140039_773457`, `low` +`20261014_141904_322634`. The sessions are not identical: `xhigh` ran first and +carried two extra turns that taught the knowledge base the light path and the +full-frame imaging, so `medium` and `low` started with that stored; `low` added +"combine into a 3 colour image and save" and "show me the final image". + +The five shared turns (red snap, "is it focused?", focus sweep, 3-colour image, +refocus and repeat), each split from the usage records into model time and the +gaps between calls (tools and hardware): + +| level | wall | generation | prompt processing | tools + hardware | hidden reasoning | +|---|---|---|---|---|---| +| `xhigh` | 1,064 s | 916 s (86 %) | 51 s | 97 s | 69,806 chars | +| `medium` | 832 s | 705 s (85 %) | 64 s | 63 s | 54,920 chars | +| `low` | 632 s | 483 s (76 %) | 69 s | 79 s | 28,288 chars | + +**The time is the model thinking, not the microscope:** tools and hardware were +6–12 % of each session. `low` cut the shared turns by ~40 % against `xhigh`; +`medium` sat between but had the slowest single turn (3-colour, 33,040 +reasoning characters, 375 s). n = 1 per level: turn-to-turn spread is as large +as the difference between levels. + +**`low` reported a dataset it never acquired.** In "combine into a 3 colour +image and save", it acquired red and green (acquisition records at 18:34:34 and +18:34:51 UTC), switched the turret to DAPI, and wrote a composite script without +acquiring blue; its reply said "Saved to disk: `red_1`, `green_1`, `blue_1` … +These are the real, verified acquisitions." The refocused set later in the +session was complete (red, green, blue at 18:38–18:39). No such error was found +in `xhigh` or `medium` — which, at n = 1, is not evidence that they cannot make +it (`R161`). + +**Channel switching.** All three sessions found Micro-Manager's `Channel` config +group empty (`get_available_channels` → no preset), so every level switched the +reflector turret with `set_device_property`, which is correct for that +configuration. `xhigh` alone built the 3-colour image as one acquisition with a +per-frame filter switch (one dataset); `medium` and `low` snapped three +unsaved frames. The operator is checking the rig's config groups; a populated +`Channel` group would let `set_channel` and multi-channel acquisitions do this +in fewer calls at any level. + +**No route to a colour composite.** Asked to combine the channels, `low` said +plainly it had no tool for it, wrote a script it flagged as unrun and +unverified, and later opened the three channels with Fiji steps for the overlay +(`R163`). + +**Decision for now:** the default stays the template's (`xhigh`); +`MICROCLAW_LLM_REASONING_EFFORT=medium` is the faster option worth using; `low` +is not recommended for real work on this evidence. Re-test after the `Channel` +group exists, two or three sessions per level. + +## 91e — change the reasoning level within a session + +**D17 — a session-only reasoning-level control beside the model picker** +(operator request, 2026-10-22). On the local path the browser offers *server +default*, `low`, `medium` and `high` — the OpenAI-style vocabulary, so Microclaw +does not anchor on one template's spelling (the measured Qwen template maps +`high` to its `xhigh`). It applies to this session only; `MICROCLAW_LLM_REASONING_EFFORT` +stays the startup default. Refused while a turn runs (as a model change is) and +offered only on a loopback bind (as the key and model editors are). Operator +configuration: there is no tool through which the model changes its own level. + +**What a change costs, disclosed, not confirmed.** The template writes the level +into the system prompt, so a change invalidates the server's prompt cache and +the next request re-processes the whole prompt — measured ~40 s for the fixed +prompt cold, plus the history at the session's measured prefill rate (583–826 +tok/s on the workstation). The control shows that estimate as an information +notice; a "no" would change nothing, so it is not a confirmation (CLAUDE.md). +The budget is recomputed on change, because the fixed prompt's size depends on +the level (33,864 at the default, 33,826 at `medium`). + +**Recorded.** Every usage record carries the level in force for that call, so a +session that switches can be analysed per level (the 2026-10-14 comparison had +to infer it from file names). + +**91e review, 2026-10-22.** One runner turn, `e55761a`: a sibling route +`POST /api/model/reasoning-effort` (the model route's guards — `403` not +editable, `409` mid-turn — and refused on the Anthropic path), one shared +`_refresh_local_store_budget` for the key and reasoning routes, `reasoning_effort` +in every local usage record, a disclosure notice. Coordinator: `9611dc9` (the +helper had been inserted between two imports; a startup level outside the four +offered, such as `xhigh`, left the select blank — now listed as the startup +setting), `e611762` (the notice states an upper bound). Live against the +workstation: per-level fixed counts 33,864 / 33,826 and each record carrying +its level; switching to `medium` re-processed the whole prompt (33,831 tokens), +**switching back re-processed 4**, because this llama.cpp build kept the earlier +prompt — so the notice's figure is a ceiling. Full suite on `e611762`: **5562 +passed, 108 skipped**. Check: `design/91-block91e-check.md`, with +`91-block91e-check.py` (NOT EXERCISED on a pre-91e session, verified). + +**91e check (workstation, 2026-10-22) — passed.** Session +`20261022_152509_587336`, 9 calls, levels recorded per call in the order the +operator switched them: `server default -> medium -> server default -> medium`. +The notice predicted "up to about 33,904 tokens (39.8 s)"; the first switch to +`medium` re-processed **33,949** (call 2). Switching back to the default +re-processed 173 (call 4) and to `medium` again 209 (call 6) — the server kept +both levels' prompts, the case the notice's "less if…" covers. Operator: ~90 s +for the first switched reply (rebuild plus generation; the notice covers the +rebuild only), ~20 s and <15 s for the later ones. E1: the control showed +Server default. + diff --git a/design/prompts.md b/design/prompts.md index 29e2e493..fdb2aa20 100644 --- a/design/prompts.md +++ b/design/prompts.md @@ -11949,3 +11949,82 @@ to `main` before adding the block's worktree. (362 passed, 9 skipped). The new test was watched failing on `2e5b7e2`'s `tools.py` (`AssertionError: {'ok': True}`) and passed 30 of 30 runs. Full suite locally: 5964 passed, 107 skipped. CI: run 39276478002 at `7bc7c21` green on both runners (Ubuntu 5964 passed, 107 skipped; Windows 5956 passed, 115 skipped). n=1. + + +## design/91 blocks 91a–91c — a local model behind the same loop (merged to the operator's fork `main`, 2026-10-06) + +**Record the dependency before the runner has to imagine it.** The runner +sandbox has no network, so the coordinator recorded llama.cpp's real responses +(streams, errors, `/props`, `/tokenize`, `/apply-template`) and committed them as +fixtures; the parser was written against those bytes. Two defects were still +outside every recording: a server started with `--api-key` refuses everything +but `/health` (all fixtures were recorded with the key), and Git for Windows +rewrote the fixtures to CRLF at checkout (selftest 3 failed / 9 errors on the +workstation). Record the *unauthenticated* shape too, and give byte-exact +fixtures a `-text` rule the day they are added. + +**A runner's green run can be a weakened test.** 84c's first turn changed three +tests' watermarks until the behaviour under test never happened — 84a's own +"checkpoint reaches the wire" regression among them — and a later full run found +six more outside its targeted files. Read every changed test as a diff of its +purpose, not its numbers; "resize, never weaken" had to be said twice. + +**Mutation-check the translation rows; it found what the suite could not.** +Breaking each D2 row in turn: 11/14 caught on 84a round 1, 19/19 after. + +**A passing gate was the place to look, twice.** Round 1 scored 8/8 while the +store committed checkpoints above high-water (three rebuilds, 366 s) — the +scorer's own limb skipped compaction records. Round 2 scored 9/9 while a +compaction landed 671 tokens under high-water and bought one round. Both were +found by reading the usage records against each other, and both were confirmed +off-rig by replaying the real history through old and new `ConversationStore` +before anyone was asked for another session. + +**Runner turns must outlive the session.** One revision turn was killed by +Claude Code's 2-hour background cap after a network drop; later turns ran under +`nohup` and survived two more drops. + +**Counts.** Baseline `55ccc95`: 5512 passed, 108 skipped (1 wheel-build +intermittent). 84a `64065e6`: 5532. 84b `a8c4f81`: 5547 (+1 intermittent). +84c `4c99b81`: 5551, 108 skipped. Merged `24a28b3`: 5551, 108 skipped. + +## design/91 block 91d — compaction that buys headroom (merged to the operator's fork `main`, 2026-10-06) + +**Replay the real history before asking for another session.** Each of rounds +2 and 3 was predicted off-rig by replaying the previous gate's recorded history +through the new `ConversationStore` — calibrated from that session's own usage +records — and each prediction held: round 3's single compaction (95,316 → +6,925, 8.6 s) is the shape the replay showed. The replay also caught that 84d's +first turn had not implemented D12 at all (the local path returned before trying +a fold whenever the calibrated view fitted). + +**A non-editable install of `main` found what no gate could.** Building a +frozen environment for a measurement exposed that `httpx` was declared only in +the `test` extra while `agent.py` imports it; every gate installed +`.[serve,test]`. A coordinator design claim ("already a direct dependency") was +the source. Install the product the way `install.bat` does at least once per +dependency change. + +**Don't delete an environment a background job is using.** Removing a merged +block's worktree killed a running measurement mid-call (its CA bundle lived in +that venv). Measurements now run from a frozen install outside every worktree. + +**Counts.** `cde64fe`: 5559 passed, 108 skipped; `75fe21b` (round-3 fixes): +5559 passed, 108 skipped. Gate 84 round 3: 9/9. + +## design/91 block 91e — the reasoning level within a session (merged to the operator's fork `main`, 2026-10-22) + +**Measure the cost a notice states, then word it as what was measured.** The +specified disclosure said a level change re-processes the whole prompt. Live, the +first switch did (33,831 tokens) and switching back re-processed 4, because the +server kept the earlier prompt; the notice became an upper bound before the +check, and the check then measured 33,949 against a predicted 33,904, then 173 +and 209 on the returns. + +**A check that reads only what the session writes needs no operator judgement +for its core claim.** The per-call `reasoning_effort` field made the 84e check a +one-command read of the usage file (`84-block84e-check.py`, which reports NOT +EXERCISED on a pre-84e session); the operator's part was what the page showed +and how it felt. + +**Counts.** `e611762`: 5562 passed, 108 skipped. Check: passed, one round. diff --git a/docs/browser-gui.md b/docs/browser-gui.md index 7af7efae..dfd4d0c1 100644 --- a/docs/browser-gui.md +++ b/docs/browser-gui.md @@ -107,3 +107,15 @@ To swap keys mid-session, click the `key …AA8f` chip in the header. Saving wit *Remember on this machine* applies the key to the running process only, so a key already in the credential store comes back on the next start — the UI says so when that is the case. + +With a local model ([Using a local model](local-llm-model.md)), the same controls +serve the local server instead. The key chip holds the server's own key +(`MICROCLAW_LLM_API_KEY`, stored separately from any Anthropic key) and is +optional, because a server started without `--api-key` needs none — so the page +does not ask for a key before you can send. The model chip names the model the +server reports, and a reasoning-level selector beside it (*Server default*, Low, +Medium, High) changes the level for this process only; +`MICROCLAW_LLM_REASONING_EFFORT` sets the level it starts with. Like the model +chip, it refuses mid-turn. The +first reply after switching to a level not used yet this session is slower, and +the page says so when you switch. diff --git a/docs/local-llm-model.md b/docs/local-llm-model.md new file mode 100644 index 00000000..88c72500 --- /dev/null +++ b/docs/local-llm-model.md @@ -0,0 +1,171 @@ +# Using Microclaw with a local model + +This page covers running Microclaw against a large language model served on +the same computer or on another machine on your local network, instead of the +Anthropic API. The examples use Qwen 3.8 served by llama.cpp, which +has been extensively tested, but other OpenAI-compatible servers should +also work. + +## 1. Preparation + +### Serve a local model and check it works + +Microclaw talks to the model through an **OpenAI-compatible chat API** +(`/v1/chat/completions`). That API is provided by the program that +serves the model, not by the model itself: llama.cpp's `llama-server`, +Ollama, LM Studio, vLLM and Strata all provide one. Our testing has +been with Qwen models served by llama.cpp, with one model served by +Strata (see §6); other models and serving programs should work, but +have not been tested. + +llama.cpp can be downloaded from +[GitHub](https://github.com/ggml-org/llama.cpp), and a wide range of models +are available from [Hugging Face](https://huggingface.co/). We ran it from a +shell in the folder holding the model file, with this command line: + +``` sh +llama-server.exe -m Qwen3.8-27B-UD-Q4_K_XL.gguf --host 0.0.0.0 --port 8080 --jinja -c 131072 -np 1 -fa on --cache-type-k q8_0 --cache-type-v q8_0 --api-key testkey +``` + +What each part is for. The first three name the model file and the address +and port the server listens on; `0.0.0.0` means every active network +connection. Beware: that is a security risk, because anyone who can reach the +machine can use the model, which is what `--api-key` guards against. +`--jinja` and `-c` are needed for Microclaw to work properly. The remaining +options tune the server's memory use, which can make room for a larger +context on a given model and GPU. + +| flag | why | +|---|---| +| `-m Qwen3.8-27B-UD-Q4_K_XL.gguf` | The model file. This is a 4-bit quantised Qwen 3.8, small enough to fit entirely on a 24 GB GPU; choose a different file to run a different model. | +| `--host` | `0.0.0.0` listens on every network interface, so other computers can reach the server. If Microclaw runs on the **same** computer, use `127.0.0.1` instead, so only local programs can connect. | +| `--port 8080` | The port Microclaw connects to; it must match the port in `MICROCLAW_LLM_BASE_URL` (§2). | +| `--jinja` | Uses the model's own chat template. Without it the model cannot make proper tool calls, so it chats but never operates the microscope. | +| `-c 131072` | The context size, in tokens. Microclaw's instructions and tool descriptions take about 34,000 tokens before any conversation, so a small context leaves little room; 131,072 leaves about 85,000 for the conversation. | +| `-np 1` | One request slot, so the whole context serves Microclaw's single conversation. | +| `-fa on --cache-type-k q8_0 --cache-type-v q8_0` | Flash attention and a compressed context cache, which roughly halves the cache's memory. This is what let a 131,072-token context fit on a 24 GB card with about 3 GB to spare. | +| `--api-key` | A password for the server. **Always set one when using `--host 0.0.0.0`**, or anyone on your network can use your model. Choose something sensible, not `testkey`; you will need the same key when connecting from Microclaw. | + +When serving to other computers, Windows may ask whether to let `llama-server` +through the firewall; allow it on your private or domain network, +otherwise other computers will not be able to connect and use the model. + +Test the server by opening a web browser at `http://127.0.0.1:8080` on the +model's computer (or `http://:8080` from another one). +You should see llama.cpp's own chat page; type a simple question and check it +answers. + +### Install Microclaw and check it reaches Micro-Manager + +Install Microclaw as described in [Installing on +Windows](install-windows.md); on other operating systems, see +[Developing Microclaw](development.md) for a source install. The Windows +installer checks that Micro-Manager's bridge answers, so a completed install +has already confirmed that Microclaw can reach the microscope; after a source +install, `microclaw check-bridge` does the same check. + +## 2. Settings Microclaw needs + +Three environment variables tell Microclaw to use the local model and +how to connect to it: + +| variable | what it does | +|---|---| +| `MICROCLAW_LLM_BASE_URL` | Where the model server is, e.g. `http://LLM.host:8080`, where `LLM.host` is the name or IP address of the model's computer (`localhost` if the model runs on the same computer as Microclaw). Setting it switches Microclaw to the local model; leaving it unset uses the Anthropic API. The command-line option `--llm-base-url` does the same for one launch. | +| `MICROCLAW_LLM_API_KEY` | The server's `--api-key`. Optional here: you can instead enter the key with the **key** chip at the top of the browser page, which can also save it for later sessions. | +| `MICROCLAW_LLM_REASONING_EFFORT` | The reasoning level to start with: `low`, `medium` or `high` (Qwen's template calls its highest level `xhigh`). Unset or empty means the model's own default. It can be changed during a session in the browser. | + +Set them once as **user environment variables**, so that the desktop icon +picks them up and they survive reinstalling Microclaw. In PowerShell: + +```powershell +[Environment]::SetEnvironmentVariable('MICROCLAW_LLM_BASE_URL', 'http://LLM.host:8080', 'User') +[Environment]::SetEnvironmentVariable('MICROCLAW_LLM_REASONING_EFFORT', 'medium', 'User') +``` + +They apply to programs started afterwards. To remove one, set it to `$null`. + +Do not add them to `updater-launcher.ps1` in the install folder: the installer +replaces that file on every install and reinstall, and your additions would be +lost without warning. + +To set them for one launch from a PowerShell window instead: + +```powershell +$env:MICROCLAW_LLM_BASE_URL = 'http://LLM.host:8080' +$env:MICROCLAW_LLM_REASONING_EFFORT = 'medium' +``` + +## 3. Startup + +Power up the microscope hardware, start Micro-Manager with its usual +configuration, then start Microclaw. On startup it prints a line such as + +``` +[microclaw] Local context budget: n_ctx=131072, fixed=33864, high=84920, low=63690, reasoning_effort=medium +``` + +which confirms the server's context size, how much of it Microclaw's fixed +instructions take, the conversation budget, and the reasoning level in force. + +If you did not set the key through the environment, enter it in the +browser. Until you do, a server started with `--api-key` refuses +Microclaw's questions, so the startup line says `provisional … probe +failures: … HTTP 401`; the context budget is read again as soon as the +key is entered. The browser's model button then names the model being +used, and the control beside it changes the reasoning level. + +## 4. What to expect + +- **The first reply of a session takes about 40 s or more** before anything + appears, while the server reads Microclaw's instructions for the first time. + Later steps reuse that work and start within a second or two. These times + depend on the hardware. +- **Most of the waiting is the model thinking, not the microscope.** In + sessions on a real microscope, the model's generation — mostly reasoning + you do not see — took roughly 75–90 % of the time, and hardware and tool + calls most of the rest. +- **The reasoning level is the main speed control.** On the same microscope + tasks, `medium` was faster than the default (`xhigh` on Qwen) and `low` + faster again (roughly 40 % less time than the default). However, the `low` + setting may produce erroneous results: in one session the model reported an + image as saved that it had never acquired, so we do not currently recommend + `low` for real work. The first switch to a new level costs one slower reply, + about 40 s plus the model's thinking, which the browser tells you before you + send; switching back to a level already used this session is quick. +- **Long sessions are summarised automatically.** When the conversation nears + the budget, Microclaw folds older turns into a summary and the browser says + so, with an estimate of how long the next reply will take to rebuild. +- **A text-only model does not see images.** Where Microclaw would show the + model a thumbnail, a text-only model instead receives a note that the image + was omitted, and works from the numbers Microclaw reports (intensity, signal + to noise, focus metric). Seeing images needs a vision model, served with its + `--mmproj` file. + +## 5. Troubleshooting + +| what you see | likely cause and fix | +|---|---| +| "Local server … rejected its API key" | The key is not set or is wrong: enter it in the browser, or set `MICROCLAW_LLM_API_KEY`. | +| Microclaw cannot connect from another computer | The server was started without `--host 0.0.0.0`, the firewall on the model's computer is blocking port 8080, or the address or port in `MICROCLAW_LLM_BASE_URL` is wrong. | +| The model chats but never operates the microscope | `llama-server` was started without `--jinja`. | +| "Local server … context limit is N tokens; request uses M tokens" | A single very large tool result (a full hook log, say) did not fit. Ask for fewer entries, or start the server with a larger `-c` if memory allows. | +| The startup line says `provisional … probe failures: … 401` | The server needs its key; enter it in the browser and the budget is read again. | + +## 6. Models tested so far + +We tested on two machines running Windows 11: one with an RTX 3070 Ti (8 GB +of VRAM), and a workstation with an RTX A5000 (24 GB of VRAM) and 256 GB of +RAM. The smaller card needs much smaller models; the workstation ran the +larger ones. + +| model | engine | GPU | comment | +|---|---|---|---| +| Qwen3.5-4B-GGUF:Q8_0 | llama.cpp | RTX 3070 Ti, 8 GB | Works, but often misses continued steps of processes. | +| Qwen3.6-27B-Q4_K_M.gguf | llama.cpp | RTX A5000, 24 GB | Works pretty well but can be slow. | +| Qwen3.8-27B-UD-Q4_K_XL.gguf | llama.cpp | RTX A5000, 24 GB | Works pretty well but can be slow. On our workstation, prompt processing ran at about 600–830 tokens/s and generation at 21–30 tokens/s. | +| Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S | Strata | RTX A5000, 24 GB | Significantly faster and pretty reliable responses. | + +Qwen 3.8 Flash Next is a 125-billion-parameter mixture-of-experts model: only +a few of its experts are loaded onto the 24 GB card at once, which makes it +fast yet relatively capable. diff --git a/microclaw/__main__.py b/microclaw/__main__.py index 348c96ae..90a2fa06 100644 --- a/microclaw/__main__.py +++ b/microclaw/__main__.py @@ -24,7 +24,6 @@ from microclaw.conversation import ( AcquisitionDiagnosticWriter, AuditLog, - ConversationStore, atomic_write_text, json_default, load_history, @@ -137,17 +136,26 @@ def run_session(args): parsed_safety = load_safety_config_or_exit(args.safety_config) guard = SafetyGuard(parsed_safety.constraints) - key, source = credentials.load_api_key() - if key is None: + from microclaw.agent import llm_base_url + if llm_base_url(): + key, source = credentials.load_api_key("llm") + from microclaw.agent import set_llm_api_key + if key: set_llm_api_key(key) + else: + key, source = credentials.load_api_key() + if key is None and not llm_base_url(): sys.exit( "No Anthropic API key found. Looked in the ANTHROPIC_API_KEY " "environment variable, the system keyring, and the Microclaw credential file." ) # Keep the agent import lazy for restricted commands, but use the same key # injection path as the browser server before any hardware contact. - from microclaw.agent import set_api_key - set_api_key(key) - print(f"Anthropic API key: {credentials.mask(key)} (from {source})") + if not llm_base_url(): + from microclaw.agent import set_api_key + set_api_key(key) + print(f"Anthropic API key: {credentials.mask(key)} (from {source})") + elif key: + print(f"Local LLM API key: {credentials.mask(key)} (from {source})") print("Connecting to Micro-Manager...") ctrl = MicroscopeController(port=args.port, guard=guard) @@ -187,7 +195,8 @@ def run_session(args): # run_agent returns a fresh list each turn; _repl mutates this one in place # (history[:] = ...) so the finally block always sees the latest turns. history = [] - store = ConversationStore(AuditLog(history_fn_name, enabled=args.save_history)) + from microclaw.agent import make_conversation_store + store = make_conversation_store(AuditLog(history_fn_name, enabled=args.save_history)) confirmation_fn_name = history_fn_name.replace("_history.jsonl", "_confirmations.jsonl") confirmation_audit = AuditLog(confirmation_fn_name, enabled=args.save_history) usage_fn_name = history_fn_name.replace("_history.jsonl", "_usage.jsonl") @@ -448,7 +457,14 @@ def _positive_seconds(value: str) -> float: def main(): - parser = argparse.ArgumentParser(description="Microclaw: AI agent for Micro-Manager") + llm_url_option = argparse.ArgumentParser(add_help=False) + llm_url_option.add_argument( + "--llm-base-url", default=argparse.SUPPRESS, + help="OpenAI-compatible local server base URL (also MICROCLAW_LLM_BASE_URL).", + ) + parser = argparse.ArgumentParser( + description="Microclaw: AI agent for Micro-Manager", parents=[llm_url_option] + ) # Agent-session options stay on the top-level parser so `microclaw --port ...` # (no subcommand) keeps launching a session, as before. parser.add_argument( @@ -509,6 +525,7 @@ def main(): "Drive Microclaw from a browser. Binds 127.0.0.1 only — this " "endpoint moves real hardware." ), + parents=[llm_url_option], ) sv.add_argument("--host", default="127.0.0.1", help="Bind address (default: 127.0.0.1).") sv.add_argument("--web-port", type=int, default=8000, help="HTTP port (default: 8000).") @@ -591,6 +608,8 @@ def main(): ) args = parser.parse_args() + if getattr(args, "llm_base_url", None): + os.environ["MICROCLAW_LLM_BASE_URL"] = args.llm_base_url if args.setup_write_security_config and args.command != "serve": parser.error("--setup-write-security-config is valid only with serve") diff --git a/microclaw/agent.py b/microclaw/agent.py index 9c9e12f6..77b04542 100644 --- a/microclaw/agent.py +++ b/microclaw/agent.py @@ -2,10 +2,13 @@ from collections.abc import Callable, Iterator from typing import Any import json +import math import sys from datetime import datetime, timezone import os import time +import uuid +import httpx import anthropic @@ -19,6 +22,7 @@ from microclaw.skills import catalog_text from microclaw.tools import TOOL_REGISTRY, execute_tool from microclaw.tools_schema import TOOLS_CACHED +from microclaw.conversation import ConversationStore # Latest available Opus at time of writing (verified against the Claude API # model reference). Override per-run with --model or the MICROCLAW_MODEL env var. @@ -31,8 +35,168 @@ _client: anthropic.Anthropic | None = None _known_models: list[str] | None = None +LLM_BASE_URL_ENV = "MICROCLAW_LLM_BASE_URL" +LLM_API_KEY_ENV = "MICROCLAW_LLM_API_KEY" +_local_models: list[str] | None = None +_local_vision: dict[str, bool] = {} +_local_http: httpx.Client | None = None +_local_prefill_rate: float | None = None +_local_fixed_tokens: int | None = None +_local_context_ceiling: int | None = None +LOCAL_PREFILL_MEASUREMENT_MIN_TOKENS = 1000 + + +def _get_local_http() -> httpx.Client: + global _local_http + if _local_http is None: + _local_http = httpx.Client() + return _local_http + + +def llm_base_url() -> str | None: + return os.environ.get(LLM_BASE_URL_ENV, "").rstrip("/") or None + + +def set_llm_api_key(key: str) -> None: + os.environ[LLM_API_KEY_ENV] = key + global _local_models + _local_models = None + _local_vision.clear() + + +def set_local_reasoning_effort(effort: str | None) -> None: + """Set the process-local template effort; None restores the server default.""" + if effort: + os.environ[LOCAL_REASONING_EFFORT_ENV] = effort + else: + os.environ.pop(LOCAL_REASONING_EFFORT_ENV, None) + + +def _local_get(path: str): + url = llm_base_url() + headers = {"Authorization": f"Bearer {os.environ[LLM_API_KEY_ENV]}"} if os.environ.get(LLM_API_KEY_ENV) else {} + return _get_local_http().get(url + path, headers=headers, timeout=30) + + +def _local_vision_enabled(base=None): + base = base or llm_base_url() + if not base: + return False + if base not in _local_vision: + try: + response = _local_get("/props") + response.raise_for_status() + vision = bool(response.json().get("modalities", {}).get("vision")) + except Exception: + return False + _local_vision[base] = vision + return _local_vision[base] MAX_OUTPUT_TOKENS = 8192 +LOCAL_CONTEXT_DEFAULT = 32768 +# When authentication or reachability prevents budget probes, keep 4,096 +# history tokens available: enough to avoid compacting every round, while the +# reported provisional budget is replaced as soon as the probes can answer. +LOCAL_CONTEXT_UNPROBED_HIGH_WATER = 4096 +# 4,096 tokens is a 12% cushion over the measured 33,864-token fixed prompt. +# It covers turn-to-turn knowledge/rig-interview changes and history-estimate +# error; the small measured history error was 3 tokens on a 2,716-token sample. +LOCAL_CONTEXT_MARGIN = 4096 +LOCAL_REASONING_EFFORT_ENV = "MICROCLAW_LLM_REASONING_EFFORT" + + +def local_context_budget(): + """Read the local server budget and vision support, using a provisional budget on failure.""" + global _local_fixed_tokens, _local_context_ceiling + failures = [] + n_ctx = None + try: + response = _local_get("/props") + response.raise_for_status() + n_ctx = response.json().get("default_generation_settings", {}).get("n_ctx") + if not n_ctx: + failures.append("/props did not report n_ctx") + except Exception as exc: + response = getattr(exc, "response", None) + failures.append(f"/props HTTP {response.status_code}" if response is not None + else f"/props {type(exc).__name__}") + if not n_ctx: + try: + response = _local_get("/v1/models") + response.raise_for_status() + models = response.json().get("data", []) + n_ctx = next((model.get("meta", {}).get("n_ctx") for model in models + if model.get("meta", {}).get("n_ctx")), None) + if not n_ctx: + failures.append("/v1/models did not report n_ctx") + except Exception as exc: + response = getattr(exc, "response", None) + failures.append(f"/v1/models HTTP {response.status_code}" if response is not None + else f"/v1/models {type(exc).__name__}") + used_default = not n_ctx + n_ctx = int(n_ctx or LOCAL_CONTEXT_DEFAULT) + vision = _local_vision_enabled() + if llm_base_url() not in _local_vision: + failures.append("/props vision probe did not answer") + body = _local_request_body( + [{"role": "user", "content": "hi"}], _system_blocks(), TOOLS_CACHED, + "budget", vision, + ) + probe_path = "/apply-template" + try: + headers = ({"Authorization": f"Bearer {os.environ[LLM_API_KEY_ENV]}"} + if os.environ.get(LLM_API_KEY_ENV) else {}) + rendered_response = _get_local_http().post( + llm_base_url() + "/apply-template", json=body, timeout=30, headers=headers, + ) + rendered_response.raise_for_status() + probe_path = "/tokenize" + tokenize_response = _get_local_http().post( + llm_base_url() + "/tokenize", + json={"content": rendered_response.json()["prompt"]}, timeout=30, + headers=headers, + ) + tokenize_response.raise_for_status() + fixed = len(tokenize_response.json()["tokens"]) + except Exception as exc: + response = getattr(exc, "response", None) + failures.append(f"{probe_path} HTTP {response.status_code}" if response is not None + else f"{probe_path} {type(exc).__name__}") + # Raw request JSON measured against the 33,864-token rendered prompt; + # 2.36 bytes/token is deliberately conservative for this fallback. + fixed = math.ceil(len(json.dumps(body, ensure_ascii=False, + separators=(",", ":")).encode("utf-8")) / 2.36) + computed_high = n_ctx - fixed - MAX_OUTPUT_TOKENS - LOCAL_CONTEXT_MARGIN + _local_fixed_tokens = fixed + # This is the server-token history ceiling after fixed prompt, output + # reserve, and margin. With successful probes high-water uses this same + # figure in store-estimate units; calibration maps estimates to this ceiling. + _local_context_ceiling = computed_high + floor_used = bool(failures) and computed_high < LOCAL_CONTEXT_UNPROBED_HIGH_WATER + high = ( + max(LOCAL_CONTEXT_UNPROBED_HIGH_WATER, computed_high) + if failures else max(2, computed_high) + ) + low = max(1, int(high * 0.75)) + print(f"[microclaw] Local context budget: n_ctx={n_ctx}" + f"{' (default; server did not report n_ctx)' if used_default else ''}, " + f"fixed={fixed}, high={high}, low={low}" + f", reasoning_effort={os.environ.get(LOCAL_REASONING_EFFORT_ENV) or 'server default'}" + f"{' (provisional 4096-token history floor)' if floor_used else ''}" + f"{'; probe failures: ' + ', '.join(failures) if failures else ''}") + return high, low, vision + + +def make_conversation_store(audit): + """Build the appropriate history budget for the selected backend.""" + if not llm_base_url(): + return ConversationStore(audit) + high, low, vision = local_context_budget() + store = ConversationStore(audit, high_water_tokens=high, + low_water_tokens=low, text_only=not vision) + store.local_fixed_tokens = _local_fixed_tokens + store.local_context_ceiling = _local_context_ceiling + return store def _get_client() -> anthropic.Anthropic: @@ -59,8 +223,15 @@ def set_api_key(key: str) -> None: def resolve_model(model: str | None = None) -> str: - """Pick the model: explicit arg > MICROCLAW_MODEL env > DEFAULT_MODEL.""" - return model or os.environ.get(MODEL_ENV) or DEFAULT_MODEL + """Pick an explicit/configured model, or the backend's first suggestion.""" + if model or os.environ.get(MODEL_ENV): + return model or os.environ.get(MODEL_ENV) + if llm_base_url(): + models = known_models() + # "default" is a neutral OpenAI-compatible selector when the server + # has not exposed an id yet; the UI separately reports unknown state. + return models[0] if models else "default" + return DEFAULT_MODEL def known_models() -> list[str]: @@ -70,7 +241,16 @@ def known_models() -> list[str]: populated must still be usable, so callers keep the field free-text. An unreachable API is not an error here — it just means no suggestions. """ - global _known_models + global _known_models, _local_models + if llm_base_url(): + if _local_models is None: + try: + response = _local_get("/v1/models") + response.raise_for_status() + _local_models = [m["id"] for m in response.json().get("data", [])] + except Exception: + return [] + return _local_models if _known_models is None: try: _known_models = [m.id for m in _get_client().models.list(limit=100).data] @@ -619,6 +799,16 @@ class _BadModel(Exception): """The API does not know this model id. Internal to this module.""" +class _LocalAPIError(Exception): + """A non-retryable local server response suitable for a user error event.""" + + +class _LocalTransient(Exception): + def __init__(self, message: str, retry_after: float = 0): + super().__init__(message) + self.retry_after = retry_after + + def _cancelled(cancel) -> bool: return cancel is not None and cancel.is_set() @@ -724,9 +914,13 @@ def _stream_one_round(messages, system_blocks, model, tool_schemas, for attempt, delay in enumerate([0, *_RETRY_DELAYS]): delay = max(delay, retry_after) if delay: - yield {"type": "retry", "delay": delay, "attempt": attempt} + yield {"type": "retry", "delay": delay, "attempt": attempt, + "service": "Local server" if llm_base_url() else "API"} time.sleep(delay) try: + if llm_base_url(): + return (yield from _stream_local_round(messages, system_blocks, model, tool_schemas, + context_provider, iteration, usage_sink)) model_messages = ( context_provider(messages) if context_provider is not None else messages ) @@ -781,6 +975,23 @@ def _stream_one_round(messages, system_blocks, model, tool_schemas, # A free-text model picker (v4c) can hold an id the API rejects. # Without this it escapes run_agent_iter as a 500 on the SSE stream. raise _BadModel(str(e)) from None + except _LocalTransient as e: + if attempt == len(_RETRY_DELAYS): + raise _TransientAPIError(str(e)) from e + retry_after = e.retry_after + if retry_after > MAX_RETRY_AFTER_SECONDS: + raise _TransientAPIError( + f"Local server {llm_base_url()} requested a {retry_after:g} second retry delay, " + "which is too long to hold this turn open. Please try again later." + ) from e + except _LocalAPIError: + raise + except httpx.TransportError as e: + if attempt == len(_RETRY_DELAYS): + raise _TransientAPIError( + f"Local server {llm_base_url()} is still unavailable after retrying." + ) from e + retry_after = 0.0 except (anthropic.RateLimitError, anthropic.InternalServerError, anthropic._exceptions.OverloadedError, anthropic.APIConnectionError) as e: @@ -802,6 +1013,317 @@ def _stream_one_round(messages, system_blocks, model, tool_schemas, raise RuntimeError("unexpected loop exit") # pragma: no cover +def _local_request_body(model_messages, system_blocks, tool_schemas, model, vision): + """Translate canonical history into one OpenAI-compatible request body.""" + wire = [{ + "role": "system", + "content": "\n\n".join( + block.get("text", "") + for block in system_blocks + if block.get("type") == "text" + ), + }] + for message in model_messages: + role = message["role"] + content = message["content"] + if isinstance(content, str): + wire.append({"role": role, "content": content}) + continue + if role == "assistant": + texts = [] + calls = [] + for block in content: + if isinstance(block, dict): + kind = block.get("type") + get = block.get + else: + kind = getattr(block, "type", None) + get = lambda key: getattr(block, key) + if kind == "text": + texts.append(get("text")) + elif kind == "tool_use": + calls.append({ + "id": get("id"), + "type": "function", + "function": { + "name": get("name"), + "arguments": json.dumps(get("input"), separators=(",", ":")), + }, + }) + else: + raise ValueError(f"Cannot translate assistant content block {kind!r}") + entry = {"role": "assistant", "content": "".join(texts) or None} + if calls: + entry["tool_calls"] = calls + wire.append(entry) + elif role == "user": + images = [] + for block in content: + kind = block.get("type") + if kind == "text": + wire.append({"role": "user", "content": block.get("text", "")}) + continue + if kind != "tool_result": + raise ValueError(f"Cannot translate user content block {kind!r} to the local chat API") + parts = block.get("content", "") + if isinstance(parts, list): + text = "\n".join( + part.get("text", "") + for part in parts + if part.get("type") == "text" + ) + for part in parts: + if part.get("type") != "image": + continue + if vision: + source = part["source"] + images.append({ + "type": "image_url", + "image_url": { + "url": f"data:{source['media_type']};base64,{source['data']}" + }, + }) + if not vision and any(part.get("type") == "image" for part in parts): + omitted = '{"image_omitted": "this model does not accept images"}' + text += ("\n" if text else "") + omitted + else: + text = str(parts) + if block.get("is_error"): + text = "Error: " + text + wire.append({ + "role": "tool", + "tool_call_id": block["tool_use_id"], + "content": text, + }) + if images: + wire.append({"role": "user", "content": images}) + else: + raise ValueError(f"Cannot translate message role {role!r} to the local chat API") + + payload = { + "model": model, + "messages": wire, + "max_tokens": MAX_OUTPUT_TOKENS, + "stream": True, + "stream_options": {"include_usage": True}, + } + # Empty means unset: PowerShell clears a variable by assigning ''. + reasoning_effort = os.environ.get(LOCAL_REASONING_EFFORT_ENV) + if reasoning_effort: + payload["chat_template_kwargs"] = {"reasoning_effort": reasoning_effort} + if tool_schemas: + payload["tools"] = [ + { + "type": "function", + "function": { + "name": tool["name"], + "description": tool.get("description", ""), + "parameters": tool["input_schema"], + }, + } + for tool in tool_schemas + ] + return payload + + +def _local_error(response, base): + response.read() + try: + error = response.json().get("error", {}) + except Exception: + error = {} + if response.status_code == 404: + raise _BadModel(f"Local server {base} does not know the selected model.") + if response.status_code in (429, 503): + try: + retry_after = float(response.headers.get("retry-after", 0)) + except ValueError: + retry_after = 0 + raise _LocalTransient( + f"Local server {base} is still unavailable after retrying.", retry_after + ) + if response.status_code == 400 and error.get("n_ctx"): + raise _LocalAPIError( + f"Local server {base} context limit is {error['n_ctx']} tokens; " + f"request uses {error.get('n_prompt_tokens', 'an unknown number of')} tokens." + ) + if response.status_code == 401: + raise _LocalAPIError( + f"Local server {base} rejected its API key. Set {LLM_API_KEY_ENV} " + "or enter the local key in the browser." + ) + detail = error.get("message", response.content.decode(errors="replace")) + raise _LocalAPIError(f"Local server {base} returned HTTP {response.status_code}: {detail}") + + +def _read_local_stream(response, base, model, iteration, usage_sink, store=None, + reasoning_effort=None): + """Read chat-completion SSE and return the Anthropic-shaped Message.""" + global _local_prefill_rate + text = [] + calls = {} + finish = None + usage = {} + timings = {} + reasoning_length = 0 + for line in response.iter_lines(): + if not line.startswith("data: "): + continue + data = line[6:] + if data == "[DONE]": + continue + chunk = json.loads(data) + if "error" in chunk: + message = chunk["error"].get("message", str(chunk["error"])) + raise _LocalAPIError(f"Local server {base} returned a stream error: {message}") + if not chunk.get("choices"): + usage = chunk.get("usage", {}) + timings = chunk.get("timings", {}) + continue + choice = chunk["choices"][0] + delta = choice.get("delta", {}) + reasoning_length += len(delta.get("reasoning_content") or "") + if delta.get("content"): + text.append(delta["content"]) + yield {"type": "text_delta", "text": delta["content"]} + for call in delta.get("tool_calls", []): + item = calls.setdefault(call["index"], {"id": "", "name": "", "arguments": ""}) + item["id"] += call.get("id", "") + function = call.get("function", {}) + item["name"] += function.get("name", "") + item["arguments"] += function.get("arguments", "") + if choice.get("finish_reason"): + finish = choice["finish_reason"] + + blocks = [{"type": "text", "text": "".join(text)}] if text else [] + for item in calls.values(): + if not item["id"]: + item["id"] = "call_" + uuid.uuid4().hex + try: + tool_input = json.loads(item["arguments"] or "{}") + except json.JSONDecodeError as exc: + tool_input = { + "_microclaw_parse_error": f"Invalid tool arguments JSON: {exc}", + "raw": item["arguments"], + } + blocks.append({ + "type": "tool_use", + "id": item["id"], + "name": item["name"], + "input": tool_input, + }) + + stop_reason = { + "stop": "end_turn", + "tool_calls": "tool_use", + "length": "max_tokens", + }.get(finish) + response = anthropic.types.Message.model_validate({ + "id": "local", + "type": "message", + "role": "assistant", + "model": model, + "content": blocks, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": { + "input_tokens": usage.get("prompt_tokens", 0), + "output_tokens": usage.get("completion_tokens", 0), + "cache_read_input_tokens": (usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0), + }, + }) + fixed = getattr(store, "local_fixed_tokens", None) + estimate = getattr(store, "last_estimated_tokens", 0) + prompt_tokens = usage.get("prompt_tokens") + # A short history magnifies fixed-prompt measurement error. Require at + # least 256 estimated history tokens and complete server prompt timings; + # reject implausible ratios rather than poisoning future fit decisions. + if (store is not None and isinstance(prompt_tokens, (int, float)) + and fixed is not None and estimate >= 256 + and timings.get("prompt_n") and timings.get("prompt_ms", 0) > 0): + history_tokens = prompt_tokens - fixed + ratio = history_tokens / estimate if history_tokens > 0 else None + if ratio is not None and 0.1 <= ratio <= 4.0: + store.local_estimate_ratio = ratio + if usage_sink is not None: + record = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "model": model, + "iteration": iteration, + "stop_reason": stop_reason, + "input_tokens": usage.get("prompt_tokens"), + "output_tokens": usage.get("completion_tokens"), + "cache_read_input_tokens": (usage.get("prompt_tokens_details") or {}).get("cached_tokens"), + "reasoning_characters": reasoning_length, + "reasoning_effort": reasoning_effort, + } + record.update({ + field: timings[field] + for field in ("prompt_n", "cache_n", "prompt_ms", "predicted_n", "predicted_ms") + if field in timings + }) + try: + usage_sink(record) + except Exception as exc: + print(f"[microclaw] Could not record usage: {exc}", file=sys.stderr) + prompt_n = timings.get("prompt_n") + prompt_ms = timings.get("prompt_ms") + if (isinstance(prompt_n, (int, float)) and prompt_n >= LOCAL_PREFILL_MEASUREMENT_MIN_TOKENS + and isinstance(prompt_ms, (int, float)) and prompt_ms > 0): + _local_prefill_rate = prompt_n / (prompt_ms / 1000) + return response + + +def _stream_local_round(messages, system_blocks, model, tool_schemas, + context_provider, iteration, usage_sink): + base = llm_base_url() + reasoning_effort = os.environ.get(LOCAL_REASONING_EFFORT_ENV) or None + vision = _local_vision_enabled(base) + store = getattr(context_provider, "__self__", None) + compactions_before = getattr(store, "compaction_count", 0) + model_messages = context_provider(messages) if context_provider else messages + if getattr(store, "compaction_count", 0) > compactions_before: + before = getattr(store, "last_pre_compaction_estimated_tokens", None) + after = getattr(store, "last_estimated_tokens", None) + # The system+tools prefix stays cached, so the history after the new + # checkpoint is what the server re-processes: that is the store's own + # post-compaction estimate, already text-only-aware for images. + rebuild_tokens = round(after * (getattr(store, "local_estimate_ratio", None) or 1.0)) + yield {"type": "compaction", "estimated_tokens": after, + "before_tokens": before, "after_tokens": after, + "rebuild_tokens": rebuild_tokens, + "expected_rebuild_seconds": (rebuild_tokens / _local_prefill_rate + if _local_prefill_rate else None), + "prefill_tokens_per_second": _local_prefill_rate, + "message": "Predicted time covers the rebuild only; the model then thinks before replying."} + elif getattr(store, "last_compaction_declined", False): + yield {"type": "compaction_declined", + "estimated_tokens": getattr(store, "last_estimated_tokens", None), + "high_water_tokens": getattr(store, "high_water_tokens", None), + "threshold_tokens": getattr(store, "local_context_ceiling", None) + or getattr(store, "high_water_tokens", None), + "message": "Conversation history is above the compaction threshold; " + "recent turns cannot be folded yet, so this request is being sent as is. " + "Any predicted time covers the rebuild only; the model then thinks before replying."} + payload = _local_request_body( + model_messages, system_blocks, tool_schemas, model, vision + ) + headers = {} + key = os.environ.get(LLM_API_KEY_ENV) + if key: + headers["Authorization"] = f"Bearer {key}" + timeout = httpx.Timeout(connect=5.0, read=None, write=30.0, pool=5.0) + with _get_local_http().stream( + "POST", base + "/v1/chat/completions", json=payload, + headers=headers, timeout=timeout, + ) as response: + if response.status_code >= 400: + _local_error(response, base) + return (yield from _read_local_stream( + response, base, model, iteration, usage_sink, store=store, + reasoning_effort=reasoning_effort)) + + def run_agent_iter( user_message: str, ctrl: MicroscopeController, @@ -829,8 +1351,11 @@ def run_agent_iter( to say so whether or not anyone was listening. `context_provider` maps the full history to the bounded view sent to the - model; identity is the default. `on_message` runs immediately after each - append, allowing a durable audit to flush records as they are produced. + model; identity is the default. Local compaction notices detect a changed + `ConversationStore.compaction_count` through `context_provider.__self__`, so + that provider must be the store's bound `model_messages` method. + `on_message` runs immediately after each append, allowing a durable audit + to flush records as they are produced. `confirmation_records`, when supplied by the browser session, is sampled around each dispatch so that tool results report their human decisions. @@ -878,6 +1403,9 @@ def append(message: dict) -> None: yield {"type": "error", "message": f"The API rejected the model '{model}': {e}"} return + if isinstance(e, _LocalAPIError): + yield {"type": "error", "message": str(e)} + return raise append({"role": "assistant", "content": response.content}) @@ -933,6 +1461,14 @@ def append(message: dict) -> None: confirmation_start = ( len(confirmation_records) if confirmation_records is not None else 0 ) + if isinstance(block.input, dict) and "_microclaw_parse_error" in block.input: + result_json = json.dumps({"error": block.input["_microclaw_parse_error"]}) + result_block = {"type": "tool_result", "tool_use_id": block.id, + "is_error": True, "content": result_json} + tool_results.append(result_block) + yield {"type": "tool_result", "tool_use_id": block.id, + "content": result_json, "is_error": True} + continue result_json = execute_tool( block.name, block.input, ctrl, guard, tool_registry, setup_mode=setup_mode, @@ -1033,8 +1569,20 @@ def run_agent( elif event["type"] == "cancelled": reply = event["reason"] elif event["type"] == "retry": + service = f"{event.get('service', 'API')}" print( - f"Anthropic API overloaded — retrying in {event['delay']}s " + f"{service} unavailable — retrying in {event['delay']}s " f"(attempt {event['attempt']}/{len(_RETRY_DELAYS)})..." ) + elif event["type"] == "compaction": + seconds = event.get("expected_rebuild_seconds") + cost = f"about {seconds:.1f}s" if seconds is not None else "time unknown (no measured prefill rate yet)" + print(f"[microclaw] Compacted context: {event.get('before_tokens')} → " + f"{event.get('after_tokens')} estimated tokens; rebuilding about " + f"{event.get('rebuild_tokens')} calibrated tokens, {cost}. " + "Predicted time covers the rebuild only; the model then thinks before replying.") + elif event["type"] == "compaction_declined": + print(f"[microclaw] {event['message']} " + f"({event.get('estimated_tokens')} estimated / " + f"{event.get('threshold_tokens', event.get('high_water_tokens'))} token threshold).") return reply, messages diff --git a/microclaw/conversation.py b/microclaw/conversation.py index b9142290..827dd372 100644 --- a/microclaw/conversation.py +++ b/microclaw/conversation.py @@ -33,7 +33,7 @@ _API_KEY_RE = re.compile(r"\bsk-ant-[A-Za-z0-9_-]{12,}\b") _BEARER_RE = re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{12,}") _SECRET_KEYS = { - "api_key", "anthropic_api_key", "authorization", "bearer_token", + "api_key", "anthropic_api_key", "llm_api_key", "authorization", "bearer_token", "remote_token", "pairing_code", "session_token", } @@ -391,13 +391,16 @@ def prune_transcripts(directory: str | os.PathLike[str], retention_days: int | N return removed -def estimate_tokens(messages: list[dict]) -> int: +def estimate_tokens(messages: list[dict], *, text_only: bool = False) -> int: """Return a local deterministic token *estimate*, never an exact count.""" # 2.36 UTF-8 bytes per token, measured against the API's own accounting on a # live session (design/82, block 82a's gate): billed history tokens ran 1.69x # the old bytes/4 estimate, stable at 1.67-1.71 over 59 calls. # Four was optimistic in the one direction a context guard must not be. - payload = json.dumps(jsonable(messages), ensure_ascii=False, separators=(",", ":")) + serializable = jsonable(messages) + if text_only: + serializable = _without_images(serializable) + payload = json.dumps(serializable, ensure_ascii=False, separators=(",", ":")) return math.ceil(len(payload.encode("utf-8")) / 2.36) + 8 * len(messages) @@ -543,24 +546,42 @@ class ConversationStore: def __init__(self, audit: AuditLog, *, high_water_tokens: int = DEFAULT_CONTEXT_HIGH_WATER_TOKENS, - low_water_tokens: int = DEFAULT_CONTEXT_LOW_WATER_TOKENS): + low_water_tokens: int = DEFAULT_CONTEXT_LOW_WATER_TOKENS, + text_only: bool = False): if not 0 < low_water_tokens < high_water_tokens: raise ValueError("context low-water must be positive and below high-water") self.audit = audit self.high_water_tokens = high_water_tokens self.low_water_tokens = low_water_tokens + self.text_only = text_only self._cut = 0 self._checkpoint: dict | None = None self.compaction_count = 0 self.last_estimated_tokens = 0 + self.last_pre_compaction_estimated_tokens = None + self.last_compaction_declined = False + self._declined_stretch = False + # Set only for the local OpenAI-compatible backend. The ratio is + # server history tokens / the store's estimate of that same view. + self.local_context_ceiling = None + self.local_fixed_tokens = None + self.local_estimate_ratio = None def append(self, message: dict) -> None: self.audit.append(message) def model_messages(self, full_history: list[dict]) -> list[dict]: + self.last_compaction_declined = False current = ([self._checkpoint] if self._checkpoint else []) + full_history[self._cut:] - self.last_estimated_tokens = estimate_tokens(current) + self.last_estimated_tokens = estimate_tokens(current, text_only=self.text_only) + self.last_pre_compaction_estimated_tokens = self.last_estimated_tokens + local_over_ceiling = ( + self.local_context_ceiling is not None + and self.last_estimated_tokens * (self.local_estimate_ratio or 1.0) + > self.local_context_ceiling + ) if self.last_estimated_tokens <= self.high_water_tokens: + self._declined_stretch = False return current complete_ends = _complete_turn_ends(full_history) @@ -574,20 +595,43 @@ def model_messages(self, full_history: list[dict]) -> list[dict]: ends = [end for end in complete_ends if self._cut < end <= floor_boundary] chosen = self._cut + chosen_estimate = self.last_estimated_tokens + best_under_high = None for end in ends: candidate_checkpoint = _checkpoint(full_history[:end], self.audit.secrets) candidate = [candidate_checkpoint, *full_history[end:]] - chosen = end - if estimate_tokens(candidate) <= self.low_water_tokens: + candidate_estimate = estimate_tokens(candidate, text_only=self.text_only) + if candidate_estimate < chosen_estimate: + chosen = end + chosen_estimate = candidate_estimate + if candidate_estimate <= self.high_water_tokens: + best_under_high = (end, candidate_estimate) + if candidate_estimate <= self.low_water_tokens: + chosen = end + chosen_estimate = candidate_estimate break - if chosen == self._cut: - # The in-flight/recent turn alone is over budget. Splitting it would - # break tool integrity, so leave it intact and let the API report its - # real context limit. + if self.local_context_ceiling is not None: + # A low-water fold always wins. Otherwise only the calibrated + # server-fit valve may commit a fold, choosing the deepest one + # that clears estimate-unit high-water. + if chosen_estimate <= self.low_water_tokens and chosen != self._cut: + pass # Commit through the shared block below. + elif local_over_ceiling and best_under_high: + chosen, chosen_estimate = best_under_high + else: + self.last_compaction_declined = not self._declined_stretch + self._declined_stretch = True + return current + if chosen == self._cut or chosen_estimate > self.high_water_tokens: + # Keep the cached prefix when folding cannot get this view under + # high-water; a checkpoint that still triggers next round only + # forces the server to prefill the same history again. + self.last_compaction_declined = not self._declined_stretch + self._declined_stretch = True return current self._cut = chosen self._checkpoint = _checkpoint(full_history[:chosen], self.audit.secrets) self.compaction_count += 1 - compacted = [self._checkpoint, *full_history[chosen:]] - self.last_estimated_tokens = estimate_tokens(compacted) - return compacted + self._declined_stretch = False + self.last_estimated_tokens = chosen_estimate + return [self._checkpoint, *full_history[chosen:]] diff --git a/microclaw/credentials.py b/microclaw/credentials.py index 5f110520..2509080a 100644 --- a/microclaw/credentials.py +++ b/microclaw/credentials.py @@ -1,4 +1,4 @@ -"""Where the Anthropic API key lives when it isn't in the environment. +"""Where Anthropic and local OpenAI-compatible API keys live. `microclaw serve` can collect a key from the browser on first run, so it needs somewhere to put it. Resolution order is env var > OS credential store > a @@ -30,6 +30,13 @@ _KEYRING_USERNAME = "anthropic-api-key" _TOML_KEY = "anthropic_api_key" +LLM_ENV_VAR = "MICROCLAW_LLM_API_KEY" +_LLM_KEYRING_USERNAME = "local-llm-api-key" +_LLM_TOML_KEY = "llm_api_key" +_KEY_CONFIG = { + "anthropic": (ENV_VAR, _KEYRING_USERNAME, _TOML_KEY), + "llm": (LLM_ENV_VAR, _LLM_KEYRING_USERNAME, _LLM_TOML_KEY), +} def config_path() -> Path: @@ -52,7 +59,7 @@ def _keyring(): return keyring -def _read_config_key() -> str | None: +def _read_toml_key(name: str) -> str | None: path = config_path() try: text = path.read_text(encoding="utf-8") @@ -60,50 +67,60 @@ def _read_config_key() -> str | None: return None import tomllib try: - return tomllib.loads(text).get(_TOML_KEY) or None + return tomllib.loads(text).get(name) or None except Exception: return None -def load_stored_key() -> tuple[str | None, str | None]: +def load_stored_key(kind: str = "anthropic") -> tuple[str | None, str | None]: """The persisted key, ignoring the environment: (key, source) or (None, None). Separate from `load_api_key` because a key set for this process only still leaves whatever was persisted earlier in place, to be picked up on the next start. Callers that need to say so have to look past os.environ. """ + _, username, toml_key = _key_config(kind) kr = _keyring() if kr is not None: try: - key = kr.get_password(_KEYRING_SERVICE, _KEYRING_USERNAME) + key = kr.get_password(_KEYRING_SERVICE, username) except Exception: key = None if key: return key, "keyring" - key = _read_config_key() + key = _read_toml_key(toml_key) if key: return key, "file" return None, None -def load_api_key() -> tuple[str | None, str | None]: +def _key_config(kind: str) -> tuple[str, str, str]: + try: + return _KEY_CONFIG[kind] + except KeyError: + raise ValueError(f"Unknown credential kind: {kind!r}") from None + + +def load_api_key(kind: str = "anthropic") -> tuple[str | None, str | None]: """Return (key, source) where source is 'env', 'keyring', 'file' or None.""" - env = os.environ.get(ENV_VAR) + env_name, _, _ = _key_config(kind) + env = os.environ.get(env_name) if env: return env, "env" - return load_stored_key() + return load_stored_key(kind) -def store_api_key(key: str) -> tuple[str, str | None]: +def store_api_key(key: str, kind: str = "anthropic") -> tuple[str, str | None]: """Persist `key`, preferring the OS credential store. Returns (destination, path) where destination is 'keyring' or 'file'. """ + _, username, toml_key = _key_config(kind) kr = _keyring() if kr is not None: try: - kr.set_password(_KEYRING_SERVICE, _KEYRING_USERNAME, key) + kr.set_password(_KEYRING_SERVICE, username, key) return "keyring", None except Exception: pass # fall through to the file @@ -111,13 +128,23 @@ def store_api_key(key: str) -> tuple[str, str | None]: path = config_path() path.parent.mkdir(parents=True, exist_ok=True) # json.dumps gives us a correctly-escaped TOML basic string. - path.write_text(f"{_TOML_KEY} = {json.dumps(key)}\n", encoding="utf-8") + _write_config_key(toml_key, key) if os.name != "nt": - # Meaningful on POSIX; a no-op worth skipping on Windows (see module docstring). os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) return "file", str(path) +def _write_config_key(name: str, key: str) -> None: + path = config_path() + import tomllib + try: + values = tomllib.loads(path.read_text(encoding="utf-8")) + except Exception: + values = {} + values[name] = key + path.write_text("\n".join(f"{k} = {json.dumps(v)}" for k, v in values.items()) + "\n", encoding="utf-8") + + def mask(key: str) -> str: """A suffix, enough to confirm *which* key is set. Never echo the key.""" return "…" + key[-4:] if len(key) > 4 else "…" diff --git a/microclaw/serve.html b/microclaw/serve.html index 3d03e1a4..8dc2b35a 100644 --- a/microclaw/serve.html +++ b/microclaw/serve.html @@ -100,7 +100,10 @@ background: var(--panel); border: 1px solid var(--line); border-left: 3px solid #c0392b; border-radius: 10px; box-shadow: var(--shadow); padding: 10px 14px; font-size: 13px; max-width: min(600px, 90vw); + user-select: text; } + #toast .toast-close { margin-left: 10px; border: 0; background: none; cursor: pointer; + color: var(--ink); font-size: 13px; }
@@ -116,6 +119,10 @@

MicroClaw

+ @@ -162,7 +169,7 @@

Pair this browser