diff --git a/README.md b/README.md index e610cc02b..2e64cf0e4 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,9 @@ bot can also read a page with `web_fetch`, which the daemon runs itself. Put `web_search = "disabled"` in `config.toml` to leave both out of new bots. A message one agent sends another reaches its model with a line naming the sender, so no agent takes it for yours; `senders = "hidden"` leaves that out. +A request Claude's safety classifiers decline ends the turn, unless +`[fallbacks]` names another model for the bot's own; that model then answers +that request alone ([CLI.md](docs/CLI.md)). ## How fast is it? diff --git a/bench/harbor_agent.py b/bench/harbor_agent.py index 9f04be655..153567785 100644 --- a/bench/harbor_agent.py +++ b/bench/harbor_agent.py @@ -115,12 +115,14 @@ class Agent(BaseInstalledAgent): def __init__(self, *args: Any, binary: str | None = None, provider: str | list[str] | None = None, reasoning: str | None = None, - chatgpt_login: str | None = None, **kwargs: Any) -> None: + fallback: str | None = None, chatgpt_login: str | None = None, + **kwargs: Any) -> None: limits = {key: kwargs.pop(key) for key in list(kwargs) if key in self._LIMITS} super().__init__(*args, **kwargs) self._binary = Path(binary or os.environ.get('AGENT_HARBOR_BINARY') or DEFAULT_BINARY) self._providers = [provider] if isinstance(provider, str) else list(provider or []) self._reasoning = reasoning + self._fallback = fallback self._limits = limits # A chatgpt/ model spends the plan through `agent login chatgpt`'s # login, unless a chatgpt spec names another endpoint, which never @@ -202,9 +204,10 @@ def _chatgpt_login(self, now: float | None = None) -> dict[str, Any]: def _command(self, instruction: str) -> str: if not self.model_name: raise ValueError('Model name is required, as PROVIDER/MODEL') - # Benchmarks keep going past a declined Anthropic request on the - # model Anthropic recommends; a fleet opts in per bot. - flags = ['--new', '--bot', BOT, '--agents', '--fallbacks', '--model', self.model_name] + flags = ['--new', '--bot', BOT, '--agents', '--model', self.model_name] + # A request a safety classifier declined goes to this model alone. + if self._fallback: + flags += ['--fallback', self._fallback] for spec in self._providers: flags += ['--provider', spec] if self._reasoning: @@ -344,7 +347,7 @@ def populate_context_post_run(self, context: AgentContext) -> None: context.metadata['bots'] = len({t['bot'] for t in turns}) # A bot the task delegated to chose its own settings. context.metadata['bot_settings'] = { - t['bot']: {'reasoning': t['reasoning'], 'fallbacks': bool(t['fallbacks'])} + t['bot']: {'reasoning': t['reasoning'], 'fallback': t['fallback']} for t in turns} # Two arms asking for one model are matched only if both ran it, so # say what was asked and what answered. Without the store only the @@ -375,7 +378,7 @@ def _store_turns(self) -> list[dict[str, Any]] | None: return [dict(row) for row in db.execute( "SELECT t.bot,COALESCE(t.model,b.provider||'/'||b.model) AS model,t.status," 't.input_tokens,t.cached_input_tokens,t.output_tokens,' - 't.model_rounds,t.retries,t.paced_ms,b.reasoning,b.fallbacks ' + 't.model_rounds,t.retries,t.paced_ms,b.reasoning,b.fallback ' 'FROM turns t JOIN bots b ON b.name=t.bot ORDER BY t.id')] except sqlite3.Error: return None diff --git a/client/src/config.rs b/client/src/config.rs index 0afdd5443..e1dde3ad8 100644 --- a/client/src/config.rs +++ b/client/src/config.rs @@ -9,6 +9,9 @@ //! web_search = "disabled" # "live", the default, lets new agents search and fetch //! senders = "hidden" # "shown", the default, labels what agents send //! +//! [fallbacks] # the model a declined request goes to, by model +//! "anthropic/claude-opus-5-5" = "anthropic/claude-opus-4-8" +//! //! [judge] //! model = "openai/gpt-6-luna" # who judges for the approver and the lookout //! effort = "low" # a model judge's effort @@ -54,6 +57,10 @@ pub struct Config { /// sent it with no line naming the sender: `senders = "hidden"`; /// `"shown"`, the default, names it. pub hide_senders: bool, + /// Each model's fallback, as `provider/model` in its family: a new + /// agent on that model that names none sends a request a safety + /// classifier declined to it, that request alone. + pub fallbacks: BTreeMap, pub judge: Judge, pub sonny: Sonny, } @@ -339,6 +346,14 @@ fn parse(text: &str) -> Result { }; continue; } + if name == "fallbacks" { + let models = value.as_table().ok_or("fallbacks must be a table")?; + for (model, fallback) in models { + let fallback = string(&format!("fallbacks.{model}"), fallback)?; + config.fallbacks.insert(model.clone(), fallback); + } + continue; + } if let Some(top) = match name.as_str() { "model" => Some(&mut config.model), "provider" => Some(&mut config.provider), @@ -686,6 +701,12 @@ mod tests { assert!(!quiet.sonny.chime && quiet.sonny.verbose); assert!(parse("senders = \"hidden\"").unwrap().hide_senders); assert!(!parse("senders = \"shown\"").unwrap().hide_senders); + let fallbacks = parse("[fallbacks]\n\"a/big\" = \"a/other\"\n") + .unwrap() + .fallbacks; + assert_eq!(fallbacks.get("a/big").map(String::as_str), Some("a/other")); + assert!(parse("[fallbacks]\n\"a/big\" = 1\n").is_err()); + assert!(parse("fallbacks = \"a/other\"").is_err()); for (text, error) in [ ( "web_search = \"cached\"", diff --git a/client/src/project.rs b/client/src/project.rs index 1fe27f3c3..692f7f2ac 100644 --- a/client/src/project.rs +++ b/client/src/project.rs @@ -297,6 +297,8 @@ pub struct Picks { pub threads: Option, /// The person's `web_search`, for a coordinator its role gives no tools. pub web_search: crate::config::WebSearch, + /// The person's fallback for each model, from their config. + pub fallbacks: std::collections::BTreeMap, } #[derive(Debug, Default)] @@ -408,6 +410,9 @@ pub async fn create( if let Some(effort) = effort { params["effort"] = json!(effort); } + if let Some(fallback) = picks.fallbacks.get(model) { + params["fallback"] = json!(fallback); + } // The client's settings, such as a sandbox, with the model's. let mut settings = policy["settings"].as_object().cloned().unwrap_or_default(); if let Some(ttl) = crate::policy::cache_ttl(model) { @@ -559,6 +564,7 @@ mod tests { in_project: true, }), web_search: Default::default(), + fallbacks: [("alpha/one".into(), "alpha/two".into())].into(), }; let made = make(&daemon, &dir, &policy, &picks).unwrap(); assert_eq!( @@ -589,6 +595,8 @@ mod tests { &json!("high") ) ); + // The person's fallback for its model. + assert_eq!(create["fallback"], "alpha/two"); assert_eq!( create["instructions"], "rules\n\nEvery task works in this folder, with no worktree of its own." @@ -773,6 +781,7 @@ mod tests { effort: Some("max".into()), threads: None, web_search: Default::default(), + fallbacks: [("alpha/one".into(), "alpha/two".into())].into(), }; let made = make(&daemon, &dir, &json!({"instructions": "rules"}), &picks).unwrap(); assert_eq!( @@ -785,6 +794,8 @@ mod tests { (&create["bot"], &create["model"], &create["effort"]), (&json!("demo.lead"), &json!("alpha/one"), &json!("low")) ); + // The fallback follows the model the file kept, not the pick. + assert_eq!(create["fallback"], "alpha/two"); // A kept file that names no effort takes the picked one. let dir = root("kept-effort"); write(&dir, "plain", "alpha/one", None, &Threads::default()).unwrap(); diff --git a/docs/BEDROCK.md b/docs/BEDROCK.md index 8be784a31..0d31b5f09 100644 --- a/docs/BEDROCK.md +++ b/docs/BEDROCK.md @@ -147,6 +147,7 @@ it bounds the sends, and a fleet cannot start every pre-read at once. | Legacy thinking budget (Haiku 4.5) | yes | **observed** on Mantle | carried | | Thinking-binding check (`anthropic-beta: thinking-binding-controls-2026-08-01`, `block_binding`) | yes (PR #12) | **observed** on both: signed blocks replayed with no drops; on Mantle `block_binding` without the beta is a 400 | carried | | Server-side fallbacks (`fallbacks`, PR #13) | yes | **documented** unsupported; **observed** on Mantle: the field and the beta are each a 400 | not carried: Bedrock requests send neither | +| Fallback credit (`fallback_credit_token`, `fallback-credit-2026-07-01`) | yes | **documented** supported; not tried on Bedrock | not carried: a refused request's retry on the bot's `--fallback` sends no credit, since an unknown beta is a 400 on Mantle | | Responses over WebSocket | yes | **documented** unsupported on either endpoint | refused for Bedrock URLs | | Rate-limit headers for pacing | yes | **documented** absent | escalating refusal backoff instead | | Legacy thinking budgets by model name | yes | ids are vendor- and profile-prefixed | carried: `us.anthropic.claude-haiku-4-5…` reads as `claude-haiku-4-5` | diff --git a/docs/CLI.md b/docs/CLI.md index 2ee84de46..1321b8fdc 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -309,6 +309,15 @@ family is trained on: `apply_patch` for a Responses model, else `edit`, and `~/.agent/config.toml` sets `web_search = "disabled"`. An existing bot keeps its own, so `--tools` while continuing one is a usage error. +`run --fallback PROVIDER/MODEL` gives a new bot a model, in its family, that +answers a request a safety classifier declined, that request alone; the next +goes to the bot's own model. Without it the bot takes its model's entry in +`[fallbacks]` in `~/.agent/config.toml`, if any: + +```toml +[fallbacks] +"anthropic/claude-opus-5-5" = "anthropic/claude-opus-4-8" +``` Other client commands require an already running daemon. `models` prints `~/.agent/models`, the models clients offer, as JSON diff --git a/docs/COMPARISON_CONTRACT.md b/docs/COMPARISON_CONTRACT.md index 8d9776d31..3c886c2b5 100644 --- a/docs/COMPARISON_CONTRACT.md +++ b/docs/COMPARISON_CONTRACT.md @@ -153,8 +153,8 @@ it asked for and what actually answered, each from that harness's own records: response, and its per-model totals, which include any auxiliary model it calls. - **Fallback policy as configured.** Our adapter creates task bots with - `--fallbacks`, so a declined Anthropic request finishes on the model - Anthropic recommends; the flag does nothing on the OpenAI or ChatGPT + `--fallback` when given one, so a request a safety classifier declined is + answered by that model; it does nothing on the OpenAI or ChatGPT providers. A bot the task delegates to sets its own, and the trial metadata's `bot_settings` records each counted bot's choice. Each baseline's flags or settings come from its adapter's source at the pinned diff --git a/docs/DAEMON_MEASUREMENTS.md b/docs/DAEMON_MEASUREMENTS.md index 48cf00556..f36818f0c 100644 --- a/docs/DAEMON_MEASUREMENTS.md +++ b/docs/DAEMON_MEASUREMENTS.md @@ -5717,3 +5717,30 @@ in every bench, skip the look. The Perf workflow's matched screen, base main Store jobs per turn did not change, and every change is within the gate's 15%. Sandboxed bots and the parse itself are not measured: it walks the command's characters once, against a list of at most 64 prefixes. + +## A declined request goes to the bot's fallback, that request alone (#349) + +Every model call now keeps its own provider and model beside the target it +sends to, and takes a parked fallback back on resume; a bot without a +fallback, as in every bench, carries `None` through both. A refusal builds a +fallback only on `stop_reason: "refusal"`. The Perf workflow's matched +screen, base main fc8560a against the PR head 7de1ff4, on the GitHub runner +(2026-10-29): + +| Bench | Measure | Base | Head | Change | +| --- | --- | ---: | ---: | ---: | +| admission | lone submit CPU, ms | 29.67 | 29.48 | -0.6% | +| admission | burst submit p99, ms | 4.079 | 3.971 | -2.7% | +| completion | release to terminal p99, ms | 9.463 | 9.056 | -4.3% | +| footprint | idle RSS, MiB | 18.861 | 19.109 | +1.3% | +| footprint | per idle bot, KiB | 1.652 | 1.372 | -16.9% | +| footprint | RSS after a turn each, MiB | 28.752 | 28.312 | -1.5% | +| footprint | CPU per turn, ms | 0.893 | 0.867 | -2.9% | +| lifecycle | CPU, s | 0.88 | 0.885 | +0.6% | +| lifecycle | p95 turn, ms | 630.609 | 620.821 | -1.6% | +| lifecycle | peak RSS, MiB | 25.453 | 25.283 | -0.7% | + +Store jobs per turn did not change (7.5 on both), and no change is a cost +past the gate's 15%; the per-idle-bot drop is the bench's noise, as the +change adds nothing to an idle bot. The refused path is not measured: it +runs once per refusal, a second request to the provider. diff --git a/docs/HARBOR.md b/docs/HARBOR.md index dac3f0c6b..fdc0772ae 100644 --- a/docs/HARBOR.md +++ b/docs/HARBOR.md @@ -220,8 +220,8 @@ the same model can drive each harness on the same tasks. empty when any model used is missing from the table, rather than reported low. The trial's metadata records the requested model, `served_calls` (billed calls per model, fallbacks and delegated bots included) and - `bot_settings` (each counted bot's reasoning level and whether it takes - fallbacks, since a delegated bot sets its own), so a comparison can check + `bot_settings` (each counted bot's reasoning level and fallback model, + since a delegated bot sets its own), so a comparison can check that both arms ran the same model the same way ([COMPARISON_CONTRACT.md](COMPARISON_CONTRACT.md#task-comparisons)). A call counts under the model the provider named in its response, each @@ -274,10 +274,12 @@ harbor run -d terminal-bench/terminal-bench-2-1 -a strands -m anthropic/claude-f ``` To run on a ChatGPT plan instead of an API key, sign in with `agent login -chatgpt` and name the model `chatgpt/MODEL`, using an id `agent models` lists. Every -task bot is created with `--fallbacks`, so a declined Anthropic request finishes -on the model Anthropic recommends instead of failing the task; a fleet opts in -per bot, and so does a bot the task delegates to. The +chatgpt` and name the model `chatgpt/MODEL`, using an id `agent models` lists. With +`--ak fallback=PROVIDER/MODEL` the task bot is created with `--fallback`, so a +request a safety classifier declined is answered by that model instead of +failing the task; runs before 2026-10-29 created it with `--fallbacks`, which +asked Anthropic for its server-side fallback. A bot the task delegates to sets +its own. The adapter copies the access token, client and expiry from `~/.agent/chatgpt.json` (or `AGENT_CHATGPT_LOGIN`) into each task container, readable only by the agent user, points `AGENT_CHATGPT_LOGIN` at it, and adds `--provider chatgpt`. The @@ -301,7 +303,7 @@ PYTHONPATH=. harbor run -d terminal-bench/terminal-bench-2-1 -a bench.harbor_age CODEX_FORCE_AUTH_JSON=1 harbor run -d terminal-bench/terminal-bench-2-1 -a codex -m MODEL -n 2 -l 5 ``` -Pass `--ak KEY=VALUE` for adapter options: `reasoning`, `max_output_tokens`, +Pass `--ak KEY=VALUE` for adapter options: `reasoning`, `fallback`, `max_output_tokens`, `stall_timeout`, `context_bytes`, `compact_at`, `binary` (another build), `chatgpt_login` (another login file for `chatgpt/` models), and `provider` (a `--provider` spec for a gateway such as Bedrock, whose named key diff --git a/docs/RUST_PROTOTYPE.md b/docs/RUST_PROTOTYPE.md index 183f748ea..dd31ea4ee 100644 --- a/docs/RUST_PROTOTYPE.md +++ b/docs/RUST_PROTOTYPE.md @@ -271,7 +271,7 @@ a process budget of two, because waiters hold nothing. ## Accounting and budgets Provider-reported usage records a durable `usage` event (with a per-model -`models` split when a provider-side fallback ran another model and on every +`models` split when a fallback ran another model and on every summarizer call, naming the summarizer's provider and model, and `cache_write_tokens` when Anthropic wrote input to its prompt cache: part of `input_tokens`, billed above the base rate, and kept only in the event since @@ -721,7 +721,7 @@ Nothing else in the request differs, since the cache is keyed on everything it renders. A refresh is paced and admitted like a call, but gives up waiting for its pool and startup admission once the cache it would refresh has expired, and after 30 seconds in any case. Usage is read per attempt, as for -a call, when a server-side fallback served it. A refused +a call. A refused refresh publishes a non-durable `keep_warm_failed` event and ends refreshes until the next model call, which pays the write it would have paid anyway. @@ -751,45 +751,52 @@ between turns may never be called again. The Responses cache needs none: the ChatGPT backend still read 7.4k of 7.7k tokens after a 20-minute gap, and it refuses `prompt_cache_retention: "24h"` with a 400. -A bot created with `--fallbacks` (`create.fallbacks`, inherited by its forks) -opts its Anthropic requests into server-side fallbacks -(`fallbacks: "default"` with the `server-side-fallback-2026-07-01` beta header); -other bots send neither, and a declined request ends their turn with -`provider_refusal` as before. It is a bot's choice because a fallback answers -with a model the caller did not pick and bills at that model's rate. The -Harbor adapter asks for it, since a benchmark task is better finished on the -recommended model than failed. +A bot created with `--fallback PROVIDER/MODEL` (`create.fallback`, a known +provider's model in the bot's family, inherited by its forks) sends a request +a safety classifier declined (`stop_reason: "refusal"`) to that model, on the +same body, and only that request: the next goes to the bot's own model again, +with the fallback's answer in its history. Other bots end the turn with +`provider_refusal` as before, and so does a refusal the fallback repeats. It +is a bot's choice because a fallback answers with a model the caller did not +pick and bills at that model's rate. A client may name one for each model: +the CLI and Sonny give a new bot that names none the person's entry for its +model from `[fallbacks]` in `~/.agent/config.toml`. + +The retry is published as a `retry` event with the refusal as its error and +`fallback` naming the model, and the answer as a non-durable `model_fallback` +event with `from` and `to`; its `usage` event names the fallback's provider +and model in `models`, so it is priced at that model's rates. On the Claude +API, when the fallback is on the bot's own provider, the retry redeems the +refusal's `stop_details.fallback_credit_token` (the +`fallback-credit-2026-07-01` beta), so the fallback's cache writes are priced +as if it had run there all along; a 400 to the credit sends the same retry +without it. Bedrock requests send no credit: the beta is documented there, but +Mantle answered a beta it did not take with a 400 +([BEDROCK.md](BEDROCK.md)), and this one has not been tried there. + +Anthropic's server-side fallback (`fallbacks: "default"`) is not asked for: +its sticky routing sends the same conversation to the fallback model for about +an hour, where this one goes back to the bot's model at once. Its +`server-side-fallback-2026-07-01` beta is still sent on the Claude API, so a +history that holds an answer it served replays: a `fallback` content block +marks each switch. Blocks before the last one are a declined attempt's partial +output: its non-empty text is kept as part of the answer, while its thinking +and tool calls are neither stored, replayed, nor run. The marker is stored in +place, since the API checks the thinking around it by its position, and the +response's per-attempt `usage.iterations`, when present, replace its top-level +usage. + The protocol described here is from Anthropic's documentation, read -2026-09-25: [Refusals and fallback](https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback) -for the stream, the `usage.iterations` entries, billing and the replay rules, -and Anthropic's model migration guide (the Claude Opus 5 and Claude Fable 5.1 -sections) for the `"default"` form and its beta header. Only model acceptance -of the field was observed live; the fallback stream itself is exercised against -a synthetic endpoint built from that description. With it, a request a safety -classifier declines is rerun, on the same stream, on the -model Anthropic recommends for that refusal category instead of ending the turn -with `provider_refusal`. A `fallback` content block marks each switch. Blocks -before the last one are a declined attempt's partial output: its non-empty text -is kept as part of the answer, while its thinking and tool calls are neither -stored, replayed, nor run. The marker is stored in place, since the API checks -the thinking around it by its position. Each switch is published as a -non-durable `model_fallback` event with `from` and `to`. When the response's -per-attempt `usage.iterations` list is present it replaces the top-level usage, -which covers only the last attempt: an attempt declined before any output is -reported but not billed, so it is not counted, and every other attempt is -billed at the rates of the model that ran it. The `usage` event then carries a -`models` list with each billed attempt, including a turn that sticky routing -(about an hour after a fallback) sent straight to the fallback model, so -clients can price them per model; its totals are their sum. A refusal that -survives the fallbacks still fails the turn with `provider_refusal`, with the -models it switched to, the refusal category and any recommended retry model as -its detail. Bedrock, -Vertex and Foundry do not offer server-side fallback, so Bedrock requests send -neither the field nor its beta header. Observed 2026-09-24: -`claude-sonnet-5`, `claude-haiku-4-5` and `claude-sonnet-4-5` accept the field, -and `claude-opus-5-5`, `claude-opus-5` and `claude-fable-5-1` answer normally -with it set; a live refusal was not reproduced, so the fallback path is covered -by parser and synthetic-endpoint tests. +2026-10-29: [Refusals and fallback](https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback) +and [Handle streaming refusals](https://platform.claude.com/docs/en/test-and-evaluate/strengthen-guardrails/handle-streaming-refusals), +which lists Bedrock. A refusal may follow partial output, which is discarded; +its `stop_details` names a `category` (kept in the error's detail and facts) +and may name a recommended model and a credit. The fallback's thinking blocks +are dropped by the API when the bot's model reads them back +(`input_transformations` reason `model_binding_mismatch`), which is not +counted as a binding drop; only `prefix_binding_mismatch` is. A live refusal +was not reproduced, so the fallback path is covered by parser and +synthetic-endpoint tests. `effort` (`low`, `medium`, `high`, `xhigh`, `max`) maps to Responses `reasoning.effort` with summaries requested, and to Anthropic adaptive thinking @@ -2780,9 +2787,7 @@ rounds (a copy and a request of its own) and, under a budget, the most those two may bill: each sending at most the view, the tools and both instructions at a token to a byte, and generating at most the summarizer's output bound; with no known output bound (a Responses provider without -`max_output_tokens`), or with server-side fallbacks on, which may bill any -number of attempts per request, a budgeted bot's summary does not run -beside. A summary beside the call that an interrupt cancels still adds its +`max_output_tokens`), a budgeted bot's summary does not run beside. A summary beside the call that an interrupt cancels still adds its retries and pacing to the turn's. A refresh does not keep warm the view of a call whose summary has written its replacement. Otherwise, and at a turn's start or as a parked turn resumes, when the diff --git a/sonny/src/calls.rs b/sonny/src/calls.rs index f2ebe9d21..23eb5eafc 100644 --- a/sonny/src/calls.rs +++ b/sonny/src/calls.rs @@ -159,7 +159,6 @@ const SWITCHES: &[&str] = &[ "--no-spawn", "--agents", "--no-compaction", - "--fallbacks", "--discover", "--all", "--any", diff --git a/sonny/src/main.rs b/sonny/src/main.rs index 1c14bd544..74eca236e 100644 --- a/sonny/src/main.rs +++ b/sonny/src/main.rs @@ -986,6 +986,9 @@ async fn create(client: &Client, options: &Options, bot: &str) -> Result<(), Str settings.insert("hide_senders".into(), json!(true)); } asked["settings"] = Value::Object(settings); + if let Some(fallback) = config.fallbacks.get(&model) { + asked["fallback"] = json!(fallback); + } let tools: Vec = tools.into_iter().map(String::from).collect(); let tools = approvals::gateable(&tools, &declare); if let Value::Object(fields) = &mut asked { diff --git a/sonny/src/project.rs b/sonny/src/project.rs index ea505e429..e759e9ec0 100644 --- a/sonny/src/project.rs +++ b/sonny/src/project.rs @@ -527,6 +527,7 @@ async fn create(client: &Client, dir: &Path, model: Option<&str>) -> Result<(), model: Some(model), effort: (config.sonny.lead_effort.clone()).or(effort.filter(|e| !e.is_empty())), web_search: config.web_search, + fallbacks: config.fallbacks, ..Default::default() }; make(client, dir, &role, &picks).await.map(|_| ()) @@ -583,6 +584,7 @@ pub async fn add(caller: &str, arguments: &Value) -> Result { picks.effort = config.sonny.lead_effort; } picks.web_search = config.web_search; + picks.fallbacks = config.fallbacks; let socket = std::env::var_os("AGENT_SOCKET") .map(PathBuf::from) .ok_or("AGENT_SOCKET is not set")?; diff --git a/src/cli.rs b/src/cli.rs index 9cc122c59..714681c80 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -21,7 +21,7 @@ const COMMANDS: &[Command] = &[ about: "Send a prompt to a bot, or make one with --new; prints the turn as it runs", usage: "run [OPTIONS] [--] PROMPT...", flags: "--bot --new --fork --allow --expires --detach --delivery --turn --model --workspace --effort --turn-budget-tokens --image --agents --profile --request-id --bot-id --pretty --no-spawn", - settings: "--tools --instructions --instructions-file --budget-tokens --compaction-instructions --compaction-instructions-file --compaction-model --no-compaction --fallbacks --approval --approve --context-bytes --context-items --note-turns --compact-at --compact-keep --compact-tokens --keep-turns --approval-hold --max-output-tokens --keep-warm --cache-ttl --sandbox --writable --network --hide-senders --unsandboxed", + settings: "--tools --instructions --instructions-file --budget-tokens --compaction-instructions --compaction-instructions-file --compaction-model --no-compaction --fallback --approval --approve --context-bytes --context-items --note-turns --compact-at --compact-keep --compact-tokens --keep-turns --approval-hold --max-output-tokens --keep-warm --cache-ttl --sandbox --writable --network --hide-senders --unsandboxed", startup: true, }, Command { @@ -392,9 +392,9 @@ fn print_flags(flags: &str) { "COMMAND", "A command, by its first words, a sandboxed bot runs outside its sandbox without asking; repeat for more", ), - "--fallbacks" => ( - "", - "A new bot's declined Anthropic requests rerun on the recommended model", + "--fallback" => ( + "PROVIDER/MODEL", + "A new bot's model for a request a safety classifier declined, that request alone; default none", ), "--grace" => ( "DURATION", @@ -590,7 +590,6 @@ pub fn prepare(args: Vec) -> Result>> { | "--newest" | "--no-compaction" | "--agents" - | "--fallbacks" | "--network" | "--hide-senders" | "--discover" @@ -717,7 +716,7 @@ pub fn prepare(args: Vec) -> Result>> { "--agents", "--profile", "--budget-tokens", - "--fallbacks", + "--fallback", "--sandbox", "--writable", "--network", diff --git a/src/client.rs b/src/client.rs index ccd9152d1..afae8e1af 100644 --- a/src/client.rs +++ b/src/client.rs @@ -77,7 +77,7 @@ struct Options { /// own, or none with --no-compaction. compaction_instructions: Option, compaction_model: Option, - fallbacks: bool, + fallback: Option, discover: bool, after: i64, /// Listing filters: a GLOB on bot names, bots with a turn running, @@ -161,7 +161,7 @@ fn parse(args: &[String]) -> Result { bot_id: None, compaction_instructions: Some(DEFAULT_COMPACTION_INSTRUCTIONS.to_owned()), compaction_model: None, - fallbacks: false, + fallback: None, discover: false, after: 0, name: None, @@ -214,7 +214,6 @@ fn parse(args: &[String]) -> Result { "--new" => options.new = true, "--agents" => options.agents = true, "--no-compaction" => options.compaction_instructions = None, - "--fallbacks" => options.fallbacks = true, "--discover" => options.discover = true, "--detach" => options.detach = true, "--all" => options.all = true, @@ -295,6 +294,7 @@ fn parse(args: &[String]) -> Result { })?) } "--compaction-model" => options.compaction_model = Some(value), + "--fallback" => options.fallback = Some(value), "--effort" => options.effort = Some(value), "--workspace" => options.workspace = Some(value.into()), "--bot" => options.bot = Some(value), @@ -1399,6 +1399,11 @@ fn run(options: &Options) -> Result { "a new bot needs a model: pass --model PROVIDER/MODEL, set AGENT_MODEL, or set model in ~/.agent/config.toml", ))?, }; + // --fallback, else the person's for its model. + let fallback = match &options.fallback { + Some(fallback) => Some(fallback.clone()), + None => config()?.fallbacks.remove(&model), + }; let instructions = composed_instructions(options, workspace, role.as_ref())?; let (created_by, created_by_id) = created_by()?; // --tools, else the role's, else the default set for its model. @@ -1427,7 +1432,7 @@ fn run(options: &Options) -> Result { "budget_tokens":options.budget_tokens,"tools":tools, "created_by":created_by,"created_by_id":created_by_id, "compaction_instructions":options.compaction_instructions, - "compaction_model":options.compaction_model,"fallbacks":options.fallbacks, + "compaction_model":options.compaction_model,"fallback":fallback, "settings":settings,"request_id":request_id}); if let Value::Object(gate) = requested_gate(options, &tools)? { // Its approver first, so a missing judge leaves no bot behind. diff --git a/src/provider.rs b/src/provider.rs index 54ca7ad53..443594c85 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -105,12 +105,13 @@ impl Refresh { } /// Betas every Anthropic API request opts into: the thinking-binding check, -/// dropping rather than failing on a mismatch (the drops are reported), and -/// server-side fallbacks, which rerun a declined request on another model -/// instead of ending the turn with a refusal, which a bot opts into. Bedrock -/// and bots that did not ask take only the first. -const ANTHROPIC_BETAS: &str = - "thinking-binding-controls-2026-08-01,server-side-fallback-2026-07-01"; +/// dropping rather than failing on a mismatch (the drops are reported); +/// fallback credit, which a refusal mints so a retry on another model is +/// priced as if it had run there all along; and server-side fallbacks, whose +/// blocks a history from before the daemon retried on its own may still +/// hold. No request asks for a server-side fallback: it sticks to the +/// fallback model for later requests. Bedrock takes only the first. +const ANTHROPIC_BETAS: &str = "thinking-binding-controls-2026-08-01,fallback-credit-2026-07-01,server-side-fallback-2026-07-01"; const BEDROCK_BETAS: &str = "thinking-binding-controls-2026-08-01"; /// HTTP connections and the startup-admission budget shared by every @@ -212,8 +213,8 @@ pub struct Provider { sockets: Option>, /// Bedrock signs every request with SigV4 instead of sending a key. aws: Option>, - /// A Bedrock endpoint, however it authenticates: it runs no server-side - /// fallbacks and serves no WebSocket. + /// A Bedrock endpoint, however it authenticates: it takes no fallback + /// credit and serves no WebSocket. bedrock: bool, /// The models this provider last listed, shared by every clone. listing: Arc, @@ -292,7 +293,8 @@ pub struct Completion { /// Replayed thinking blocks the provider dropped because the history /// before them changed. The runtime avoids this, so any is a bug. pub thinking_dropped: usize, - /// Each model switch a provider-side fallback made, as (from, to). + /// Each model switch a fallback made, as (from, to): one a history from + /// before the daemon retried on its own still holds, or the daemon's. pub fallbacks: Vec<(Option, String)>, /// The provider paused a long run of its own tools (Anthropic's /// `pause_turn`): sending the history again, reply included, resumes it. @@ -356,8 +358,9 @@ pub struct Request<'a> { pub allow_tool_calls: bool, /// Offer the provider's web search beside `tools`, as it declares it. pub web_search: bool, - /// Anthropic server-side fallbacks for a declined request; Bedrock has none. - pub fallbacks: bool, + /// The credit a refusal minted, sent with this retry of the refused + /// request on another model. Bedrock takes none. + pub fallback_credit: Option<&'a str>, /// Groups calls that share a prefix for the Responses prompt cache. All /// bots share their leading instructions, so without a key their calls /// route by that prefix alone, pile onto the same cache machines and @@ -721,11 +724,8 @@ impl Provider { thinking: Option, #[serde(skip_serializing_if = "Option::is_none")] output_config: Option, - /// A request a safety classifier declines is rerun on the model - /// Anthropic recommends for that refusal category. Bedrock does - /// not run server-side fallbacks. #[serde(skip_serializing_if = "Option::is_none")] - fallbacks: Option<&'static str>, + fallback_credit_token: Option<&'a str>, } let search = request .web_search @@ -824,7 +824,7 @@ impl Provider { .reasoning .filter(|_| !legacy_thinking(request.model)) .map(|level| json!({"effort":level})), - fallbacks: (!self.bedrock && request.fallbacks).then_some("default"), + fallback_credit_token: request.fallback_credit.filter(|_| !self.bedrock), })?, &b",\"messages\":["[..], ), @@ -901,7 +901,7 @@ impl Provider { .header("content-length", len) .header("accept", "application/json") .body(body); - let http = self.anthropic_headers(http, self.key.as_ref(), request.fallbacks); + let http = self.anthropic_headers(http, self.key.as_ref()); if !refresh.send() { return fail("keep_warm_cancelled"); } @@ -977,14 +977,12 @@ impl Provider { &self, http: reqwest::RequestBuilder, key: Option<&String>, - fallbacks: bool, ) -> reqwest::RequestBuilder { let http = http.header("anthropic-version", "2023-06-01").header( "anthropic-beta", - if self.bedrock || !fallbacks { - BEDROCK_BETAS - } else { - ANTHROPIC_BETAS + match self.bedrock { + true => BEDROCK_BETAS, + false => ANTHROPIC_BETAS, }, ); match key { @@ -1124,7 +1122,7 @@ impl Provider { .body(body); http = match (self.family, key) { (Family::Responses, Some(key)) => http.bearer_auth(key), - (Family::Anthropic, key) => self.anthropic_headers(http, key, request.fallbacks), + (Family::Anthropic, key) => self.anthropic_headers(http, key), (Family::Responses, None) => http, }; // The ChatGPT Codex endpoint takes cache affinity from this header, not @@ -2046,7 +2044,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2078,7 +2076,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: true, + fallback_credit: Some("credit"), cache_key: Some("k"), items: Items::empty(), chain: None, @@ -2091,8 +2089,10 @@ mod tests { assert!(text.ends_with(",\"messages\":[")); assert!(text.contains("\"type\":\"adaptive\"")); assert!(text.contains("\"block_binding\":{\"prefix_mismatch_behavior\":\"drop_block\"}")); - assert!(text.contains("\"fallbacks\":\"default\"")); - // Off unless the bot asked: neither the field nor its beta header. + // A retry on another model redeems the refusal's credit; no request + // asks for a server-side fallback, which would stick to its model. + assert!(text.contains("\"fallback_credit_token\":\"credit\"")); + assert!(!text.contains("fallbacks"), "{text}"); let plain = String::from_utf8( provider .prefix(&Request { @@ -2102,7 +2102,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2112,7 +2112,7 @@ mod tests { .unwrap(), ) .unwrap(); - assert!(!plain.contains("fallbacks"), "{plain}"); + assert!(!plain.contains("fallback"), "{plain}"); assert_eq!(text.matches("\"cache_control\"").count(), 2); assert!(text.contains("\"effort\":\"low\"")); let legacy = provider @@ -2123,7 +2123,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: true, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2136,7 +2136,6 @@ mod tests { assert!(legacy.contains("\"max_tokens\":64000")); assert!(text.contains("\"max_tokens\":128000")); assert!(legacy.contains("\"prefix_mismatch_behavior\":\"drop_block\"")); - assert!(legacy.contains("\"fallbacks\":\"default\"")); assert!(!legacy.contains("output_config")); assert!(!text.contains("budget_tokens")); let mut empty_prefix = provider @@ -2147,7 +2146,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2174,7 +2173,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: Some("k"), items: Items::empty(), chain: None, @@ -2210,7 +2209,7 @@ mod tests { tools, allow_tool_calls: allow, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2293,7 +2292,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2369,10 +2368,10 @@ mod tests { ); } - /// Bedrock runs no server-side fallbacks, so its requests do not ask for - /// them, whichever endpoint and however they authenticate. + /// Bedrock takes no fallback credit, so a retry there redeems none, + /// whichever endpoint and however it authenticates. #[test] - fn bedrock_requests_ask_for_no_server_side_fallbacks() { + fn bedrock_requests_redeem_no_fallback_credit() { let transport = Transport::new(64, 1).unwrap(); for (url, key) in [ ( @@ -2393,7 +2392,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: true, + fallback_credit: Some("credit"), cache_key: None, items: Items::empty(), chain: None, @@ -2402,7 +2401,7 @@ mod tests { }) .unwrap(); let text = String::from_utf8(prefix).unwrap(); - assert!(!text.contains("fallbacks"), "{text}"); + assert!(!text.contains("fallback"), "{text}"); assert!(text.contains("\"block_binding\""), "{text}"); } } @@ -2427,7 +2426,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2470,7 +2469,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2537,7 +2536,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2714,7 +2713,7 @@ mod tests { ) .unwrap(); let request = provider - .anthropic_headers(reqwest::Client::new().post(base), Some(&key), false) + .anthropic_headers(reqwest::Client::new().post(base), Some(&key)) .build() .unwrap(); assert_eq!(request.headers()[header], value, "{base}"); @@ -2753,7 +2752,7 @@ mod tests { tools: &none(), allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items, chain: None, @@ -2844,7 +2843,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2912,7 +2911,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -2981,7 +2980,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -3055,7 +3054,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -3164,7 +3163,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: Some(Chain { @@ -3252,7 +3251,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: Some(Chain { @@ -3304,7 +3303,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -3355,7 +3354,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -3410,7 +3409,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, @@ -3463,7 +3462,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, items: Items::empty(), chain: None, route: None, @@ -3531,7 +3530,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, items: Items::empty(), chain: None, route: None, @@ -3581,7 +3580,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, items: Items::empty(), chain: None, route: None, @@ -3627,7 +3626,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, items: Items::empty(), chain: None, route: None, @@ -3664,7 +3663,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, items: Items::empty(), chain: None, route: None, @@ -3713,7 +3712,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search: false, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, diff --git a/src/provider/anthropic.rs b/src/provider/anthropic.rs index af894efe6..1e1ef8d97 100644 --- a/src/provider/anthropic.rs +++ b/src/provider/anthropic.rs @@ -104,6 +104,9 @@ pub struct State { thinking_dropped: usize, fallbacks: Vec<(Option, String)>, stop_details: Option, + /// What a refusal names for a retry: its category and the credit that + /// prices a retry on another model as if it had run there all along. + refusal: serde_json::Map, /// The request cached for an hour: see [`usage`]. hour: bool, } @@ -287,6 +290,11 @@ impl State { .flatten() .collect(); self.stop_details = (!parts.is_empty()).then(|| parts.join("; ")); + for key in ["category", "fallback_credit_token"] { + if let Some(value) = details[key].as_str() { + self.refusal.insert(key.to_owned(), json!(value)); + } + } } // Sent again after a server-side fallback. self.dropped(&event["input_transformations"]); @@ -369,9 +377,13 @@ impl State { /// The thinking blocks the request lost to the binding check. fn dropped(&mut self, transformations: &Value) { if let Some(list) = transformations.as_array() { + // Only an edit under replayed thinking; a block another model + // wrote, as after a fallback, is dropped as the API means it to. self.thinking_dropped = list .iter() - .filter(|t| t["type"] == "thinking_dropped") + .filter(|t| { + t["type"] == "thinking_dropped" && t["reason"] == "prefix_binding_mismatch" + }) .count(); } } @@ -396,10 +408,14 @@ impl State { .map(|(_, to)| format!("fell back to {to}")) .chain(self.stop_details) .collect(); - return match parts.is_empty() { - true => fail("provider_refusal"), - false => fail_with("provider_refusal", parts.join("; ")), + let error = match parts.is_empty() { + true => Error::new("provider_refusal"), + false => Error::with("provider_refusal", parts.join("; ")), }; + return Err(match self.refusal.is_empty() { + true => error, + false => error.facts(Value::Object(self.refusal)), + }); } other => { return fail_with( @@ -674,15 +690,17 @@ mod tests { } #[test] fn dropped_thinking_is_counted_from_the_latest_report() { - let start = r#"{"type":"message_start","message":{"usage":{"input_tokens":1},"input_transformations":[{"type":"thinking_dropped","message_index":1,"block_index":0},{"type":"other"}]}}"#; + let start = r#"{"type":"message_start","message":{"usage":{"input_tokens":1},"input_transformations":[{"type":"thinking_dropped","path":"messages.1.content.0","reason":"prefix_binding_mismatch"},{"type":"other"}]}}"#; let text = r#"{"type":"content_block_start","index":0,"content_block":{"type":"text","text":"ok"}}"#; let delta = r#"{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}"#; let stop = r#"{"type":"message_stop"}"#; let mut state = State::default(); feed(&mut state, &[start, text, delta, stop]); assert_eq!(state.finish().unwrap().thinking_dropped, 1); - // After a fallback the delta repeats the whole list for the final request. - let fallback = r#"{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1},"input_transformations":[{"type":"thinking_dropped"},{"type":"thinking_dropped"}]}"#; + // After a fallback the delta repeats the whole list for the final + // request. A block another model wrote is dropped as the API means + // to, and an unknown reason is not counted either. + let fallback = r#"{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1},"input_transformations":[{"type":"thinking_dropped","reason":"prefix_binding_mismatch"},{"type":"thinking_dropped","reason":"prefix_binding_mismatch"},{"type":"thinking_dropped","reason":"model_binding_mismatch"},{"type":"thinking_dropped","reason":"later"}]}"#; let mut state = State::default(); feed(&mut state, &[start, text, fallback, stop]); assert_eq!(state.finish().unwrap().thinking_dropped, 2); @@ -798,6 +816,36 @@ mod tests { error.detail.as_deref(), Some("category cyber; retry on claude-opus-4-8") ); + assert_eq!( + serde_json::to_value(error.facts.unwrap()).unwrap(), + json!({"category":"cyber"}) + ); + // The credit for a retry elsewhere is a fact, not part of the detail. + let mut state = State::default(); + feed( + &mut state, + &[ + r#"{"type":"message_delta","delta":{"stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber","fallback_credit_token":"credit","fallback_has_prefill_claim":false}},"usage":{"output_tokens":0}}"#, + r#"{"type":"message_stop"}"#, + ], + ); + let error = state.finish().unwrap_err(); + assert_eq!(error.detail.as_deref(), Some("category cyber")); + assert_eq!( + serde_json::to_value(error.facts.unwrap()).unwrap(), + json!({"category":"cyber","fallback_credit_token":"credit"}) + ); + // A refusal naming nothing has no facts. + let mut state = State::default(); + feed( + &mut state, + &[ + r#"{"type":"message_delta","delta":{"stop_reason":"refusal","stop_details":{"type":"refusal","category":null}},"usage":{"output_tokens":0}}"#, + r#"{"type":"message_stop"}"#, + ], + ); + let error = state.finish().unwrap_err(); + assert_eq!((error.detail, error.facts), (None, None)); // A chain that switched models and still declined names the switch. let mut state = State::default(); feed( diff --git a/src/provider/search.rs b/src/provider/search.rs index fe9352935..5f7f959ac 100644 --- a/src/provider/search.rs +++ b/src/provider/search.rs @@ -308,7 +308,7 @@ mod tests { tools: &tools, allow_tool_calls: true, web_search, - fallbacks: false, + fallback_credit: None, cache_key: None, items: Items::empty(), chain: None, diff --git a/src/server/mod.rs b/src/server/mod.rs index 012b6bea2..52ca2501d 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -110,6 +110,7 @@ fn admission_bound(command: &Command, id: &Value, deliveries: usize) -> Bound { created_by, compaction_instructions, compaction_model, + fallback, declare, settings, .. @@ -126,6 +127,7 @@ fn admission_bound(command: &Command, id: &Value, deliveries: usize) -> Bound { created_by, compaction_instructions, compaction_model, + fallback, )) .map(|bytes| { bytes @@ -218,9 +220,9 @@ enum Command { /// the client's; without instructions the bot never compacts. compaction_instructions: Option, compaction_model: Option, - /// Anthropic server-side fallbacks for this bot; off unless asked. - #[serde(default)] - fallbacks: bool, + /// The model, as `provider/model` in the bot's family, that answers + /// a request a safety classifier declined, that request alone. + fallback: Option, /// A gate: tools whose calls wait for a verdict, the approver tag /// that answers them, and how long a call may wait for it. approve: Option>, @@ -2469,7 +2471,7 @@ impl Service { created_by_id, compaction_instructions, compaction_model, - fallbacks, + fallback, approve, approver, approve_expire_ms, @@ -2549,12 +2551,13 @@ impl Service { } // The summarizer must speak the bot's family: its items are // stored in that encoding and go to the summarizer as they are. - if let Some(reference) = &compaction_model { - let (summarizer, _) = split_model(reference)?; + // So must a fallback, which is sent the same request. + for reference in compaction_model.iter().chain(&fallback) { + let (other, _) = split_model(reference)?; let known = self .providers - .get(summarizer) - .ok_or(Error::with("provider_unavailable", summarizer))?; + .get(other) + .ok_or(Error::with("provider_unavailable", other))?; if known.family() != family { return fail_with("provider_family_mismatch", reference.clone()); } @@ -2578,7 +2581,7 @@ impl Service { created_by_id, compaction_instructions: compaction_instructions.as_deref(), compaction_model: compaction_model.as_deref(), - fallbacks, + fallback: fallback.as_deref(), gate: gate.as_ref(), settings: *settings, declared: declare.as_deref(), @@ -4146,7 +4149,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -4290,7 +4293,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -4507,7 +4510,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -4747,7 +4750,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -5043,7 +5046,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -5130,7 +5133,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -5278,7 +5281,7 @@ mod tests { let bob = running(&store, &["Bob".into()]).await[0].1; store .call(move |db| { - db.suspend_paced(bob, 0, 0, 0, 0, 0, 0, false, None, None, None) + db.suspend_paced(bob, 0, 0, 0, 0, 0, 0, false, None, None, None, None) .map(|_| ()) }) .await @@ -5376,7 +5379,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, approve: None, approver: None, approve_expire_ms: None, @@ -5960,7 +5963,7 @@ mod tests { created_by_id: creator.as_ref().map(|c| c.bot_id), compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: Some(&gate), settings: Default::default(), declared: None, @@ -6241,7 +6244,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: Some(&gate), settings: Default::default(), declared: None, diff --git a/src/server/turn.rs b/src/server/turn.rs index b65c0285d..abecd3eb3 100644 --- a/src/server/turn.rs +++ b/src/server/turn.rs @@ -283,6 +283,28 @@ impl TokenTotals { } } +/// The model a refused request is sent to next: the bot's fallback. +struct Fallback { + provider: Arc, + /// Its provider binding and model, which its usage is priced at. + name: String, + model: String, + /// The refusal's credit, when the retry stays on the same provider. + credit: Option, + /// Chosen, and not yet sent. + pending: bool, +} + +impl Fallback { + /// As a call parked on its pool keeps it. + fn parked(&self) -> agent_runtime::store::ParkedFallback { + agent_runtime::store::ParkedFallback { + model: format!("{}/{}", self.name, self.model), + credit: self.credit.clone(), + } + } +} + /// The last model call of a turn, kept so its prompt cache can be refreshed /// while the calls it planned run: see [`Turn::run_tool`]. struct Warm<'a> { @@ -294,8 +316,6 @@ struct Warm<'a> { context: &'a Context, /// The items the call sent without thinking: a refresh sends the same. thinking: Strip, - /// The call's fallback choice: a refresh renders the same request. - fallbacks: bool, /// Whether the call offered web search; a refresh offers the same. web_search: bool, after: std::time::Duration, @@ -330,6 +350,8 @@ struct Accounting { /// Attempts and non-pacing time spent by the unfinished model call. call_attempts: u32, call_spent_ms: u64, + /// The fallback the unfinished call went to. + call_fallback: Option, /// Set when a call parked the turn on a closed pool. parked_until: u64, /// Persist the call's phase with its existing park transaction. @@ -511,6 +533,7 @@ impl Turn { let turn = self.turn; let flushed = if let Ok(Round::Paced(at)) = &result { let (at, attempts, spent) = (*at, accounting.call_attempts, accounting.call_spent_ms); + let fallback = accounting.call_fallback.take(); let compaction = accounting.compaction; let copied = accounting.copied.take().filter(|_| compaction); let route = accounting.route.get().cloned(); @@ -529,6 +552,7 @@ impl Turn { copied, route.as_deref(), summary_retry_ms, + fallback, ) .map(|_| ()) }) @@ -1151,7 +1175,7 @@ impl Turn { /// each sends at most `view` bytes, the tools and both instructions, a /// token to a byte at most, and generates at most the summarizer's /// output bound. Nothing without a budget; `None` when that bound is - /// unknown, as with fallbacks on, and then the summary goes before the call. + /// unknown, and then the summary goes before the call. fn summary_bound( &self, record: &agent_runtime::store::Bot, @@ -1161,10 +1185,6 @@ impl Turn { if record.ceiling().is_none() { return Some(0); } - // Server-side fallbacks may bill any number of attempts per request. - if record.fallbacks { - return None; - } let reference = summarizer(record); let (name, model) = split_model(&reference).ok()?; let provider = shaped(self.providers.get(name)?, self.settings()).ok()?; @@ -1535,9 +1555,7 @@ impl Turn { .saturating_add(usage.input_tokens) .saturating_add(usage.output_tokens); } - let usage = completion - .usage - .map(|usage| summarizer_usage(usage, name, model)); + let usage = completion.usage.map(|usage| priced_at(usage, name, model)); let summary = completion_text(&completion.items); // A copy offers the bot's tools; a call there is not a summary. let invalid = if !completion.calls.is_empty() { @@ -2001,6 +2019,7 @@ impl Turn { if waiting.paced_since_ms.is_some() { accounting.call_attempts = waiting.call_attempts; accounting.call_spent_ms = waiting.call_spent_ms; + accounting.call_fallback = waiting.fallback.clone(); resume_window = !waiting.compaction; copied = waiting.copied.clone().filter(|_| waiting.compaction); } else { @@ -2577,7 +2596,6 @@ impl Turn { tools: &tools, context: &context, thinking: context.thinking, - fallbacks: record.fallbacks, web_search: searches(&record), after, read_at, @@ -2704,6 +2722,29 @@ impl Turn { Ok(steered) } + /// The bot's fallback `reference` names, unsent and with no credit, when + /// its provider is known and still speaks the bot's encoding. + fn fallback( + &self, + record: &agent_runtime::store::Bot, + reference: &str, + ) -> Result> { + let (name, model) = split_model(reference)?; + let Some(known) = self.providers.get(name) else { + return Ok(None); + }; + if known.family() != record.family()? { + return Ok(None); + } + Ok(Some(Fallback { + provider: Arc::new(shaped(known, self.settings())?.into_owned()), + name: name.to_owned(), + model: model.to_owned(), + credit: None, + pending: false, + })) + } + /// One model call with retries. Each attempt streams the same immutable /// nodes and shares its prepared prefix; a /// refusal for pace holds the provider's pool rather than this turn. @@ -2780,6 +2821,16 @@ impl Turn { // Replayed thinking a 400 may have come from, withheld from this // call's later attempts. let mut withheld = None; + // The bot's fallback model, once a classifier declined this request: + // it answers this request alone, and the next goes to the bot's model. + // A call parked on the fallback's pool resumes there. + let mut fallback = match std::mem::take(&mut accounting.call_fallback) { + Some(parked) => self.fallback(record, &parked.model)?.map(|to| Fallback { + credit: parked.credit, + ..to + }), + None => None, + }; let mut tool_calls = tool_calls; loop { // A billed attempt that failed counts as a round: the retry that @@ -2803,6 +2854,13 @@ impl Turn { accounting.begin(attempt > 0); (accounting.warm_read_at, accounting.warm_stopped) = (None, false); let sent = agent_runtime::provider::Sent::default(); + // The call's own provider and model, and where this attempt goes. + let (own_provider, own_model) = (provider, model); + let target = (fallback.as_ref()).map(|to| (to.provider.clone(), to.model.clone())); + let (provider, model) = match &target { + Some((to, model)) => (to.as_ref(), model.as_str()), + None => (provider, model), + }; let call = provider.complete_accounted( ModelRequest { model, @@ -2813,7 +2871,7 @@ impl Turn { // A summary offers it too, so its request reads // the cache the bot's calls write. web_search: searches(record), - fallbacks: record.fallbacks, + fallback_credit: fallback.as_ref().and_then(|to| to.credit.as_deref()), cache_key: Some(&cache_key), items, // A summary beside the turn's call keeps off the @@ -2836,7 +2894,8 @@ impl Turn { }, &mut accounting.report, ); - let result = match (warm_after, body) { + // The fallback's answer is not the request a refresh renders. + let result = match (warm_after.filter(|_| fallback.is_none()), body) { (Some(after), Body::Window(context)) => { let mut warm = Warm { provider, @@ -2846,7 +2905,6 @@ impl Turn { tools, context, thinking: withheld.unwrap_or(context.thinking), - fallbacks: record.fallbacks, web_search: searches(record), after, read_at: tokio::time::Instant::now(), @@ -2871,6 +2929,13 @@ impl Turn { let sent_ms = sent.get().map_or(0, epoch_ms); let error = match result { Ok(mut completion) => { + if let Some(to) = &fallback { + completion.usage = (completion.usage.take()) + .map(|usage| priced_at(usage, &to.name, &to.model)); + completion + .fallbacks + .push((Some(own_model.to_owned()), to.model.clone())); + } if let Some(usage) = &mut completion.usage { usage.sent_ms = sent_ms; self.tokens.add(usage); @@ -2913,8 +2978,41 @@ impl Turn { } return Ok(Some(completion)); } - Err(error) => error, + Err(mut error) => { + // A refusal's credit is for the retry below, not the record. + let credit = error + .facts + .as_mut() + .and_then(|facts| facts.remove("fallback_credit_token")); + if error.facts.as_ref().is_some_and(|facts| facts.is_empty()) { + error.facts = None; + } + let credit = credit.and_then(|token| token.as_str().map(str::to_owned)); + // A classifier declined: the bot's fallback model answers + // this request, on the same body. The refusal's credit + // prices the retry as if it had run there all along, + // when it goes to the endpoint that declined it. + if fallback.is_none() + && error.code == "provider_refusal" + && matches!(body, Body::Window(_)) + && let Some(reference) = &record.fallback + { + let reference = reference.clone(); + fallback = self.fallback(record, &reference)?.map(|to| Fallback { + credit: credit.filter(|_| to.provider.url() == own_provider.url()), + pending: true, + ..to + }); + } + error + } }; + // What the fallback's own attempt billed is priced at its model. + if let Some(to) = fallback.as_ref().filter(|to| !to.pending) + && let Some(usage) = accounting.report.usage.take() + { + accounting.report.usage = Some(priced_at(usage, &to.name, &to.model)); + } let paced_ms = accounting.totals().1 - paced_before; let spent = prior_spent + started @@ -2924,6 +3022,7 @@ impl Turn { // No HTTP call took place. Keep its retry state without // incrementing attempts or cumulative dispatched retries. accounting.call_attempts = attempt; + accounting.call_fallback = fallback.as_ref().map(Fallback::parked); accounting.call_spent_ms = spent.as_millis() as u64; accounting.parked_until = now_ms() + block.as_millis() as u64; return Ok(None); @@ -2942,7 +3041,7 @@ impl Turn { if summary { if let Some(usage) = usage { let reference = summarizer(record); - let usage = summarizer_usage(usage, split_model(&reference)?.0, model); + let usage = priced_at(usage, split_model(&reference)?.0, model); self.store .op("compaction_usage", move |db| { db.compaction_usage(turn, Some(&usage)) @@ -2953,7 +3052,21 @@ impl Turn { self.failed_usage(usage).await?; } let mut resend = false; - if let Body::Window(context) = body + let mut retry = json!({"thinking_withheld":true}); + if let Some(to) = fallback.as_mut().filter(|to| to.pending) { + // Sent next on the fallback model. + to.pending = false; + resend = true; + retry = json!({"fallback":format!("{}/{}", to.name, to.model)}); + } else if let Some(to) = (fallback.as_mut()) + .filter(|to| to.credit.is_some() && error.code == "provider_http_400") + { + // A credit the provider would not take: the same retry + // without it, before anything else the 400 could be. + to.credit = None; + resend = true; + retry = json!({"fallback_credit":false}); + } else if let Body::Window(context) = body && withheld.is_none() && record.thinking_prefix.is_some() && let Some(strip) = withhold_thinking(context, &error) @@ -2973,14 +3086,15 @@ impl Turn { } else { MAX_ATTEMPTS }; - // Sent again at once without the thinking, within the same - // attempt and time limits as any retry. + // Sent again at once, without the thinking or on the fallback + // model, within the same attempt and time limits as any retry. if resend && capped.is_none() && attempt < cap && spent < RETRY_BUDGET { - self.retried( - json!({"attempt":attempt,"error":error.code,"detail":error.detail, - "delay_ms":0,"parked":false,"thinking_withheld":true}), - ) - .await?; + let mut data = json!({"attempt":attempt,"error":error.code,"detail":error.detail, + "delay_ms":0,"parked":false}); + if let (Some(data), Value::Object(retry)) = (data.as_object_mut(), retry) { + data.extend(retry); + } + self.retried(data).await?; continue; } // Final provider errors are kept. @@ -3035,6 +3149,7 @@ impl Turn { }; accounting.call_attempts = attempt; accounting.call_spent_ms = (spent + backoff).as_millis() as u64; + accounting.call_fallback = fallback.as_ref().map(Fallback::parked); accounting.parked_until = now_ms() + block.as_millis() as u64; return Ok(None); } @@ -3262,7 +3377,7 @@ impl Turn { let (model, instructions) = (warm.model.to_owned(), warm.instructions.to_owned()); let reasoning = warm.reasoning.map(str::to_owned); let tools = warm.tools.to_owned(); - let (fallbacks, web_search) = (warm.fallbacks, warm.web_search); + let web_search = warm.web_search; let items = self.items(warm.context, warm.thinking); let sent = refresh.clone(); let expires = warm.read_at + agent_runtime::provider::CACHE_LIFETIME; @@ -3274,7 +3389,7 @@ impl Turn { tools: &tools, allow_tool_calls: true, web_search, - fallbacks, + fallback_credit: None, cache_key: None, items, chain: None, @@ -5056,10 +5171,11 @@ fn summarizer(record: &agent_runtime::store::Bot) -> String { .unwrap_or_else(|| format!("{}/{}", record.provider, record.model)) } -/// A summary is charged to the turn that needed it, but the summarizer may -/// be another model on another provider; name both, as a fallback names its -/// attempts, so the call is priced at that model's rates. -fn summarizer_usage( +/// A call another model may have answered on another provider binding: a +/// summary, charged to the turn that needed it, or a refused request's +/// fallback. Name both, as a server-side fallback names its attempts, so +/// the call is priced at that model's rates. +fn priced_at( mut usage: agent_runtime::provider::Usage, provider: &str, model: &str, @@ -5180,7 +5296,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, diff --git a/src/store.rs b/src/store.rs index 80f630a3a..76223c35a 100644 --- a/src/store.rs +++ b/src/store.rs @@ -17,9 +17,9 @@ pub use context::{ContextPrefix, ContextUsage, pinned_item, thinking_bytes, with pub use db::{ Absorbed, Answered, Binding, Bot, CacheTtl, CallAnswer, CatchUp, CompactionPlan, CompactionView, CopiedCall, DENIAL_WINDOW, Database, Decision, Delivery, ElisionPlan, - FOLDERS_JSON, Fork, Gate, Gated, MAX_GATES, Planning, Publication, Served, Settings, Started, - Strip, TURN_VIEW_ACCOUNTING, TurnBudget, TurnContext, TurnOptions, Waiting, Wake, Window, - cache_hit, merge_gates, unserved, + FOLDERS_JSON, Fork, Gate, Gated, MAX_GATES, ParkedFallback, Planning, Publication, Served, + Settings, Started, Strip, TURN_VIEW_ACCOUNTING, TurnBudget, TurnContext, TurnOptions, Waiting, + Wake, Window, cache_hit, merge_gates, unserved, }; type ReadJob = Box; @@ -853,7 +853,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -1010,7 +1010,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -1167,7 +1167,7 @@ mod tests { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, diff --git a/src/store/db.rs b/src/store/db.rs index 2f0bb8f87..bf8c01a0a 100644 --- a/src/store/db.rs +++ b/src/store/db.rs @@ -98,10 +98,10 @@ pub struct Bot { pub thinking: Strip, #[serde(skip)] pub thinking_elided: i64, - /// Anthropic server-side fallbacks: a declined request is rerun on the - /// model Anthropic recommends instead of failing the turn. The client's - /// choice at creation; forks inherit it. - pub fallbacks: bool, + /// The model, as `provider/model` in the bot's family, that answers a + /// request a safety classifier declined, that request alone, instead of + /// the turn failing. The client's choice at creation; forks inherit it. + pub fallback: Option, /// Tools whose calls wait for a verdict, per approver tag. Fixed at /// creation, except a root bot's through `configure`: a bot keeps every /// gate it descends from. @@ -580,8 +580,8 @@ pub struct Binding<'a> { /// client's; with no instructions the bot never compacts. pub compaction_instructions: Option<&'a str>, pub compaction_model: Option<&'a str>, - /// Anthropic server-side fallbacks for this bot. - pub fallbacks: bool, + /// The model that answers a request a classifier declined. + pub fallback: Option<&'a str>, /// The bot's own gate; its creator's, if any, is added by the store. pub gate: Option<&'a Gate>, pub settings: Settings, @@ -752,6 +752,18 @@ pub struct Waiting { /// tried again, so the turn resumed before then does not wait on it. #[serde(default, skip_serializing_if = "Option::is_none")] pub summary_retry_ms: Option, + /// The fallback a declined request went to, so the call parked on its + /// pool resumes there. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fallback: Option, +} +/// A declined request's fallback model, as `provider/model`, and the credit +/// its retry redeems. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, serde::Deserialize)] +pub struct ParkedFallback { + pub model: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credit: Option, } /// The call a summary copies: the elision floor its window was read under, /// the node that window starts at, the node it ends at when the copy had @@ -1559,7 +1571,7 @@ impl Database { /// Stored schema version, kept in `PRAGMA user_version`. Stores created /// before versioning and stores from newer binaries are rejected; an older /// versioned store is migrated forward, one version at a time, at open. - pub const SCHEMA: i32 = 59; + pub const SCHEMA: i32 = 60; /// Verbatim user prompts a compaction keeps: per-prompt text, and the /// total text plus `(ordinal, String)` entry metadata. Empty entries cost /// space too, so the retained list cannot grow with conversation length. @@ -1689,7 +1701,7 @@ impl Database { compaction_instructions TEXT, compaction_model TEXT, cache_bot INTEGER, thinking_prefix INTEGER, thinking_floor INTEGER NOT NULL DEFAULT 0, - fallbacks INTEGER NOT NULL DEFAULT 0, + fallback TEXT, elision INTEGER REFERENCES elisions(node), thinking_from INTEGER NOT NULL DEFAULT 0, thinking_to INTEGER NOT NULL DEFAULT 0, @@ -2105,7 +2117,7 @@ impl Database { to: r.get(28)?, }, thinking_elided: r.get(29)?, - fallbacks: r.get::<_, i64>(25)? != 0, + fallback: r.get(25)?, elision: r.get(26)?, gates: match r.get::<_, Option>(30)? { None => Vec::new(), @@ -2164,7 +2176,7 @@ impl Database { }, }) } - const COLUMNS: &str = "name,head,workspace,status,running_turn,provider,family,model,instructions,reasoning,budget_tokens,tokens_used,tools,input_tokens,cached_input_tokens,id,created_by,created_by_id,note,compaction,compaction_instructions,compaction_model,cache_bot,thinking_prefix,thinking_floor,fallbacks,elision,thinking_from,thinking_to,thinking_elided,gates,denials,allowed,closed,settings,expire_ms,expires_at,sent_bytes,sent_tokens,compact_asked,declared"; + const COLUMNS: &str = "name,head,workspace,status,running_turn,provider,family,model,instructions,reasoning,budget_tokens,tokens_used,tools,input_tokens,cached_input_tokens,id,created_by,created_by_id,note,compaction,compaction_instructions,compaction_model,cache_bot,thinking_prefix,thinking_floor,fallback,elision,thinking_from,thinking_to,thinking_elided,gates,denials,allowed,closed,settings,expire_ms,expires_at,sent_bytes,sent_tokens,compact_asked,declared"; /// Validate the caller's captured identity in the same transaction that /// creates the child. Never resolve a stale shell's name to a new bot. fn creator_id( @@ -2569,7 +2581,7 @@ impl Database { "provider":binding.provider,"model":binding.model,"effort":binding.effort, "budget_tokens":binding.budget_tokens,"tools":binding.tools,"created_by":binding.created_by, "created_by_id":binding.created_by_id,"compaction_model":binding.compaction_model, - "fallbacks":binding.fallbacks,"gate":binding.gate,"settings":binding.settings, + "fallback":binding.fallback,"gate":binding.gate,"settings":binding.settings, "declare":binding.declared})); if let Some(bot) = self.resent(name, binding.request_id, request.as_ref())? { if bot.compaction_instructions.as_deref() != binding.compaction_instructions { @@ -2605,7 +2617,7 @@ impl Database { None => binding.settings.clone(), }; tx.execute( - "INSERT INTO bots(name,id,head,workspace,status,running_turn,provider,family,model,instructions,reasoning,budget_tokens,tokens_used,context_start,pruned_cursor,tools,created_by,created_by_id,compaction_instructions,compaction_model,fallbacks,gates,settings,creation,declared) VALUES (?,?,NULL,?,'idle',NULL,?,?,?,?,?,?,0,NULL,0,?,?,?,?,?,?,?,?,?,?)", + "INSERT INTO bots(name,id,head,workspace,status,running_turn,provider,family,model,instructions,reasoning,budget_tokens,tokens_used,context_start,pruned_cursor,tools,created_by,created_by_id,compaction_instructions,compaction_model,fallback,gates,settings,creation,declared) VALUES (?,?,NULL,?,'idle',NULL,?,?,?,?,?,?,0,NULL,0,?,?,?,?,?,?,?,?,?,?)", params![ name, id, @@ -2621,7 +2633,7 @@ impl Database { created_by_id, binding.compaction_instructions, binding.compaction_model, - binding.fallbacks, + binding.fallback, gates, settings.stored()?, creation(binding.request_id, request), @@ -5941,6 +5953,7 @@ impl Database { answer: None, copied: None, summary_retry_ms, + fallback: None, }; let tx = self.conn.savepoint()?; flush(&tx, turn, self.live.get(&turn))?; @@ -6054,6 +6067,7 @@ impl Database { answer: Some(tag.clone()), copied: None, summary_retry_ms, + fallback: None, }; let tx = self.conn.savepoint()?; flush(&tx, turn, self.live.get(&turn))?; @@ -7096,6 +7110,7 @@ impl Database { answer: None, copied: None, summary_retry_ms, + fallback: None, }; let tx = self.conn.savepoint()?; // A verdict for a later call of the round waits in the store. @@ -7128,6 +7143,7 @@ impl Database { copied: Option, route: Option<&str>, summary_retry_ms: Option, + fallback: Option, ) -> Result { let bot = self.active(turn)?; if bot.status != "running" { @@ -7150,6 +7166,7 @@ impl Database { answer: None, copied, summary_retry_ms, + fallback, }; let tx = self.conn.savepoint()?; flush(&tx, turn, self.live.get(&turn))?; @@ -7735,7 +7752,7 @@ impl Database { None => parent.settings.clone(), }; tx.execute( - "INSERT INTO bots(name,id,head,workspace,status,running_turn,provider,family,model,instructions,reasoning,budget_tokens,tokens_used,context_start,pruned_cursor,tools,created_by,created_by_id,compaction_instructions,compaction_model,cache_bot,thinking_prefix,thinking_floor,fallbacks,thinking_from,thinking_to,thinking_elided,gates,allowed,settings,creation,expire_ms,expires_at,declared) VALUES (?,?,?,?,'idle',NULL,?,?,?,?,?,?,0,NULL,0,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + "INSERT INTO bots(name,id,head,workspace,status,running_turn,provider,family,model,instructions,reasoning,budget_tokens,tokens_used,context_start,pruned_cursor,tools,created_by,created_by_id,compaction_instructions,compaction_model,cache_bot,thinking_prefix,thinking_floor,fallback,thinking_from,thinking_to,thinking_elided,gates,allowed,settings,creation,expire_ms,expires_at,declared) VALUES (?,?,?,?,'idle',NULL,?,?,?,?,?,?,0,NULL,0,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", params![ name, id, @@ -7760,7 +7777,7 @@ impl Database { // its own context against it. thinking.0, thinking.1.below, - parent.fallbacks, + parent.fallback, thinking.1.from, thinking.1.to, thinking.2, @@ -10316,6 +10333,19 @@ fn migrate(conn: &Connection, from: i32) -> Result<()> { // kind of item. No custom tool was declared before. conn.execute_batch("ALTER TABLE tools ADD COLUMN custom INTEGER NOT NULL DEFAULT 0;")?; } + if !conn.query_row( + "SELECT EXISTS(SELECT 1 FROM pragma_table_info('bots') WHERE name='fallback')", + [], + |r| r.get::<_, bool>(0), + )? { + // 59 -> 60: a bot names the model a declined request falls back to, + // retried by the daemon, rather than asking Anthropic for its own + // fallback, which sticks to that model. No bot names one yet. + conn.execute_batch( + "ALTER TABLE bots ADD COLUMN fallback TEXT; + ALTER TABLE bots DROP COLUMN fallbacks;", + )?; + } Ok(()) } /// 53 -> 54: a turn's denial count became a window of its last decided diff --git a/tests/store_contract.rs b/tests/store_contract.rs index 9d8164bc6..3e6b62d3a 100644 --- a/tests/store_contract.rs +++ b/tests/store_contract.rs @@ -46,7 +46,7 @@ fn binding() -> Binding<'static> { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, @@ -356,7 +356,7 @@ fn unfinished_tools_are_answered_truthfully_without_disabling_the_bot() { family, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, ..binding() }, @@ -466,7 +466,7 @@ fn restart_repairs_unanswered_tools_once_including_previously_blocked_bots() { family, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, ..binding() }, @@ -4485,7 +4485,7 @@ fn tool_selection_migration_rejects_unknown_policy_without_changing_data() { tools: &tools, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, ..binding() }, @@ -5522,7 +5522,7 @@ fn anthropic_forks_check_the_whole_tool_batch_after_a_checkpoint() { family: Family::Anthropic, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, ..binding() }, @@ -6032,7 +6032,7 @@ fn history_normalizes_multiline_items_without_changing_fields_or_replay() { family, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, ..binding() }, @@ -9776,26 +9776,30 @@ fn schema_27_migrates_cache_lineage_and_thinking_sizes() { } #[test] -fn a_store_keeps_its_identity_and_forks_inherit_fallbacks() { +fn a_store_keeps_its_identity_and_forks_inherit_a_fallback() { let dir = std::env::temp_dir().join(format!("agent-identity-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let path = dir.join("state.sqlite"); let identity = { let mut db = Database::initialize(Connection::open(&path).unwrap()).unwrap(); let identity = db.store_identity().unwrap(); - // Off unless the client asks; asked for, forks keep it. + // None unless the client names one; named, forks keep it. let plain = db.create("plain", None, binding()).unwrap().0; - assert!(!plain.fallbacks); + assert_eq!(plain.fallback, None); let mut asked = binding(); - asked.fallbacks = true; + asked.fallback = Some("anthropic/claude-opus-4-8"); let bot = db.create("bot", None, asked).unwrap().0; - assert!(bot.fallbacks); - assert!(db.fork("bot", "fork", Fork::default()).unwrap().0.fallbacks); - assert!( - !db.fork("plain", "plain-fork", Fork::default()) + assert_eq!(bot.fallback.as_deref(), Some("anthropic/claude-opus-4-8")); + assert_eq!( + (db.fork("bot", "fork", Fork::default()).unwrap().0.fallback).as_deref(), + Some("anthropic/claude-opus-4-8") + ); + assert_eq!( + db.fork("plain", "plain-fork", Fork::default()) .unwrap() .0 - .fallbacks + .fallback, + None ); identity }; diff --git a/tests/store_payloads.rs b/tests/store_payloads.rs index ee58ab144..76b620936 100644 --- a/tests/store_payloads.rs +++ b/tests/store_payloads.rs @@ -36,7 +36,7 @@ fn binding(family: Family) -> Binding<'static> { created_by_id: None, compaction_instructions: None, compaction_model: None, - fallbacks: false, + fallback: None, gate: None, settings: Default::default(), declared: None, diff --git a/tests/test_accounting.py b/tests/test_accounting.py index 59432173f..000d37f46 100644 --- a/tests/test_accounting.py +++ b/tests/test_accounting.py @@ -276,6 +276,31 @@ def test_unversioned_and_newer_stores_are_refused_with_clear_codes(self): @unittest.skipUnless(os.environ.get('AGENT_TEST_RUNTIME') == '1', 'set AGENT_TEST_RUNTIME=1 after a Rust release build') class IdleExitTests(ModelFixture): + def test_a_new_bot_takes_the_person_s_fallback_for_its_model(self): + self.model.models = ('synthetic-model', 'synthetic-other') + (self.path / '.agent').mkdir() + (self.path / '.agent/config.toml').write_text( + '[fallbacks]\n"openai/synthetic-model" = "openai/synthetic-backup"\n') + store = self.path / 'state.sqlite' + common = ['--store', str(store), '--provider', f'openai=responses,{self.url}', '--detach'] + env = {**clean_env(), 'HOME': str(self.path)} + + def made(bot, *flags): + run = subprocess.run([str(self.binary), 'run', *common, '--new', '--bot', bot, *flags, 'hello'], + env=env, capture_output=True, text=True, timeout=30, cwd=self.path) + self.assertEqual(run.returncode, 0, run.stderr) + inspected = subprocess.run([str(self.binary), 'inspect', '--store', str(store), '--bot', bot], + env=env, capture_output=True, text=True, timeout=10) + return json.loads(inspected.stdout)['fallback'] + + self.addCleanup(subprocess.run, [str(self.binary), 'shutdown', '--store', str(store)], + env=env, capture_output=True, timeout=5) + # --fallback wins; a model the table does not name has none. + self.assertEqual(made('Bob', '--model', 'openai/synthetic-model'), 'openai/synthetic-backup') + self.assertEqual(made('Ann', '--model', 'openai/synthetic-model', '--fallback', 'openai/other'), + 'openai/other') + self.assertIsNone(made('Cy', '--model', 'openai/synthetic-other')) + def test_socket_daemon_exits_when_idle_and_parked_turns_survive(self): store = self.path / 'state.sqlite' socket = self.path / 'state.sqlite.sock' diff --git a/tests/test_harbor_agent.py b/tests/test_harbor_agent.py index f7a117401..73570826c 100644 --- a/tests/test_harbor_agent.py +++ b/tests/test_harbor_agent.py @@ -38,13 +38,13 @@ def store(path, turns): """The columns of a store's bots and turns that accounting reads.""" with sqlite3.connect(path) as db: db.execute('CREATE TABLE bots(name TEXT, provider TEXT, model TEXT, reasoning TEXT, ' - 'fallbacks INT)') + 'fallback TEXT)') db.execute('CREATE TABLE turns(id INTEGER PRIMARY KEY, bot TEXT, model TEXT, status TEXT, ' 'input_tokens INT, cached_input_tokens INT, output_tokens INT, ' 'model_rounds INT, retries INT, paced_ms INT)') # The task bot as the adapter creates it; others as a model might. db.executemany('INSERT INTO bots VALUES (?,?,?,?,?)', - {(t[0], 'gw', 'm', *(('high', 1) if t[0] == 'task' else (None, 0))) + {(t[0], 'gw', 'm', *(('high', 'gw/backup') if t[0] == 'task' else (None, None))) for t in turns}) db.executemany('INSERT INTO turns(bot,model,status,input_tokens,cached_input_tokens,' 'output_tokens,model_rounds,retries,paced_ms) VALUES (?,?,?,?,?,?,2,1,5)', @@ -63,9 +63,11 @@ def agent(self, logs, **kwargs): def test_command_quotes_the_instruction_and_keeps_the_turn_status(self): with tempfile.TemporaryDirectory() as logs: - command = self.agent(logs, reasoning='high', max_output_tokens=4096)._command( + command = self.agent(logs, reasoning='high', fallback='gw/backup', + max_output_tokens=4096)._command( "it's \"quoted\"; rm -rf /\n$(x)") self.assertIn("-- 'it'\"'\"'s \"quoted\"; rm -rf /\n$(x)' < /dev/null", command) + self.assertIn('--model gw/m --fallback gw/backup', command) self.assertIn('--effort high --max-output-tokens 4096 --', command) self.assertTrue(command.endswith('| tee /logs/agent/agent.jsonl; exit ${PIPESTATUS[0]}')) @@ -142,8 +144,8 @@ def test_each_model_is_priced_at_its_own_rates(self): self.assertEqual(context.metadata, {'model_rounds': 6, 'retries': 3, 'paced_ms': 15, 'status': ['completed', 'interrupted'], 'bots': 2, 'bot_settings': { - 'task': {'reasoning': 'high', 'fallbacks': True}, - 'helper': {'reasoning': None, 'fallbacks': False}}, + 'task': {'reasoning': 'high', 'fallback': 'gw/backup'}, + 'helper': {'reasoning': None, 'fallback': None}}, 'requested_model': 'gw/m', 'served_calls': {}}) self.assertIsNone(unpriced.cost_usd) self.assertEqual(unpriced.n_output_tokens, 300) @@ -183,8 +185,8 @@ def test_a_fallback_attempt_is_priced_at_the_model_that_ran_it(self): # as the provider named it: each attempt of a fallback, and the dated # snapshot behind the requested name. self.assertEqual((context.metadata['requested_model'], context.metadata['served_calls'], - context.metadata['bot_settings']['task']['fallbacks']), - ('gw/m', {'gw/m': 1, 'gw/backup': 1, 'gw/m-2026-09-26': 1}, True)) + context.metadata['bot_settings']['task']['fallback']), + ('gw/m', {'gw/m': 1, 'gw/backup': 1, 'gw/m-2026-09-26': 1}, 'gw/backup')) self.assertNotIn('unrecorded_input_tokens', context.metadata) self.assertNotIn('unnamed_calls', context.metadata) self.assertEqual((short.metadata['served_calls'], short.metadata['unrecorded_input_tokens']), @@ -384,7 +386,7 @@ def test_accounting_reads_a_real_store(self): (listed[0]['input_tokens'], listed[0]['output_tokens'], ['completed'])) self.assertEqual(list(context.model_usage), ['openai/synthetic-model']) self.assertEqual(context.metadata['bot_settings'], - {'task': {'reasoning': None, 'fallbacks': False}}) + {'task': {'reasoning': None, 'fallback': None}}) # The daemon's count and the stored usage events agree, and every # call names the dated snapshot the test provider answers with. self.assertEqual(context.metadata['served_calls'], diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 4cbf9f7af..d3bb274e9 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -603,14 +603,15 @@ def do_POST(self): assert self.path == '/v1/messages' assert self.headers.get('x-api-key') == 'synthetic-anthropic-key' assert self.headers.get('anthropic-version') == '2023-06-01' - # Server-side fallbacks are a bot's choice: the field and its beta - # header travel together, or not at all. - if 'fallbacks' in request: - assert request['fallbacks'] == 'default' - assert self.headers.get('anthropic-beta') == ( - 'thinking-binding-controls-2026-08-01,server-side-fallback-2026-07-01') - else: - assert self.headers.get('anthropic-beta') == 'thinking-binding-controls-2026-08-01' + # No request asks for a server-side fallback, which would stick; + # a refused request's own retry redeems the refusal's credit. + assert 'fallbacks' not in request + assert self.headers.get('anthropic-beta') == ( + 'thinking-binding-controls-2026-08-01,fallback-credit-2026-07-01,' + 'server-side-fallback-2026-07-01') + if 'fallback_credit_token' in request: + assert request['fallback_credit_token'] == 'credit-1' and request['model'] == 'synthetic-claude-2' + self.server.credits = getattr(self.server, 'credits', 0) + 1 if 'thinking' in request: assert request['thinking']['block_binding'] == {'prefix_mismatch_behavior': 'drop_block'} # A cache refresh is the same request with no output and no stream. @@ -682,6 +683,17 @@ def do_POST(self): # for an api_error after the stream began. An optional failure # count also models a transient outage with the same request, # and `unreadable_any` a failure whatever the request replays. + # The fallback model's pool closes long enough to park the turn. + if request['model'] == 'synthetic-claude-2' and getattr(self.server, 'limit_fallback', 0): + self.server.limit_fallback -= 1 + body = b'{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}' + self.send_response(429) + self.send_header('Retry-After', '1.2') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + self.wfile.flush() + return if getattr(self.server, 'rate_limits', 0): self.server.rate_limits -= 1 body = b'{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}' @@ -737,6 +749,7 @@ def do_POST(self): return signature = thinking_binding(request, request['messages'], self.headers.get('Host')) blocks = [{'type': 'thinking', 'thinking': 'plan', 'signature': signature}] + details = None prompt = next((b['text'] for m in request['messages'] if m['role'] == 'user' for b in m['content'] if b['type'] == 'text' and b['text'].startswith('long:')), '') if prompt and not summary: @@ -794,6 +807,16 @@ def do_POST(self): 'title': 'Example', 'encrypted_content': 'opaque'}]}, {'type': 'text', 'text': 'found'}] stop = 'pause_turn' + elif user.startswith('refuse:') and (request['model'] == 'synthetic-claude' + or user.startswith('refuse:all')): + # A classifier declines mid-output: the partial is + # discarded, and the refusal names its category and, + # on the first model, a credit for the retry. + blocks.append({'type': 'text', 'text': 'Partial '}) + stop = 'refusal' + details = {'type': 'refusal', 'category': 'cyber', 'explanation': None} + if request['model'] == 'synthetic-claude': + details['fallback_credit_token'] = 'credit-1' elif user.startswith('fallback:'): # A classifier declines mid-output and another model # finishes: the declined partial stays in the stream. @@ -811,7 +834,8 @@ def do_POST(self): 'usage': {'input_tokens': 5, 'cache_read_input_tokens': 2, **getattr(self.server, 'start_usage', {})}} if getattr(self.server, 'report_drops', 0): - start['input_transformations'] = [{'type': 'thinking_dropped', 'message_index': 1, 'block_index': 0} + start['input_transformations'] = [{'type': 'thinking_dropped', 'message_index': 1, 'block_index': 0, + 'reason': 'prefix_binding_mismatch'} for _ in range(self.server.report_drops)] events = [('message_start', {'message': start})] for index, block in enumerate(blocks): @@ -840,7 +864,8 @@ def do_POST(self): 'cache_creation_input_tokens': 0, 'output_tokens': 4}, {'type': 'fallback_message', 'model': 'synthetic-fallback', 'input_tokens': 6, 'cache_read_input_tokens': 0, 'cache_creation_input_tokens': 1, 'output_tokens': 7}] - events.append(('message_delta', {'delta': {'stop_reason': stop}, 'usage': usage})) + delta = {'stop_reason': stop, **({'stop_details': details} if details else {})} + events.append(('message_delta', {'delta': delta, 'usage': usage})) events.append(('message_stop', {})) # An overload after the stream began, as Anthropic sends one. if getattr(self.server, 'overloads', 0): @@ -1084,9 +1109,10 @@ def test_a_short_tool_call_sends_no_refresh(self): model.requests.get(timeout=1))], [True, True]) self.assertTrue(model.requests.empty()) - def test_a_fallback_answer_replays_without_the_declined_attempt(self): + def test_a_server_side_fallback_answer_replays_without_the_declined_attempt(self): + # No request asks for one now, but a history may hold one. client, model, path = self.start() - client.request('create', bot='Bob', workspace=str(path), effort='low', fallbacks=True) + client.request('create', bot='Bob', workspace=str(path), effort='low') turn = client.request('submit', bot='Bob', request_id='f1', prompt='fallback:kept')['result']['turn'] self.assertEqual(client.finished(turn)['data']['status'], 'completed') fallback = [m for m in client.saved if m.get('event') == 'model_fallback'] @@ -1114,6 +1140,59 @@ def test_a_fallback_answer_replays_without_the_declined_attempt(self): self.assertEqual(second['messages'][2]['content'], [{'type': 'tool_result', 'tool_use_id': 'toolu_1', 'content': 'kept'}]) + def test_a_refused_request_alone_goes_to_the_fallback_model(self): + client, model, path = self.start() + # The fallback's pool parks the turn once: it resumes on the fallback. + model.limit_fallback = 1 + client.request('create', bot='Bob', workspace=str(path), effort='low', + fallback='anthropic/synthetic-claude-2') + turn = client.request('submit', bot='Bob', request_id='f1', prompt='refuse:once')['result']['turn'] + self.assertEqual(client.finished(turn)['data']['status'], 'completed') + refused, limited, answered = (model.requests.get(timeout=1) for _ in range(3)) + self.assertEqual((limited['model'], limited['fallback_credit_token']), ('synthetic-claude-2', 'credit-1')) + # The same body on the fallback model, with the refusal's credit. + self.assertEqual((refused['model'], answered['model']), ('synthetic-claude', 'synthetic-claude-2')) + self.assertNotIn('fallback_credit_token', refused) + self.assertEqual(answered['fallback_credit_token'], 'credit-1') + self.assertEqual(answered['messages'], refused['messages']) + retried = [m['data'] for m in client.saved if m.get('event') == 'retry'] + self.assertEqual([(r['error'], r.get('fallback'), r['parked']) for r in retried], + [('provider_refusal', 'anthropic/synthetic-claude-2', False), + ('provider_http_429', None, True)]) + self.assertEqual(retried[0]['detail'], 'category cyber') + fallback = [m for m in client.saved if m.get('event') == 'model_fallback'] + self.assertEqual([(m['from'], m['to']) for m in fallback], [('synthetic-claude', 'synthetic-claude-2')]) + usage = [m['data'] for m in client.saved if m.get('event') == 'usage'] + self.assertEqual(usage[-1]['served_model'], 'synthetic-claude-2-20260926') + # The next request goes back to the bot's own model, with the + # fallback's answer in its history and the declined partial not. + turn = client.request('submit', bot='Bob', request_id='f2', prompt='hello')['result']['turn'] + self.assertEqual(client.finished(turn)['data']['status'], 'completed') + after = model.requests.get(timeout=1) + self.assertEqual(after['model'], 'synthetic-claude') + self.assertNotIn('fallback_credit_token', after) + self.assertEqual([b.get('text') for b in after['messages'][1]['content'] if b['type'] == 'text'], + ['reply:refuse:once']) + self.assertTrue(model.requests.empty()) + + def test_a_refusal_the_fallback_repeats_fails_the_turn(self): + client, model, path = self.start() + client.request('create', bot='Bob', workspace=str(path), fallback='anthropic/synthetic-claude-2') + client.request('create', bot='Solo', workspace=str(path)) + turn = client.request('submit', bot='Bob', request_id='f1', prompt='refuse:all')['result']['turn'] + finished = client.finished(turn)['data'] + self.assertEqual((finished['error'], finished['detail']), ('provider_refusal', 'category cyber')) + self.assertEqual([model.requests.get(timeout=1)['model'] for _ in range(2)], + ['synthetic-claude', 'synthetic-claude-2']) + # With no fallback, a refusal ends the turn at once. + turn = client.request('submit', bot='Solo', request_id='s1', prompt='refuse:all')['result']['turn'] + self.assertEqual(client.finished(turn)['data']['error'], 'provider_refusal') + self.assertEqual(model.requests.get(timeout=1)['model'], 'synthetic-claude') + self.assertTrue(model.requests.empty()) + # A fallback is a known provider's model in the bot's family. + self.assertEqual(client.request('create', bot='Bad', workspace=str(path), + fallback='openai/x')['error'], 'provider_unavailable') + def test_a_hosted_search_is_declared_kept_and_its_paused_turn_resumed(self): client, model, path = self.start() self.assertTrue(client.ready['providers']['anthropic']['web_search']) @@ -1186,7 +1265,8 @@ def test_a_lone_overload_mid_stream_backs_off_its_own_call(self): def test_messages_family_round_trips_thinking_tools_and_usage(self): client, model, path = self.start() - client.request('create', bot='Bob', workspace=str(path), effort='low', fallbacks=True) + client.request('create', bot='Bob', workspace=str(path), effort='low', + fallback='anthropic/synthetic-claude-2') before = time.time() * 1000 turn = client.request('submit', bot='Bob', request_id='r1', prompt='tool:shared')['result']['turn'] finished = client.finished(turn)