diff --git a/docs/api.md b/docs/api.md index 08fe7fb..f23c1e6 100644 --- a/docs/api.md +++ b/docs/api.md @@ -147,6 +147,26 @@ billed. Measured across twelve calls with a 27k-token prefix, exactly one reported a write, on `/v1/messages`. The dashboard hides the column outright when nothing wrote. +### Cache lifetime + +A `cache_control` breakpoint with no `ttl` gets the five-minute tier. That is +short enough to be self-defeating on long turns: the entry is written during +prefill, so a turn that itself runs longer than five minutes has already +outlived its own cache by the time it finishes, and the next turn pays a full +cold prefill of the whole conversation. + +Setting `extend_cache_ttl: true` promotes breakpoints that carry no explicit +`ttl` to the one-hour tier. An explicit `ttl` is always left as the client sent +it. Copilot honours both tiers and accounts for them separately, in +`cache_creation.ephemeral_5m_input_tokens` and `ephemeral_1h_input_tokens`. + +It is off by default because it is not free. Extended writes bill at a higher +rate than five-minute ones while reads cost the same, so the premium is charged +on *every* write and the saving only lands on an expiry that would otherwise +have happened. On a conversation that does many small incremental writes +between rare expiries it costs more than it saves; it pays off when turns +routinely run past five minutes. + A model with no cache activity at all is usually not a fault either: Copilot needs a minimum cacheable prefix before any of the prompt is eligible. `claude-haiku-4.5` was observed caching a 6902-token prefix but not a diff --git a/docs/configuration.md b/docs/configuration.md index f038072..c9eb8cc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -200,6 +200,7 @@ Every config field has a `GHC_PROXY_*` override: | `GHC_PROXY_MAX_CONNECTION_RETRIES` | Max connection retries | | `GHC_PROXY_UPSTREAM_READ_TIMEOUT` | Max seconds of upstream silence (`0` disables) | | `GHC_PROXY_REDIRECT_ANTHROPIC` | Always translate Anthropic via chat completions | +| `GHC_PROXY_EXTEND_CACHE_TTL` | Promote `cache_control` breakpoints to the 1h tier (`true`/`1`) | | `GHC_PROXY_SHOW_TOKEN` | Log tokens on refresh (`true`/`1`) | | `GHC_PROXY_DYNAMIC_VSCODE_VERSION` | Fetch latest VS Code version (`true`/`1`) | | `GHC_PROXY_AUTO_UPGRADE` | Auto-upgrade app on startup (`true`/`1`); set `0` to disable | diff --git a/src/anthropic.rs b/src/anthropic.rs index 2b41a53..04b2ae3 100644 --- a/src/anthropic.rs +++ b/src/anthropic.rs @@ -773,13 +773,19 @@ pub fn uses_context_management(req: &Value) -> bool { .unwrap_or(false) } -fn clean_cache_control(block: &mut Value) { +fn clean_cache_control(block: &mut Value, extend_ttl: bool) { if let Some(cc) = block.get_mut("cache_control") { if cc.get("type").and_then(|t| t.as_str()) == Some("ephemeral") { if let Some(obj) = cc.as_object_mut() { if obj.contains_key("scope") { obj.remove("scope"); } + // Only fill a gap. An explicit `ttl` is the client's decision + // and overriding it would bill the extended rate against a + // choice someone deliberately made. + if extend_ttl && !obj.contains_key("ttl") { + obj.insert("ttl".to_string(), Value::String("1h".to_string())); + } } } } @@ -797,8 +803,10 @@ fn is_empty_text_block(block: &Value) -> bool { } /// Filters an Anthropic request down to the allowed keys and strips the -/// unsupported `scope` field from ephemeral `cache_control` blocks. -pub fn sanitize_anthropic_request(req: &Value) -> Value { +/// unsupported `scope` field from ephemeral `cache_control` blocks. When +/// `extend_ttl` is set, breakpoints left without a `ttl` are promoted to the +/// one-hour tier. +pub fn sanitize_anthropic_request(req: &Value, extend_ttl: bool) -> Value { let mut out = Map::new(); if let Some(obj) = req.as_object() { for (k, v) in obj { @@ -811,12 +819,12 @@ pub fn sanitize_anthropic_request(req: &Value) -> Value { if let Some(tools) = out.get_mut("tools").and_then(|t| t.as_array_mut()) { for t in tools { - clean_cache_control(t); + clean_cache_control(t, extend_ttl); } } if let Some(system) = out.get_mut("system").and_then(|s| s.as_array_mut()) { for s in system { - clean_cache_control(s); + clean_cache_control(s, extend_ttl); } } if let Some(messages) = out.get_mut("messages").and_then(|m| m.as_array_mut()) { @@ -828,7 +836,7 @@ pub fn sanitize_anthropic_request(req: &Value) -> Value { } if let Some(content) = msg.get_mut("content").and_then(|c| c.as_array_mut()) { for block in content.iter_mut() { - clean_cache_control(block); + clean_cache_control(block, extend_ttl); } content.retain(|block| !is_empty_text_block(block)); } @@ -1362,7 +1370,7 @@ mod tests { "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]} }); assert!(uses_context_management(&req)); - let out = sanitize_anthropic_request(&req); + let out = sanitize_anthropic_request(&req, false); assert!(out.get("context_management").is_some()); } @@ -1472,7 +1480,7 @@ mod tests { #[test] fn sanitize_drops_unknown_keys() { let req = json!({"model": "m", "messages": [], "foo": "bar"}); - let out = sanitize_anthropic_request(&req); + let out = sanitize_anthropic_request(&req, false); assert!(out.get("foo").is_none()); assert_eq!(out["model"], "m"); } @@ -1480,10 +1488,75 @@ mod tests { #[test] fn sanitize_keeps_output_config() { let req = json!({"model": "m", "messages": [], "output_config": {"effort": "high"}}); - let out = sanitize_anthropic_request(&req); + let out = sanitize_anthropic_request(&req, false); assert_eq!(out["output_config"]["effort"], "high"); } + fn ttl_probe_request() -> Value { + json!({ + "model": "m", + "tools": [{"name": "t", "cache_control": {"type": "ephemeral"}}], + "system": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}], + "messages": [{ + "role": "user", + "content": [{"type": "text", "text": "u", "cache_control": {"type": "ephemeral"}}] + }] + }) + } + + /// Off by default: extended writes bill at a higher rate. + #[test] + fn cache_ttl_is_left_alone_unless_asked_for() { + let out = sanitize_anthropic_request(&ttl_probe_request(), false); + assert!(out["tools"][0]["cache_control"].get("ttl").is_none()); + assert!(out["system"][0]["cache_control"].get("ttl").is_none()); + assert!(out["messages"][0]["content"][0]["cache_control"] + .get("ttl") + .is_none()); + } + + #[test] + fn cache_ttl_is_extended_at_every_breakpoint() { + let out = sanitize_anthropic_request(&ttl_probe_request(), true); + assert_eq!(out["tools"][0]["cache_control"]["ttl"], "1h"); + assert_eq!(out["system"][0]["cache_control"]["ttl"], "1h"); + assert_eq!( + out["messages"][0]["content"][0]["cache_control"]["ttl"], + "1h" + ); + } + + /// A client that names a ttl has priced the trade itself. + #[test] + fn an_explicit_cache_ttl_is_never_overridden() { + let req = json!({ + "model": "m", + "system": [{ + "type": "text", "text": "s", + "cache_control": {"type": "ephemeral", "ttl": "5m"} + }], + "messages": [] + }); + let out = sanitize_anthropic_request(&req, true); + assert_eq!(out["system"][0]["cache_control"]["ttl"], "5m"); + } + + /// `scope` is rejected upstream, so it has to go even while a ttl goes in. + #[test] + fn extending_the_ttl_still_strips_scope() { + let req = json!({ + "model": "m", + "system": [{ + "type": "text", "text": "s", + "cache_control": {"type": "ephemeral", "scope": "global"} + }], + "messages": [] + }); + let out = sanitize_anthropic_request(&req, true); + assert!(out["system"][0]["cache_control"].get("scope").is_none()); + assert_eq!(out["system"][0]["cache_control"]["ttl"], "1h"); + } + #[test] fn sanitize_drops_empty_text_blocks() { let req = json!({ @@ -1496,7 +1569,7 @@ mod tests { ] }] }); - let out = sanitize_anthropic_request(&req); + let out = sanitize_anthropic_request(&req, false); let blocks = out["messages"][0]["content"] .as_array() .cloned() @@ -1515,7 +1588,7 @@ mod tests { {"role": "user", "content": [{"type": "text", "text": "hello"}]} ] }); - let out = sanitize_anthropic_request(&req); + let out = sanitize_anthropic_request(&req, false); let messages = out["messages"].as_array().cloned().unwrap_or_default(); assert_eq!(messages.len(), 1); assert_eq!(messages[0]["role"], "user"); diff --git a/src/config.rs b/src/config.rs index 984437b..7e510c2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -164,6 +164,17 @@ pub struct Config { /// translate Anthropic requests through the OpenAI chat completions API. #[serde(default)] pub redirect_anthropic: bool, + /// When true, give `cache_control` breakpoints that carry no explicit `ttl` + /// the one-hour tier instead of the five-minute default. + /// + /// Worth it only when turns regularly run longer than five minutes: an + /// entry is written during prefill, so a turn that takes longer than the + /// TTL outlives its own cache and the next turn pays a full cold prefill. + /// The trade is that every write bills at the higher extended rate, which + /// on a workload of many small incremental writes costs more than the + /// occasional expiry it prevents. + #[serde(default)] + pub extend_cache_ttl: bool, /// When true, log the GitHub and Copilot tokens whenever they are resolved /// or refreshed. Useful for debugging; keep disabled in shared environments. #[serde(default)] @@ -260,6 +271,7 @@ impl Default for Config { max_connection_retries: default_max_retries(), upstream_read_timeout_seconds: default_read_timeout(), redirect_anthropic: false, + extend_cache_ttl: false, show_token: false, dynamic_vscode_version: false, auto_upgrade: true, @@ -539,6 +551,15 @@ pub fn render_config_yaml(cfg: &Config) -> String { ); let _ = writeln!(s, "redirect_anthropic: {}", cfg.redirect_anthropic); } + if cfg.extend_cache_ttl { + s.push('\n'); + s.push_str("# Promote cache_control breakpoints without an explicit ttl to the 1h tier.\n"); + s.push_str( + "# Helps when turns run longer than 5m and expire their own cache; costs more\n", + ); + s.push_str("# per write, so it loses on workloads of many small incremental writes.\n"); + let _ = writeln!(s, "extend_cache_ttl: {}", cfg.extend_cache_ttl); + } if cfg.show_token || cfg.dynamic_vscode_version || cfg.rate_limit_seconds.is_some() @@ -753,6 +774,13 @@ pub fn load_config_with_options(write_back_on_migration: bool) -> Config { cfg.redirect_anthropic ); } + if let Ok(val) = std::env::var("GHC_PROXY_EXTEND_CACHE_TTL") { + cfg.extend_cache_ttl = val.eq_ignore_ascii_case("true") || val == "1"; + tracing::info!( + "✓ Overriding extend_cache_ttl from GHC_PROXY_EXTEND_CACHE_TTL: {}", + cfg.extend_cache_ttl + ); + } if let Ok(val) = std::env::var("GHC_PROXY_SHOW_TOKEN") { cfg.show_token = val.eq_ignore_ascii_case("true") || val == "1"; tracing::info!( diff --git a/src/server.rs b/src/server.rs index efa975d..6aff552 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1838,7 +1838,10 @@ async fn messages_direct( let mut current = req.clone(); let mut thinking_adapted = false; for _ in 0..4 { - let mut sanitized = anthropic::sanitize_anthropic_request(¤t); + let mut sanitized = anthropic::sanitize_anthropic_request( + ¤t, + state.config_snapshot().extend_cache_ttl, + ); sanitized = anthropic::adjust_thinking_budget(&sanitized); let payload = serde_json::to_vec(&sanitized).unwrap_or_default(); log_debug_request(&state, "/v1/messages", &sanitized); @@ -2251,8 +2254,9 @@ async fn count_tokens( ) .await; let url = format!("{}/v1/messages/count_tokens", state.copilot_base_url()); - let payload = - serde_json::to_vec(&anthropic::sanitize_anthropic_request(&req)).unwrap_or_default(); + // Counting writes no cache entry, so the ttl is irrelevant here. + let payload = serde_json::to_vec(&anthropic::sanitize_anthropic_request(&req, false)) + .unwrap_or_default(); // Deliberately not retried: clients call this before every turn, so a // backoff here stalls the turn only to arrive at the local estimate // anyway.