From e9c55faa30bc300b996ab3c079244f9efec603bb Mon Sep 17 00:00:00 2001 From: Sergei Notevskii Date: Mon, 26 Oct 2026 01:45:50 +0200 Subject: [PATCH] Preserve known Claude usage when cache fields are null Accept only the documented nullable Claude cache fields as unknown so native transcripts retain measured sibling counters and attribution. Keep other Claude and Codex counter validation strict, with regression coverage for partial totals and repeated responses. --- crates/warp_harness_usage/src/claude.rs | 9 +- crates/warp_harness_usage/src/claude_tests.rs | 133 ++++++++++++++++++ crates/warp_harness_usage/src/codex.rs | 2 +- crates/warp_harness_usage/src/codex_tests.rs | 23 +++ crates/warp_harness_usage/src/counters.rs | 12 +- 5 files changed, 174 insertions(+), 5 deletions(-) diff --git a/crates/warp_harness_usage/src/claude.rs b/crates/warp_harness_usage/src/claude.rs index 063bf4d2444..6c0d05d5361 100644 --- a/crates/warp_harness_usage/src/claude.rs +++ b/crates/warp_harness_usage/src/claude.rs @@ -272,12 +272,17 @@ pub fn extract_claude<'a>( fn parse_usage(value: &Value, findings: &mut Findings) -> Option> { if value .get("cache_creation") - .is_some_and(|partitions| !partitions.is_object()) + .is_some_and(|partitions| !partitions.is_null() && !partitions.is_object()) { findings.token(ReasonCode::InvalidData); return None; } - let usage = Counters::parse(value, PATHS, findings)?; + let usage = Counters::parse( + value, + PATHS, + &["/cache_read_input_tokens", "/cache_creation_input_tokens"], + findings, + )?; let [_, _, _, aggregate, short, long] = usage.values; if let (Some(aggregate), Some(short), Some(long)) = (aggregate, short, long) && short.checked_add(long) != Some(aggregate) diff --git a/crates/warp_harness_usage/src/claude_tests.rs b/crates/warp_harness_usage/src/claude_tests.rs index 66b3eb49362..76ccb5f2a3c 100644 --- a/crates/warp_harness_usage/src/claude_tests.rs +++ b/crates/warp_harness_usage/src/claude_tests.rs @@ -91,6 +91,139 @@ fn response(id: &str, usage: Value, content: Value) -> Value { json!({"type":"assistant","message":{"id":id,"model":"claude-a","usage":usage,"content":content}}) } +#[test] +fn nullable_cache_fields_preserve_known_usage_attribution_and_tools() { + for (field, expected) in [ + ( + "cache_creation", + json!({"input_tokens":100,"output_tokens":0,"cache_read_input_tokens":4096,"cache_creation_input_tokens":0}), + ), + ( + "cache_read_input_tokens", + json!({"input_tokens":100,"output_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0}}), + ), + ( + "cache_creation_input_tokens", + json!({"input_tokens":100,"output_tokens":0,"cache_read_input_tokens":4096,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0}}), + ), + ] { + let mut usage = json!({ + "input_tokens":100,"output_tokens":0,"cache_read_input_tokens":4096, + "cache_creation_input_tokens":0, + "cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0}, + "service_tier":"standard" + }); + usage[field] = Value::Null; + let snapshot = capture(&[response( + "a", + usage, + json!([{"type":"tool_use","id":"t","name":"Read"}]), + )]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap(), + json!({ + "usage":expected, + "attribution":[{"model":"claude-a","service_tier":"standard","usage":expected}], + "toolCalls":{"total":1,"byName":{"Read":1}} + }), + "{field}" + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Known); + assert_eq!(snapshot.coverage.tool_status, CoverageStatus::Known); + assert!(snapshot.diagnostics.reasons.is_empty(), "{field}"); + } +} + +#[test] +fn null_cache_counter_in_one_response_makes_its_observed_total_partial() { + let snapshot = capture(&[ + response( + "a", + json!({"input_tokens":10,"output_tokens":1,"cache_read_input_tokens":100}), + json!([]), + ), + response( + "b", + json!({"input_tokens":20,"output_tokens":2,"cache_read_input_tokens":null}), + json!([]), + ), + ]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap()["usage"], + json!({"input_tokens":30,"output_tokens":3,"cache_read_input_tokens":100}) + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Partial); + assert_eq!(snapshot.coverage.tool_status, CoverageStatus::Known); + assert_eq!( + snapshot.diagnostics.reasons, + [(ReasonCode::IncompleteInput, 1)].into() + ); +} + +#[test] +fn later_null_cache_fields_do_not_replace_a_complete_response() { + let known = json!({ + "input_tokens":100,"output_tokens":2,"cache_read_input_tokens":4096, + "cache_creation_input_tokens":15, + "cache_creation":{"ephemeral_5m_input_tokens":10,"ephemeral_1h_input_tokens":5} + }); + for field in [ + "cache_creation", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ] { + let mut later = known.clone(); + later[field] = Value::Null; + let snapshot = capture(&[ + response("a", known.clone(), json!([])), + response("a", later, json!([])), + ]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap()["usage"], + json!({ + "input_tokens":100,"output_tokens":2,"cache_read_input_tokens":4096, + "cache_creation_input_tokens":15, + "cache_creation":{"ephemeral_5m_input_tokens":10,"ephemeral_1h_input_tokens":5} + }), + "{field}" + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Known); + assert!(snapshot.diagnostics.reasons.is_empty(), "{field}"); + } +} + +#[test] +fn nullable_cache_fields_do_not_relax_other_counter_validation() { + for (path, value) in [ + ("/cache_read_input_tokens", json!("0")), + ("/cache_read_input_tokens", json!(-1)), + ("/cache_creation_input_tokens", json!("0")), + ("/cache_creation_input_tokens", json!(-1)), + ("/cache_creation", json!("0")), + ("/cache_creation", json!(-1)), + ("/input_tokens", Value::Null), + ("/output_tokens", Value::Null), + ("/cache_creation/ephemeral_5m_input_tokens", Value::Null), + ("/cache_creation/ephemeral_1h_input_tokens", Value::Null), + ] { + let mut usage = json!({ + "input_tokens":100,"output_tokens":0,"cache_read_input_tokens":4096, + "cache_creation_input_tokens":0, + "cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0} + }); + *usage.pointer_mut(path).unwrap() = value; + let snapshot = capture(&[response("a", usage, json!([]))]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap(), + json!({"toolCalls":{"total":0,"byName":{}}}), + "{path}" + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Unavailable); + assert_eq!(snapshot.coverage.tool_status, CoverageStatus::Known); + assert_eq!(snapshot.diagnostics.reasons[&ReasonCode::InvalidData], 1); + } +} + #[test] fn evolving_responses_do_not_deduplicate_distinct_tool_blocks() { let entries = parse_jsonl( diff --git a/crates/warp_harness_usage/src/codex.rs b/crates/warp_harness_usage/src/codex.rs index 380a572e940..1f2428a48dc 100644 --- a/crates/warp_harness_usage/src/codex.rs +++ b/crates/warp_harness_usage/src/codex.rs @@ -257,7 +257,7 @@ fn observe_checkpoint( } fn parse_usage(value: &Value, findings: &mut Findings) -> Option> { - let usage = Counters::parse(value, PATHS, findings)?; + let usage = Counters::parse(value, PATHS, &[], findings)?; let [input, cached, cache_write, output, reasoning, total] = usage.values; let invalid = matches!((input, cached), (Some(input), Some(cached)) if cached > input) || matches!((input, cache_write), (Some(input), Some(cache_write)) if cache_write > input) diff --git a/crates/warp_harness_usage/src/codex_tests.rs b/crates/warp_harness_usage/src/codex_tests.rs index 000e52e59ac..fce70f46172 100644 --- a/crates/warp_harness_usage/src/codex_tests.rs +++ b/crates/warp_harness_usage/src/codex_tests.rs @@ -346,6 +346,29 @@ fn counter_bounds_and_absence_survive_serialization() { ); } +#[test] +fn null_counters_preserve_the_valid_prefix() { + for field in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_output_tokens", + "total_tokens", + ] { + let mut invalid = checkpoint(100, 90); + invalid["payload"]["info"]["total_token_usage"][field] = Value::Null; + let snapshot = capture(&[checkpoint(10, 10), invalid]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap()["usage"], + json!({"total_tokens":10}), + "{field}" + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Partial); + assert_eq!(snapshot.diagnostics.reasons[&ReasonCode::InvalidData], 1); + } +} + #[test] fn excessive_cache_write_totals_preserve_the_valid_prefix() { for input in [0, 100, i64::MAX - 1] { diff --git a/crates/warp_harness_usage/src/counters.rs b/crates/warp_harness_usage/src/counters.rs index 700470c9a48..161524d9597 100644 --- a/crates/warp_harness_usage/src/counters.rs +++ b/crates/warp_harness_usage/src/counters.rs @@ -21,8 +21,13 @@ impl Default for Counters { } impl Counters { - /// Parse known nonnegative counters while preserving absent fields as unknown. - pub(crate) fn parse(value: &Value, paths: [&str; N], findings: &mut Findings) -> Option { + /// Parse nonnegative counters, preserving absent and explicitly nullable fields as unknown. + pub(crate) fn parse( + value: &Value, + paths: [&str; N], + nullable_paths: &[&str], + findings: &mut Findings, + ) -> Option { if !value.is_object() { findings.token(ReasonCode::InvalidData); return None; @@ -30,6 +35,9 @@ impl Counters { let mut result = Self::default(); for (index, path) in paths.into_iter().enumerate() { if let Some(value) = value.pointer(path) { + if value.is_null() && nullable_paths.contains(&path) { + continue; + } match value.as_i64().filter(|count| *count >= 0) { Some(count) => result.values[index] = Some(count), None => {