diff --git a/crates/warp_harness_usage/src/claude.rs b/crates/warp_harness_usage/src/claude.rs index 063bf4d2444..6c0d05d5361 100644 --- a/crates/warp_harness_usage/src/claude.rs +++ b/crates/warp_harness_usage/src/claude.rs @@ -272,12 +272,17 @@ pub fn extract_claude<'a>( fn parse_usage(value: &Value, findings: &mut Findings) -> Option> { if value .get("cache_creation") - .is_some_and(|partitions| !partitions.is_object()) + .is_some_and(|partitions| !partitions.is_null() && !partitions.is_object()) { findings.token(ReasonCode::InvalidData); return None; } - let usage = Counters::parse(value, PATHS, findings)?; + let usage = Counters::parse( + value, + PATHS, + &["/cache_read_input_tokens", "/cache_creation_input_tokens"], + findings, + )?; let [_, _, _, aggregate, short, long] = usage.values; if let (Some(aggregate), Some(short), Some(long)) = (aggregate, short, long) && short.checked_add(long) != Some(aggregate) diff --git a/crates/warp_harness_usage/src/claude_tests.rs b/crates/warp_harness_usage/src/claude_tests.rs index 66b3eb49362..76ccb5f2a3c 100644 --- a/crates/warp_harness_usage/src/claude_tests.rs +++ b/crates/warp_harness_usage/src/claude_tests.rs @@ -91,6 +91,139 @@ fn response(id: &str, usage: Value, content: Value) -> Value { json!({"type":"assistant","message":{"id":id,"model":"claude-a","usage":usage,"content":content}}) } +#[test] +fn nullable_cache_fields_preserve_known_usage_attribution_and_tools() { + for (field, expected) in [ + ( + "cache_creation", + json!({"input_tokens":100,"output_tokens":0,"cache_read_input_tokens":4096,"cache_creation_input_tokens":0}), + ), + ( + "cache_read_input_tokens", + json!({"input_tokens":100,"output_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0}}), + ), + ( + "cache_creation_input_tokens", + json!({"input_tokens":100,"output_tokens":0,"cache_read_input_tokens":4096,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0}}), + ), + ] { + let mut usage = json!({ + "input_tokens":100,"output_tokens":0,"cache_read_input_tokens":4096, + "cache_creation_input_tokens":0, + "cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0}, + "service_tier":"standard" + }); + usage[field] = Value::Null; + let snapshot = capture(&[response( + "a", + usage, + json!([{"type":"tool_use","id":"t","name":"Read"}]), + )]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap(), + json!({ + "usage":expected, + "attribution":[{"model":"claude-a","service_tier":"standard","usage":expected}], + "toolCalls":{"total":1,"byName":{"Read":1}} + }), + "{field}" + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Known); + assert_eq!(snapshot.coverage.tool_status, CoverageStatus::Known); + assert!(snapshot.diagnostics.reasons.is_empty(), "{field}"); + } +} + +#[test] +fn null_cache_counter_in_one_response_makes_its_observed_total_partial() { + let snapshot = capture(&[ + response( + "a", + json!({"input_tokens":10,"output_tokens":1,"cache_read_input_tokens":100}), + json!([]), + ), + response( + "b", + json!({"input_tokens":20,"output_tokens":2,"cache_read_input_tokens":null}), + json!([]), + ), + ]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap()["usage"], + json!({"input_tokens":30,"output_tokens":3,"cache_read_input_tokens":100}) + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Partial); + assert_eq!(snapshot.coverage.tool_status, CoverageStatus::Known); + assert_eq!( + snapshot.diagnostics.reasons, + [(ReasonCode::IncompleteInput, 1)].into() + ); +} + +#[test] +fn later_null_cache_fields_do_not_replace_a_complete_response() { + let known = json!({ + "input_tokens":100,"output_tokens":2,"cache_read_input_tokens":4096, + "cache_creation_input_tokens":15, + "cache_creation":{"ephemeral_5m_input_tokens":10,"ephemeral_1h_input_tokens":5} + }); + for field in [ + "cache_creation", + "cache_read_input_tokens", + "cache_creation_input_tokens", + ] { + let mut later = known.clone(); + later[field] = Value::Null; + let snapshot = capture(&[ + response("a", known.clone(), json!([])), + response("a", later, json!([])), + ]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap()["usage"], + json!({ + "input_tokens":100,"output_tokens":2,"cache_read_input_tokens":4096, + "cache_creation_input_tokens":15, + "cache_creation":{"ephemeral_5m_input_tokens":10,"ephemeral_1h_input_tokens":5} + }), + "{field}" + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Known); + assert!(snapshot.diagnostics.reasons.is_empty(), "{field}"); + } +} + +#[test] +fn nullable_cache_fields_do_not_relax_other_counter_validation() { + for (path, value) in [ + ("/cache_read_input_tokens", json!("0")), + ("/cache_read_input_tokens", json!(-1)), + ("/cache_creation_input_tokens", json!("0")), + ("/cache_creation_input_tokens", json!(-1)), + ("/cache_creation", json!("0")), + ("/cache_creation", json!(-1)), + ("/input_tokens", Value::Null), + ("/output_tokens", Value::Null), + ("/cache_creation/ephemeral_5m_input_tokens", Value::Null), + ("/cache_creation/ephemeral_1h_input_tokens", Value::Null), + ] { + let mut usage = json!({ + "input_tokens":100,"output_tokens":0,"cache_read_input_tokens":4096, + "cache_creation_input_tokens":0, + "cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0} + }); + *usage.pointer_mut(path).unwrap() = value; + let snapshot = capture(&[response("a", usage, json!([]))]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap(), + json!({"toolCalls":{"total":0,"byName":{}}}), + "{path}" + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Unavailable); + assert_eq!(snapshot.coverage.tool_status, CoverageStatus::Known); + assert_eq!(snapshot.diagnostics.reasons[&ReasonCode::InvalidData], 1); + } +} + #[test] fn evolving_responses_do_not_deduplicate_distinct_tool_blocks() { let entries = parse_jsonl( diff --git a/crates/warp_harness_usage/src/codex.rs b/crates/warp_harness_usage/src/codex.rs index 380a572e940..1f2428a48dc 100644 --- a/crates/warp_harness_usage/src/codex.rs +++ b/crates/warp_harness_usage/src/codex.rs @@ -257,7 +257,7 @@ fn observe_checkpoint( } fn parse_usage(value: &Value, findings: &mut Findings) -> Option> { - let usage = Counters::parse(value, PATHS, findings)?; + let usage = Counters::parse(value, PATHS, &[], findings)?; let [input, cached, cache_write, output, reasoning, total] = usage.values; let invalid = matches!((input, cached), (Some(input), Some(cached)) if cached > input) || matches!((input, cache_write), (Some(input), Some(cache_write)) if cache_write > input) diff --git a/crates/warp_harness_usage/src/codex_tests.rs b/crates/warp_harness_usage/src/codex_tests.rs index 000e52e59ac..fce70f46172 100644 --- a/crates/warp_harness_usage/src/codex_tests.rs +++ b/crates/warp_harness_usage/src/codex_tests.rs @@ -346,6 +346,29 @@ fn counter_bounds_and_absence_survive_serialization() { ); } +#[test] +fn null_counters_preserve_the_valid_prefix() { + for field in [ + "input_tokens", + "cached_input_tokens", + "cache_write_input_tokens", + "output_tokens", + "reasoning_output_tokens", + "total_tokens", + ] { + let mut invalid = checkpoint(100, 90); + invalid["payload"]["info"]["total_token_usage"][field] = Value::Null; + let snapshot = capture(&[checkpoint(10, 10), invalid]); + assert_eq!( + serde_json::to_value(&snapshot.payload).unwrap()["usage"], + json!({"total_tokens":10}), + "{field}" + ); + assert_eq!(snapshot.coverage.token_status, CoverageStatus::Partial); + assert_eq!(snapshot.diagnostics.reasons[&ReasonCode::InvalidData], 1); + } +} + #[test] fn excessive_cache_write_totals_preserve_the_valid_prefix() { for input in [0, 100, i64::MAX - 1] { diff --git a/crates/warp_harness_usage/src/counters.rs b/crates/warp_harness_usage/src/counters.rs index 700470c9a48..161524d9597 100644 --- a/crates/warp_harness_usage/src/counters.rs +++ b/crates/warp_harness_usage/src/counters.rs @@ -21,8 +21,13 @@ impl Default for Counters { } impl Counters { - /// Parse known nonnegative counters while preserving absent fields as unknown. - pub(crate) fn parse(value: &Value, paths: [&str; N], findings: &mut Findings) -> Option { + /// Parse nonnegative counters, preserving absent and explicitly nullable fields as unknown. + pub(crate) fn parse( + value: &Value, + paths: [&str; N], + nullable_paths: &[&str], + findings: &mut Findings, + ) -> Option { if !value.is_object() { findings.token(ReasonCode::InvalidData); return None; @@ -30,6 +35,9 @@ impl Counters { let mut result = Self::default(); for (index, path) in paths.into_iter().enumerate() { if let Some(value) = value.pointer(path) { + if value.is_null() && nullable_paths.contains(&path) { + continue; + } match value.as_i64().filter(|count| *count >= 0) { Some(count) => result.values[index] = Some(count), None => {