From a14e1027e7de7507e6ff82b3844a916ff19a26b9 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Sun, 25 Oct 2026 19:18:13 +0800 Subject: [PATCH 1/3] cua-s1: replay language Graphs with resident vision features --- recipe/cua_s1/native_screenshot.md | 6 +- recipe/cua_s1/native_vision_graph.md | 22 ++++-- src/models/qwen3_5/native/src/model.rs | 12 +++- tests/qwen3_5/multimodal_graph.rs | 98 ++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 10 deletions(-) diff --git a/recipe/cua_s1/native_screenshot.md b/recipe/cua_s1/native_screenshot.md index d591a6df..45eb094e 100644 --- a/recipe/cua_s1/native_screenshot.md +++ b/recipe/cua_s1/native_screenshot.md @@ -76,11 +76,11 @@ multimodal adapter and reports prompt-token usage with zero generated tokens. ## Execution controls and limits -`CUA_S1_GRAPH=1` enables host-feature language Graph replay; +`CUA_S1_GRAPH=1` enables language Graph replay for host or resident features; `CUA_S1_VISION_GRAPH=1` independently enables vision Graph replay. `CUA_S1_DEVICE_FEATURES=1` keeps vision features on the GPU and copies image rows -directly into language scratch. Its language path remains eager; it can be paired -with either vision Graph setting. All three controls default off. See the +directly into language scratch before capture/replay. It can be paired with +either vision Graph setting. All three controls default off. See the [resident-feature contract](native_vision_graph.md#gpu-resident-vision-features) for strict control parsing, synchronization and reproduction commands. diff --git a/recipe/cua_s1/native_vision_graph.md b/recipe/cua_s1/native_vision_graph.md index 7675280f..16b921b5 100644 --- a/recipe/cua_s1/native_vision_graph.md +++ b/recipe/cua_s1/native_vision_graph.md @@ -104,8 +104,12 @@ export CUA_S1_DEVICE_FEATURES=1 ``` This switch is independent of `CUA_S1_VISION_GRAPH`. Vision can run eagerly or -replay a Graph; the resident-feature language path remains eager in both modes. -The host-feature language path retains the independent `CUA_S1_GRAPH` control. +replay a Graph; `CUA_S1_GRAPH=1` independently enables language capture/replay +for both host and resident features. Current token IDs, T/H/W positions and +image rows are staged before capture/replay into language-owned scratch. Captures +do not retain the borrowed vision allocation; host/device calls with the same +sequence length share the existing multimodal cache. Scratch growth and capture +failure retain the existing invalidation and eager-fallback behavior. The resident path reuses one vision output across all questions in the request and copies contiguous image-placeholder rows directly into language scratch. It avoids the full feature download and per-question feature upload. It still @@ -142,14 +146,24 @@ The finite-scan test needs only `CUA_S1_CUDA_LIB`; the other tests need all four ```sh cargo test --release --locked -p omni-qwen3-5-native --lib device_tests \ -- --ignored --test-threads=1 -CUA_S1_VISION_GRAPH=0 cargo test --release --locked -p omni-cua-s1-native \ +CUA_S1_GRAPH=1 CUA_S1_VISION_GRAPH=0 cargo test --release --locked -p omni-cua-s1-native \ --lib device_feature_tests -- --ignored --test-threads=1 -CUA_S1_VISION_GRAPH=1 cargo test --release --locked -p omni-cua-s1-native \ +CUA_S1_GRAPH=1 CUA_S1_VISION_GRAPH=1 cargo test --release --locked -p omni-cua-s1-native \ --lib device_feature_tests -- --ignored --test-threads=1 cargo test --release --locked -p omni-qwen3-5-native --lib device_vision_graph_ \ -- --ignored --test-threads=1 ``` +The language-Graph regression also runs with the real library: + +```sh +CUA_S1_TEST_MODEL="$CUA_S1_MODEL" cargo test --release --locked \ + -p omni-qwen3-5-native --lib device_multimodal_graph_ \ + -- --ignored --test-threads=1 --nocapture +``` + +It checks changed tokens/features/positions, host/device interleaving after +producer release, scratch growth, and actual recording-error eager fallback. These tests compare feature bits, hidden states, logits, probabilities and complete responses across changed pixels/goals, repeated questions, different grids with equal patch counts, scratch growth and short-request recovery. diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index b47f4489..95b888f1 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -1293,6 +1293,13 @@ impl Model { self.upload_positions(s, input.position_ids)?; self.embed_tokens(s, input.token_ids)?; self.overwrite_image_rows(s, input, 0)?; + self.run_multimodal(t) + } + + /// Inputs are staged in owned scratch before capture; producer addresses + /// never enter the graph, so host and device inputs can share a capture. + fn run_multimodal(&mut self, t: usize) -> Result> { + let s = self.scratch.as_ref().unwrap(); if self.graph_enabled { if let Some((_, graph)) = self .mm_graphs @@ -1536,7 +1543,7 @@ impl Model { Ok(logprobs) } - /// Eager multimodal execution using borrowed device-resident image rows. + /// Multimodal execution using borrowed device-resident image rows. /// The producer must finish writes before entry. All queued consumer work is /// completed on every exit path, before the external allocation can be reused. pub fn forward_multimodal_device( @@ -1582,8 +1589,7 @@ impl Model { } begin = end; } - self.run(s, &[t], true, false)?; - self.last_hidden(s, t) + self.run_multimodal(t) })(); let synced = self.synchronize(); synced?; diff --git a/tests/qwen3_5/multimodal_graph.rs b/tests/qwen3_5/multimodal_graph.rs index 7b9f7771..40ae99dc 100644 --- a/tests/qwen3_5/multimodal_graph.rs +++ b/tests/qwen3_5/multimodal_graph.rs @@ -364,3 +364,101 @@ fn graph_recording_failure_keeps_eager_result_and_disables_both_modes() -> Resul } Ok(()) } + +#[test] +#[ignore = "requires the Cua-S1 language checkpoint and real CUDA library; run serially"] +fn device_multimodal_graph_uses_current_rows_and_releases_external_storage() -> Result<()> { + let mut model = load_model()?; + let image_token = model.cfg.image_token_id.context("image_token_id")?; + let text = text_tokens(&model, 4)?; + let mut last = Vec::new(); + for seed in 0..4 { + let ids = [text[seed], image_token, image_token, text[(seed + 1) % 4]]; + let indices = [1, 2]; + let temporal = [0, 1 + seed as i64, 1 + seed as i64, 5]; + let height = [0, 1, 2 + seed as i64, 5]; + let width = [0, 2 + seed as i64, 1, 5]; + let features: Vec = (0..2 * model.cfg.hidden) + .map(|i| bf16::from_f32(((i + seed * 7) % 19) as f32 / 18.0 - 0.5)) + .collect(); + let input = MultimodalInput { + token_ids: &ids, + image_token_indices: &indices, + image_embeddings: &features, + position_ids: [&temporal, &height, &width], + }; + model.graph_enabled = false; + let expected = model.forward_multimodal(&input)?; + if !last.is_empty() { + assert_ne!(expected, last, "changed input must exercise new data"); + } + let bytes: Vec<_> = features.iter().flat_map(|v| v.to_le_bytes()).collect(); + let source = DeviceBuffer::new(bytes.len())?; + unsafe { cuda::upload(source.at(0), &bytes, model.stream)? }; + model.synchronize()?; + let device = DeviceMultimodalInput { + token_ids: &ids, + image_token_indices: &indices, + image_embeddings: source.bf16_matrix(2, model.cfg.hidden)?, + position_ids: input.position_ids, + }; + model.graph_enabled = true; + assert_hidden_eq( + &model.forward_multimodal_device(&device)?, + &expected, + "device capture/replay", + ); + assert_eq!( + model.mm_graphs.len(), + 1, + "device path must cache the shared MM graph" + ); + assert_eq!( + model.graph_stats().captures, + 1, + "same shape must not recapture" + ); + assert_eq!(model.graph_stats().replays, 2 * seed as u64); + // A capture must not retain the external producer address. Releasing it + // and using host input must replay the same graph safely. + drop(source); + assert_hidden_eq( + &model.forward_multimodal(&input)?, + &expected, + "host replay after producer release", + ); + assert_eq!(model.graph_stats().replays, 2 * seed as u64 + 1); + last = expected; + } + model.prepare_scratch(model.scratch.as_ref().unwrap().cap + 1)?; + assert!(model.mm_graphs.is_empty(), "growth must retire captures"); + let ids = [text[0], image_token, image_token, text[1]]; + let positions = [0, 1, 1, 2]; + let source = DeviceBuffer::new(2 * model.cfg.hidden * BF16)?; + let bytes = vec![0; 2 * model.cfg.hidden * BF16]; + unsafe { cuda::upload(source.at(0), &bytes, model.stream)? }; + model.synchronize()?; + let device = DeviceMultimodalInput { + token_ids: &ids, + image_token_indices: &[1, 2], + image_embeddings: source.bf16_matrix(2, model.cfg.hidden)?, + position_ids: [&positions; 3], + }; + let captured = model.forward_multimodal_device(&device)?; + assert_eq!(model.mm_graphs.len(), 1); + assert_eq!(model.graph_stats().captures, 2); + let failed = cuda::Graph::capture(model.stream, || { + anyhow::bail!("injected device record failure") + }); + assert!(failed.is_err()); + model.cache_graph(&[4], true, failed)?; + assert!(!model.graph_enabled); + assert!(model.mm_graphs.is_empty()); + assert_hidden_eq( + &model.forward_multimodal_device(&device)?, + &captured, + "device eager recovery", + ); + assert!(!model.graph_enabled); + Ok(()) +} From 911faa0d22753be56c8bf513a23741dfa6d00593 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Sun, 25 Oct 2026 21:08:27 +0800 Subject: [PATCH 2/3] docs(cua-s1): qualify Graph warmup and cache measurements --- recipe/cua_s1/native_vision_graph.md | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/recipe/cua_s1/native_vision_graph.md b/recipe/cua_s1/native_vision_graph.md index 16b921b5..04b83cc4 100644 --- a/recipe/cua_s1/native_vision_graph.md +++ b/recipe/cua_s1/native_vision_graph.md @@ -65,6 +65,37 @@ For optional cache and capture/replay messages, set `CUA_S1_GRAPH_TRACE=1`. Fail are unconditional. Requests execute serially through the existing worker runtime; this feature does not add batching or concurrent forwards. +## Warming the complete working set + +Language scratch growth invalidates earlier language Graphs. Exercise the largest +input first, then warm every measured sequence length and vision grid. Five +warmups alone need not cover a heterogeneous workload. With tracing enabled, +check for timed replays and no new captures or fallback before describing a run +as stable replay. If the real workload exceeds the cache, include and report +its capture/eviction costs instead. + +Entry and byte limits must both fit the intended working set. The recorded +8-payload synthetic matrix uses six grids: its 1024-square image alone requests +262537216 bytes, leaving too little of a 256 MiB budget for the smallest grid. +The original timed rounds therefore contained 15 vision captures (3/6/6), plus +5 language captures after scratch growth in the first round. The original +aggregate latency comparison is not a pure warmed-replay measurement. + +For that bounded matrix, explicitly setting `CUA_S1_VISION_CACHE_ENTRIES=8` and +`CUA_S1_VISION_CACHE_BYTES=1073741824` eliminated timed vision captures after full +warming; these are workload-specific settings, not new defaults or a total GPU +memory limit. On RTX 4090 / SM8.9, matched resident eager/Graph p50 was +116.891/116.399 ms and p95 was 271.109/269.011 ms (24 timed requests per mode, +3 rounds, concurrency 1). Sampled process maxima were 10138/10202 MiB. This small +sample does not establish a general acceleration: Graphs reduce submission +work, while GEMMs, input staging/checks, CPU preprocessing/readout and stream +synchronization remain. Defaults stay off. + +The [follow-up qualification](https://github.com/Levius-Fubuki/system1-omni/releases/download/cua-mm-qualification-20261025/cua-mm-followup-qualification-20261025.tar.gz) +preserves both cache settings, raw/excluded samples, capture traces, source pins +and the full-warming protocol. This follow-up uses the unchanged runtime from +`a14e1027e7de7507e6ff82b3844a916ff19a26b9`. + ## Build and launch Use the pinned base and multimodal adapter revisions listed in the From 781730089520ffd5735b9eac02d1d26b222a1efb Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Sun, 1 Nov 2026 08:47:51 +0800 Subject: [PATCH 3/3] test: qualify OmniJev window Graph replay --- recipe/cua_s1/native_screenshot.md | 2 +- recipe/omnijev/native.md | 26 +++ src/models/omnijev/README.md | 6 + src/models/omnijev/native/src/executor.rs | 4 + tests/omnijev/window_graph.rs | 184 ++++++++++++++++++++++ 5 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 tests/omnijev/window_graph.rs diff --git a/recipe/cua_s1/native_screenshot.md b/recipe/cua_s1/native_screenshot.md index 064928fa..7b225d8a 100644 --- a/recipe/cua_s1/native_screenshot.md +++ b/recipe/cua_s1/native_screenshot.md @@ -251,7 +251,7 @@ export CUA_S1_GRAPH=1 CUA_S1_VISION_GRAPH=1 # Keep prefix=off as the ordinary complete-request numerical control. ``` -The matched implementation is shared with PR184; this extension combines it +The matched implementation is shared with [PR #184](https://github.com/InferMatrixLab/system1-omni/pull/184); this extension combines it with the resident language-Graph path. Prefix reuse removes repeated computation across questions; any combined speedup must be attributed to that work as well as Graph submission savings. A single question or incompatible GEMM plans still diff --git a/recipe/omnijev/native.md b/recipe/omnijev/native.md index b6467c99..f8716919 100644 --- a/recipe/omnijev/native.md +++ b/recipe/omnijev/native.md @@ -19,6 +19,16 @@ cargo build --release --locked -p omni-omnijev-native OMNIJEV_MODEL=$PWD/omnijev-export target/release/omni-omnijev-native ``` +`CUA_S1_GRAPH=1` also enables language window Graphs in this worker, through the +shared `Model::readout_window` implementation. It defaults off. Request and +question prefixes and option continuations can capture and replay; current token, +position and image inputs are staged before replay. Graph entries retain their +prefix source and capture destination allocations. The shared 128-entry FIFO and +scratch growth drain dependent work before retiring affected Graphs; recording +failure disables language Graphs and preserves the eager +result. This switch does not enable vision Graphs or Cua's matched-prefix mode. +The Cua-S1 latency measurements do not establish an OmniJev speedup. + `OMNIJEV_HOST` and `OMNIJEV_PORT` default to `127.0.0.1` and `8000`. At start the worker checks every file of the export against its manifest, loads the weights and answers one request with all three question types before it listens, so `/health` @@ -63,6 +73,22 @@ QWEN3_5_CHECKPOINT=$PWD/omnijev-export CUA_S1_CUDA_LIB=$PWD/target/release/libqw cargo test --release --locked -p omni-qwen3-5-native --test readout -- --ignored --test-threads=1 ``` +The direct OmniJev window-Graph regression selects eager and Graph models without +changing process-wide environment variables. It compares raw FP32 head bits and +answers (excluding timing fields), requires actual capture and warm replay, and +covers same-layout new pixels, changed questions/options, geometry and scratch +growth, returning to an earlier request and malformed-input recovery: + +```sh +OMNIJEV_EXPORT=$PWD/omnijev-export OMNIJEV_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ + cargo test --release --locked -p omni-omnijev-native --lib window_graph_tests \ + -- --ignored --test-threads=1 --nocapture +``` + +Also repeat the `--test worker` command above with `CUA_S1_GRAPH=0` and `1` to +exercise both modes through the production loader. Qualification applies to the +tested model, GPU and CUDA library combination; recheck parity on another stack. + ## Comparison with the reference [`validate.py`](validate.py) runs the reference and the worker on the same images and diff --git a/src/models/omnijev/README.md b/src/models/omnijev/README.md index e614f21b..a61a5e6d 100644 --- a/src/models/omnijev/README.md +++ b/src/models/omnijev/README.md @@ -115,6 +115,12 @@ tokens that remain before it, each as a continuation of the state before it (`Model::readout_window`). The passes use the fixed-algorithm GEMMs and split at multiples of 64, so every answer equals that of one plain pass per (question, option) row over the prefix and that row, which `Executor::execute_rows` keeps for the checks. +The shared language executor also honors `CUA_S1_GRAPH=1` for these windows; +it defaults off. Current inputs are staged before replay, and cached Graphs retain +their recorded prefix allocations until synchronized retirement. See the +[native recipe](../../../recipe/omnijev/native.md#tests) for the direct eager/Graph +parity and capture/replay regression. Cua-S1 performance results do not qualify +OmniJev latency. The passes read the final-norm hidden states at `zq` and `u`, which is what the reference's `hidden_states[-1]` holds, and the option-text tokens' log-probabilities: BF16 logits from the tied embedding over the diff --git a/src/models/omnijev/native/src/executor.rs b/src/models/omnijev/native/src/executor.rs index 170034a5..21d09b31 100644 --- a/src/models/omnijev/native/src/executor.rs +++ b/src/models/omnijev/native/src/executor.rs @@ -19,6 +19,10 @@ use crate::export; use crate::heads::{self, Heads}; use crate::processing::{IMAGE_TOKEN, PreparedRequest, Processor, Row}; +#[cfg(test)] +#[path = "../../../../../tests/omnijev/window_graph.rs"] +mod window_graph_tests; + pub struct Executor { vision: VisionModel, language: Model, diff --git a/tests/omnijev/window_graph.rs b/tests/omnijev/window_graph.rs new file mode 100644 index 00000000..625feae9 --- /dev/null +++ b/tests/omnijev/window_graph.rs @@ -0,0 +1,184 @@ +//! Exercise OmniJev's production prefix/option routing with language Graphs. +use super::*; + +#[path = "common.rs"] +mod common; + +// Replace the model only after dropping its prefix states and the old model. +// This selects both modes without mutating process-wide environment variables. +fn language_mode(executor: Executor, dir: &Path, library: &Path, graph: bool) -> Executor { + let Executor { + vision, + language, + heads, + calibration, + request, + question, + } = executor; + drop((request, question, language)); + Executor { + vision, + language: Model::load_with_graph(dir, library, graph).unwrap(), + heads, + calibration, + request: None, + question: None, + } +} + +fn request( + processor: &Processor, + width: u32, + height: u32, + image_changed: bool, + question_changed: bool, + growth: bool, +) -> PreparedRequest { + let colour = if image_changed { + [30, 180, 90] + } else { + [200, 40, 40] + }; + let instruction = if growth { + format!( + "Look{} at the image. Which colour dominates?", + " again".repeat(1200) + ) + } else if question_changed { + "Which colour occupies most of this picture?".into() + } else { + "Which colour dominates the image?".into() + }; + let options = if question_changed { + json!([{"key": "green", "text": "green"}, {"key": "red", "text": "red"}, + {"key": "blue", "text": "blue"}]) + } else { + json!([{"key": "red", "text": "red"}, {"key": "green", "text": "green"}]) + }; + let raw = serde_json::to_vec(&json!({ + "model": contract::MODEL_ID, + "state": {"images": [common::png_url(width, height, colour)]}, + "questions": { + "colour": {"type": "choice", "instructions": instruction, "options": options}, + "red": {"type": "noul", "instructions": "The image is mostly red."}, + "warm": {"type": "score", "instructions": "How warm is the colour?", + "levels": ["cold", "neutral", "warm", "hot"]} + } + })) + .unwrap(); + processor.prepare(&raw).unwrap() +} + +fn bits(outputs: &[Vec]) -> Vec { + outputs.iter().flatten().map(|v| v.to_bits()).collect() +} + +fn decisions(mut answers: Vec) -> Vec { + for answer in &mut answers { + let answer = answer.as_object_mut().unwrap(); + answer.remove("latency_s"); + answer.remove("latency_total_s"); + } + answers +} + +#[test] +#[ignore = "requires CUDA, OMNIJEV_EXPORT and OMNIJEV_CUDA_LIB; run serially"] +fn window_graph_matches_eager_across_requests_growth_and_recovery() { + let dir = + std::path::PathBuf::from(std::env::var_os("OMNIJEV_EXPORT").expect("set OMNIJEV_EXPORT")); + let library = std::path::PathBuf::from( + std::env::var_os("OMNIJEV_CUDA_LIB").expect("set OMNIJEV_CUDA_LIB"), + ); + let (executor, processor) = Executor::load(&dir, &library).unwrap(); + let cases = [ + request(&processor, 320, 200, false, false, false), + request(&processor, 320, 200, true, false, false), + request(&processor, 320, 200, true, true, false), + request(&processor, 1280, 720, false, false, true), + ]; + let mut eager = language_mode(executor, &dir, &library, false); + let expected: Vec<_> = cases + .iter() + .map(|prepared| { + let shared = eager.head_outputs(prepared, true).unwrap(); + let plain = eager.head_outputs(prepared, false).unwrap(); + assert_eq!(bits(&shared), bits(&plain), "eager prefix/row parity"); + (bits(&shared), decisions(eager.execute(prepared).unwrap())) + }) + .collect(); + assert_ne!( + expected[0].0, expected[1].0, + "changed request must change head outputs" + ); + assert_eq!(eager.language.graph_stats().captures, 0); + let mut graph = language_mode(eager, &dir, &library, true); + assert_eq!(cases[0].inputs[0].token_ids, cases[1].inputs[0].token_ids); + let mut previous_captures = 0; + for (visit, case) in [0, 1, 2, 0, 3, 0].into_iter().enumerate() { + let prepared = &cases[case]; + let before = graph.language.graph_stats(); + let actual = bits(&graph.head_outputs(prepared, true).unwrap()); + assert_eq!( + actual, expected[case].0, + "Graph/eager head parity, visit {visit}" + ); + let captured = graph.language.graph_stats(); + assert!(captured.enabled, "Graph unexpectedly disabled"); + assert!( + captured.captures > 0, + "OmniJev windows must capture a Graph" + ); + assert_eq!(captured.fallbacks, 0, "unexpected capture failure"); + if case == 1 { + assert_eq!( + captured.captures, previous_captures, + "same-layout new pixels must reuse Graphs" + ); + } + // Growth can retire the request-prefix Graph midway through the first + // pass. Stabilize once before asserting warm-cache reuse. + assert_eq!(bits(&graph.head_outputs(prepared, true).unwrap()), actual); + let stable = graph.language.graph_stats(); + assert_eq!(bits(&graph.head_outputs(prepared, true).unwrap()), actual); + let replayed = graph.language.graph_stats(); + assert!( + replayed.replays > stable.replays, + "warm OmniJev windows must replay" + ); + assert_eq!( + replayed.captures, stable.captures, + "warm windows must reuse captures" + ); + let answers = decisions(graph.execute(prepared).unwrap()); + assert_eq!( + answers, expected[case].1, + "Graph/eager decisions, visit {visit}" + ); + if case == 3 { + assert!( + replayed.invalidations > before.invalidations, + "large request must retire scratch Graphs" + ); + } + previous_captures = graph.language.graph_stats().captures; + eprintln!( + "OMNIJEV_WINDOW_GRAPH {}", + json!({ + "visit": visit, "case": case, "head_bits": actual, "answers": answers, + "captures": replayed.captures, "replays": replayed.replays, + "invalidations": replayed.invalidations, "fallbacks": replayed.fallbacks + }) + ); + } + let mut malformed = request(&processor, 320, 200, false, false, false); + malformed.inputs[0].token_ids[0] = u32::MAX; + assert!(graph.head_outputs(&malformed, true).is_err()); + assert_eq!( + bits(&graph.head_outputs(&cases[0], true).unwrap()), + expected[0].0 + ); + assert_eq!(decisions(graph.execute(&cases[0]).unwrap()), expected[0].1); + assert!(graph.language.graph_stats().enabled); + assert_eq!(graph.language.graph_stats().fallbacks, 0); +}