From f6cfefe5d7df8e31884b9396701245b934590bd6 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 1 Oct 2026 03:20:00 +0000 Subject: [PATCH 1/9] Expose training rollout scores and full policy publication Signed-off-by: 0z5a --- benchmarks/nixl_preflight.py | 73 + benchmarks/pd_training_contract.py | 172 +++ benchmarks/training_contract.py | 94 ++ docs/training_contract.md | 82 ++ .../training-contract-a100-20261001/README.md | 77 ++ .../raw/contract-cpu-final.log | 10 + .../raw/cpu-regression-final.log | 9 + .../raw/lint-final.log | 4 + .../raw/nixl-preflight-ipc.log | 51 + .../raw/nixl-preflight-shm.log | 51 + .../raw/nixl-preflight-tcp.log | 6 + .../raw/nixl-preflight.log | 12 + .../raw/pd-full-graph-matched.json | 1189 +++++++++++++++++ .../raw/pd-full-graph-matched.log | 11 + .../raw/pd-full-graph-tcp.log | 16 + .../raw/pd-tiny-eager-tcp.json | 1160 ++++++++++++++++ .../raw/pd-tiny-eager-tcp.log | 11 + .../raw/pd-tiny-graph-tcp.json | 1163 ++++++++++++++++ .../raw/pd-tiny-graph-tcp.log | 11 + .../raw/runtime.json | 13 + .../training-contract-thor-20261001/README.md | 35 + .../raw/shared-full-rollout-benchmark.json | 98 ++ tests/__init__.py | 1 + tests/test_pd.py | 1 + tests/test_pd_weight_control.py | 91 ++ tests/test_speculative.py | 2 +- tests/test_training_contract.py | 186 +++ vllm_rlt/core/kv_cache_manager.py | 6 + vllm_rlt/engine/llm_engine.py | 94 +- vllm_rlt/entrypoints/llm.py | 28 + vllm_rlt/pd/engine.py | 107 +- vllm_rlt/pd/worker.py | 33 +- vllm_rlt/request.py | 14 + vllm_rlt/sampling_params.py | 20 +- vllm_rlt/worker/model_runner.py | 3 + vllm_rlt/worker/sampler.py | 6 +- vllm_rlt/worker/sampling.py | 13 +- vllm_rlt/worker/speculative.py | 14 +- 38 files changed, 4952 insertions(+), 15 deletions(-) create mode 100644 benchmarks/nixl_preflight.py create mode 100644 benchmarks/pd_training_contract.py create mode 100644 benchmarks/training_contract.py create mode 100644 docs/training_contract.md create mode 100644 docs/validation/training-contract-a100-20261001/README.md create mode 100644 docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/lint-final.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/runtime.json create mode 100644 docs/validation/training-contract-thor-20261001/README.md create mode 100644 docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json create mode 100644 tests/__init__.py create mode 100644 tests/test_pd_weight_control.py create mode 100644 tests/test_training_contract.py diff --git a/benchmarks/nixl_preflight.py b/benchmarks/nixl_preflight.py new file mode 100644 index 00000000..a56156b9 --- /dev/null +++ b/benchmarks/nixl_preflight.py @@ -0,0 +1,73 @@ +import multiprocessing as mp +import time + +import torch +from nixl._api import nixl_agent + + +def worker(device, pipe): + torch.cuda.set_device(device) + tensor = ( + torch.arange(32, dtype=torch.float32, device=f"cuda:{device}") + if device == 0 + else torch.zeros(32, device=f"cuda:{device}") + ) + torch.cuda.synchronize() + name = f"ouro-nixl-{device}" + agent = nixl_agent(name) + registration = agent.register_memory(tensor) + pipe.send( + dict(name=name, metadata=agent.get_agent_metadata(), ptr=tensor.data_ptr(), device=device) + ) + peer = pipe.recv() + agent.add_remote_agent(peer["metadata"]) + pipe.send("ready") + pipe.recv() + if device == 0: + handle = agent.initialize_xfer( + "WRITE", + agent.get_xfer_descs(tensor), + agent.get_xfer_descs([(peer["ptr"], 128, peer["device"])], "VRAM"), + peer["name"], + notif_msg=b"done", + backends=["UCX"], + ) + agent.transfer(handle) + while agent.check_xfer_state(handle) != "DONE": + time.sleep(0.001) + agent.release_xfer_handle(handle) + else: + while not agent.get_new_notifs(): + time.sleep(0.001) + pipe.send(tensor.cpu().tolist()) + pipe.recv() + agent.remove_remote_agent(peer["name"]) + agent.deregister_memory(registration) + pipe.close() + + +if __name__ == "__main__": + context = mp.get_context("spawn") + peers = [] + for device in range(2): + parent, child = context.Pipe() + process = context.Process(target=worker, args=(device, child)) + process.start() + child.close() + peers.append((process, parent)) + infos = [p.recv() for _, p in peers] + for i, (_, p) in enumerate(peers): + p.send(infos[1 - i]) + assert all(p.recv() == "ready" for _, p in peers) + for _, p in peers: + p.send("go") + values = [p.recv() for _, p in peers] + print("VALUES", values, flush=True) + for _, p in peers: + p.send("stop") + for process, p in peers: + process.join() + p.close() + assert process.exitcode == 0 + assert values[0] == values[1] + print("MULTIPROCESS_NIXL_PASSED", flush=True) diff --git a/benchmarks/pd_training_contract.py b/benchmarks/pd_training_contract.py new file mode 100644 index 00000000..38830d64 --- /dev/null +++ b/benchmarks/pd_training_contract.py @@ -0,0 +1,172 @@ +"""Two spawned CUDA owners: NIXL KV, policy publication, and local-engine parity.""" + +import argparse +import json +import time +from dataclasses import asdict, replace +from pathlib import Path + +import torch + +from vllm_rlt import LLM, CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig +from vllm_rlt.models import OuroConfig, OuroForCausalLM +from vllm_rlt.pd.config import PDConfig +from vllm_rlt.pd.engine import PDEngine + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("output", type=Path) + parser.add_argument("--model", type=Path) + parser.add_argument("--cuda-graphs", action="store_true") + parser.add_argument("--temperature", type=float, default=0.9) + parser.add_argument("--max-num-seqs", type=int, default=4) + args = parser.parse_args() + torch.set_num_threads(4) + torch.manual_seed(42) + dtype = torch.bfloat16 if args.model else torch.float32 + source = args.model or OuroConfig( + vocab_size=64, + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=8, + max_position_embeddings=128, + total_ut_steps=4, + bos_token_id=0, + eos_token_id=0, + ) + native = ( + OuroForCausalLM.from_pretrained(source, device="cuda:0", dtype=dtype) + if args.model + else OuroForCausalLM(source).to(device="cuda:0", dtype=dtype) + ) + execution = ExecutionConfig(cuda_graphs=args.cuda_graphs, async_scheduling=True) + cache = CacheConfig(num_blocks=512, enable_prefix_caching=True) + scheduler = SchedulerConfig( + max_num_seqs=args.max_num_seqs, max_num_batched_tokens=64, prefill_chunk_size=16 + ) + common = dict( + execution_config=execution, + exit_config=ExitConfig("ouro_delayed"), + attention_backend="triton", + ) + local = LLM(native, cache_config=cache, scheduler_config=scheduler, **common) + pd = PDEngine( + source, + dtype="bfloat16" if args.model else "float32", + pd_config=PDConfig( + max_control_messages=1, startup_timeout=600, request_timeout=600, shutdown_timeout=120 + ), + prefill_cache_config=cache, + decode_cache_config=cache, + prefill_scheduler_config=scheduler, + decode_scheduler_config=scheduler, + terminate_workers_on_failure=False, + **common, + ) + disaggregated = LLM(pd) + prompts = [[3, 4, 5] * 9 + [6], [3, 4, 5] * 9 + [7]] + report = dict(model=asdict(native.config), cuda_graphs=args.cuda_graphs, iterations=[]) + report["max_num_seqs"] = args.max_num_seqs + try: + for version, loops in enumerate((2, 3, 4), 1): + with torch.no_grad(): + native.lm_head.weight.mul_(0.98) + for arm in (local, disaggregated): + arm.start_weight_update(version) + weights = list(native.named_parameters()) + for offset in range(0, len(weights), 8): + arm.update_weights(weights[offset : offset + 8]) + arm.finish_weight_update() + assert arm.get_weight_version() == version + for mode in ("raw", "processed"): + params = [ + SamplingParams( + max_tokens=6, + max_loops=loops, + seed=11 + i, + temperature=args.temperature, + top_k=8, + top_p=0.8, + ignore_eos=True, + logprobs=0, + logprobs_mode=mode, + ) + for i in range(len(prompts)) + ] + outputs, timings = {}, {} + for label, arm in (("local", local), ("pd", disaggregated)): + started = time.monotonic() + outputs[label] = arm.generate(prompts, params) + timings[label] = time.monotonic() - started + errors = [] + for expected, result in zip(outputs["local"], outputs["pd"], strict=True): + assert expected.token_ids == result.token_ids + assert expected.exit_depths == result.exit_depths + assert result.weight_version == version + assert result.sampling_params == expected.sampling_params + delta = ( + torch.tensor(expected.log_probs) - torch.tensor(result.log_probs) + ).abs() + errors.append(delta.max().item()) + torch.testing.assert_close( + torch.tensor(expected.log_probs), + torch.tensor(result.log_probs), + atol=2e-5, + rtol=2e-5, + ) + row = dict( + version=version, + loops=loops, + mode=mode, + seconds=timings, + max_logprob_error=max(errors), + outputs=[asdict(o) for o in outputs["pd"]], + ) + report["iterations"].append(row) + print("PARITY", version, loops, mode, timings, max(errors), flush=True) + # A failed chunk cannot expose a mixed policy; a newer full epoch recovers. + disaggregated.start_weight_update(4) + try: + disaggregated.update_weights([("missing_parameter", torch.zeros(1))]) + except RuntimeError as error: + report["rejected_chunk"] = str(error) + else: + raise AssertionError("Invalid publication unexpectedly succeeded") + disaggregated.start_weight_update(5) + disaggregated.update_weights(native.named_parameters()) + disaggregated.finish_weight_update() + recovered = disaggregated.generate(prompts[:1], params[:1])[0] + assert recovered.weight_version == 5 + stopped = disaggregated.generate( + prompts[:1], [replace(params[0], stop_token_ids=(recovered.token_ids[0],))] + )[0] + assert len(stopped.token_ids) == len(stopped.log_probs) == 1 + assert stopped.finish_reason == "stop" + pd.add_request("abort", prompts[0], params[0]) + while not pd.transfers[next(iter(pd.transfers))].latest_output: + pd.step() + aborted = pd.abort_request("abort") + assert aborted.finish_reason == "abort" + assert len(aborted.log_probs) == len(aborted.token_ids) + while pd.has_unfinished_requests(): + pd.step() + disaggregated.reset_prefix_cache() + report.update(recovered_version=5, stop=asdict(stopped), abort=asdict(aborted)) + finally: + disaggregated.close() + local.close() + report["workers"] = pd.worker_metrics + assert all( + m["used_blocks"] == 0 and m["weight_version"] == 5 for m in pd.worker_metrics.values() + ) + assert sum(m["bytes_sent"] for m in pd.worker_metrics.values()) > 0 + args.output.write_text(json.dumps(report, indent=2)) + print("PD_CONTRACT_PASSED", args.output, flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/training_contract.py b/benchmarks/training_contract.py new file mode 100644 index 00000000..0bd6fbe0 --- /dev/null +++ b/benchmarks/training_contract.py @@ -0,0 +1,94 @@ +"""Alternating-order warmed rollout timings, including returned log probabilities.""" + +import argparse +import json +import statistics +import time +from pathlib import Path + +import torch +from transformers import AutoTokenizer + +from vllm_rlt import ( + LLM, + CacheConfig, + ExecutionConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.models import OuroForCausalLM + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("model", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--repeats", type=int, default=5) + args = parser.parse_args() + torch.set_num_threads(4) + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False) + prompts = [tokenizer.encode(text) for text in ("Compute 17 + 25.\n", "Compute 9 * 8.\n")] + native = OuroForCausalLM.from_pretrained(args.model, device="cuda", dtype=torch.bfloat16) + report = {} + for speculative in (False, True): + arms = { + mode: LLM( + native, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=512), + scheduler_config=SchedulerConfig(max_num_seqs=8, max_num_batched_tokens=128), + execution_config=ExecutionConfig(cuda_graphs=mode == "graph"), + speculative_config=SpeculativeConfig(3) if speculative else None, + ) + for mode in ("eager", "graph") + } + for loops in (4,) if speculative else (2, 3, 4): + params = [ + SamplingParams( + max_tokens=32, + max_loops=loops, + temperature=0.9, + seed=31 + i, + logprobs=0, + ignore_eos=True, + ) + for i in range(len(prompts)) + ] + for arm in arms.values(): + arm.generate(prompts, params) + arm.start_weight_update() + arm.update_weights(native.named_parameters()) + arm.finish_weight_update() + version = arms["eager"].get_weight_version() + times = {mode: [] for mode in arms} + for index in range(args.repeats): + outputs = {} + for mode in ("eager", "graph") if index % 2 == 0 else ("graph", "eager"): + torch.cuda.synchronize() + started = time.monotonic() + outputs[mode] = arms[mode].generate(prompts, params) + torch.cuda.synchronize() + times[mode].append(time.monotonic() - started) + for eager, graph in zip(outputs["eager"], outputs["graph"], strict=True): + assert eager.token_ids == graph.token_ids + assert eager.exit_depths == graph.exit_depths + assert eager.log_probs == graph.log_probs + assert eager.weight_version == graph.weight_version == version + medians = {mode: statistics.median(values) for mode, values in times.items()} + name = f"speculative-{loops}" if speculative else f"native-{loops}" + report[name] = { + "seconds": times, + "medians": medians, + "speedup": medians["eager"] / medians["graph"], + "tokens_and_logprobs_exact": True, + } + print(name, report[name], flush=True) + for arm in arms.values(): + arm.close() + del arms + args.output.write_text(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/training_contract.md b/docs/training_contract.md new file mode 100644 index 00000000..9345cbf6 --- /dev/null +++ b/docs/training_contract.md @@ -0,0 +1,82 @@ +# Training rollout and full-weight publication + +`LLM` and `LLMEngine` expose the same full-update operations. `LLM` also accepts +an existing `LLMEngine` or `PDEngine`, so framework adapters use the same generation +and publication calls with either backend. + +```python +llm.start_weight_update(version=1) +for chunk in converted_hf_weights: + llm.update_weights(chunk) # Iterable[(physical_parameter_name, torch.Tensor)] +llm.finish_weight_update() +outputs = llm.generate(prompts, SamplingParams( + max_tokens=128, temperature=1.0, seed=42, logprobs=0, +)) +``` + +Each output contains aligned selected-token `log_probs`, `exit_depths`, the +committed `weight_version`, and its effective `sampling_params`. `logprobs=None` +keeps the ordinary inference path. Reported parameters resolve the model's +default `max_loops` and cap `top_k` at its vocabulary size. `logprobs=0` requests only the emitted token's +probability; top-N alternatives are not implemented. + +`logprobs_mode="raw"` is the default: FP32 log-softmax of unscaled model logits. +`"processed"` returns probabilities after positive temperature and top-k/top-p +filtering. Greedy requests return the raw score in either mode. Speculative +outputs use the verified **target** policy's score, including correction/bonus +tokens; proposal or rejection-residual probabilities are never exposed as the +policy score. Explicit `stop_token_ids` apply even with `ignore_eos=True`. +`seed=None` creates an independent entropy-seeded request generator. + +Publication requires an idle engine. Each chunk is checked for duplicate names, +unknown names and mismatched shapes before that chunk is copied. `finish_weight_update` +requires every physical parameter exactly once. Copies preserve parameter addresses +and captured graphs; a fresh transaction invalidates all cached prefix KV. If a +transfer fails, generation stays blocked until a fresh full transaction succeeds. +Starting again discards an unfinished transaction. `pause_generation` stops +admission while existing requests drain; `resume_generation` requires publication +to be complete. `abort_request` remains available to cancel requests before an update. + +PDEngine waits for all transfers to release, broadcasts CPU-staged weight chunks +to every P/D worker, and commits a version only after every peer acknowledges it. +Recovery uses a version newer than **all attempted** PD versions because some +workers may already have committed. With `terminate_workers_on_failure=False`, fatal workers retain their registered +memory rather than receive forced process signals; normal `close()` remains +cooperative. No CUDA tensors change ownership over IPC; +NIXL's existing GPU KV path and registrations remain intact. Prefix invalidation +does not replace registered KV storage. PD worker control errors return replies +without terminating the worker. + +Existing exit, async and speculative configuration rules still apply. In particular, +upstream self-speculation uses a full-depth target and synchronous execution; +upstream PD does not implement self-speculation. Publication adds no such restriction +to ordinary early-exit or async rollouts. Trainers must replay full-depth prompt +prefill and the returned decode depths rather than assume the entire sequence used K. + +## Validation + +```bash +python -m unittest tests.test_training_contract tests.test_pd_weight_control -v +CONTRACT_DEVICE=cuda python -m unittest tests.test_training_contract -v +python benchmarks/training_contract.py /models/Ouro-1.4B timings.json +UCX_TLS=tcp,cuda_copy python benchmarks/nixl_preflight.py +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py pd.json --cuda-graphs +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py full-pd.json \ + --model /models/Ouro-1.4B --cuda-graphs --max-num-seqs 1 +``` + +CPU and Thor CUDA tests cover raw/processed probabilities against an independent +dense oracle, speculative correction/bonus alignment, publication, pointer stability, +prefix invalidation, early exit, async stop/abort and entropy seeds. The PD control +test drives two real CPU engines and checks recovery after one peer commits and +another fails. It uses in-process control delivery; it does **not** validate IPC or +NIXL transport. The separate two-A100 run uses real spawned workers and NIXL: +FP32 multi-request and full BF16 matched-batch checks cover three publications, +raw/processed scores, graph replay, prefix reuse, explicit stop/abort and recovery +to version five. See [PD evidence](validation/training-contract-a100-20261001/README.md). +No PD workers were launched on single-GPU Thor, where NIXL is absent. + +This is a draft implementation of the +[Phase 1 contract proposed in issue 70](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5888101273) +on upstream `ecb1f8b505b7e831815b40aec3b4598619cca23a`; formal upstream agreement +remains outside the scope of this contributor draft. diff --git a/docs/validation/training-contract-a100-20261001/README.md b/docs/validation/training-contract-a100-20261001/README.md new file mode 100644 index 00000000..40473595 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/README.md @@ -0,0 +1,77 @@ +# Training contract with two A100 workers — 2026-10-01 + +Two A100-SXM4-40GB GPUs, driver 580.105.08, the existing `/venv/main` runtime, +PyTorch 2.13.0+cu130, Triton 3.7.1 and NIXL 1.4.1. Separate spawned P/D owners +register their GPU KV/hidden pools and exchange real NIXL WRITE transfers. +The public `LLM` interface stages physical policy chunks through CPU IPC and waits +for every worker to commit. No package, driver or service changes are made. + +| Case | Active slots | Maximum logprob error vs local | Published/recovered version | D graph captures / replays | P bytes transferred | +|---|---:|---:|---:|---:|---:| +| Random FP32, eager, K=2/3/4 | 4 | 4.77e-7 | 5 | Disabled | 300,928 | +| Random FP32, graphs, K=2/3/4 | 4 | 4.77e-7 | 5 | 2 / 116 | 300,928 | +| Full BF16 Ouro-1.4B, graphs, K=2/3/4 | 1 | 0 | 5 | 1 / 206 | 229,699,584 | + +Each case compares raw and processed scores at temperature 0.9, top-k 8 and +nucleus 0.8, with two prompts and six response tokens. Token IDs, physical exit +depths, effective parameters and policy versions agree. Prompt prefill is full +K=4; decode budgets change after each complete publication. Prefix reuse, +chunked updates, a rejected update, recovery into version five, explicit stop +and active abort are exercised. Both workers finish with zero used blocks and +version five. Graph captures remain valid through publication and recovery. + +The full-model source is `ByteDance/Ouro-1.4B` at +`574fa66cb8bf5abdc979642d01cf2b79b16bfab1`, safetensors SHA256 +`58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af`. +See [full outputs/counters](raw/pd-full-graph-matched.json), +[tiny eager](raw/pd-tiny-eager-tcp.json), [tiny graphs](raw/pd-tiny-graph-tcp.json) +and [runtime](raw/runtime.json). + +## Transport and numerical limits + +Default UCX GPU IPC reports completion but leaves the destination zero in an +independent two-process, 128-byte probe. Explicit IPC and shared-memory-only +configurations also fail on this container. `UCX_TLS=tcp,cuda_copy` passes the +probe and every case above with the installed library. Transfers are host staged; +this report does not establish direct GPU IPC/RDMA performance. The workaround +is scoped to each command. [Probe source](../../../benchmarks/nixl_preflight.py), +[default failure](raw/nixl-preflight.log) and [passing probe](raw/nixl-preflight-tcp.log) +are retained. No processes are forcibly terminated; successful workers use the +normal stop/acknowledgment protocol. + +A full BF16 comparison with four slots has a maximum observed selected-score +difference of 0.11199 when local and PD prefill batches differ. The matched +one-slot run above is exact; the FP32 four-slot run is within 4.77e-7. +Different BF16 batching is therefore not claimed to be bitwise equivalent. See +[the rejected strict comparison](raw/pd-full-graph-tcp.log). +Other jobs were already running on both GPUs. Timings in these parity probes +include startup/capture/prefix-state differences and contention, and are not a +controlled throughput comparison. The [Thor table](../training-contract-thor-20261001/README.md) +contains alternating-order warmed eager/graph timings. + +## Reproduction + +From the repository root, in an existing CUDA/NIXL environment: + +```bash +UCX_TLS=tcp,cuda_copy python benchmarks/nixl_preflight.py +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py tiny.json +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py tiny-graph.json --cuda-graphs +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py full.json \ + --model /models/Ouro-1.4B --cuda-graphs --max-num-seqs 1 +``` + +The VIME training/save/resume and general HF export checks are recorded in +[0z5a/vime#5](https://github.com/0z5a/vime/pull/5). + +## Repository checks + +The complete default CPU regression suite passes: **421 passed, 141 skipped** +in 19.41 s; GPU tests, including the upstream forced-peer-failure test, are skipped +by the repository's ordinary collection rules. See +[regression log](raw/cpu-regression-final.log) and +[focused contract tests](raw/contract-cpu-final.log). +Ruff lint and format checks pass for all 88 Python files. +The existing runtimes lack `pre_commit`, so its three `language: system` hooks +are run directly without installing it: Python-file lint, format check and the +full default CPU suite. See [lint log](raw/lint-final.log). diff --git a/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log b/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log new file mode 100644 index 00000000..885048c7 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log @@ -0,0 +1,10 @@ +test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok +test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok +test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok +test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok +test_all_workers_commit_and_partial_commit_requires_new_epoch (tests.test_pd_weight_control.PDWeightControlTest.test_all_workers_commit_and_partial_commit_requires_new_epoch) ... ok + +---------------------------------------------------------------------- +Ran 5 tests in 2.095s + +OK diff --git a/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log b/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log new file mode 100644 index 00000000..d10641bc --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log @@ -0,0 +1,9 @@ +.......s.s.s.sssssssssssssssssss....sssssssssssssss.s................... [ 12%] +............................................ssssss......s..............s [ 25%] +ssssssss.sssssssssssssssssssssssss...................................... [ 38%] +....ssssssss.ssss....sssssss...............................s............ [ 51%] +..sssssssssssssssss.................sss................................. [ 64%] +..ssssss..sss.....ss...ss............................................... [ 76%] +........................................................................ [ 89%] +.............ss.ss...............................sssss.... [100%] +421 passed, 141 skipped in 19.41s diff --git a/docs/validation/training-contract-a100-20261001/raw/lint-final.log b/docs/validation/training-contract-a100-20261001/raw/lint-final.log new file mode 100644 index 00000000..3d3e3c6d --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/lint-final.log @@ -0,0 +1,4 @@ +Required pre-commit hooks use language: system. pre_commit is absent; no installation performed. +ruff 0.16.8 +All checks passed! +88 files already formatted diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log new file mode 100644 index 00000000..11287164 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log @@ -0,0 +1,51 @@ +[1790822017.515227] [25f216012e8e:21943:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy, cuda_ipc/cuda - no am bcopy +W1001 02:33:37.515292 21943 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable +E1001 02:33:37.515377 21943 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep +E1001 02:33:37.515387 21943 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND +Process SpawnProcess-1: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker + agent = nixl_agent(name) + ^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ + self.create_backend(bknd, init) + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend + self.backends[backend] = self.agent.createBackend(backend, initParams) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND +[1790822017.583102] [25f216012e8e:21944:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy, cuda_ipc/cuda - no am bcopy +W1001 02:33:37.583151 21944 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable +E1001 02:33:37.583303 21944 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep +E1001 02:33:37.583311 21944 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND +Process SpawnProcess-2: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker + agent = nixl_agent(name) + ^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ + self.create_backend(bknd, init) + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend + self.backends[backend] = self.agent.createBackend(backend, initParams) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND +Traceback (most recent call last): + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 42, in + infos=[p.recv() for _,p in peers] + ^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 250, in recv + buf = self._recv_bytes() + ^^^^^^^^^^^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes + buf = self._recv(4) + ^^^^^^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 399, in _recv + raise EOFError +EOFError diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log new file mode 100644 index 00000000..d2072821 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log @@ -0,0 +1,51 @@ +[1790822145.345936] [25f216012e8e:23279:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy +W1001 02:35:45.345994 23279 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable +E1001 02:35:45.346127 23279 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep +E1001 02:35:45.346146 23279 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND +Process SpawnProcess-1: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker + agent = nixl_agent(name) + ^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ + self.create_backend(bknd, init) + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend + self.backends[backend] = self.agent.createBackend(backend, initParams) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND +[1790822145.362990] [25f216012e8e:23280:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy +W1001 02:35:45.363052 23280 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable +E1001 02:35:45.363210 23280 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep +E1001 02:35:45.363225 23280 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND +Process SpawnProcess-2: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker + agent = nixl_agent(name) + ^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ + self.create_backend(bknd, init) + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend + self.backends[backend] = self.agent.createBackend(backend, initParams) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND +Traceback (most recent call last): + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 42, in + infos=[p.recv() for _,p in peers] + ^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 250, in recv + buf = self._recv_bytes() + ^^^^^^^^^^^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes + buf = self._recv(4) + ^^^^^^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 399, in _recv + raise EOFError +EOFError diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log new file mode 100644 index 00000000..c5bfffe1 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log @@ -0,0 +1,6 @@ +2026-10-01 02:12:24 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:12:24 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-1 +2026-10-01 02:12:24 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:12:24 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-0 +VALUES [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0]] +MULTIPROCESS_NIXL_PASSED diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log new file mode 100644 index 00000000..ce2d6594 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log @@ -0,0 +1,12 @@ +2026-10-01 02:11:20 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:11:20 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-0 +2026-10-01 02:11:20 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:11:20 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-1 +VALUES [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]] +[1790820681.044462] [25f216012e8e:15986:0] cuda_ctx.c:23 UCX ERROR cuDevicePrimaryCtxGetState(cuda_device, &flags, &active) failed: unrecognized error code 4 +[1790820681.060647] [25f216012e8e:15985:0] cuda_ctx.c:23 UCX ERROR cuDevicePrimaryCtxGetState(cuda_device, &flags, &active) failed: unrecognized error code 4 +Traceback (most recent call last): + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 50, in + assert values[0]==values[1] + ^^^^^^^^^^^^^^^^^^^^ +AssertionError diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json new file mode 100644 index 00000000..03da93de --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json @@ -0,0 +1,1189 @@ +{ + "model": { + "vocab_size": 49152, + "hidden_size": 2048, + "intermediate_size": 5632, + "num_hidden_layers": 24, + "num_attention_heads": 16, + "num_key_value_heads": 16, + "head_dim": 128, + "hidden_act": "silu", + "max_position_embeddings": 65536, + "initializer_range": 0.02, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "total_ut_steps": 4, + "early_exit_threshold": 1.0, + "bos_token_id": 0, + "eos_token_id": 0, + "pad_token_id": null, + "tie_word_embeddings": false, + "attention_dropout": 0.0, + "rope_scaling": null, + "use_sliding_window": false, + "sliding_window": null, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ] + }, + "cuda_graphs": true, + "iterations": [ + { + "version": 1, + "loops": 2, + "mode": "raw", + "seconds": { + "local": 1.501969379838556, + "pd": 15.95300066191703 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "0", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.4585607051849365, + -1.3445959091186523, + -0.6607269048690796, + -0.4440094828605652, + -0.288143515586853, + -0.2284386157989502 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "1", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 4, + 4, + 4, + 198 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.4905070662498474, + -1.852401614189148, + -1.8474926948547363, + -4.226284027099609, + -2.1031107902526855, + -3.4267778396606445 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 1, + "loops": 2, + "mode": "processed", + "seconds": { + "local": 0.544591479934752, + "pd": 6.968707168009132 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "2", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.5379835367202759, + -0.22262510657310486, + 0.0, + 0.0, + 0.0 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "3", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 4, + 4, + 4, + 198 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.5063599348068237, + -1.3956828117370605, + -1.77802312374115, + -1.5563204288482666, + -0.19239495694637299 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "raw", + "seconds": { + "local": 0.7185850627720356, + "pd": 14.518414614256471 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "4", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.5055561065673828, + -0.5541938543319702, + -0.48413512110710144, + -0.3641066551208496, + -0.22259598970413208, + -0.19062571227550507 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "5", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 4, + 3, + 4, + 3 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.54900062084198, + -0.6617550849914551, + -1.9724218845367432, + -0.34641382098197937, + -1.458582878112793, + -0.42156851291656494 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "processed", + "seconds": { + "local": 0.66817781701684, + "pd": 7.080546109005809 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "6", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.1622927039861679, + -0.15219146013259888, + 0.0, + 0.0, + 0.0 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "7", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 4, + 3, + 4, + 3 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.2678171992301941, + -1.781590223312378, + 0.0, + -1.1942179203033447, + 0.0 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "raw", + "seconds": { + "local": 0.849096744786948, + "pd": 14.640745197888464 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "8", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.5625201463699341, + -0.37419480085372925, + -0.2975218594074249, + -0.2469334453344345, + -0.20516522228717804, + -0.1743372529745102 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "9", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.616585373878479, + -0.5554823875427246, + -0.43136879801750183, + -0.33480215072631836, + -0.2979239821434021, + -0.46127814054489136 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "processed", + "seconds": { + "local": 0.7937315646559, + "pd": 7.206241376232356 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "10", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "11", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.1622924953699112, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + } + ], + "max_num_seqs": 1, + "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", + "recovered_version": 5, + "stop": { + "request_id": "13", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "stop", + "text": "", + "log_probs": [ + 0.0 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [ + 3 + ], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "abort": { + "request_id": "abort", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "abort", + "text": "", + "log_probs": [ + 0.0 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "workers": { + "cae75e7035744e8795024750bd65b22e-decode-1": { + "kind": "stopped", + "prefill_tokens": 0, + "bytes_sent": 0, + "transfers": 0, + "transfer_seconds": 0.0, + "used_blocks": 0, + "weight_version": 5, + "graph_captures": 1, + "graph_replays": 206, + "graph_fallbacks": 0 + }, + "cae75e7035744e8795024750bd65b22e-prefill-0": { + "kind": "stopped", + "prefill_tokens": 244, + "bytes_sent": 229699584, + "transfers": 22, + "transfer_seconds": 207.36763043235987, + "used_blocks": 0, + "weight_version": 5 + } + } +} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log new file mode 100644 index 00000000..d37a3c92 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log @@ -0,0 +1,11 @@ +2026-10-01 02:30:31 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:30:31 NIXL INFO _api.py:285 Initialized NIXL agent: cae75e7035744e8795024750bd65b22e-decode-1 +2026-10-01 02:30:31 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:30:31 NIXL INFO _api.py:285 Initialized NIXL agent: cae75e7035744e8795024750bd65b22e-prefill-0 +PARITY 1 2 raw {'local': 1.501969379838556, 'pd': 15.95300066191703} 0.0 +PARITY 1 2 processed {'local': 0.544591479934752, 'pd': 6.968707168009132} 0.0 +PARITY 2 3 raw {'local': 0.7185850627720356, 'pd': 14.518414614256471} 0.0 +PARITY 2 3 processed {'local': 0.66817781701684, 'pd': 7.080546109005809} 0.0 +PARITY 3 4 raw {'local': 0.849096744786948, 'pd': 14.640745197888464} 0.0 +PARITY 3 4 processed {'local': 0.7937315646559, 'pd': 7.206241376232356} 0.0 +PD_CONTRACT_PASSED pd-full-graph-matched.json diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log new file mode 100644 index 00000000..03b98612 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log @@ -0,0 +1,16 @@ +2026-10-01 02:19:01 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:19:01 NIXL INFO _api.py:285 Initialized NIXL agent: 35f3775d875246919e95de82111ac017-prefill-0 +2026-10-01 02:19:01 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:19:01 NIXL INFO _api.py:285 Initialized NIXL agent: 35f3775d875246919e95de82111ac017-decode-1 +Traceback (most recent call last): + File "/workspace/0z5a/work/ouro-vime-contract-20261001/rlt/benchmarks/pd_training_contract.py", line 168, in + main() + File "/workspace/0z5a/work/ouro-vime-contract-20261001/rlt/benchmarks/pd_training_contract.py", line 111, in main + torch.testing.assert_close( + File "/usr/local/lib/python3.12/dist-packages/torch/testing/_comparison.py", line 1631, in assert_close + raise error_metas[0].to_error(msg) +AssertionError: Tensor-likes are not close! + +Mismatched elements: 6 / 6 (100.0%) +Greatest absolute difference: 0.11199069023132324 at index (1,) (up to 2e-05 allowed) +Greatest relative difference: 0.07553135603666306 at index (1,) (up to 2e-05 allowed) diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json new file mode 100644 index 00000000..c5c64fd2 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json @@ -0,0 +1,1160 @@ +{ + "model": { + "vocab_size": 64, + "hidden_size": 32, + "intermediate_size": 64, + "num_hidden_layers": 2, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 8, + "hidden_act": "silu", + "max_position_embeddings": 128, + "initializer_range": 0.02, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "total_ut_steps": 4, + "early_exit_threshold": 1.0, + "bos_token_id": 0, + "eos_token_id": 0, + "pad_token_id": null, + "tie_word_embeddings": false, + "attention_dropout": 0.0, + "rope_scaling": null, + "use_sliding_window": false, + "sliding_window": null, + "layer_types": null + }, + "cuda_graphs": false, + "iterations": [ + { + "version": 1, + "loops": 2, + "mode": "raw", + "seconds": { + "local": 1.1838269280269742, + "pd": 2.0369991376064718 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "0", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 42, + 42, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.992061138153076, + -3.8517885208129883, + -3.85848069190979, + -3.9884941577911377, + -3.8614566326141357, + -3.8722712993621826 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "1", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 51, + 15, + 55 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.861236810684204, + -3.9610671997070312, + -3.9557578563690186, + -4.008734226226807, + -3.9762279987335205, + -4.012908458709717 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 1, + "loops": 2, + "mode": "processed", + "seconds": { + "local": 0.23476281203329563, + "pd": 0.41898630978539586 + }, + "max_logprob_error": 1.1920928955078125e-07, + "outputs": [ + { + "request_id": "2", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 42, + 42, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.969618320465088, + -1.8275089263916016, + -1.835660457611084, + -1.980717658996582, + -1.8372858762741089, + -1.8491119146347046 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "3", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 51, + 15, + 55 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8221769332885742, + -1.9443289041519165, + -1.942601203918457, + -2.004945993423462, + -1.9664429426193237, + -2.008206367492676 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "raw", + "seconds": { + "local": 0.21088168676942587, + "pd": 0.45077179512009025 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "4", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 12, + 61, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.9952800273895264, + -4.035950660705566, + -3.7910518646240234, + -3.8775575160980225, + -3.849231719970703, + -3.8591208457946777 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "5", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 56, + 15, + 56 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.8670716285705566, + -3.976059913635254, + -3.975245237350464, + -4.008485317230225, + -3.941457986831665, + -4.011442184448242 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "processed", + "seconds": { + "local": 0.24939796887338161, + "pd": 0.38997333077713847 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "6", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 12, + 61, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.9691096544265747, + -2.072470188140869, + -1.8017675876617432, + -1.8991870880126953, + -1.8665740489959717, + -1.8762420415878296 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "7", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 56, + 15, + 56 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8246119022369385, + -2.0073468685150146, + -2.00469970703125, + -2.0388402938842773, + -1.96330726146698, + -2.036628007888794 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "raw", + "seconds": { + "local": 0.22700949804857373, + "pd": 0.45369746908545494 + }, + "max_logprob_error": 4.76837158203125e-07, + "outputs": [ + { + "request_id": "8", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 60, + 9, + 25, + 60, + 42 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.9984395503997803, + -4.025251865386963, + -4.024652004241943, + -3.9303245544433594, + -4.02800178527832, + -4.010705947875977 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "9", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 60, + 60, + 9, + 15, + 56 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.8727941513061523, + -4.021127223968506, + -4.020061016082764, + -4.025237083435059, + -3.993036985397339, + -4.017432689666748 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "processed", + "seconds": { + "local": 0.25585177494212985, + "pd": 0.42146871984004974 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "10", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 60, + 9, + 25, + 60, + 42 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.9686124324798584, + -1.9942327737808228, + -1.9923925399780273, + -1.8824831247329712, + -1.9928045272827148, + -1.9732145071029663 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "11", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 60, + 60, + 9, + 15, + 56 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8269999027252197, + -1.9946293830871582, + -1.9918233156204224, + -1.995558738708496, + -1.9558247327804565, + -1.983024001121521 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + } + ], + "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", + "recovered_version": 5, + "stop": { + "request_id": "13", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "stop", + "text": "", + "log_probs": [ + -1.9686124324798584 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [ + 42 + ], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "abort": { + "request_id": "abort", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "abort", + "text": "", + "log_probs": [ + -1.9686124324798584 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "workers": { + "e7b52c471c5a4a7383476001ca2853cc-decode-1": { + "kind": "stopped", + "prefill_tokens": 0, + "bytes_sent": 0, + "transfers": 0, + "transfer_seconds": 0.0, + "used_blocks": 0, + "weight_version": 5 + }, + "e7b52c471c5a4a7383476001ca2853cc-prefill-0": { + "kind": "stopped", + "prefill_tokens": 292, + "bytes_sent": 300928, + "transfers": 22, + "transfer_seconds": 2.774511670228094, + "used_blocks": 0, + "weight_version": 5 + } + } +} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log new file mode 100644 index 00000000..9ae443a5 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log @@ -0,0 +1,11 @@ +2026-10-01 02:14:34 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:14:34 NIXL INFO _api.py:285 Initialized NIXL agent: e7b52c471c5a4a7383476001ca2853cc-decode-1 +2026-10-01 02:14:34 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:14:34 NIXL INFO _api.py:285 Initialized NIXL agent: e7b52c471c5a4a7383476001ca2853cc-prefill-0 +PARITY 1 2 raw {'local': 1.1838269280269742, 'pd': 2.0369991376064718} 2.384185791015625e-07 +PARITY 1 2 processed {'local': 0.23476281203329563, 'pd': 0.41898630978539586} 1.1920928955078125e-07 +PARITY 2 3 raw {'local': 0.21088168676942587, 'pd': 0.45077179512009025} 2.384185791015625e-07 +PARITY 2 3 processed {'local': 0.24939796887338161, 'pd': 0.38997333077713847} 2.384185791015625e-07 +PARITY 3 4 raw {'local': 0.22700949804857373, 'pd': 0.45369746908545494} 4.76837158203125e-07 +PARITY 3 4 processed {'local': 0.25585177494212985, 'pd': 0.42146871984004974} 2.384185791015625e-07 +PD_CONTRACT_PASSED pd-tiny-eager-tcp.json diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json new file mode 100644 index 00000000..e95c674c --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json @@ -0,0 +1,1163 @@ +{ + "model": { + "vocab_size": 64, + "hidden_size": 32, + "intermediate_size": 64, + "num_hidden_layers": 2, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 8, + "hidden_act": "silu", + "max_position_embeddings": 128, + "initializer_range": 0.02, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "total_ut_steps": 4, + "early_exit_threshold": 1.0, + "bos_token_id": 0, + "eos_token_id": 0, + "pad_token_id": null, + "tie_word_embeddings": false, + "attention_dropout": 0.0, + "rope_scaling": null, + "use_sliding_window": false, + "sliding_window": null, + "layer_types": null + }, + "cuda_graphs": true, + "iterations": [ + { + "version": 1, + "loops": 2, + "mode": "raw", + "seconds": { + "local": 1.2744349539279938, + "pd": 2.329219114035368 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "0", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 42, + 42, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.992061138153076, + -3.8517885208129883, + -3.85848069190979, + -3.9884941577911377, + -3.8614566326141357, + -3.8722712993621826 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "1", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 51, + 15, + 55 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.861236810684204, + -3.9610671997070312, + -3.9557578563690186, + -4.008734226226807, + -3.9762279987335205, + -4.012908458709717 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 1, + "loops": 2, + "mode": "processed", + "seconds": { + "local": 0.23766471119597554, + "pd": 0.436434387229383 + }, + "max_logprob_error": 1.1920928955078125e-07, + "outputs": [ + { + "request_id": "2", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 42, + 42, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.969618320465088, + -1.8275089263916016, + -1.835660457611084, + -1.980717658996582, + -1.8372858762741089, + -1.8491119146347046 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "3", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 51, + 15, + 55 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8221769332885742, + -1.9443289041519165, + -1.942601203918457, + -2.004945993423462, + -1.9664429426193237, + -2.008206367492676 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "raw", + "seconds": { + "local": 0.22012542001903057, + "pd": 0.4495909558609128 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "4", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 12, + 61, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.9952800273895264, + -4.035950660705566, + -3.7910518646240234, + -3.8775575160980225, + -3.849231719970703, + -3.8591208457946777 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "5", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 56, + 15, + 56 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.8670716285705566, + -3.976059913635254, + -3.975245237350464, + -4.008485317230225, + -3.941457986831665, + -4.011442184448242 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "processed", + "seconds": { + "local": 0.24550614319741726, + "pd": 0.40186773892492056 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "6", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 12, + 61, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.9691096544265747, + -2.072470188140869, + -1.8017675876617432, + -1.8991870880126953, + -1.8665740489959717, + -1.8762420415878296 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "7", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 56, + 15, + 56 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8246119022369385, + -2.0073468685150146, + -2.00469970703125, + -2.0388402938842773, + -1.96330726146698, + -2.036628007888794 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "raw", + "seconds": { + "local": 0.23163969395682216, + "pd": 0.4639628282748163 + }, + "max_logprob_error": 4.76837158203125e-07, + "outputs": [ + { + "request_id": "8", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 60, + 9, + 25, + 60, + 42 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.9984395503997803, + -4.025251865386963, + -4.024652004241943, + -3.9303245544433594, + -4.02800178527832, + -4.010705947875977 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "9", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 60, + 60, + 9, + 15, + 56 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.8727941513061523, + -4.021127223968506, + -4.020061016082764, + -4.025237083435059, + -3.993036985397339, + -4.017432689666748 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "processed", + "seconds": { + "local": 0.2783969510346651, + "pd": 0.46485115215182304 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "10", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 60, + 9, + 25, + 60, + 42 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.9686124324798584, + -1.9942327737808228, + -1.9923925399780273, + -1.8824831247329712, + -1.9928045272827148, + -1.9732145071029663 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "11", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 60, + 60, + 9, + 15, + 56 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8269999027252197, + -1.9946293830871582, + -1.9918233156204224, + -1.995558738708496, + -1.9558247327804565, + -1.983024001121521 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + } + ], + "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", + "recovered_version": 5, + "stop": { + "request_id": "13", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "stop", + "text": "", + "log_probs": [ + -1.9686124324798584 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [ + 42 + ], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "abort": { + "request_id": "abort", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "abort", + "text": "", + "log_probs": [ + -1.9686124324798584 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "workers": { + "0759283ec0764cee8d1bd7369fa54644-prefill-0": { + "kind": "stopped", + "prefill_tokens": 292, + "bytes_sent": 300928, + "transfers": 22, + "transfer_seconds": 2.949465547222644, + "used_blocks": 0, + "weight_version": 5 + }, + "0759283ec0764cee8d1bd7369fa54644-decode-1": { + "kind": "stopped", + "prefill_tokens": 0, + "bytes_sent": 0, + "transfers": 0, + "transfer_seconds": 0.0, + "used_blocks": 0, + "weight_version": 5, + "graph_captures": 2, + "graph_replays": 116, + "graph_fallbacks": 0 + } + } +} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log new file mode 100644 index 00000000..e391a5ae --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log @@ -0,0 +1,11 @@ +2026-10-01 02:16:37 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:16:37 NIXL INFO _api.py:285 Initialized NIXL agent: 0759283ec0764cee8d1bd7369fa54644-prefill-0 +2026-10-01 02:16:37 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:16:37 NIXL INFO _api.py:285 Initialized NIXL agent: 0759283ec0764cee8d1bd7369fa54644-decode-1 +PARITY 1 2 raw {'local': 1.2744349539279938, 'pd': 2.329219114035368} 2.384185791015625e-07 +PARITY 1 2 processed {'local': 0.23766471119597554, 'pd': 0.436434387229383} 1.1920928955078125e-07 +PARITY 2 3 raw {'local': 0.22012542001903057, 'pd': 0.4495909558609128} 2.384185791015625e-07 +PARITY 2 3 processed {'local': 0.24550614319741726, 'pd': 0.40186773892492056} 2.384185791015625e-07 +PARITY 3 4 raw {'local': 0.23163969395682216, 'pd': 0.4639628282748163} 4.76837158203125e-07 +PARITY 3 4 processed {'local': 0.2783969510346651, 'pd': 0.46485115215182304} 2.384185791015625e-07 +PD_CONTRACT_PASSED pd-tiny-graph-tcp.json diff --git a/docs/validation/training-contract-a100-20261001/raw/runtime.json b/docs/validation/training-contract-a100-20261001/raw/runtime.json new file mode 100644 index 00000000..f4813a5e --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/runtime.json @@ -0,0 +1,13 @@ +{ + "torch": "2.13.0+cu130", + "triton": "3.7.1", + "nixl": "1.4.1", + "transformers": "5.17.0", + "numpy": "2.3.5", + "safetensors": "0.8.0", + "gpus": [ + "NVIDIA A100-SXM4-40GB, 580.105.08, 40960 MiB", + "NVIDIA A100-SXM4-40GB, 580.105.08, 40960 MiB" + ], + "UCX_TLS": "tcp,cuda_copy" +} \ No newline at end of file diff --git a/docs/validation/training-contract-thor-20261001/README.md b/docs/validation/training-contract-thor-20261001/README.md new file mode 100644 index 00000000..52920253 --- /dev/null +++ b/docs/validation/training-contract-thor-20261001/README.md @@ -0,0 +1,35 @@ +# Training contract on Thor — 2026-10-01 + +These timings use full BF16 `ByteDance/Ouro-1.4B` at revision +`574fa66cb8bf5abdc979642d01cf2b79b16bfab1`, with the public `LLM` rollout/update +API on upstream `ecb1f8b505b7e831815b40aec3b4598619cca23a`. + +| Rollout | Eager (s) | CUDA graphs (s) | Speedup | Time saved | +|---|---:|---:|---:|---:| +| Native K=2 decode | 1.0689 | 0.9939 | 1.076× | 7.02% | +| Native K=3 decode | 1.5190 | 1.4207 | 1.069× | 6.47% | +| Native K=4 decode | 1.9414 | 1.8234 | 1.065× | 6.08% | +| Self-speculative target K=4, draft K=2, 3 candidates | 1.4952 | 1.3741 | 1.088× | 8.11% | + +Five alternating-order pairs after per-arm warmup and full-weight publication; +two prompts, 32 response tokens each, temperature=0.9 and fixed per-request seeds. +Returned selected-token raw logprobs, tokens, exit depths and weight versions +match exactly between eager and graph arms in every pair. Each native prompt +prefill uses full K=4. Measurements include logprob delivery and exclude model +loading and initial capture. Each speculative comparison uses the same speculative +algorithm in both arms. See [raw timing pairs](raw/shared-full-rollout-benchmark.json). + +CPU tests pass all five checks; CUDA/Triton tests pass all four applicable checks. +The independent FP32 oracle covers processed probabilities with temperature and +top-k/top-p, speculative target scores after publication, early exit, async +stop/abort, prefix invalidation, stable parameter addresses and failed publication +recovery. The PD control test confirms a newer complete epoch recovers from a +partial peer commit; delivery is in-process, so IPC and NIXL are not covered. + +Full-model VIME training and fresh-process checkpoint/HF export parity are +reported in [0z5a/vime#5](https://github.com/0z5a/vime/pull/5). +Real multi-GPU PD publication and rollout checks are recorded in the +[A100 report](../training-contract-a100-20261001/README.md); Thor has one GPU and no NIXL. +No packages, CUDA drivers or environments were updated. GPU runs use the existing +performance lock and finish naturally. Completed experiment weights/checkpoints +are removed after preserving logs and tensor hashes. diff --git a/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json b/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json new file mode 100644 index 00000000..22ae874c --- /dev/null +++ b/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json @@ -0,0 +1,98 @@ +{ + "native-2": { + "seconds": { + "eager": [ + 1.068922233011108, + 1.0962036999990232, + 1.0150359050021507, + 1.0776073549932335, + 1.0171921259898227 + ], + "graph": [ + 0.99387969498639, + 1.031959738000296, + 0.9904317960026674, + 1.0318653039867058, + 0.9868912409874611 + ] + }, + "medians": { + "eager": 1.068922233011108, + "graph": 0.99387969498639 + }, + "speedup": 1.0755046495096627, + "tokens_and_logprobs_exact": true + }, + "native-3": { + "seconds": { + "eager": [ + 1.5189770860015415, + 1.5267343849991448, + 1.4750143490091432, + 1.5399230089969933, + 1.4762638669926673 + ], + "graph": [ + 1.4206840230035596, + 1.4595294760074466, + 1.3996001509949565, + 1.4518495189840905, + 1.4113349709950853 + ] + }, + "medians": { + "eager": 1.5189770860015415, + "graph": 1.4206840230035596 + }, + "speedup": 1.0691871390164396, + "tokens_and_logprobs_exact": true + }, + "native-4": { + "seconds": { + "eager": [ + 1.9413879640051164, + 1.9600694489781745, + 1.8969329710234888, + 1.9612911650037859, + 1.8895282850135118 + ], + "graph": [ + 1.7855259570060298, + 1.8588317520043347, + 1.8060117769928183, + 1.8618553779961076, + 1.8234225899796002 + ] + }, + "medians": { + "eager": 1.9413879640051164, + "graph": 1.8234225899796002 + }, + "speedup": 1.0646944787641552, + "tokens_and_logprobs_exact": true + }, + "speculative-4": { + "seconds": { + "eager": [ + 1.4952338639996015, + 1.523701586003881, + 1.4740307310130447, + 1.5373682600038592, + 1.4766599040012807 + ], + "graph": [ + 1.3729902119957842, + 1.415411785012111, + 1.373799616994802, + 1.416570646018954, + 1.3740531750081573 + ] + }, + "medians": { + "eager": 1.4952338639996015, + "graph": 1.3740531750081573 + }, + "speedup": 1.0881921392821823, + "tokens_and_logprobs_exact": true + } +} \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..f4a55800 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Repository tests; keep imports separate from installed test packages.""" diff --git a/tests/test_pd.py b/tests/test_pd.py index 9b07ff14..71999078 100644 --- a/tests/test_pd.py +++ b/tests/test_pd.py @@ -239,6 +239,7 @@ def test_engine_yields_while_waiting_for_remote_kv(async_scheduling): from vllm_rlt.request import Request, Stage engine = object.__new__(LLMEngine) + engine._update_version = None engine.profiling = Profiler("cpu") engine.preemption = PreemptionManager(engine) engine.execution_config = ExecutionConfig(async_scheduling=async_scheduling) diff --git a/tests/test_pd_weight_control.py b/tests/test_pd_weight_control.py new file mode 100644 index 00000000..523efbbb --- /dev/null +++ b/tests/test_pd_weight_control.py @@ -0,0 +1,91 @@ +"""PD publication control with two real CPU engines; no NIXL transport claim.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from tests.helpers import tiny_ouro_config +from vllm_rlt import LLM, SamplingParams +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.models import OuroForCausalLM +from vllm_rlt.pd.config import PDConfig +from vllm_rlt.pd.engine import PDEngine +from vllm_rlt.pd.worker import PDWorker + + +def control_pair(): + coordinator = PDEngine.__new__(PDEngine) + coordinator.config = PDConfig() + coordinator.closed, coordinator.failure = False, None + coordinator.requests, coordinator.transfers = {}, {} + coordinator._weight_version, coordinator._update_version = 0, None + coordinator._update_failed, coordinator._paused = False, False + coordinator._weight_replies = {} + coordinator.peers = { + role: SimpleNamespace(name=role, role=role) for role in ("prefill", "decode") + } + workers = {} + for role, peer in coordinator.peers.items(): + worker = PDWorker.__new__(PDWorker) + worker.engine = LLMEngine(OuroForCausalLM(tiny_ouro_config())) + worker.work = {} + worker.send = lambda kind, peer=peer, **fields: coordinator._message( + peer, dict(kind=kind, **fields) + ) + workers[role] = worker + coordinator._send = lambda name, kind, **fields: workers[name].command( + dict(kind=kind, **fields) + ) + return coordinator, workers + + +class PDWeightControlTest(unittest.TestCase): + def test_all_workers_commit_and_partial_commit_requires_new_epoch(self): + coordinator, workers = control_pair() + facade = LLM(coordinator) + weights = list(workers["prefill"].engine.model.named_parameters()) + pointers = { + role: [p.data_ptr() for p in worker.engine.model.parameters()] + for role, worker in workers.items() + } + facade.start_weight_update() + facade.update_weights(weights) + facade.finish_weight_update() + self.assertEqual(facade.get_weight_version(), 1) + coordinator.start_weight_update() + coordinator.update_weights(weights) + with patch.object( + workers["decode"].engine, + "finish_weight_update", + side_effect=RuntimeError("copy failure"), + ): + with self.assertRaisesRegex(RuntimeError, "copy failure"): + coordinator.finish_weight_update() + self.assertEqual(workers["prefill"].engine.get_weight_version(), 2) + self.assertEqual(workers["decode"].engine.get_weight_version(), 1) + with self.assertRaises(RuntimeError): + coordinator.step() + with self.assertRaises(RuntimeError): + coordinator.add_request("blocked", [3]) + coordinator.start_weight_update() + coordinator.update_weights(weights) + coordinator.finish_weight_update() + self.assertEqual(coordinator.get_weight_version(), 3) + for role, worker in workers.items(): + self.assertEqual(worker.engine.get_weight_version(), 3) + self.assertEqual( + pointers[role], [p.data_ptr() for p in worker.engine.model.parameters()] + ) + output = LLM(worker.engine).generate( + [[3, 4]], SamplingParams(max_tokens=2, logprobs=0) + )[0] + self.assertEqual(output.weight_version, 3) + self.assertEqual(len(output.log_probs), len(output.token_ids)) + coordinator.reset_prefix_cache() + + +if __name__ == "__main__": + torch.set_num_threads(1) + unittest.main() diff --git a/tests/test_speculative.py b/tests/test_speculative.py index 412a8b13..162f54ec 100644 --- a/tests/test_speculative.py +++ b/tests/test_speculative.py @@ -395,7 +395,7 @@ def test_greedy_round_reads_back_once_with_ragged_requests(): # Draft IDs stay on the device; every speculative round, including ragged and # budget-shortened ones, synchronizes with the host exactly once. torch.manual_seed(123) - m = OuroForCausalLM(OuroConfig.tiny(hidden_size=256, head_dim=64)).to( + m = OuroForCausalLM(tiny_ouro_config(hidden_size=256, head_dim=64)).to( device="cuda", dtype=torch.bfloat16 ) e = engine(m, k=3, cache_config=CacheConfig(128, 16), attention_backend="triton") diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py new file mode 100644 index 00000000..803b8381 --- /dev/null +++ b/tests/test_training_contract.py @@ -0,0 +1,186 @@ +"""Training contract checks runnable with unittest on CPU or CUDA/Triton.""" + +import os +import unittest +from dataclasses import replace + +import torch + +from tests.reference import dense_reference +from vllm_rlt import ( + LLM, + CacheConfig, + ExecutionConfig, + ExitConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.models import OuroConfig, OuroForCausalLM +from vllm_rlt.request import Request +from vllm_rlt.worker.sampling import generator_for, probabilities + +DEVICE = os.environ.get("CONTRACT_DEVICE", "cpu") + + +def model(): + torch.manual_seed(17) + config = OuroConfig( + vocab_size=64, + hidden_size=256 if DEVICE == "cuda" else 32, + intermediate_size=512 if DEVICE == "cuda" else 64, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=64 if DEVICE == "cuda" else 8, + max_position_embeddings=128, + total_ut_steps=4, + eos_token_id=0, + ) + return OuroForCausalLM(config).to(DEVICE) + + +def llm(native, *, graphs=False, asynchronous=False, speculative=False, prefix=False): + return LLM( + native, + attention_backend="triton" if DEVICE == "cuda" else "torch", + cache_config=CacheConfig(num_blocks=128, block_size=2, enable_prefix_caching=prefix), + scheduler_config=SchedulerConfig(max_num_seqs=4, max_num_batched_tokens=32), + exit_config=ExitConfig("ouro_delayed" if asynchronous else "ouro"), + execution_config=ExecutionConfig(cuda_graphs=graphs, async_scheduling=asynchronous), + speculative_config=SpeculativeConfig(3) if speculative else None, + ) + + +class TrainingContractTest(unittest.TestCase): + def test_selected_probabilities_speculative_target_and_post_update(self): + native = model() + prompt = [3, 4, 5] + for graphs in [False, True] if DEVICE == "cuda" else [False]: + for speculative in (False, True): + rollout = llm(native, graphs=graphs, speculative=speculative) + pointers = [p.data_ptr() for p in native.parameters()] + for version in (0, 1): + if version: + weights = [(n, p.detach() * 0.97) for n, p in native.named_parameters()] + rollout.start_weight_update(version) + rollout.update_weights(weights[:5]) + rollout.update_weights(weights[5:]) + rollout.finish_weight_update() + for mode in ("raw", "processed"): + params = SamplingParams( + max_tokens=8, + temperature=0.8, + top_k=13, + top_p=0.9, + logprobs=0, + logprobs_mode=mode, + seed=29, + ignore_eos=True, + ) + output = rollout.generate([prompt], params)[0] + logits = dense_reference(native, prompt + output.token_ids[:-1], 4)[-1][2] + rows = logits[len(prompt) - 1 :] + expected = torch.stack( + [ + probabilities(row, params)[token].log() + if mode == "processed" + else row.float().log_softmax(-1)[token] + for row, token in zip(rows, output.token_ids, strict=True) + ] + ) + torch.testing.assert_close( + torch.tensor(output.log_probs, device=DEVICE), + expected, + atol=4e-5, + rtol=4e-5, + ) + self.assertEqual(output.weight_version, version) + self.assertEqual(output.sampling_params, replace(params, max_loops=4)) + self.assertEqual(pointers, [p.data_ptr() for p in native.parameters()]) + if graphs: + bank = ( + rollout.engine.speculative_runner.graphs + if speculative + else rollout.engine.model_runner.graphs + ) + self.assertGreater(bank.replays, 0) + + def test_early_exit_async_stop_and_abort(self): + for graphs in [False, True] if DEVICE == "cuda" else [False]: + for asynchronous in (False, True): + rollout = llm(model(), graphs=graphs, asynchronous=asynchronous) + rollout.start_weight_update(2) + rollout.update_weights(rollout.engine.model.named_parameters()) + rollout.finish_weight_update() + params = SamplingParams(max_tokens=5, exit_threshold=0, logprobs=0, ignore_eos=True) + output = rollout.generate([[3, 4]], params)[0] + self.assertEqual(output.weight_version, 2) + self.assertEqual(output.exit_depths, [4] + [3 if asynchronous else 2] * 4) + self.assertEqual(len(output.log_probs), 5) + stopped = rollout.generate( + [[3, 4]], replace(params, stop_token_ids=(output.token_ids[0],)) + )[0] + self.assertEqual(stopped.token_ids, output.token_ids[:1]) + self.assertEqual(len(stopped.log_probs), 1) + self.assertEqual(stopped.finish_reason, "stop") + rollout.engine.add_request("abort", [3, 4], params) + while not rollout.engine.step(): + pass + aborted = rollout.engine.abort_request("abort") + self.assertEqual(len(aborted.log_probs), len(aborted.token_ids)) + self.assertEqual(aborted.finish_reason, "abort") + + def test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy(self): + rollout = llm(model(), prefix=True) + native = rollout.engine.model + params = SamplingParams(max_tokens=3, logprobs=0, ignore_eos=True) + rollout.generate([[3, 4, 5, 6]], params) + self.assertTrue(rollout.engine.cache_manager.lookup_prefix([3, 4, 5, 6])) + rollout.pause_generation() + with self.assertRaises(RuntimeError): + rollout.generate([[3]], params) + rollout.resume_generation() + rollout.engine.add_request("active", [3], params) + with self.assertRaises(RuntimeError): + rollout.start_weight_update() + rollout.engine.abort_request("active") + rollout.start_weight_update(7) + self.assertEqual(rollout.engine.cache_manager.num_used_blocks, 0) + self.assertFalse(rollout.engine.cache_manager.lookup_prefix([3, 4, 5, 6])) + names = list(native.named_parameters()) + before = names[0][1].detach().clone() + with self.assertRaises(ValueError): + rollout.update_weights( + [(names[0][0], torch.zeros_like(before)), ("invalid", torch.zeros(1))] + ) + torch.testing.assert_close(before, names[0][1], rtol=0, atol=0) + with self.assertRaises(RuntimeError): + rollout.finish_weight_update() + with self.assertRaises(RuntimeError): + rollout.generate([[3]], params) + rollout.start_weight_update(7) + rollout.update_weights(names[:1]) + with self.assertRaises(ValueError): + rollout.finish_weight_update() + rollout.update_weights(names[1:]) + rollout.finish_weight_update() + self.assertEqual(rollout.generate([[3]], params)[0].weight_version, 7) + rollout.reset_prefix_cache() + self.assertEqual(rollout.engine.cache_manager.num_used_blocks, 0) + + def test_seed_none_and_validation(self): + params = SamplingParams(seed=None, temperature=1) + requests = [Request(str(i), [3], params) for i in range(2)] + seeds = [generator_for(r, DEVICE).initial_seed() for r in requests] + self.assertNotEqual(*seeds) + for values in ({"logprobs": 1}, {"logprobs_mode": "invalid"}, {"stop_token_ids": (-1,)}): + with self.assertRaises(ValueError): + SamplingParams(**values) + with self.assertRaises(ValueError): + llm(model()).generate([[3]], SamplingParams(stop_token_ids=(64,))) + + +if __name__ == "__main__": + torch.set_num_threads(1) + unittest.main() diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 968932ac..77fd211f 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -250,6 +250,12 @@ def lookup_prefix(self, tokens): found.append(blocks) return tuple(found) + def reset_prefix_cache(self): + self._pending_prefixes.clear() + for blocks in self._prefixes.values(): + self._drop_refs(blocks) + self._prefixes.clear() + def allocate(self, request_id: str, max_tokens: int, *, initial_tokens=None, prefix=()) -> bool: """Reserve a logical capacity; acquire physical pages for the current frontier.""" if not isinstance(request_id, str) or not request_id: diff --git a/vllm_rlt/engine/llm_engine.py b/vllm_rlt/engine/llm_engine.py index 8a7bb559..0f374d6d 100644 --- a/vllm_rlt/engine/llm_engine.py +++ b/vllm_rlt/engine/llm_engine.py @@ -1,5 +1,8 @@ +from collections.abc import Iterable from dataclasses import replace +import torch + from vllm_rlt.config import CacheConfig, ExecutionConfig, ExitConfig, SchedulerConfig from vllm_rlt.core.kv_cache_manager import KVCacheManager from vllm_rlt.core.memory import plan_cache @@ -28,6 +31,11 @@ def __init__( speculative_config=None, ): self.model = model + self._weight_version = 0 + self._update_version: int | None = None + self._update_failed = False + self._updated_names: set[str] = set() + self._paused = False self.profiling = Profiler(next(model.parameters()).device) cache_config = cache_config or CacheConfig() scheduler_config = scheduler_config or SchedulerConfig() @@ -132,6 +140,8 @@ def add_request( *, trace_id: str | None = None, ): + if self._update_version is not None or self._paused: + raise RuntimeError("Generation admission is paused") if not isinstance(request_id, str) or not request_id: raise ValueError("request_id must be a nonempty string") params = sampling_params or SamplingParams() @@ -140,9 +150,12 @@ def add_request( raise ValueError("prompt must contain at least one token") if any(type(t) is not int or not 0 <= t < config.vocab_size for t in prompt_token_ids): raise ValueError("prompt token IDs must be integers within the model vocabulary") + if any(t >= config.vocab_size for t in params.stop_token_ids): + raise ValueError("stop token IDs must be within the model vocabulary") max_loops = params.max_loops or config.total_ut_steps if max_loops > config.total_ut_steps or params.min_loops > max_loops: raise ValueError("requested loop bounds exceed the model's supported depth") + params = replace(params, max_loops=max_loops, top_k=min(params.top_k, config.vocab_size)) if self.speculative_config is not None and ( max_loops != self.speculative_config.target_loops or params.exit_threshold != 1.0 ): @@ -180,9 +193,76 @@ def add_request( "increase num_blocks or reduce prompt/max_tokens" ) self.scheduler.add_request( - Request(request_id, list(prompt_token_ids), params, exit_trace=trace) + Request( + request_id, + list(prompt_token_ids), + params, + exit_trace=trace, + weight_version=self._weight_version, + ) ) + def get_weight_version(self) -> int: + return self._weight_version + + def pause_generation(self): + # Existing requests can drain through step(); no new requests are admitted. + self._paused = True + + def resume_generation(self): + if self._update_version is not None: + raise RuntimeError("Complete the weight update before resuming generation") + self._paused = False + + def reset_prefix_cache(self): + if self.has_unfinished_requests(): + raise RuntimeError("Finish active requests before resetting cached KV") + self.model_runner.synchronize() + self.cache_manager.reset_prefix_cache() + + def start_weight_update(self, version: int | None = None): + if self.has_unfinished_requests(): + raise RuntimeError("Weight updates require an idle engine") + version = self._weight_version + 1 if version is None else version + if type(version) is not int or version <= self._weight_version: + raise ValueError("Weight version must increase") + self.reset_prefix_cache() + self._update_version, self._update_failed = version, False + self._updated_names.clear() + + @torch.no_grad() + def update_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + if self._update_version is None or self._update_failed: + raise RuntimeError("Start a full weight update first") + parameters = dict(self.model.named_parameters()) + weights = list(weights) + names = [name for name, _ in weights] + try: + if len(set(names)) != len(names) or self._updated_names.intersection(names): + raise ValueError("Duplicate weight in full update") + for name, value in weights: + if name not in parameters or value.shape != parameters[name].shape: + raise ValueError(f"Weight name or shape mismatch: {name}") + for name, value in weights: + parameters[name].copy_(value) + self._updated_names.update(names) + except Exception: + # A failed transfer may have copied some tensors. A fresh full update + # is the only recovery; generation never sees that mixed policy. + self._update_failed = True + raise + + def finish_weight_update(self): + if self._update_version is None or self._update_failed: + raise RuntimeError("No successful weight update to finish") + if self._updated_names != dict(self.model.named_parameters()).keys(): + raise ValueError("Full update must include every physical parameter") + device = next(self.model.parameters()).device + if device.type == "cuda": + torch.cuda.synchronize(device) + self._weight_version, self._update_version = self._update_version, None + self._updated_names.clear() + def has_unfinished_requests(self) -> bool: return self.scheduler.has_unfinished_requests @@ -208,6 +288,8 @@ def close(self): self.profiling.close() def step(self) -> list[RequestOutput]: + if self._update_version is not None: + raise RuntimeError("Generation is unavailable during a weight update") if not self.profiling.recording: return self._step() try: @@ -263,9 +345,11 @@ def _update_speculative(self, batch, results): reason = None for token in result.token_ids: request.generated_token_ids.append(token) + if params.logprobs is not None: + request.log_probs.append(result.log_probs[emitted]) request.exit_depths.append(self.speculative_config.target_loops) emitted += 1 - if token in eos_ids and not params.ignore_eos: + if token in params.stop_token_ids or (token in eos_ids and not params.ignore_eos): reason = FinishReason.STOP break if len(request.generated_token_ids) >= params.max_tokens: @@ -333,7 +417,9 @@ def _update(self, batch, result) -> list[RequestOutput]: request.exit_depths.append(request.loops_done) eos = self.model.config.eos_token_id eos_ids = eos if isinstance(eos, (tuple, list)) else [eos] - if token_id in eos_ids and not params.ignore_eos: + if token_id in params.stop_token_ids or ( + token_id in eos_ids and not params.ignore_eos + ): self._finish(request, FinishReason.STOP) elif len(request.generated_token_ids) >= params.max_tokens: self._finish(request, FinishReason.LENGTH) @@ -409,7 +495,7 @@ def _deliver_coda(self, ticket): params = request.sampling_params eos = self.model.config.eos_token_id eos_ids = eos if isinstance(eos, (tuple, list)) else [eos] - if token_id in eos_ids and not params.ignore_eos: + if token_id in params.stop_token_ids or (token_id in eos_ids and not params.ignore_eos): self._finish(request, FinishReason.STOP) elif len(request.generated_token_ids) >= params.max_tokens: self._finish(request, FinishReason.LENGTH) diff --git a/vllm_rlt/entrypoints/llm.py b/vllm_rlt/entrypoints/llm.py index a154477f..0895d0d1 100644 --- a/vllm_rlt/entrypoints/llm.py +++ b/vllm_rlt/entrypoints/llm.py @@ -1,3 +1,4 @@ +from collections.abc import Iterable from dataclasses import replace import torch @@ -9,6 +10,7 @@ resolve_model_config, resolve_model_source, ) +from vllm_rlt.pd.engine import PDEngine from vllm_rlt.sampling_params import SamplingParams @@ -31,6 +33,11 @@ def __init__( speculative_config=None, ): self._tokenizer_source = None + if isinstance(model, (LLMEngine, PDEngine)): + self.tokenizer = tokenizer + self.engine = model + self._next_request_id = 0 + return if isinstance(model, str): tokenizer_source = resolve_model_source(model) model_name, revision, _ = resolve_model_config(model, revision=revision) @@ -104,6 +111,27 @@ def generate(self, prompts, sampling_params=None): def start_profile(self, config, *, scheduled=False): return self.engine.start_profile(config, scheduled=scheduled) + def start_weight_update(self, version: int | None = None): + self.engine.start_weight_update(version) + + def update_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + self.engine.update_weights(weights) + + def finish_weight_update(self): + self.engine.finish_weight_update() + + def get_weight_version(self) -> int: + return self.engine.get_weight_version() + + def reset_prefix_cache(self): + self.engine.reset_prefix_cache() + + def pause_generation(self): + self.engine.pause_generation() + + def resume_generation(self): + self.engine.resume_generation() + def stop_profile(self): return self.engine.stop_profile() diff --git a/vllm_rlt/pd/engine.py b/vllm_rlt/pd/engine.py index 59b5da94..3fadd635 100644 --- a/vllm_rlt/pd/engine.py +++ b/vllm_rlt/pd/engine.py @@ -1,17 +1,20 @@ """CPU coordinator for independent prefill/decode GPU pools. D reservations precede P computation. Control channels carry only metadata and -outputs; NIXL moves KV directly between registered worker allocations. +outputs and CPU-staged full-weight chunks; NIXL moves KV between GPU pools. """ import multiprocessing as mp import time import uuid from collections import deque +from collections.abc import Iterable from dataclasses import asdict, dataclass, field, replace from pathlib import Path from types import SimpleNamespace +import torch + from vllm_rlt.config import CacheConfig, ExecutionConfig, ExitConfig, SchedulerConfig from vllm_rlt.models import OuroConfig, resolve_model_config from vllm_rlt.profiling import ProfileConfig @@ -55,6 +58,7 @@ class Pending: d_released: bool = False finished: bool = False timings: dict = field(default_factory=dict) + latest_output: RequestOutput | None = None class PDEngine: @@ -74,6 +78,7 @@ def __init__( revision=None, seed=0, speculative_config=None, + terminate_workers_on_failure: bool = True, ): if speculative_config is not None: raise ValueError("speculative decoding is not yet supported by PD") @@ -105,6 +110,12 @@ def __init__( self.outputs = deque() self.closed = False self.failure = None + self.terminate_workers_on_failure = terminate_workers_on_failure + self._weight_version = 0 + self._update_version: int | None = None + self._update_failed = False + self._paused = False + self._weight_replies = {} self._profile_session = None self._profile_replies = {} self._profile_statuses = {} @@ -190,6 +201,11 @@ def _poll(self): def _message(self, peer, m): kind = m["kind"] + if kind == "weight_reply": + replies = self._weight_replies.get(m["control_id"]) + if replies is not None: + replies[peer.name] = m + return if kind == "profile_reply": pending = self._profile_replies.get(m.get("control_id")) if pending is not None and m.get("session_id") == self._profile_session: @@ -249,6 +265,9 @@ def _message(self, peer, m): elif kind == "output": if not w.cancelled: output = replace(m["output"], request_id=w.request.request_id) + if output.weight_version != w.request.weight_version: + raise RuntimeError("PD workers returned a different policy version") + w.latest_output = output w.request.generated_token_ids = list(output.token_ids) w.request.exit_depths = list(output.exit_depths) w.timings.setdefault("first_token", now) @@ -296,6 +315,8 @@ def _blocks(self, peer, tokens): def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, trace_id=None): if self.closed or self.failure: raise RuntimeError("PD engine is unavailable") + if self._paused or self._update_version is not None or self._update_failed: + raise RuntimeError("Generation admission is paused") if not isinstance(request_id, str) or not request_id or request_id in self.requests: raise ValueError("request ID must be nonempty and unique among active requests") if len(self.transfers) >= self.config.max_pending_requests: @@ -306,9 +327,12 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra type(t) is not int or not 0 <= t < cfg.vocab_size for t in prompt_token_ids ): raise ValueError("prompt must contain valid model token IDs") + if any(t >= cfg.vocab_size for t in params.stop_token_ids): + raise ValueError("stop token IDs must be within the model vocabulary") maximum = params.max_loops or cfg.total_ut_steps if maximum > cfg.total_ut_steps or params.min_loops > maximum: raise ValueError("requested loop bounds exceed model depth") + params = replace(params, max_loops=maximum, top_k=min(params.top_k, cfg.vocab_size)) if len(prompt_token_ids) + params.max_tokens - 1 > cfg.max_position_embeddings: raise ValueError("prompt plus decode exceeds context capacity") if trace_id is not None and (not isinstance(trace_id, str) or not trace_id): @@ -335,7 +359,9 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra if p.role == role ): raise ValueError(f"request exceeds {role} KV pool capacity") - request = Request(request_id, list(prompt_token_ids), params) + request = Request( + request_id, list(prompt_token_ids), params, weight_version=self._weight_version + ) w = Pending(request, uuid.uuid4().hex, trace_id) self.requests[request_id] = request self.transfers[w.tid] = w @@ -412,14 +438,88 @@ def abort_request(self, request_id): self._send(w.p, "cancel", tid=w.tid) request.stage = Stage.FINISHED request.finish_reason = FinishReason.ABORT + if w.latest_output is not None: + return replace(w.latest_output, finished=True, finish_reason=FinishReason.ABORT.value) return RequestOutput.from_request(request) + def get_weight_version(self) -> int: + return self._weight_version + + def pause_generation(self): + self._paused = True + + def resume_generation(self): + if self._update_version is not None or self._update_failed: + raise RuntimeError("Complete the weight update before resuming generation") + self._paused = False + + def _weight_control(self, action, **arguments): + control_id = uuid.uuid4().hex + replies = self._weight_replies[control_id] = {} + try: + for name in self.peers: + self._send( + name, + "weight_control", + control_id=control_id, + action=action, + arguments=arguments, + ) + self._wait( + lambda: len(replies) == len(self.peers), + time.monotonic() + self.config.request_timeout, + ) + errors = [reply["error"] for reply in replies.values() if "error" in reply] + if errors: + raise RuntimeError("PD weight control failed: " + "; ".join(errors)) + return [reply["version"] for reply in replies.values()] + except Exception: + # Some peers may already have copied or committed the new policy. + # Keep admission blocked until every peer completes a newer full update. + self._update_failed = True + raise + finally: + del self._weight_replies[control_id] + + def reset_prefix_cache(self): + if self.has_unfinished_requests(): + raise RuntimeError("Finish active PD transfers before resetting cached KV") + self._weight_control("reset_prefix_cache") + + def start_weight_update(self, version: int | None = None): + if self.closed or self.failure or self.has_unfinished_requests(): + raise RuntimeError("Weight updates require an available, idle PD engine") + latest = max(self._weight_version, self._update_version or 0) + version = latest + 1 if version is None else version + if type(version) is not int or version <= latest: + raise ValueError("PD weight version must exceed all previous update attempts") + self._update_version, self._update_failed = version, False + self._weight_control("start_weight_update", version=version) + + def update_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + if self._update_version is None or self._update_failed: + raise RuntimeError("Start a full PD weight update first") + # CPU staging avoids CUDA tensor ownership across spawned worker processes. + weights = [(name, tensor.detach().cpu()) for name, tensor in weights] + self._weight_control("update_weights", weights=weights) + + def finish_weight_update(self): + if self._update_version is None or self._update_failed: + raise RuntimeError("No successful PD weight update to finish") + versions = self._weight_control("finish_weight_update") + if any(version != self._update_version for version in versions): + self._update_failed = True + raise RuntimeError("PD workers committed different policy versions") + self._weight_version, self._update_version = self._update_version, None + def has_unfinished_requests(self): return bool(self.transfers) def step(self): if self.closed: raise RuntimeError("PD engine is closed") + if self._update_version is not None or self._update_failed: + raise RuntimeError("Generation is unavailable during a PD weight update") try: self._poll() if any( @@ -573,6 +673,9 @@ def wait_for_profile_artifacts(self, timeout=None): return {"recording": False, "ranks": {}} def _terminate(self): + if not self.terminate_workers_on_failure: + self.closed = True + return if self._profile_session: self._profile_recording = False for name in self.peers: diff --git a/vllm_rlt/pd/worker.py b/vllm_rlt/pd/worker.py index db1df13d..7131c8d2 100644 --- a/vllm_rlt/pd/worker.py +++ b/vllm_rlt/pd/worker.py @@ -1,4 +1,4 @@ -"""One CUDA owner process per P/D worker. Control messages never carry tensors.""" +"""One CUDA owner per P/D worker; full updates use staged CPU tensor chunks.""" import hashlib import json @@ -170,6 +170,25 @@ def accept(self, command): def command(self, command): kind = command["kind"] + if kind == "weight_control": + try: + if self.work: + raise RuntimeError("Finish PD transfers before updating weights") + operations = { + "start_weight_update": self.engine.start_weight_update, + "update_weights": self.engine.update_weights, + "finish_weight_update": self.engine.finish_weight_update, + "reset_prefix_cache": self.engine.reset_prefix_cache, + } + operations[command["action"]](**command["arguments"]) + self.send( + "weight_reply", + control_id=command["control_id"], + version=self.engine.get_weight_version(), + ) + except Exception as error: + self.send("weight_reply", control_id=command["control_id"], error=str(error)) + return if kind.startswith("profile_"): try: action = kind.removeprefix("profile_") @@ -408,6 +427,16 @@ def run(self): if not self.work and not self.channel.poll(): time.sleep(0.001) self.engine.model_runner.synchronize() + graphs = self.engine.model_runner.graphs + graph_stats = ( + dict( + graph_captures=graphs.captures, + graph_replays=graphs.replays, + graph_fallbacks=graphs.fallbacks, + ) + if graphs is not None + else {} + ) self.engine.close() self.connector.close() self.send( @@ -417,6 +446,8 @@ def run(self): transfers=self.connector.transfers, transfer_seconds=self.connector.transfer_seconds, used_blocks=self.cache.num_used_blocks, + weight_version=self.engine.get_weight_version(), + **graph_stats, ) diff --git a/vllm_rlt/request.py b/vllm_rlt/request.py index c78eb51c..e8394589 100644 --- a/vllm_rlt/request.py +++ b/vllm_rlt/request.py @@ -44,6 +44,8 @@ class Request: generator: torch.Generator | None = field(default=None, repr=False) finish_reason: FinishReason | None = None exit_trace: tuple[int, ...] = field(default=(), repr=False) + weight_version: int = 0 + log_probs: list[torch.Tensor] = field(default_factory=list, repr=False) @property def num_scheduled_outputs(self) -> int: @@ -67,6 +69,9 @@ class RequestOutput: finished: bool finish_reason: str | None = None text: str = "" + log_probs: list[float] | None = None + weight_version: int = 0 + sampling_params: SamplingParams | None = None @classmethod def from_request(cls, request: Request): @@ -79,4 +84,13 @@ def from_request(cls, request: Request): finish_reason=request.finish_reason.value if request.finish_reason is not None else None, + log_probs=( + torch.stack(request.log_probs[: len(request.generated_token_ids)]).tolist() + if request.generated_token_ids + else [] + ) + if request.sampling_params.logprobs is not None + else None, + weight_version=request.weight_version, + sampling_params=request.sampling_params, ) diff --git a/vllm_rlt/sampling_params.py b/vllm_rlt/sampling_params.py index 30009bde..c5f75096 100644 --- a/vllm_rlt/sampling_params.py +++ b/vllm_rlt/sampling_params.py @@ -8,7 +8,10 @@ class SamplingParams: temperature: float = 0.0 top_p: float = 1.0 top_k: int = -1 - seed: int = 0 + seed: int | None = 0 + logprobs: int | None = None + logprobs_mode: str = "raw" + stop_token_ids: tuple[int, ...] = () min_loops: int = 2 max_loops: int | None = None exit_threshold: float = 1.0 @@ -18,8 +21,11 @@ class SamplingParams: def __post_init__(self): if type(self.priority) is not int: raise ValueError("priority must be an integer") - for name in ("max_tokens", "min_loops", "max_loops"): - value = getattr(self, name) + for name, value in ( + ("max_tokens", self.max_tokens), + ("min_loops", self.min_loops), + ("max_loops", self.max_loops), + ): if name == "max_loops" and value is None: continue if type(value) is not int or value < 1: @@ -34,5 +40,11 @@ def __post_init__(self): raise ValueError("top_k must be -1 or positive") if not 0 <= self.exit_threshold <= 1: raise ValueError("exit_threshold must be in [0, 1]") - if type(self.seed) is not int or not 0 <= self.seed < 2**63: + if self.seed is not None and (type(self.seed) is not int or not 0 <= self.seed < 2**63): raise ValueError("seed must be an integer in [0, 2**63)") + if self.logprobs is not None and (type(self.logprobs) is not int or self.logprobs != 0): + raise ValueError("logprobs=0 returns the selected-token probability") + if self.logprobs_mode not in ("raw", "processed"): + raise ValueError("logprobs_mode must be raw or processed") + if any(type(token) is not int or token < 0 for token in self.stop_token_ids): + raise ValueError("stop_token_ids must contain nonnegative integer IDs") diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index b0025137..640782e0 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -13,6 +13,7 @@ from vllm_rlt.worker.buffers import Workspace from vllm_rlt.worker.cuda_graph import RecurrentGraphs from vllm_rlt.worker.sampler import Sampler +from vllm_rlt.worker.sampling import selected_logprob @dataclass @@ -603,4 +604,6 @@ def _sample_tensor(self, logits: torch.Tensor, request: Request): # this method; the algorithm lives in Sampler. token, generator = self.sampler.sample(logits, request.sampling_params, request.generator) request.generator = generator + if request.sampling_params.logprobs is not None: + request.log_probs.append(selected_logprob(logits, token, request.sampling_params)) return token diff --git a/vllm_rlt/worker/sampler.py b/vllm_rlt/worker/sampler.py index 4bd8f31b..30506bf6 100644 --- a/vllm_rlt/worker/sampler.py +++ b/vllm_rlt/worker/sampler.py @@ -41,5 +41,9 @@ def sample( remove[0] = False logits = logits.scatter(0, indices, sorted_logits.masked_fill(remove, -torch.inf)) if generator is None: - generator = torch.Generator(device=self.device).manual_seed(params.seed) + generator = torch.Generator(device=self.device) + if params.seed is None: + generator.seed() + else: + generator.manual_seed(params.seed) return torch.multinomial(logits.softmax(-1), 1, generator=generator).squeeze(0), generator diff --git a/vllm_rlt/worker/sampling.py b/vllm_rlt/worker/sampling.py index 3aabb84d..54f59a89 100644 --- a/vllm_rlt/worker/sampling.py +++ b/vllm_rlt/worker/sampling.py @@ -21,10 +21,21 @@ def probabilities(logits, params): def generator_for(request, device): if request.generator is None: - request.generator = torch.Generator(device=device).manual_seed(request.sampling_params.seed) + request.generator = torch.Generator(device=device) + if request.sampling_params.seed is None: + request.generator.seed() + else: + request.generator.manual_seed(request.sampling_params.seed) return request.generator +def selected_logprob(logits, token, params): + if params.logprobs_mode == "processed" and params.temperature > 0: + return probabilities(logits, params)[token].log() + # Integer speculative tokens otherwise retain a view of the vocabulary row. + return logits.float().log_softmax(-1)[token].clone() + + def draw(probs, generator): return torch.multinomial(probs, 1, generator=generator).squeeze(0) diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index 9542ee70..0e5e940a 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -4,7 +4,7 @@ outputs; the engine applies returned tokens and owns KV commit/truncation. """ -from dataclasses import dataclass +from dataclasses import dataclass, field import torch @@ -14,6 +14,7 @@ generator_for, probabilities, rejection_sample, + selected_logprob, ) @@ -22,6 +23,7 @@ class SpeculativeResult: token_ids: list[int] accepted_count: int draft_count: int + log_probs: list[torch.Tensor] = field(default_factory=list) @dataclass @@ -163,6 +165,11 @@ def execute(self, batch): ) start += item.token_count results.append(SpeculativeResult(tokens, accepted, len(candidates[i]))) + if request.sampling_params.logprobs is not None: + results[-1].log_probs.extend( + selected_logprob(row, token, request.sampling_params) + for row, token in zip(rows, tokens) + ) continue start += item.token_count # Sampling keeps sequential rejection; RNG consumption order is unchanged. @@ -183,6 +190,11 @@ def execute(self, batch): ) tokens.append(int(token.item())) results.append(SpeculativeResult(tokens, accepted, len(candidates[i]))) + if request.sampling_params.logprobs is not None: + results[-1].log_probs.extend( + selected_logprob(row, token, request.sampling_params) + for row, token in zip(rows, tokens) + ) self.stats.rounds += len(items) self.stats.drafted_tokens += sum(len(c) for c in candidates) self.stats.verified_rows += len(ids) From e591bdab74df7e1482a36bd12fffce62bfad6bd0 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 1 Oct 2026 11:32:57 +0000 Subject: [PATCH 2/9] Report entropy seeds for reproducible training rollouts Signed-off-by: 0z5a --- docs/training_contract.md | 5 +- .../README.md | 51 ++++++++++++++++++ .../raw/minhao-cpu.log | 12 +++++ .../raw/minhao-seed-before.log | 14 +++++ tests/test_training_contract.py | 52 +++++++++++++++++++ vllm_rlt/request.py | 7 ++- 6 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 docs/validation/minhao-coordination-thor-20261001/README.md create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log diff --git a/docs/training_contract.md b/docs/training_contract.md index 9345cbf6..fa1d2b84 100644 --- a/docs/training_contract.md +++ b/docs/training_contract.md @@ -26,7 +26,10 @@ filtering. Greedy requests return the raw score in either mode. Speculative outputs use the verified **target** policy's score, including correction/bonus tokens; proposal or rejection-residual probabilities are never exposed as the policy score. Explicit `stop_token_ids` apply even with `ignore_eos=True`. -`seed=None` creates an independent entropy-seeded request generator. +`seed=None` resolves an independent seed in `[0, 2**63)` when the request is +created. The effective seed is returned in `sampling_params`, including for an +abort before sampling. Reusing those parameters replays the same request on the +same policy and execution configuration. The default remains `seed=0`. Publication requires an idle engine. Each chunk is checked for duplicate names, unknown names and mismatched shapes before that chunk is copied. `finish_weight_update` diff --git a/docs/validation/minhao-coordination-thor-20261001/README.md b/docs/validation/minhao-coordination-thor-20261001/README.md new file mode 100644 index 00000000..142e2799 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/README.md @@ -0,0 +1,51 @@ +# Phase 1 / VIME coordination checks — 2026-10-01 + +These checks exercise the semantics in +[MinhaoLi0318's PR 1 proposal](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5927860064) +on the existing Thor runtime. They test the contributor implementation, not an +unpublished Phase 1 branch. The +[coordination reply](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5930413915) +records the interface differences and division of upstream work. + +The replay requirement exposed a real omission: an entropy-seeded request +returned `sampling_params.seed=None`. The +[unfixed regression](raw/minhao-seed-before.log) fails on that missing effective +seed. `Request` now resolves an independent 63-bit seed when it is created and +retains it through output, preemption, abort and existing PD parameter transport. +Default `seed=0` is unchanged. An abort before any sampling also reports the seed; +the caller's immutable `SamplingParams(seed=None)` is not modified. + +| Thor CPU check | Result | +|---|---| +| Entropy seed reporting and exact token/score/depth replay | Ordinary, async and speculative paths pass | +| Abort before sampling | Effective seed retained; zero selected scores | +| Preemption | Token IDs, selected scores, depths and RNG continuity match uninterrupted execution | +| Existing probability/publication/cache/stop/abort checks | All pass | +| Full focused suite including in-process PD publication | **7 tests pass, 1.736 s** | + +See [CPU log](raw/minhao-cpu.log). Independent VIME FP32 mixed-depth replay, +recomputed gradients and generic physical-parameter conversion are checked +separately. No runtime packages are installed or updated. + +At the time of the initial coordination reply, new CUDA checks are queued behind +an existing Qwen-Image training reservation; they are not counted as passed here. +Full-model score-mismatch and selected-score-overhead measurements will be added +after that reservation permits execution. Earlier full-model/real-PD evidence +is retained in the [Thor](../training-contract-thor-20261001/README.md) and +[A100](../training-contract-a100-20261001/README.md) reports. + +## Interface coordination + +| Proposed Phase 1 interface | Current reference draft | VIME consumer plan | +|---|---|---| +| `RequestOutput.logprobs` | `RequestOutput.log_probs` | Use the upstream spelling once its PR 1 branch is available | +| Engine-level `raw_logprobs` / `processed_logprobs` | Request-level `raw` / `processed` | Configure processed scores for the trainer; avoid a second compatibility API | +| Effective seed reported in resolved parameters | Fixed by this follow-up | Keep seeded reproduction and opt-in entropy groups | +| Full-depth first token and executed decode depths | Already present | Full four-loop prefill and differentiable `last_exited` trace replay | +| PR 2 versioned full publication | Validated reference implementation | Reuse one agreed upstream contract | + +The old `RLEngine.set_loop_budget` mutated `total_ut_steps`; the current VIME +adapter supersedes it and does not need reduced-depth prefill. No new transport +layer is needed for the pilot: the consumer is in-process `LLM`, and existing PD +worker IPC remains inside the engine. MinhaoLi0318 and IDEA-V retain the proposed +Phase 1 upstream roles; this draft supplies reusable implementation and checks. diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log new file mode 100644 index 00000000..665bd66e --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log @@ -0,0 +1,12 @@ +test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok +test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok +test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok +test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok +test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok +test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok +test_all_workers_commit_and_partial_commit_requires_new_epoch (tests.test_pd_weight_control.PDWeightControlTest.test_all_workers_commit_and_partial_commit_requires_new_epoch) ... ok + +---------------------------------------------------------------------- +Ran 7 tests in 1.736s + +OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log new file mode 100644 index 00000000..1a90a902 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log @@ -0,0 +1,14 @@ +test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... FAIL + +====================================================================== +FAIL: test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/home/jwipc/experiments/ouro-vime-20260930/rlt-training-contract-20261001/tests/test_training_contract.py", line 190, in test_entropy_seed_is_reported_and_replays_the_same_policy + self.assertIsInstance(output.sampling_params.seed, int) +AssertionError: None is not an instance of + +---------------------------------------------------------------------- +Ran 1 test in 0.067s + +FAILED (failures=1) diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py index 803b8381..b536eaa5 100644 --- a/tests/test_training_contract.py +++ b/tests/test_training_contract.py @@ -16,6 +16,7 @@ SchedulerConfig, SpeculativeConfig, ) +from vllm_rlt.engine.llm_engine import LLMEngine from vllm_rlt.models import OuroConfig, OuroForCausalLM from vllm_rlt.request import Request from vllm_rlt.worker.sampling import generator_for, probabilities @@ -180,6 +181,57 @@ def test_seed_none_and_validation(self): with self.assertRaises(ValueError): llm(model()).generate([[3]], SamplingParams(stop_token_ids=(64,))) + def test_entropy_seed_is_reported_and_replays_the_same_policy(self): + params = SamplingParams( + max_tokens=8, temperature=0.9, seed=None, logprobs=0, ignore_eos=True + ) + for options in ({}, {"asynchronous": True}, {"speculative": True}): + rollout = llm(model(), graphs=DEVICE == "cuda", **options) + output = rollout.generate([[3, 4, 5]], params)[0] + self.assertIsInstance(output.sampling_params.seed, int) + self.assertLess(output.sampling_params.seed, 2**63) + replay = rollout.generate([[3, 4, 5]], output.sampling_params)[0] + self.assertEqual(output.token_ids, replay.token_ids) + self.assertEqual(output.log_probs, replay.log_probs) + self.assertEqual(output.exit_depths, replay.exit_depths) + rollout.engine.add_request("unstarted", [3], replace(params, temperature=0)) + aborted = rollout.engine.abort_request("unstarted") + self.assertIsInstance(aborted.sampling_params.seed, int) + self.assertEqual(aborted.log_probs, []) + rollout.close() + self.assertIsNone(params.seed) + + def test_preemption_preserves_selected_scores_and_rng(self): + native = model() + params = SamplingParams( + max_tokens=8, temperature=0.8, seed=45, logprobs=0, exit_threshold=0, ignore_eos=True + ) + results = [] + for suspend in (False, True): + engine = LLMEngine( + native, + cache_config=CacheConfig(64, 2), + scheduler_config=SchedulerConfig(enable_preemption=True), + attention_backend="triton" if DEVICE == "cuda" else "torch", + ) + engine.add_request("a", [1, 2, 3], params) + while len(engine.scheduler.requests["a"].generated_token_ids) < 3: + engine.step() + if suspend: + engine.add_request("b", [5], SamplingParams(max_tokens=1)) + engine.scheduler.selected_request_ids.clear() + self.assertTrue(engine.preemption.preempt(engine.scheduler.requests["b"])) + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.request_id == "a" and output.finished: + results.append(output) + if suspend: + self.assertEqual(engine.preemption.resumptions, 1) + engine.close() + self.assertEqual(results[0].token_ids, results[1].token_ids) + self.assertEqual(results[0].log_probs, results[1].log_probs) + self.assertEqual(results[0].exit_depths, results[1].exit_depths) + if __name__ == "__main__": torch.set_num_threads(1) diff --git a/vllm_rlt/request.py b/vllm_rlt/request.py index e8394589..e4d7ae21 100644 --- a/vllm_rlt/request.py +++ b/vllm_rlt/request.py @@ -1,5 +1,6 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from enum import Enum +from secrets import randbits import torch @@ -47,6 +48,10 @@ class Request: weight_version: int = 0 log_probs: list[torch.Tensor] = field(default_factory=list, repr=False) + def __post_init__(self): + if self.sampling_params.seed is None: + self.sampling_params = replace(self.sampling_params, seed=randbits(63)) + @property def num_scheduled_outputs(self) -> int: return len(self.generated_token_ids) + self.num_output_placeholders From bdce2876e3f78ac7efaa6cbf5783c185da2e9d19 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 1 Oct 2026 12:23:11 +0000 Subject: [PATCH 3/9] Measure selected-score cost and replay semantics on Thor Signed-off-by: 0z5a --- benchmarks/logprob_overhead.py | 125 + .../README.md | 93 +- .../raw/minhao-cleanup-smoke-removed.json | 4 + .../raw/minhao-cleanup-smoke.json | 2669 ++++++++++++++++ .../raw/minhao-cleanup-smoke.log | 40 + .../raw/minhao-cleanup.json | 7 + .../raw/minhao-cuda-parallel.log | 11 + .../raw/minhao-cuda.log | 11 + .../raw/minhao-driver-stack-wait.log | 19 + .../raw/minhao-driver-wait.log | 42 + .../raw/minhao-gpu-before.csv | 2 + .../raw/minhao-logprob-overhead.json | 2741 +++++++++++++++++ .../raw/minhao-logprob-overhead.log | 40 + .../raw/minhao-slow-cleanup.json | 4 + .../raw/minhao-tiny-overhead.json | 2669 ++++++++++++++++ 15 files changed, 8472 insertions(+), 5 deletions(-) create mode 100644 benchmarks/logprob_overhead.py create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json diff --git a/benchmarks/logprob_overhead.py b/benchmarks/logprob_overhead.py new file mode 100644 index 00000000..cdf5bdbd --- /dev/null +++ b/benchmarks/logprob_overhead.py @@ -0,0 +1,125 @@ +"""Alternate unprofiled score modes; report profiler operator snapshots separately.""" + +import argparse +import json +import statistics +import time +from dataclasses import replace +from pathlib import Path + +import torch +from transformers import AutoTokenizer + +from vllm_rlt import ( + LLM, + CacheConfig, + ExecutionConfig, + ExitConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.models import OuroForCausalLM + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("model", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--repeats", type=int, default=5) + args = parser.parse_args() + torch.set_num_threads(4) + native = OuroForCausalLM.from_pretrained(args.model, device="cuda", dtype=torch.bfloat16) + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False) + prompts = [tokenizer.encode(text) for text in ("Compute 17 + 25.\n", "Compute 9 * 8.\n")] + base = [ + SamplingParams(max_tokens=32, temperature=0.9, seed=31 + i, ignore_eos=True) + for i in range(2) + ] + modes = { + "none": base, + **{ + name: [replace(p, logprobs=0, logprobs_mode=name) for p in base] + for name in ("raw", "processed") + }, + } + report = {} + for path in ("sync", "async", "speculative"): + engine = LLM( + native, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=512), + scheduler_config=SchedulerConfig(max_num_seqs=2), + exit_config=ExitConfig("ouro_delayed" if path == "async" else "ouro"), + execution_config=ExecutionConfig(cuda_graphs=True, async_scheduling=path == "async"), + speculative_config=SpeculativeConfig(3) if path == "speculative" else None, + ) + for params in modes.values(): + engine.generate(prompts, params) + times = {name: [] for name in modes} + memory = {name: [] for name in modes} + for index in range(args.repeats): + names = list(modes) + names = names[index % 3 :] + names[: index % 3] + outputs = {} + for name in names: + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + baseline = torch.cuda.memory_allocated() + started = time.monotonic() + outputs[name] = engine.generate(prompts, modes[name]) + torch.cuda.synchronize() + times[name].append(time.monotonic() - started) + memory[name].append(torch.cuda.max_memory_allocated() - baseline) + assert all(len(o.token_ids) == 32 for o in outputs["none"]) + assert all(o.log_probs is None for o in outputs["none"]) + for name in ("raw", "processed"): + assert [o.token_ids for o in outputs[name]] == [ + o.token_ids for o in outputs["none"] + ] + assert all(len(o.log_probs) == len(o.token_ids) for o in outputs[name]) + medians = {name: statistics.median(values) for name, values in times.items()} + profiles = {} + for name in ("none", "raw", "processed"): + for detail in ("ops", "full"): + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + record_shapes=detail == "full", + with_stack=detail == "full", + profile_memory=detail == "full", + ) as profile: + engine.generate(prompts, modes[name]) + torch.cuda.synchronize() + events = sorted( + profile.key_averages(), key=lambda e: e.self_cpu_time_total, reverse=True + ) + profiles[f"{name}-{detail}"] = [ + { + "operator": e.key, + "count": e.count, + "self_cpu_us": e.self_cpu_time_total, + "self_device_us": e.self_device_time_total, + "self_device_memory_bytes": e.self_device_memory_usage, + } + for e in events[:20] + ] + report[path] = { + "seconds": times, + "median_seconds": medians, + "tokens_per_second": {name: 64 / value for name, value in medians.items()}, + "peak_extra_allocated_bytes": memory, + "profiles": profiles, + "tokens_unchanged": True, + } + engine.close() + print(path, medians, flush=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + del engine + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/docs/validation/minhao-coordination-thor-20261001/README.md b/docs/validation/minhao-coordination-thor-20261001/README.md index 142e2799..59c4dce1 100644 --- a/docs/validation/minhao-coordination-thor-20261001/README.md +++ b/docs/validation/minhao-coordination-thor-20261001/README.md @@ -27,13 +27,96 @@ See [CPU log](raw/minhao-cpu.log). Independent VIME FP32 mixed-depth replay, recomputed gradients and generic physical-parameter conversion are checked separately. No runtime packages are installed or updated. -At the time of the initial coordination reply, new CUDA checks are queued behind -an existing Qwen-Image training reservation; they are not counted as passed here. -Full-model score-mismatch and selected-score-overhead measurements will be added -after that reservation permits execution. Earlier full-model/real-PD evidence -is retained in the [Thor](../training-contract-thor-20261001/README.md) and +The user authorized concurrent short CUDA validation while an existing +Qwen-Image training reservation remained active. **All six CUDA contract tests +pass in 135.474 s**, including entropy-seed replay with graphs, stop/abort, +preemption, speculative target scores and publication recovery; see +[CUDA log](raw/minhao-cuda-parallel.log). Startup is included in the suite time. The originally queued CUDA suite also +completed naturally after the reservation was released: **6/6 in 3.989 s**, +[log](raw/minhao-cuda.log). +Independent VIME FP32 mixed-depth replay/recomputed gradients and generic +physical-parameter conversion pass **two CPU tests in 0.302 s**. No existing +training process or reservation was interrupted. + +Earlier full-model training/save/resume and real-PD evidence is retained in the +[Thor](../training-contract-thor-20261001/README.md) and [A100](../training-contract-a100-20261001/README.md) reports. +## Selected-score cost + +`benchmarks/logprob_overhead.py` warms each mode and rotates the measurement +order over five repeats. Each unprofiled window generates two prompts with 32 +response tokens each, T=0.9, fixed per-request seeds, no filtering and CUDA +graphs enabled. Sync and speculative paths use Ouro exit; async uses delayed +Ouro exit. Token sequences must match across score modes; selected scores must +align with every committed token. Initialization and capture are excluded. + +Separately, all three modes receive an ops-only and a full-detail CPU/CUDA +profiler window (`record_shapes`, stacks and memory enabled for full detail). +The raw JSON retains the 20 operators with largest self CPU time per window, +with counts, self device time and memory. These are operator summaries, not +exported timeline traces. Profiled runs are excluded from throughput medians. +The [tiny smoke JSON](raw/minhao-tiny-overhead.json) checks all three paths and +18 profiler windows with one repeat; its timings are not full-model results. + +### Full Ouro-1.4B shared-GPU observations + +The full run overlaps another Qwen-Image training phase. GPU utilization +before this run was **98%**; [snapshot](raw/minhao-gpu-before.csv). +These measurements include resource contention and are not isolated throughput +results. Each mode has five unprofiled repeats. All three paths complete; +token IDs match across modes and selected scores align with committed tokens. + +| Path | None median (s) | Raw median (s) | Processed median (s) | Raw time overhead | Processed time overhead | +|---|---:|---:|---:|---:|---:| +| sync | 1.8017 | 1.8280 | 1.8281 | +1.46% | +1.47% | +| async | 1.6974 | 1.7849 | 1.7894 | +5.15% | +5.42% | +| speculative | 1.4366 | 1.4462 | 1.4486 | +0.67% | +0.83% | + +| Path / score mode | Tokens/s | Speed vs None | Median extra PyTorch allocation (KiB) | +|---|---:|---:|---:| +| sync / none | 35.52 | 1.000× | 933.0 | +| sync / raw | 35.01 | 0.986× | 965.0 | +| sync / processed | 35.01 | 0.986× | 933.0 | +| async / none | 37.70 | 1.000× | 965.0 | +| async / raw | 35.86 | 0.951× | 963.5 | +| async / processed | 35.77 | 0.949× | 965.0 | +| speculative / none | 44.55 | 1.000× | 2733.5 | +| speculative / raw | 44.25 | 0.993× | 2793.5 | +| speculative / processed | 44.18 | 0.992× | 2761.5 | + +[Raw timings / all 18 profiler operator summaries](raw/minhao-logprob-overhead.json), +[benchmark log](raw/minhao-logprob-overhead.log). Extra allocation is +per-window PyTorch peak above its starting allocation; it excludes resident +model/cache/graph storage and other processes. + +Between completed paths, the process temporarily waited in NVIDIA UVM external +allocation mapping (`os_acquire_rwlock_read/write`); an independent GPU status +query also waited. The [captured stack](raw/minhao-driver-stack-wait.log) and +[kernel log](raw/minhao-driver-wait.log) record concurrent `NV_ERR_NO_MEMORY` +allocation errors. Both waits resolved naturally; all paths completed without +process termination, driver reset or environment changes. Initialization waits +are outside the warmed measurement windows. The harness now drops the previous +engine and clears its allocator cache between paths to avoid retaining that +engine while constructing the next one. The recorded full run predates this +inter-path cleanup; its timed body is unchanged. +The updated cleanup harness then passes a separate tiny-model smoke across +all three paths and 18 profiler windows ([JSON](raw/minhao-cleanup-smoke.json), +[log](raw/minhao-cleanup-smoke.log)); its [57,039,421-byte model is removed](raw/minhao-cleanup-smoke-removed.json). + +All five full-model depth/effective-seed replay cases also complete, including +delayed async. The [VIME score-error table](https://github.com/0z5a/vime/blob/feat/ouro-rollout-cuda-graphs/docs/validation/minhao-coordination-thor-20261001/README.md) +records the remaining training/inference recomputation differences; full-depth +K=4 also has a nonzero residual. Exact seed replay and score-recomputation +agreement are separate checks. + +The [completed full-model copy and replacement shard](raw/minhao-cleanup.json) +are removed (3,232,344,846 bytes). The 358,667,054-byte local transfer copy +was also removed. The original slower download completed naturally; its +[2,873,677,792-byte model copy was removed](raw/minhao-slow-cleanup.json) by the +existing queue, without repeating GPU tests. All four task-owned model folders +and both replacement transfer copies are absent after validation. + ## Interface coordination | Proposed Phase 1 interface | Current reference draft | VIME consumer plan | diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json new file mode 100644 index 00000000..1f9931bd --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json @@ -0,0 +1,4 @@ +{ + "removed": "models/Ouro-minhao-cleanup-smoke", + "bytes": 57039421 +} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json new file mode 100644 index 00000000..ed52a20b --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json @@ -0,0 +1,2669 @@ +{ + "sync": { + "seconds": { + "none": [ + 0.08066848499993284 + ], + "raw": [ + 0.0914349139998194 + ], + "processed": [ + 0.08466345700026068 + ] + }, + "median_seconds": { + "none": 0.08066848499993284, + "raw": 0.0914349139998194, + "processed": 0.08466345700026068 + }, + "tokens_per_second": { + "none": 793.3705461315318, + "raw": 699.9514430573688, + "processed": 755.9341688563809 + }, + "peak_extra_allocated_bytes": { + "none": [ + 758784 + ], + "raw": [ + 823808 + ], + "processed": [ + 791040 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 24741.71000000011, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 14785.07000000005, + "self_device_us": 6.143999999963853, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2004, + "self_cpu_us": 10337.619999999994, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2020, + "self_cpu_us": 6443.9159999996045, + "self_device_us": 2353.745999999865, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1348, + "self_cpu_us": 4180.128999999671, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Activity Buffer Request", + "count": 3, + "self_cpu_us": 3175.8589999999667, + "self_device_us": 15.329000000011547, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 1981.3640000000255, + "self_device_us": 966.4389999999644, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 208, + "self_cpu_us": 1895.6409999999596, + "self_device_us": 293.95500000013635, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1556, + "self_cpu_us": 1581.7819999997378, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 716, + "self_cpu_us": 1457.2760000001927, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1335.993999999857, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1012, + "self_cpu_us": 1159.8320000000067, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 962.9140000004063, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 879.5000000000755, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 867.1529999999693, + "self_device_us": 227.0119999999879, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 830.8970000000882, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 824.2710000000934, + "self_device_us": 553.6069999999727, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2415, + "self_cpu_us": 754.1939999999772, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 744.7460000000592, + "self_device_us": 198.56199999995624, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 116, + "self_cpu_us": 645.1380000000013, + "self_device_us": 236.03399999999692, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 51961.9490000005, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 16149.892999999873, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2004, + "self_cpu_us": 10752.365000000542, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9578.167000000489, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 7612.398999999911, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2020, + "self_cpu_us": 7304.089999999687, + "self_device_us": 5244.179000000046, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1348, + "self_cpu_us": 4891.489000000262, + "self_device_us": 0, + "self_device_memory_bytes": 26818560 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 4097.280999999177, + "self_device_us": 0.0, + "self_device_memory_bytes": 162304 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 3075.5700000002166, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1556, + "self_cpu_us": 3032.1670000008203, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 716, + "self_cpu_us": 2489.1329999999634, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 2429.843999999812, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2326.7890000000816, + "self_device_us": 1660.2469999999175, + "self_device_memory_bytes": 6881280 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 2264.487000000183, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "aten::cat", + "count": 208, + "self_cpu_us": 2253.055999999902, + "self_device_us": 608.0070000000824, + "self_device_memory_bytes": 305152 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 2143.693999999712, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 2037.7939999994514, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1728.5120000003517, + "self_device_us": 0.0, + "self_device_memory_bytes": -12845056 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 1563.099999999995, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 1466.244999999848, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 119556.1379999991, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 23820.948999999957, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2194, + "self_cpu_us": 14371.875999999655, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2212, + "self_cpu_us": 11208.853000000732, + "self_device_us": 9572.278999999877, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 9940.423999999859, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1540, + "self_cpu_us": 7332.918000000681, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 3733.2030000000573, + "self_device_us": 1393.6519999997668, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 2742.5369999992313, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 2586.1810000003697, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2453.102000000058, + "self_device_us": 2902.2410000000755, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2360.366000000333, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2117.0190000001458, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1898.4860000001345, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 624, + "self_cpu_us": 1705.4810000000034, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3537, + "self_cpu_us": 1688.5239999999105, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1140, + "self_cpu_us": 1436.9140000001375, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1312.8229999998948, + "self_device_us": 794.6019999998643, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1280.3379999999306, + "self_device_us": 2046.0730000000694, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1207.4139999993881, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 1164.5660000002226, + "self_device_us": 719.3059999997731, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 115092.6299999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 29600.60599999951, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 25946.991000000064, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 17345.223999999165, + "self_device_us": 0.0, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cudaLaunchKernel", + "count": 2194, + "self_cpu_us": 15940.768999999744, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 13333.437000000138, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2212, + "self_cpu_us": 13191.647000002034, + "self_device_us": 8704.526000000209, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1540, + "self_cpu_us": 8662.159000000243, + "self_device_us": 0, + "self_device_memory_bytes": 39434240 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 7366.031999998762, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 4815.7890000007, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 4814.504000000853, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 4703.688000000431, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 4365.908999999834, + "self_device_us": 1169.2249999997575, + "self_device_memory_bytes": 337920 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 4116.904000000264, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 4072.4490000006735, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 3742.0259999994387, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 3527.933999998534, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 3314.13099999951, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 624, + "self_cpu_us": 3090.420999999369, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 3038.0740000001883, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 105856.23600000067, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 23095.000000000022, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2322, + "self_cpu_us": 15003.759999999633, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2148, + "self_cpu_us": 10396.17099999906, + "self_device_us": 9188.07700000014, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 8986.292999999905, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1476, + "self_cpu_us": 6711.9949999997025, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 3563.392000000484, + "self_device_us": 1348.502999999796, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 2623.3650000003126, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2498.1059999999925, + "self_device_us": 2908.510000000058, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 2357.0779999990714, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2048.535, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2041.7819999998937, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3537, + "self_cpu_us": 1853.5020000003638, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1701.4389999999712, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 192, + "self_cpu_us": 1573.1770000001234, + "self_device_us": 1020.2990000001446, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 1523.2590000001437, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1140, + "self_cpu_us": 1365.1570000002835, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1229.6910000001153, + "self_device_us": 799.3670000000275, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1220.0819999998166, + "self_device_us": 2031.6709999999912, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1216.561000000036, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 155410.87200000056, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 26514.28400000018, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 22682.278999999973, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2322, + "self_cpu_us": 17120.490999999194, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 16900.313999997426, + "self_device_us": 0.0, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 13033.013999998899, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2148, + "self_cpu_us": 12944.766999999709, + "self_device_us": 12005.580999999922, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1476, + "self_cpu_us": 8355.110999999586, + "self_device_us": 0, + "self_device_memory_bytes": 39401472 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 7274.417999999696, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 4893.769999999835, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 4797.883999998665, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 4710.996000000819, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 4480.034000000242, + "self_device_us": 1730.5530000000113, + "self_device_memory_bytes": 337920 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 4113.781999999035, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 3937.3690000012284, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 3691.073999999741, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 3555.606999998805, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 3436.052999998901, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 3407.9150000012155, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2940.6499999998923, + "self_device_us": 0.0, + "self_device_memory_bytes": -12845056 + } + ] + }, + "tokens_unchanged": true + }, + "async": { + "seconds": { + "none": [ + 0.1108772420002424 + ], + "raw": [ + 0.1287477310002032 + ], + "processed": [ + 0.12239944100019784 + ] + }, + "median_seconds": { + "none": 0.1108772420002424, + "raw": 0.1287477310002032, + "processed": 0.12239944100019784 + }, + "tokens_per_second": { + "none": 577.214934692009, + "raw": 497.0961391148632, + "processed": 522.8782049739635 + }, + "peak_extra_allocated_bytes": { + "none": [ + 790016 + ], + "raw": [ + 822272 + ], + "processed": [ + 822272 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 15284.012000000086, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 1850, + "self_cpu_us": 8500.083000000152, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 6639.298000000122, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1439, + "self_cpu_us": 5531.486999999959, + "self_device_us": 1906.9919999998415, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 4616.57799999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2769, + "self_cpu_us": 4578.959000000186, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 601, + "self_cpu_us": 2438.2479999999, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2059.4019999999846, + "self_device_us": 778.1539999999927, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3851, + "self_cpu_us": 1576.196000000137, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 629, + "self_cpu_us": 1306.6099999999087, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1285.6129999997502, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 1154.325999999974, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 996.3809999998794, + "self_device_us": 225.15500000009706, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 798.8650000000216, + "self_device_us": 569.2259999999969, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 751.5409999999792, + "self_device_us": 210.21199999991222, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::record_stream", + "count": 436, + "self_cpu_us": 730.9929999999513, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 116, + "self_cpu_us": 650.3899999999953, + "self_device_us": 229.37999999999693, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventQuery", + "count": 513, + "self_cpu_us": 642.4939999999581, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventRecordWithFlags", + "count": 698, + "self_cpu_us": 624.0679999998281, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 281, + "self_cpu_us": 616.0749999999405, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 16433.368999999864, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10018.040999999977, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9557.15300000049, + "self_device_us": 0.0, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cudaLaunchKernel", + "count": 1850, + "self_cpu_us": 9201.422999999402, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8294.379000000605, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 7561.7619999999515, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2769, + "self_cpu_us": 7053.931000000085, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6636.100999999369, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1439, + "self_cpu_us": 6178.882000000486, + "self_device_us": 6708.658999999896, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5287.671999999828, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 4708.536999999357, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4653.8080000003965, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4520.836999999436, + "self_device_us": 0.0, + "self_device_memory_bytes": -79872 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4280.746000000392, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 3811.028999999038, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3031.376999999363, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 601, + "self_cpu_us": 2754.339000000004, + "self_device_us": 0, + "self_device_memory_bytes": 26515968 + }, + { + "operator": "", + "count": 904, + "self_cpu_us": 2668.428999999229, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/__init__.py(1235): current_stream", + "count": 1126, + "self_cpu_us": 2653.2650000008134, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/_utils.py(869): _get_device_index", + "count": 1504, + "self_cpu_us": 2575.899000002003, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 42234.66600000008, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 17010.12299999994, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 11070.726999999686, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2040, + "self_cpu_us": 10325.780999999826, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1631, + "self_cpu_us": 6525.653000000375, + "self_device_us": 8328.703999999532, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 4947.713000000557, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 4864.119999999892, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 793, + "self_cpu_us": 3114.437000000318, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2199.381999999973, + "self_device_us": 3114.2829999999713, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2073.5459999992754, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1458.7159999996475, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 693, + "self_cpu_us": 1455.3750000004352, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 116, + "self_cpu_us": 1195.895999999781, + "self_device_us": 692.2980000000898, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1053.418000000198, + "self_device_us": 860.6630000000077, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 1023.9610000000209, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1004.951999999932, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 409, + "self_cpu_us": 958.3949999998126, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 96, + "self_cpu_us": 888.4549999998671, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 859.6940000000086, + "self_device_us": 2180.226000000088, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 837.9099999999289, + "self_device_us": 781.5500000001175, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 22104.463000000447, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 17926.967000000168, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 14532.418000000032, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2040, + "self_cpu_us": 10800.322000000502, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10283.571000000464, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9719.242999999962, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8455.242999999638, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 7985.128999999445, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1631, + "self_cpu_us": 7096.114999998659, + "self_device_us": 7530.1170000006005, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6655.402000001546, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5625.204000000713, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 4933.312999999274, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4771.646000000193, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4725.08799999866, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4402.655000000255, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 3951.7359999998625, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "aten::empty_strided", + "count": 793, + "self_cpu_us": 3523.918000000247, + "self_device_us": 0, + "self_device_memory_bytes": 39131648 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3151.4589999996992, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2998.801000000943, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 904, + "self_cpu_us": 2673.5620000006493, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 43409.94599999997, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 17950.804999999928, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 15988.948999999806, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2168, + "self_cpu_us": 11112.410000000078, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1567, + "self_cpu_us": 6483.044000000588, + "self_device_us": 8276.219999999763, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 4952.348999999742, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 4921.737000000263, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 729, + "self_cpu_us": 2952.603999999926, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2189.6210000000665, + "self_device_us": 3186.7999999999442, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2120.180999999159, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 693, + "self_cpu_us": 1494.2490000004798, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1457.0889999997416, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 116, + "self_cpu_us": 1239.0429999999242, + "self_device_us": 708.8769999999126, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 192, + "self_cpu_us": 1220.1639999996369, + "self_device_us": 1123.057999999999, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1135.0230000010429, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 1087.3439999998745, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1070.1700000001292, + "self_device_us": 874.7320000000436, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 409, + "self_cpu_us": 998.1089999999881, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 96, + "self_cpu_us": 941.3419999995094, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 902.430999999955, + "self_device_us": 2233.6359999999077, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 23466.177000000756, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 18052.96899999999, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 12057.866999999911, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2168, + "self_cpu_us": 11316.609999999708, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10142.197000000597, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9814.517999998712, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8444.050000000905, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 7757.864000000339, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1567, + "self_cpu_us": 6859.583000000366, + "self_device_us": 7261.939000000045, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6667.205999998861, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5536.804000000327, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 4946.131999999503, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4856.363000000383, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4753.402000000731, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4567.222000000331, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 3955.984000000677, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "aten::empty_strided", + "count": 729, + "self_cpu_us": 3271.6339999996685, + "self_device_us": 0, + "self_device_memory_bytes": 39098880 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3136.034000000277, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 3133.5739999998086, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 904, + "self_cpu_us": 2658.5939999993807, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + }, + "speculative": { + "seconds": { + "none": [ + 0.06240307000007306 + ], + "raw": [ + 0.0662938270002087 + ], + "processed": [ + 0.06682750300024054 + ] + }, + "median_seconds": { + "none": 0.06240307000007306, + "raw": 0.0662938270002087, + "processed": 0.06682750300024054 + }, + "tokens_per_second": { + "none": 1025.590567898744, + "raw": 965.3990860988991, + "processed": 957.6895309072021 + }, + "peak_extra_allocated_bytes": { + "none": [ + 2750976 + ], + "raw": [ + 2790912 + ], + "processed": [ + 2774528 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 14082.55900000005, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2245, + "self_cpu_us": 9969.27999999969, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1128, + "self_cpu_us": 8384.83099999996, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1277, + "self_cpu_us": 3738.911000000112, + "self_device_us": 1777.0809999999603, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 143, + "self_cpu_us": 2681.666000000013, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 592, + "self_cpu_us": 2131.4490000001315, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1464.813999999862, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1219, + "self_cpu_us": 1242.6320000000528, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1164.5480000000407, + "self_device_us": 348.58399999995163, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 977.9220000000182, + "self_device_us": 302.44099999999275, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 969.8469999999941, + "self_device_us": 148.32200000000057, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 893.5579999999991, + "self_device_us": 672.4589999999498, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 814.1800000000003, + "self_device_us": 175.17400000007547, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 797.685999999996, + "self_device_us": 602.407000000032, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2113, + "self_cpu_us": 754.8929999999518, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 77, + "self_cpu_us": 751.3600000000063, + "self_device_us": 112.54500000000348, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 743.7420000000911, + "self_device_us": 1537.3079999999263, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 742.4579999999497, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 348, + "self_cpu_us": 714.7729999999433, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_softmax", + "count": 119, + "self_cpu_us": 704.4650000000765, + "self_device_us": 1838.0370000000748, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaStreamSynchronize", + "count": 143, + "self_cpu_us": 19864.37699999997, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 14970.81799999996, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2245, + "self_cpu_us": 10701.993000000055, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1128, + "self_cpu_us": 10585.129000000112, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9116.944000000163, + "self_device_us": 0.0, + "self_device_memory_bytes": -1536 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6067.3340000002245, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 4490.072999999706, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1277, + "self_cpu_us": 4200.721000000049, + "self_device_us": 2472.8799999996913, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 592, + "self_cpu_us": 2475.1309999999894, + "self_device_us": 0, + "self_device_memory_bytes": 52925952 + }, + { + "operator": "aten::slice", + "count": 1219, + "self_cpu_us": 2209.2680000001774, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2199.3709999999483, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1887.626000000164, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1604.1319999999032, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1586.2180000000506, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1252.9259999999376, + "self_device_us": 522.2780000000967, + "self_device_memory_bytes": 25165824 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1165.056000000348, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + }, + { + "operator": "aten::_to_copy", + "count": 348, + "self_cpu_us": 1149.622000000063, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1149.0220000000054, + "self_device_us": 234.2420000000002, + "self_device_memory_bytes": 786432 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1108.9360000000352, + "self_device_us": 466.69400000002315, + "self_device_memory_bytes": 1404416 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 1083.3829999999834, + "self_device_us": 1011.7250000000568, + "self_device_memory_bytes": 79872 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1219, + "self_cpu_us": 32065.39100000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 15680.841000000019, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2394, + "self_cpu_us": 11380.852999999855, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 4496.739999999833, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1428, + "self_cpu_us": 4368.5809999999665, + "self_device_us": 4428.808999999856, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 743, + "self_cpu_us": 2724.119000000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1590.5000000001273, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 1249.5310000003237, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1218.5759999999027, + "self_device_us": 777.1980000000331, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 100, + "self_cpu_us": 1051.9999999999136, + "self_device_us": 428.0050000000215, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1017.5759999999709, + "self_device_us": 1260.8519999999708, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1012.5349999999967, + "self_device_us": 733.8019999999997, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 959.7169999995767, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 930.4999999999745, + "self_device_us": 1327.9919999999292, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 913.5249999998941, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 857.5730000000585, + "self_device_us": 328.99800000001414, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 831.2279999999759, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 829.5409999999483, + "self_device_us": 1179.376999999913, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 778.833999999988, + "self_device_us": 2838.9859999999917, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 736.6149999998129, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1219, + "self_cpu_us": 17222.49800000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 15855.267000000127, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 11819.789999999986, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2394, + "self_cpu_us": 11519.752999999928, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9310.95399999983, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6072.020999999813, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1428, + "self_cpu_us": 4727.487999999833, + "self_device_us": 4655.986000000061, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 4696.012000000468, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::empty_strided", + "count": 743, + "self_cpu_us": 3005.7680000001283, + "self_device_us": 0, + "self_device_memory_bytes": 65541632 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 2301.55100000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2113.589000000433, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1948.2030000000268, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 1625.992999999985, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1563.8129999999483, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1353.5050000000588, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1338.518000000211, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1312.731999999898, + "self_device_us": 807.4049999999152, + "self_device_memory_bytes": 25165824 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 1273.8249999999016, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1184.5459999999466, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1142.3969999999777, + "self_device_us": 341.6040000000071, + "self_device_memory_bytes": 786432 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1155, + "self_cpu_us": 30886.555999999768, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 15259.450999999935, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2522, + "self_cpu_us": 11694.530000000093, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 4923.203000000066, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1364, + "self_cpu_us": 4200.7139999999345, + "self_device_us": 4325.325999999722, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 679, + "self_cpu_us": 2475.386999999806, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1579.0149999997666, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 270, + "self_cpu_us": 1526.4960000000283, + "self_device_us": 958.5179999999909, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 1211.8220000002698, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1046.327999999994, + "self_device_us": 727.1810000000069, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 100, + "self_cpu_us": 1044.046000000038, + "self_device_us": 422.95099999992226, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_softmax", + "count": 183, + "self_cpu_us": 1043.5719999998073, + "self_device_us": 4847.890000000012, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1012.0560000000614, + "self_device_us": 1260.6269999999504, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 924.9700000000994, + "self_device_us": 1308.5010000000257, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 921.9240000003183, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 911.6380000003298, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 842.8640000000414, + "self_device_us": 329.63399999997273, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 816.9300000000476, + "self_device_us": 1178.6429999999946, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 807.9510000000835, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 778.4430000000539, + "self_device_us": 2829.236000000021, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1155, + "self_cpu_us": 42451.45200000021, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 16473.516999999978, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2522, + "self_cpu_us": 12428.987999999705, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9528.780999999806, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 7053.55099999987, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6385.314000000195, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 4901.061999999998, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1364, + "self_cpu_us": 4642.906999999959, + "self_device_us": 6678.909000000535, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 679, + "self_cpu_us": 2844.3409999998976, + "self_device_us": 0, + "self_device_memory_bytes": 65508864 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 2370.445000000463, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2189.775000000467, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 2057.2630000004756, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "aten::div", + "count": 270, + "self_cpu_us": 1714.8240000003643, + "self_device_us": 1486.07199999988, + "self_device_memory_bytes": 37748736 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 1671.7550000000952, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1589.7839999998428, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1412.5379999998395, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1349.7980000001553, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "vllm_rlt/worker/sampling.py(6): probabilities", + "count": 181, + "self_cpu_us": 1349.6579999995265, + "self_device_us": 0.0, + "self_device_memory_bytes": -35586048 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 1343.5690000002296, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1245.2050000000427, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + } + ] + }, + "tokens_unchanged": true + } +} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log new file mode 100644 index 00000000..17965cd3 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log @@ -0,0 +1,40 @@ +[transformers] You are using a model of type `ouro` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating. +USDT:2026-10-01 20:16:37 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:37 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:16:39 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:39 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:16:46 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:47 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:16:50 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:50 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:16:59 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:59 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:02 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:02 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +sync {'none': 0.08066848499993284, 'raw': 0.0914349139998194, 'processed': 0.08466345700026068} +USDT:2026-10-01 20:17:12 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:12 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:16 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:16 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:31 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:31 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:34 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:35 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:52 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:52 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:56 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:56 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +async {'none': 0.1108772420002424, 'raw': 0.1287477310002032, 'processed': 0.12239944100019784} +USDT:2026-10-01 20:18:14 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:14 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:16 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:16 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:22 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:22 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:25 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:25 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:32 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:32 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:35 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:35 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +speculative {'none': 0.06240307000007306, 'raw': 0.0662938270002087, 'processed': 0.06682750300024054} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json new file mode 100644 index 00000000..8d18ff49 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json @@ -0,0 +1,7 @@ +{ + "removed": [ + "models/Ouro-minhao-full", + "minhao-fast-part-2" + ], + "bytes": 3232344846 +} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log new file mode 100644 index 00000000..8eef87ad --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log @@ -0,0 +1,11 @@ +test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok +test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok +test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok +test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok +test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok +test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 135.474s + +OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log new file mode 100644 index 00000000..ae3542fd --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log @@ -0,0 +1,11 @@ +test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok +test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok +test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok +test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok +test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok +test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 3.989s + +OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log new file mode 100644 index 00000000..855a3b10 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log @@ -0,0 +1,19 @@ +Kernel stack captured from PID 54942 during the observed driver wait on 2026-10-01: +[<0>] os_acquire_rwlock_write+0x38/0x64 [nvidia] +[<0>] portSyncRwLockAcquireWrite+0x10/0x30 [nvidia] +[<0>] rmapiLockAcquire+0x23c/0x340 [nvidia] +[<0>] _nvGpuOpsLocksAcquireWithClientLockFlags+0x98/0x1f0 [nvidia] +[<0>] dupMemory+0x88/0x690 [nvidia] +[<0>] nvUvmInterfaceDupMemory+0x34/0x40 [nvidia] +[<0>] uvm_map_external_allocation_on_gpu+0x1f0/0x3fc [nvidia_uvm] +[<0>] uvm_api_map_external_allocation+0x474/0x4e4 [nvidia_uvm] +[<0>] uvm_ioctl+0x117c/0x16a0 [nvidia_uvm] +[<0>] uvm_unlocked_ioctl_entry.part.0+0xdc/0x148 [nvidia_uvm] +[<0>] uvm_unlocked_ioctl_entry+0x78/0x9c [nvidia_uvm] +[<0>] __arm64_sys_ioctl+0xac/0xf0 +[<0>] invoke_syscall+0x48/0x114 +[<0>] el0_svc_common.constprop.0+0xc0/0xe0 +[<0>] do_el0_svc+0x1c/0x28 +[<0>] el0_svc+0x30/0xd8 +[<0>] el0t_64_sync_handler+0x120/0x12c +[<0>] el0t_64_sync+0x194/0x198 diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log new file mode 100644 index 00000000..b59b66e8 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log @@ -0,0 +1,42 @@ +Thu Oct 1 12:03:16 UTC 2026 +[<0>] futex_wait_queue+0x6c/0x98 +[<0>] __futex_wait+0xcc/0x160 +[<0>] futex_wait+0x88/0x110 +[<0>] do_futex+0xf8/0x1a0 +[<0>] __arm64_sys_futex+0xe0/0x13c +[<0>] invoke_syscall+0x48/0x114 +[<0>] el0_svc_common.constprop.0+0xc0/0xe0 +[<0>] do_el0_svc+0x1c/0x28 +[<0>] el0_svc+0x30/0xd8 +[<0>] el0t_64_sync_handler+0x120/0x12c +[<0>] el0t_64_sync+0x194/0x198 +[ 1630.645566] NVRM: GPU0 refcntRequestReference_IMPL: Failed to enter state 1 (current state: 0, status: 0x00000056) +[ 2474.164009] NVRM: GPU0 refcntRequestReference_IMPL: Failed to enter state 1 (current state: 0, status: 0x00000056) +[ 2478.973583] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2479.334052] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2479.459146] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2480.205368] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2480.300412] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2480.606304] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2480.823594] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2481.003826] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2481.261606] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2481.325949] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2482.079631] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2482.152349] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2482.562933] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2482.686092] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2483.701671] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2483.788894] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2484.193315] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2484.306663] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2484.419459] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.152876] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.231614] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.661010] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.775288] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.971025] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2486.389956] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2486.451899] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2486.841379] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2486.904428] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv new file mode 100644 index 00000000..9c1a4867 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv @@ -0,0 +1,2 @@ +name, driver_version, memory.used [MiB], utilization.gpu [%] +NVIDIA Thor, 595.78, [N/A], 98 % diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json new file mode 100644 index 00000000..9ca1eca6 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json @@ -0,0 +1,2741 @@ +{ + "sync": { + "seconds": { + "none": [ + 1.797056486999736, + 1.8016931980000663, + 1.7812917080000261, + 1.8116626850001012, + 1.8313193210001373 + ], + "raw": [ + 1.8279988220001542, + 1.8234451930002251, + 1.810924204999992, + 1.8776579790001051, + 1.8599433469998985 + ], + "processed": [ + 1.8894610560000729, + 1.7770136750000347, + 1.8161140520001027, + 1.8662560849998044, + 1.8281205210000735 + ] + }, + "median_seconds": { + "none": 1.8016931980000663, + "raw": 1.8279988220001542, + "processed": 1.8281205210000735 + }, + "tokens_per_second": { + "none": 35.52214110096099, + "raw": 35.01096348080393, + "processed": 35.00863278149123 + }, + "peak_extra_allocated_bytes": { + "none": [ + 955392, + 955392, + 955392, + 955392, + 955392 + ], + "raw": [ + 988160, + 955392, + 988160, + 988160, + 955392 + ], + "processed": [ + 955392, + 955392, + 988160, + 955392, + 955392 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 3115292.1020000014, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 297542.58400000003, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 47216.64400000275, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6232, + "self_cpu_us": 46928.53299999721, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2724, + "self_cpu_us": 25592.631000006077, + "self_device_us": 7924.548000013914, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 10, + "self_cpu_us": 19917.72599999947, + "self_device_us": 2.1440000000875443, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16707.02599999922, + "self_device_us": 69237.73299999899, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2140, + "self_cpu_us": 16282.546999996724, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Activity Buffer Request", + "count": 11, + "self_cpu_us": 11960.367999999446, + "self_device_us": 3.072000000017397, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 9020.897999999828, + "self_device_us": 3486.1089999998353, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 384, + "self_cpu_us": 8604.2179999959, + "self_device_us": 1033.1409999988641, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1420, + "self_cpu_us": 7515.799999997356, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 800, + "self_cpu_us": 5476.198000001276, + "self_device_us": 164.86600000051476, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 5247.854999999913, + "self_device_us": 1053.979999999935, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2084, + "self_cpu_us": 5008.964999994238, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3880.791000000514, + "self_device_us": 2027.154000000004, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3850.2629999997826, + "self_device_us": 547.1479999999751, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 2068, + "self_cpu_us": 3678.7860000009327, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3572.775999995909, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4087, + "self_cpu_us": 3312.0390000030416, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 1714448.007000006, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 312503.2759999983, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 88477.1809999905, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaLaunchKernel", + "count": 6232, + "self_cpu_us": 45877.57399999528, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2724, + "self_cpu_us": 30653.4359999993, + "self_device_us": 8406.053999994654, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 17, + "self_cpu_us": 21553.933999999936, + "self_device_us": 113.12300000005052, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 21004.616999993712, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2140, + "self_cpu_us": 18041.15100000197, + "self_device_us": 0, + "self_device_memory_bytes": 120670208 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 15299.454999999656, + "self_device_us": 186853.4359999999, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 13051.557000005414, + "self_device_us": 0.0, + "self_device_memory_bytes": -1185792 + }, + { + "operator": "aten::_to_copy", + "count": 1420, + "self_cpu_us": 10733.110999994191, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 10510.103999998966, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 384, + "self_cpu_us": 9402.348000001257, + "self_device_us": 4816.147000000074, + "self_device_memory_bytes": 15486976 + }, + { + "operator": "aten::slice", + "count": 2084, + "self_cpu_us": 8692.440000006642, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 566, + "self_cpu_us": 8643.991000002294, + "self_device_us": 0.0, + "self_device_memory_bytes": -8029696 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 7680, + "self_cpu_us": 8545.6210000067, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 8124.672999999564, + "self_device_us": 15076.379999999674, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "Activity Buffer Request", + "count": 10, + "self_cpu_us": 7915.628000000057, + "self_device_us": 0.47999999999956344, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 7900.098000011858, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 7111.301000002073, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 1687563.1620000014, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 297471.37799999845, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6422, + "self_cpu_us": 38591.12499999725, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2916, + "self_cpu_us": 19720.34099999547, + "self_device_us": 4969.288999992985, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 11, + "self_cpu_us": 13895.465999999578, + "self_device_us": 2.6239999998360872, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2332, + "self_cpu_us": 13841.695999991633, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 12845.257000000787, + "self_device_us": 85966.08499999985, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 11226.992999998267, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 448, + "self_cpu_us": 7124.344999999742, + "self_device_us": 1127.6500000008473, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 6689.6719999999805, + "self_device_us": 3976.72400000012, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1548, + "self_cpu_us": 5447.389000004618, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 800, + "self_cpu_us": 4433.354999998611, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2086, + "self_cpu_us": 4181.640999996726, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 3946.4330000000346, + "self_device_us": 1249.5749999998784, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 5209, + "self_cpu_us": 3355.9950000011663, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 3335.78000000483, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3175.459000005998, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 2957.2929999998587, + "self_device_us": 2255.626000000015, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 2879.41300000012, + "self_device_us": 626.137999999959, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 2196, + "self_cpu_us": 2788.2449999992214, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 1712470.881999998, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 303259.42099999957, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 84949.75999997946, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 58914.68900000312, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 38, + "self_cpu_us": 55984.85400000001, + "self_device_us": 141.47600000002421, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6422, + "self_cpu_us": 39227.52699999942, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2916, + "self_cpu_us": 20689.506999994737, + "self_device_us": 8596.109999999157, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 17150.899999998324, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2332, + "self_cpu_us": 15443.806000003227, + "self_device_us": 0, + "self_device_memory_bytes": 133285888 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 14933.654000000508, + "self_device_us": 216397.0109999997, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 10250.292999994825, + "self_device_us": 0.0, + "self_device_memory_bytes": -1185792 + }, + { + "operator": "aten::_to_copy", + "count": 1548, + "self_cpu_us": 8689.791999999334, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 7680, + "self_cpu_us": 7914.010000004513, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2086, + "self_cpu_us": 7845.527000000181, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 7781.584000000366, + "self_device_us": 14088.113000000061, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "aten::cat", + "count": 448, + "self_cpu_us": 7187.911999997445, + "self_device_us": 1075.74999999817, + "self_device_memory_bytes": 15519744 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 6351.897999999905, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 566, + "self_cpu_us": 6237.029000000981, + "self_device_us": 0.0, + "self_device_memory_bytes": -8029696 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 6197.680000002206, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 7680, + "self_cpu_us": 5868.171000009923, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 1785390.3010000014, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 287422.5859999989, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 54, + "self_cpu_us": 132692.09599999993, + "self_device_us": 137.35000000000582, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 63230.2860000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6550, + "self_cpu_us": 41654.006999997655, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2852, + "self_cpu_us": 20811.043999990427, + "self_device_us": 5011.1320000056, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2268, + "self_cpu_us": 14158.613000006142, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 13688.341999998862, + "self_device_us": 237601.12799999924, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 448, + "self_cpu_us": 7512.926999999517, + "self_device_us": 1107.5330000010308, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 6855.801000000043, + "self_device_us": 3760.478000000774, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1548, + "self_cpu_us": 5959.337000000479, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 800, + "self_cpu_us": 4528.025000001053, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2086, + "self_cpu_us": 4388.27499998413, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4043.9099999990876, + "self_device_us": 6436.210999999603, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 5209, + "self_cpu_us": 3740.4920000093116, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3374.305000002234, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 3299.6420000069775, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3031.1940000002096, + "self_device_us": 5960.095999999843, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3006.850000000014, + "self_device_us": 629.8799999999992, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 2196, + "self_cpu_us": 3004.803000003427, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 1681418.688000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 313165.76199999935, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 89656.83900000324, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 55077.46799999963, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 46, + "self_cpu_us": 49129.840000000055, + "self_device_us": 177.34899999998743, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6550, + "self_cpu_us": 45109.648999988836, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2852, + "self_cpu_us": 23547.901000007907, + "self_device_us": 22245.57399999522, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 18967.310000011475, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2268, + "self_cpu_us": 16894.335999999996, + "self_device_us": 0, + "self_device_memory_bytes": 133253120 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16466.671999998278, + "self_device_us": 191312.69199999896, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 11592.744999999559, + "self_device_us": 0.0, + "self_device_memory_bytes": -1185792 + }, + { + "operator": "aten::_to_copy", + "count": 1548, + "self_cpu_us": 9718.056999999331, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 7680, + "self_cpu_us": 8876.85000001891, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 448, + "self_cpu_us": 8632.131000004554, + "self_device_us": 1104.7309999994868, + "self_device_memory_bytes": 15519744 + }, + { + "operator": "aten::slice", + "count": 2086, + "self_cpu_us": 8531.177000011521, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 7987.785000000618, + "self_device_us": 16808.661000000444, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 566, + "self_cpu_us": 7880.714999992513, + "self_device_us": 0.0, + "self_device_memory_bytes": -8029696 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 7180.626999999522, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 6991.26000000355, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 7680, + "self_cpu_us": 6713.435000007084, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + }, + "async": { + "seconds": { + "none": [ + 1.695322036000107, + 1.6974444380002751, + 1.7046040849995734, + 1.6946781169999667, + 1.7272834199998215 + ], + "raw": [ + 1.7682591719999436, + 1.7628239639998355, + 1.7849022200002764, + 1.7954341879999447, + 1.8388650159999997 + ], + "processed": [ + 1.7741621370000757, + 1.770364053999856, + 1.7893874629999118, + 1.8360151079996285, + 1.8289056349999555 + ] + }, + "median_seconds": { + "none": 1.6974444380002751, + "raw": 1.7849022200002764, + "processed": 1.7893874629999118 + }, + "tokens_per_second": { + "none": 37.70373778796383, + "raw": 35.856305898924866, + "processed": 35.766429196225545 + }, + "peak_extra_allocated_bytes": { + "none": [ + 988160, + 988160, + 988160, + 988160, + 988160 + ], + "raw": [ + 986624, + 988160, + 986624, + 986624, + 988160 + ], + "processed": [ + 988160, + 988160, + 986624, + 988160, + 988160 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaEventSynchronize", + "count": 502, + "self_cpu_us": 1432410.3530000027, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 191290.37999999942, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 44, + "self_cpu_us": 34835.79299999999, + "self_device_us": 351.92999999994936, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6078, + "self_cpu_us": 33813.03999999954, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16364.334999999637, + "self_device_us": 183683.25199999977, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 11429.879000001887, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2143, + "self_cpu_us": 11020.502999999995, + "self_device_us": 4541.596999996408, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 10356.752000000413, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 8018.803000000195, + "self_device_us": 3866.8100000002287, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3297, + "self_cpu_us": 6736.454999988346, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1393, + "self_cpu_us": 6702.344000001056, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4626.323999999812, + "self_device_us": 9519.472000000073, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 5523, + "self_cpu_us": 3716.362000009337, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3477.173000000108, + "self_device_us": 619.724000000142, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3425.1280000001398, + "self_device_us": 12465.012999999697, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 1, + "self_cpu_us": 3281.159, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 985, + "self_cpu_us": 2562.969999997975, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 981, + "self_cpu_us": 2548.3720000009953, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2241.14700000608, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 228, + "self_cpu_us": 2224.8430000000544, + "self_device_us": 774.4480000006351, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaEventSynchronize", + "count": 502, + "self_cpu_us": 1334775.7140000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 213300.00299999965, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 37214.26000000528, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6078, + "self_cpu_us": 31441.57300000226, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16664.894000000244, + "self_device_us": 184634.85100000046, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "Command Buffer Full", + "count": 34, + "self_cpu_us": 16638.358999999953, + "self_device_us": 83.08099999994738, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 658, + "self_cpu_us": 14962.963000003772, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3297, + "self_cpu_us": 12126.546999992546, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 11609.655999997276, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 11161.335999998177, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2143, + "self_cpu_us": 10901.848999998805, + "self_device_us": 8184.701999998565, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 10622.565999999239, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 8326.869000005598, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 8074.22300000026, + "self_device_us": 0.0, + "self_device_memory_bytes": -79872 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 7691.808999999961, + "self_device_us": 7839.375999999642, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "", + "count": 658, + "self_cpu_us": 7110.883000002672, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1393, + "self_cpu_us": 7029.449000000028, + "self_device_us": 0, + "self_device_memory_bytes": 119249408 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 6454.799000000135, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 7680, + "self_cpu_us": 5952.708999990169, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 5775.1769999995595, + "self_device_us": 0.0, + "self_device_memory_bytes": -169984 + } + ], + "raw-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 865738.5560000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 440, + "self_cpu_us": 513537.4239999985, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 269982.26400000206, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6268, + "self_cpu_us": 37557.50699999495, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 19496.538000000193, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 15068.10300000109, + "self_device_us": 89085.99599999975, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2335, + "self_cpu_us": 15040.314000005079, + "self_device_us": 4926.857000000852, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 11644.150999997, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3299, + "self_cpu_us": 9157.908999995387, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1585, + "self_cpu_us": 9157.521999997385, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 7889.426999999985, + "self_device_us": 3726.571000000061, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 6645, + "self_cpu_us": 4706.0270000029495, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4578.1169999999665, + "self_device_us": 1194.1009999999906, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 1045, + "self_cpu_us": 3393.951000005082, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1113, + "self_cpu_us": 3387.474999996231, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3383.8789999999763, + "self_device_us": 2079.0089999998263, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 292, + "self_cpu_us": 3383.5519999986764, + "self_device_us": 885.724000001198, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3352.952999999814, + "self_device_us": 585.2520000001759, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3030.2580000000307, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::rsqrt", + "count": 388, + "self_cpu_us": 2112.1670000002123, + "self_device_us": 558.6339999998054, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 894514.4109999985, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 438, + "self_cpu_us": 356624.633000003, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 282386.8210000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 61673.15100000108, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 57640.45999999429, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6268, + "self_cpu_us": 37907.111000003744, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 658, + "self_cpu_us": 20129.23500000021, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 17286.161000000488, + "self_device_us": 209501.60500000042, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "Command Buffer Full", + "count": 24, + "self_cpu_us": 15990.03599999995, + "self_device_us": 94.47799999994459, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2335, + "self_cpu_us": 15931.423000005114, + "self_device_us": 5127.309000006906, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 15294.889999997744, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3299, + "self_cpu_us": 15165.398000015914, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 12728.342999995075, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 11771.589000000844, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1585, + "self_cpu_us": 10317.796999996372, + "self_device_us": 0, + "self_device_memory_bytes": 131865088 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 9568.225999996474, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "", + "count": 658, + "self_cpu_us": 8962.633000000906, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 8289.213999999814, + "self_device_us": 9734.966999999577, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 7695.529999998893, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 7675.931000003035, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 920932.4780000017, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 439, + "self_cpu_us": 482884.0049999996, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 305185.86299999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 73606.65199999898, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 58, + "self_cpu_us": 70443.25700000004, + "self_device_us": 161.2559999999503, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6396, + "self_cpu_us": 39018.222000004345, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2271, + "self_cpu_us": 15692.35799999762, + "self_device_us": 4989.233000002182, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 14275.767999999229, + "self_device_us": 203367.5870000003, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 12591.590999998225, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3299, + "self_cpu_us": 10243.788999999848, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1521, + "self_cpu_us": 8787.994000000632, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 6978.011999999464, + "self_device_us": 7834.872999999812, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 6645, + "self_cpu_us": 5005.443999995054, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4159.8260000002265, + "self_device_us": 1241.8309999997346, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 1045, + "self_cpu_us": 3796.858999999785, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1113, + "self_cpu_us": 3539.445999995534, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 292, + "self_cpu_us": 3381.941999998534, + "self_device_us": 891.852000000601, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3376.4410000027856, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3101.857999999801, + "self_device_us": 2076.3349999998336, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3023.192999999952, + "self_device_us": 613.0609999998796, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 860792.1709999986, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 439, + "self_cpu_us": 413039.0830000005, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 255455.0850000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 49905.24699999363, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6396, + "self_cpu_us": 37927.29899999841, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 658, + "self_cpu_us": 17880.09100000289, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16554.555999998465, + "self_device_us": 89435.19800000006, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 15447.242000002167, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2271, + "self_cpu_us": 13614.053999995802, + "self_device_us": 4926.6889999996565, + "self_device_memory_bytes": -1024 + }, + { + "operator": "aten::slice", + "count": 3299, + "self_cpu_us": 13604.380000005436, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 13420.60000000347, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 12200.938999997265, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 10094.503999997636, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1521, + "self_cpu_us": 8711.722999999149, + "self_device_us": 0, + "self_device_memory_bytes": 131832320 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 8518.236999999934, + "self_device_us": 3812.4549999999153, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 8272.147000002355, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "", + "count": 658, + "self_cpu_us": 8045.141999997961, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 6979.725999999505, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 6839.281000003541, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 6694.447999993816, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + }, + "speculative": { + "seconds": { + "none": [ + 1.4405976819998614, + 1.4354998989997512, + 1.4359760199999982, + 1.43785706799963, + 1.436613651999778 + ], + "raw": [ + 1.446159684000122, + 1.4462237470002037, + 1.4429731870000069, + 1.447812623000118, + 1.447334557999966 + ], + "processed": [ + 1.4485182270000223, + 1.4616416380004011, + 1.445917856000051, + 1.449624680999932, + 1.4485653670003558 + ] + }, + "median_seconds": { + "none": 1.436613651999778, + "raw": 1.4462237470002037, + "processed": 1.4485653670003558 + }, + "tokens_per_second": { + "none": 44.54920772255747, + "raw": 44.25318014086722, + "processed": 44.181644444895994 + }, + "peak_extra_allocated_bytes": { + "none": [ + 2799104, + 2799104, + 2799104, + 2799104, + 2799104 + ], + "raw": [ + 2860544, + 2827776, + 2860544, + 2860544, + 2827776 + ], + "processed": [ + 2827776, + 2827776, + 2860544, + 2827776, + 2827776 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 980, + "self_cpu_us": 1013133.5880000024, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 329889.5450000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6440, + "self_cpu_us": 44799.10400000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 14830.692999999728, + "self_device_us": 57957.05499999999, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1858, + "self_cpu_us": 11904.329999998092, + "self_device_us": 3719.296999995492, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 10062.069999999801, + "self_device_us": 3642.09899999938, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1349, + "self_cpu_us": 8686.187999998147, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 5437.313999999343, + "self_device_us": 1142.796999999984, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 5328.228999999939, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 4195.546000000275, + "self_device_us": 2040.650999999802, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 4130.804000000057, + "self_device_us": 541.9709999999882, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1036, + "self_cpu_us": 3498.193000000953, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 3478.147999996989, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1598, + "self_cpu_us": 3344.8489999989542, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 247, + "self_cpu_us": 3218.2849999998366, + "self_device_us": 773.1339999996012, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3629, + "self_cpu_us": 2757.4049999993445, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 204, + "self_cpu_us": 2495.4719999990484, + "self_device_us": 344.9210000009334, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::rsqrt", + "count": 388, + "self_cpu_us": 2447.126000000311, + "self_device_us": 568.7279999999582, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_index_put_impl_", + "count": 192, + "self_cpu_us": 2249.8519999999753, + "self_device_us": 1397.437999999971, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 134, + "self_cpu_us": 2245.1280000006864, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 980, + "self_cpu_us": 1004743.099000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 341942.8800000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 98, + "self_cpu_us": 65974.60500000956, + "self_device_us": 0.0, + "self_device_memory_bytes": -122880 + }, + { + "operator": "cudaLaunchKernel", + "count": 6440, + "self_cpu_us": 44354.85499999292, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 16, + "self_cpu_us": 43468.00100000008, + "self_device_us": 2872.282999999996, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 9, + "self_cpu_us": 17052.98900000006, + "self_device_us": 2.4319999997969717, + "self_device_memory_bytes": 122880 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 16737.56800000057, + "self_device_us": 133078.352, + "self_device_memory_bytes": 74514432 + }, + { + "operator": "aten::copy_", + "count": 1858, + "self_cpu_us": 12988.228999999212, + "self_device_us": 7612.869000000108, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 53, + "self_cpu_us": 12837.078999994766, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 10, + "self_cpu_us": 11424.281000001734, + "self_device_us": 0.0, + "self_device_memory_bytes": -41080832 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 10870.521999998786, + "self_device_us": 14274.18699999967, + "self_device_memory_bytes": 133641216 + }, + { + "operator": "aten::empty_strided", + "count": 1349, + "self_cpu_us": 9551.200000001178, + "self_device_us": 0, + "self_device_memory_bytes": 146153984 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 8481.077000000107, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 7766.478999998973, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "aten::slice", + "count": 1598, + "self_cpu_us": 7758.273000003454, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 9024, + "self_cpu_us": 6788.078000001582, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 6336, + "self_cpu_us": 6656.0759999922775, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", + "count": 96, + "self_cpu_us": 6457.112000000272, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 134, + "self_cpu_us": 6235.574999999869, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1036, + "self_cpu_us": 6220.435999998934, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1070, + "self_cpu_us": 1064068.1040000021, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 361931.5290000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 48, + "self_cpu_us": 104276.37, + "self_device_us": 164.09299999989162, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 158, + "self_cpu_us": 50917.7819999979, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6588, + "self_cpu_us": 44249.33099999894, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 9, + "self_cpu_us": 15985.770999999775, + "self_device_us": 3.1359999999403954, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2008, + "self_cpu_us": 13097.013999997875, + "self_device_us": 14456.707000001214, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 11822.664999999748, + "self_device_us": 173903.50500000032, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1499, + "self_cpu_us": 9125.855000001111, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 8022.662999998281, + "self_device_us": 14237.222000000518, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 4792.803000000362, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 4564.072000000597, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4263.941999999985, + "self_device_us": 1244.1820000001771, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1122, + "self_cpu_us": 3775.3999999982043, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 269, + "self_cpu_us": 3679.534000000115, + "self_device_us": 4876.952999999159, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1600, + "self_cpu_us": 3253.3939999918493, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3200.0180000000037, + "self_device_us": 5880.0210000000625, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3172.9669999997113, + "self_device_us": 606.8459999999995, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4139, + "self_cpu_us": 3093.79700000462, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 204, + "self_cpu_us": 2634.8199999991048, + "self_device_us": 346.6839999984659, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1070, + "self_cpu_us": 921827.2790000035, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 348470.8909999982, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 98, + "self_cpu_us": 82054.98900000838, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaLaunchKernel", + "count": 6588, + "self_cpu_us": 43856.03299999783, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 14546.498999999756, + "self_device_us": 58689.897, + "self_device_memory_bytes": 74514432 + }, + { + "operator": "aten::copy_", + "count": 2008, + "self_cpu_us": 13351.863999997171, + "self_device_us": 4054.520999999536, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 53, + "self_cpu_us": 13121.481000001226, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 10, + "self_cpu_us": 11731.000000006607, + "self_device_us": 0.0, + "self_device_memory_bytes": -41080832 + }, + { + "operator": "aten::empty_strided", + "count": 1499, + "self_cpu_us": 10030.803999998225, + "self_device_us": 0, + "self_device_memory_bytes": 158769664 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 9924.678000001335, + "self_device_us": 3753.6830000002865, + "self_device_memory_bytes": 133641216 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 7820.23400000037, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "aten::slice", + "count": 1600, + "self_cpu_us": 6671.56400000199, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 6336, + "self_cpu_us": 6460.941999995901, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1122, + "self_cpu_us": 6371.330999999439, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 9024, + "self_cpu_us": 6215.4499999884265, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", + "count": 96, + "self_cpu_us": 6019.520000000246, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/nn/modules/module.py(1782): _call_impl", + "count": 1397, + "self_cpu_us": 5992.312000000209, + "self_device_us": 0.0, + "self_device_memory_bytes": -99717120 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 5270.968999999644, + "self_device_us": 1170.8770000002514, + "self_device_memory_bytes": 28510208 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 5207.893999999975, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 4842.231000002561, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1006, + "self_cpu_us": 1086286.7079999999, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 366847.06800000044, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 48, + "self_cpu_us": 81405.13199999991, + "self_device_us": 132.25700000009965, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 158, + "self_cpu_us": 49747.239000002395, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6716, + "self_cpu_us": 44376.60800000187, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 9, + "self_cpu_us": 15859.352999999945, + "self_device_us": 2.464000000152737, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1944, + "self_cpu_us": 13164.344999999565, + "self_device_us": 7839.070999998783, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 12993.734999999626, + "self_device_us": 152600.5239999997, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1435, + "self_cpu_us": 8864.449999998058, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 8686.781999999766, + "self_device_us": 25866.910000000396, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 6254.039000003366, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4607.503999999629, + "self_device_us": 1228.5170000003036, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 4396.7750000001415, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1122, + "self_cpu_us": 4098.707000003469, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 269, + "self_cpu_us": 3794.4410000006155, + "self_device_us": 858.2720000006775, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3510.004999999982, + "self_device_us": 5943.891000000016, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1600, + "self_cpu_us": 3501.2590000032596, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3423.5350000000344, + "self_device_us": 600.6419999997925, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 268, + "self_cpu_us": 3386.080000000744, + "self_device_us": 443.71500000028755, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4139, + "self_cpu_us": 3143.4790000002977, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1006, + "self_cpu_us": 923970.0769999987, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 349496.5669999999, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 98, + "self_cpu_us": 79725.7660000025, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaLaunchKernel", + "count": 6716, + "self_cpu_us": 45678.692999995874, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 15432.809000000572, + "self_device_us": 59057.668000000325, + "self_device_memory_bytes": 74514432 + }, + { + "operator": "aten::copy_", + "count": 1944, + "self_cpu_us": 13113.157000004554, + "self_device_us": 3976.40699999536, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 53, + "self_cpu_us": 12922.643000007964, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 10, + "self_cpu_us": 11522.10300000626, + "self_device_us": 0.0, + "self_device_memory_bytes": -41080832 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 10494.68399999901, + "self_device_us": 3770.354000000312, + "self_device_memory_bytes": 133641216 + }, + { + "operator": "aten::empty_strided", + "count": 1435, + "self_cpu_us": 9768.620999997758, + "self_device_us": 0, + "self_device_memory_bytes": 158736896 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 8493.670999999964, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", + "count": 96, + "self_cpu_us": 6569.988999999679, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1600, + "self_cpu_us": 6520.582000002687, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1122, + "self_cpu_us": 6445.759999999112, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 117, + "self_cpu_us": 6329.600000002482, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 6336, + "self_cpu_us": 6263.172000000794, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/nn/modules/module.py(1782): _call_impl", + "count": 1397, + "self_cpu_us": 6239.434000000041, + "self_device_us": 0.0, + "self_device_memory_bytes": -99717120 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 9024, + "self_cpu_us": 6183.9280000038, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 5625.210000000348, + "self_device_us": 1181.320999999678, + "self_device_memory_bytes": 28510208 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 5410.53999999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + } +} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log new file mode 100644 index 00000000..b108ece5 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log @@ -0,0 +1,40 @@ +[transformers] You are using a model of type `ouro` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating. +USDT:2026-10-01 19:52:11 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:52:14 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:52:32 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:52:45 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:53:16 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:53:18 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:53:32 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:53:36 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:54:03 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:54:20 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:54:40 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:54:45 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +sync {'none': 1.8016931980000663, 'raw': 1.8279988220001542, 'processed': 1.8281205210000735} +USDT:2026-10-01 20:02:51 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:03:17 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:03:43 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:03:49 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:04:25 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:04:51 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:05:18 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:05:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:06:14 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:06:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:06:52 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:07:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +async {'none': 1.6974444380002751, 'raw': 1.7849022200002764, 'processed': 1.7893874629999118} +USDT:2026-10-01 20:11:40 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:12:07 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:12:35 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:12:37 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:13:07 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:13:09 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:13:50 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:14:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:14:49 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:14:51 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:15:33 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:01 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +speculative {'none': 1.436613651999778, 'raw': 1.4462237470002037, 'processed': 1.4485653670003558} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json new file mode 100644 index 00000000..b0c05fe2 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json @@ -0,0 +1,4 @@ +{ + "removed": "models/Ouro-1.4B-parallel", + "bytes": 2873677792 +} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json new file mode 100644 index 00000000..96acb36b --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json @@ -0,0 +1,2669 @@ +{ + "sync": { + "seconds": { + "none": [ + 0.09519798499991339 + ], + "raw": [ + 0.11991658300007657 + ], + "processed": [ + 0.11633255899982942 + ] + }, + "median_seconds": { + "none": 0.09519798499991339, + "raw": 0.11991658300007657, + "processed": 0.11633255899982942 + }, + "tokens_per_second": { + "none": 672.2831370859186, + "raw": 533.7043334528564, + "processed": 550.1469283426821 + }, + "peak_extra_allocated_bytes": { + "none": [ + 758784 + ], + "raw": [ + 823808 + ], + "processed": [ + 791040 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 33297.56899999986, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 19800.59699999992, + "self_device_us": 11.039000000033411, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2004, + "self_cpu_us": 19130.309000000445, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2020, + "self_cpu_us": 9735.057999999846, + "self_device_us": 3466.580000000014, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1348, + "self_cpu_us": 6787.842999999959, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Activity Buffer Request", + "count": 3, + "self_cpu_us": 4397.798999999917, + "self_device_us": 17.24599999999691, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1556, + "self_cpu_us": 2589.577999999135, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2440.773999999882, + "self_device_us": 1127.5460000000476, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 208, + "self_cpu_us": 2438.501999999814, + "self_device_us": 416.11600000010685, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 2014.048999999999, + "self_device_us": 681.3790000000336, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 716, + "self_cpu_us": 2006.413000000081, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1787.3960000001898, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 1310.3229999998512, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1012, + "self_cpu_us": 1157.6350000001703, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 1153.9850000001802, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1078.1530000000166, + "self_device_us": 288.4610000001521, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1031.721999999896, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2415, + "self_cpu_us": 989.9190000005906, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 935.0519999999015, + "self_device_us": 286.7849999999162, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 64, + "self_cpu_us": 901.8880000001336, + "self_device_us": 541.9650000000875, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 96740.21499999995, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 29828.6660000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 19172.10899999903, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2004, + "self_cpu_us": 16537.040000000692, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 15321.057999998613, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2020, + "self_cpu_us": 13304.200000000319, + "self_device_us": 9814.611000000457, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 9246.910000000378, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1348, + "self_cpu_us": 8824.270000000583, + "self_device_us": 0, + "self_device_memory_bytes": 26818560 + }, + { + "operator": "(2): __init__", + "count": 780, + "self_cpu_us": 7972.586999999223, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 7952.3179999991335, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "torch/autograd/grad_mode.py(290): __enter__", + "count": 312, + "self_cpu_us": 5820.944000000607, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1556, + "self_cpu_us": 5617.165999999186, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 156, + "self_cpu_us": 5495.474999999722, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", + "count": 8, + "self_cpu_us": 4696.361000000001, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 716, + "self_cpu_us": 4452.799999999923, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 4376.564999998955, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 4297.467999999419, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(287): _make_scheduled_item", + "count": 376, + "self_cpu_us": 4069.740999999527, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Activity Buffer Request", + "count": 2, + "self_cpu_us": 4064.9590000000353, + "self_device_us": 8.286999999976615, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 3937.9390000010226, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 108719.8060000003, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 24011.792999999805, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2194, + "self_cpu_us": 15252.708000000432, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 13507.98999999966, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2212, + "self_cpu_us": 11690.644999998814, + "self_device_us": 8416.399999999961, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1540, + "self_cpu_us": 7301.129000000332, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 4101.939999999672, + "self_device_us": 1255.3850000000248, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 2997.545000000003, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2510.9329999999395, + "self_device_us": 2639.002999999942, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 2464.008999998726, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2282.966000000388, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2134.255999999961, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1791.3630000006428, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3537, + "self_cpu_us": 1679.8659999990111, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 624, + "self_cpu_us": 1648.1559999993642, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1140, + "self_cpu_us": 1481.7340000004865, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1324.1299999998773, + "self_device_us": 683.1499999999578, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1257.5680000000211, + "self_device_us": 1779.3249999999916, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1212.7460000002857, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 1187.963000000027, + "self_device_us": 634.3850000001239, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 101858.63600000048, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 19925.880999999914, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2194, + "self_cpu_us": 13766.511999999813, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 13020.369000001338, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "aten::copy_", + "count": 2212, + "self_cpu_us": 10548.945999998652, + "self_device_us": 9764.56400000026, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 10036.862999999756, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 8070.916999999871, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1540, + "self_cpu_us": 6907.167000000017, + "self_device_us": 0, + "self_device_memory_bytes": 39434240 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 5716.320000000458, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 3870.769999999612, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 3773.934000000846, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 3767.767999999496, + "self_device_us": 1359.2739999999485, + "self_device_memory_bytes": 337920 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 3737.9029999998384, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 3170.4539999998087, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 3001.5509999997157, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 2973.905000000177, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 2791.1890000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2571.306999999335, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2479.8980000000774, + "self_device_us": 2988.58499999993, + "self_device_memory_bytes": 6881280 + }, + { + "operator": "aten::clone", + "count": 624, + "self_cpu_us": 2234.690999999897, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 97750.7149999996, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 22683.500000000073, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2322, + "self_cpu_us": 20361.571000000113, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 13013.803999999898, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2148, + "self_cpu_us": 11927.50900000012, + "self_device_us": 8301.904999999215, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1476, + "self_cpu_us": 6909.296999999731, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 4398.034999999721, + "self_device_us": 1229.2659999998846, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 3254.1110000007093, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2565.4240000000195, + "self_device_us": 2622.8850000000243, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 2342.3889999999733, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2110.396999999586, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2086.5779999999377, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3537, + "self_cpu_us": 1777.3179999996833, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 192, + "self_cpu_us": 1635.8050000000076, + "self_device_us": 899.7680000000564, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1635.431000000337, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1140, + "self_cpu_us": 1513.9589999994594, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 1491.8950000000768, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1314.6709999997838, + "self_device_us": 686.7049999998162, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1258.967999999892, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1243.792000000154, + "self_device_us": 1745.1790000000492, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 103497.714000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 20846.46299999978, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2322, + "self_cpu_us": 15129.085999998966, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 13175.480000000473, + "self_device_us": 0.0, + "self_device_memory_bytes": -1024 + }, + { + "operator": "aten::copy_", + "count": 2148, + "self_cpu_us": 10448.986999999888, + "self_device_us": 10015.051999999127, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 10255.892999999633, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 8545.498999999993, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1476, + "self_cpu_us": 6776.009000000365, + "self_device_us": 0, + "self_device_memory_bytes": 39401472 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 5816.4070000001375, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 3927.1879999986068, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 3912.425999998626, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 3884.787000000045, + "self_device_us": 1454.8620000000992, + "self_device_memory_bytes": 337920 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 3826.4770000002713, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 3287.5009999989925, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 3132.696000000161, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 3023.1649999991896, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2872.8629999997174, + "self_device_us": 3133.3969999999745, + "self_device_memory_bytes": 6881280 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 2871.759000000642, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2640.3929999992906, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 2423.540000000653, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + }, + "async": { + "seconds": { + "none": [ + 0.11112063099994884 + ], + "raw": [ + 0.12377583400007097 + ], + "processed": [ + 0.12324414399995476 + ] + }, + "median_seconds": { + "none": 0.11112063099994884, + "raw": 0.12377583400007097, + "processed": 0.12324414399995476 + }, + "tokens_per_second": { + "none": 575.9506531242562, + "raw": 517.0637751466356, + "processed": 519.2944502095247 + }, + "peak_extra_allocated_bytes": { + "none": [ + 790016 + ], + "raw": [ + 822272 + ], + "processed": [ + 822272 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 16279.262000000004, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 1850, + "self_cpu_us": 8762.216999999695, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 6619.0009999997255, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1439, + "self_cpu_us": 5515.489000000218, + "self_device_us": 2130.588000000116, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2769, + "self_cpu_us": 4635.025999999778, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 4584.934999999944, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 601, + "self_cpu_us": 2461.7410000001278, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2147.010999999959, + "self_device_us": 809.0290000000505, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3851, + "self_cpu_us": 1570.322000000206, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1314.6100000000988, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 629, + "self_cpu_us": 1276.088999999649, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 990.0759999999482, + "self_device_us": 233.5900000000729, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 940.6050000001692, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 864.0409999999702, + "self_device_us": 601.2849999999398, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 782.1960000001072, + "self_device_us": 233.108000000002, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::record_stream", + "count": 436, + "self_cpu_us": 745.8910000001142, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 116, + "self_cpu_us": 660.8229999999921, + "self_device_us": 266.3260000000107, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 281, + "self_cpu_us": 619.7590000002324, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventRecordWithFlags", + "count": 698, + "self_cpu_us": 613.4480000000897, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 64, + "self_cpu_us": 609.95099999999, + "self_device_us": 486.4459999999799, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 17218.51800000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 9886.790999998972, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9667.236999999815, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 1850, + "self_cpu_us": 9417.455000000573, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8362.155000000257, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 7480.238000000069, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2769, + "self_cpu_us": 7326.50600000022, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6628.762999999737, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1439, + "self_cpu_us": 6172.138999999667, + "self_device_us": 7529.890999999854, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5232.246999999894, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 4782.417000000076, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4636.270000000101, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4530.803999999982, + "self_device_us": 0.0, + "self_device_memory_bytes": -79872 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4478.493999999459, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 3908.4060000003046, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3038.3940000000453, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 601, + "self_cpu_us": 2837.559999999785, + "self_device_us": 0, + "self_device_memory_bytes": 26515968 + }, + { + "operator": "", + "count": 904, + "self_cpu_us": 2774.022999999871, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/__init__.py(1235): current_stream", + "count": 1126, + "self_cpu_us": 2606.9919999999374, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2567.98200000016, + "self_device_us": 3231.842000000015, + "self_device_memory_bytes": 6881280 + } + ], + "raw-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 35043.07600000008, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 27261.345000000074, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 26212.683000000343, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2040, + "self_cpu_us": 17226.433999998942, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 9567.590000000226, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1631, + "self_cpu_us": 7822.165999999748, + "self_device_us": 7616.087999999582, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 5809.952999998892, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 793, + "self_cpu_us": 3797.753000000337, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2782.3519999998025, + "self_device_us": 2836.7000000000608, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2576.1720000005644, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1843.8050000004368, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 693, + "self_cpu_us": 1734.2529999999033, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 116, + "self_cpu_us": 1597.180999999936, + "self_device_us": 645.6439999999984, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 409, + "self_cpu_us": 1295.4429999995746, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1207.970999999644, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1207.7140000000436, + "self_device_us": 744.3319999997984, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 1078.0610000000052, + "self_device_us": 684.6229999998395, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 96, + "self_cpu_us": 1074.36400000082, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1062.3739999999489, + "self_device_us": 1878.7700000000477, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 1000.1569999997646, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 20903.29400000006, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 18533.75499999998, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 11627.489999999798, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2040, + "self_cpu_us": 11486.406999999133, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10235.82400000188, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9755.430000000004, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 8791.236000000776, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 8558.017999999998, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8461.214000000378, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1631, + "self_cpu_us": 7077.14399999875, + "self_device_us": 7515.524999999724, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6816.739999999478, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5592.861000000013, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4809.259999999384, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4741.460000002217, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4577.247000000707, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 4044.835999999861, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "aten::empty_strided", + "count": 793, + "self_cpu_us": 3699.4469999996372, + "self_device_us": 0, + "self_device_memory_bytes": 39131648 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 3092.7339999997967, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3059.046000000157, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2879.6480000001075, + "self_device_us": 2809.507000000074, + "self_device_memory_bytes": 6881280 + } + ], + "processed-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 40731.565000000046, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 18052.8380000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2168, + "self_cpu_us": 11742.620000000232, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 10820.819999999676, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1567, + "self_cpu_us": 6499.077000000206, + "self_device_us": 8344.468000000634, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5100.179999999666, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 4953.838000000167, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 729, + "self_cpu_us": 2996.4449999999683, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2412.3269999999034, + "self_device_us": 3132.7200000000075, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2234.7580000002854, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1542.5300000003408, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 693, + "self_cpu_us": 1528.0179999995985, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 192, + "self_cpu_us": 1305.0059999999976, + "self_device_us": 1111.3880000001209, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 116, + "self_cpu_us": 1274.8779999999783, + "self_device_us": 721.4030000000266, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1105.1659999998597, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1061.8390000001273, + "self_device_us": 851.0689999999304, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 409, + "self_cpu_us": 1038.5240000001413, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 956.9540000000325, + "self_device_us": 2173.455000000007, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 96, + "self_cpu_us": 929.1470000001209, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 908.5779999999498, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 19562.78000000015, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 12463.96400000043, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2168, + "self_cpu_us": 11787.197999999762, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10282.645999998535, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9901.121999998875, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 9359.13800000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8651.15399999867, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 7960.596999999834, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1567, + "self_cpu_us": 6933.717000000461, + "self_device_us": 7324.400999999833, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6819.291999999174, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5430.379000000365, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 5054.977999999375, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4980.245000000294, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4827.424999998944, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4674.382999999931, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 4062.3459999991246, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "aten::empty_strided", + "count": 729, + "self_cpu_us": 3426.792000000013, + "self_device_us": 0, + "self_device_memory_bytes": 39098880 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3211.9410000003777, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 3122.0590000002553, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2800.17300000012, + "self_device_us": 2851.515000000054, + "self_device_memory_bytes": 6881280 + } + ] + }, + "tokens_unchanged": true + }, + "speculative": { + "seconds": { + "none": [ + 0.0662261480001689 + ], + "raw": [ + 0.06795481500012102 + ], + "processed": [ + 0.06800570499990499 + ] + }, + "median_seconds": { + "none": 0.0662261480001689, + "raw": 0.06795481500012102, + "processed": 0.06800570499990499 + }, + "tokens_per_second": { + "none": 966.385663859489, + "raw": 941.8022843544792, + "processed": 941.0975152759523 + }, + "peak_extra_allocated_bytes": { + "none": [ + 2750976 + ], + "raw": [ + 2790912 + ], + "processed": [ + 2774528 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 14250.309000000101, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2245, + "self_cpu_us": 10531.197000000051, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1128, + "self_cpu_us": 8562.952999999969, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1277, + "self_cpu_us": 3761.6589999999446, + "self_device_us": 1806.0290000000139, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 143, + "self_cpu_us": 2708.568000000006, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 592, + "self_cpu_us": 2201.785999999958, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1515.5650000001278, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1219, + "self_cpu_us": 1244.1550000002997, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1168.2059999999747, + "self_device_us": 362.9779999999464, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1026.8629999999969, + "self_device_us": 156.73599999999442, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 995.331999999973, + "self_device_us": 317.34799999996153, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 917.5149999999649, + "self_device_us": 683.0130000000008, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 843.7139999999272, + "self_device_us": 172.83400000001166, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 790.9519999999757, + "self_device_us": 621.252999999977, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 77, + "self_cpu_us": 775.0059999999964, + "self_device_us": 124.67199999998161, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2113, + "self_cpu_us": 760.4689999999889, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 759.0639999997917, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 756.3700000000226, + "self_device_us": 1531.8530000000064, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 348, + "self_cpu_us": 708.1709999999481, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_softmax", + "count": 119, + "self_cpu_us": 698.9639999999526, + "self_device_us": 1821.2669999999434, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1128, + "self_cpu_us": 50386.84900000042, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 18524.925000000065, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2245, + "self_cpu_us": 12642.707000000097, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 10299.16400000074, + "self_device_us": 0.0, + "self_device_memory_bytes": -1536 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6868.614000000468, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 143, + "self_cpu_us": 6569.547999999943, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 5634.887000000228, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1277, + "self_cpu_us": 4802.597999998776, + "self_device_us": 6307.480999999992, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 592, + "self_cpu_us": 3066.6099999999415, + "self_device_us": 0, + "self_device_memory_bytes": 52925952 + }, + { + "operator": "aten::slice", + "count": 1219, + "self_cpu_us": 2758.779000000186, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2436.3480000002855, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 2412.122000000076, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 2321.6109999999353, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1901.2210000000298, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2113, + "self_cpu_us": 1567.9659999997698, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1554.9709999999468, + "self_device_us": 1259.2209999998413, + "self_device_memory_bytes": 25165824 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1469.638999999961, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + }, + { + "operator": "aten::_to_copy", + "count": 348, + "self_cpu_us": 1457.5210000002992, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1289.5729999998493, + "self_device_us": 1402.8940000000239, + "self_device_memory_bytes": 1404416 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 1265.5860000001667, + "self_device_us": 2400.320000000047, + "self_device_memory_bytes": 79872 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1219, + "self_cpu_us": 32791.54100000013, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 17470.00599999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2394, + "self_cpu_us": 12090.505000000023, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 4819.1389999999865, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1428, + "self_cpu_us": 4432.517000000222, + "self_device_us": 4649.904999999926, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 743, + "self_cpu_us": 2811.108999999853, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1714.3729999998868, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 1327.7300000001896, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1264.6509999999944, + "self_device_us": 774.6999999999425, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 100, + "self_cpu_us": 1105.010999999929, + "self_device_us": 438.56399999991436, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1051.4090000000483, + "self_device_us": 1278.6749999999424, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1050.9589999995637, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1045.2189999999996, + "self_device_us": 741.7150000000001, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 980.1500000000906, + "self_device_us": 1307.7370000000465, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 938.0559999998939, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 902.338000000078, + "self_device_us": 333.67199999995864, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 879.014999999983, + "self_device_us": 1192.532999999974, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 833.2999999999356, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 790.916999999994, + "self_device_us": 2837.7279999999555, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 769.0820000001117, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 17014.585000000065, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1219, + "self_cpu_us": 15616.989000000358, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2394, + "self_cpu_us": 12305.728000000006, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9651.556000000543, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6259.108999999342, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 4917.951999999987, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1428, + "self_cpu_us": 4794.027999999778, + "self_device_us": 4887.427999999511, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 4222.749999999923, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 743, + "self_cpu_us": 3183.7439999999515, + "self_device_us": 0, + "self_device_memory_bytes": 65541632 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 2523.371999999762, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2202.935999999663, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 2086.883999999831, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 1783.7669999996692, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1756.5959999997158, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1639.7709999998492, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1436.0750000000262, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 1346.087000000095, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1336.519999999975, + "self_device_us": 813.8999999999978, + "self_device_memory_bytes": 25165824 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1293.4269999999624, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1222.6059999999607, + "self_device_us": 738.8599999998873, + "self_device_memory_bytes": 1404416 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1155, + "self_cpu_us": 30564.47400000023, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 16297.44999999991, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2522, + "self_cpu_us": 12829.769999999564, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 6427.685000000046, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1364, + "self_cpu_us": 4445.575999999577, + "self_device_us": 4287.241000000021, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 679, + "self_cpu_us": 2640.5459999999166, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1695.7159999999421, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 270, + "self_cpu_us": 1634.4419999999154, + "self_device_us": 895.1859999999397, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 1283.0529999999708, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 100, + "self_cpu_us": 1112.1259999999893, + "self_device_us": 597.6139999999978, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_softmax", + "count": 183, + "self_cpu_us": 1092.33499999993, + "self_device_us": 4761.238999999965, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1076.8700000000263, + "self_device_us": 1272.6010000000606, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1041.7010000000118, + "self_device_us": 734.4430000000102, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1010.8409999999494, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 1003.2569999999541, + "self_device_us": 1286.9970000000249, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 993.8319999999167, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 919.8220000000947, + "self_device_us": 318.9200000000674, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 879.4340000000484, + "self_device_us": 1155.8990000000413, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 858.7320000000709, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 807.774000000074, + "self_device_us": 2808.9659999999603, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1155, + "self_cpu_us": 20902.823999999953, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 16891.75100000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2522, + "self_cpu_us": 15932.245000000374, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9363.272999999954, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6491.891000000185, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 5420.91799999993, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 5071.345999999881, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1364, + "self_cpu_us": 4659.201999999755, + "self_device_us": 5583.503000000237, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 679, + "self_cpu_us": 3039.1240000001817, + "self_device_us": 0, + "self_device_memory_bytes": 65508864 + }, + { + "operator": "aten::bitwise_and", + "count": 87, + "self_cpu_us": 2845.459999999992, + "self_device_us": 366.18600000001607, + "self_device_memory_bytes": 44544 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 2434.940000000655, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2247.8800000001356, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::eq", + "count": 87, + "self_cpu_us": 2212.9000000000815, + "self_device_us": 339.5000000001128, + "self_device_memory_bytes": 44544 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 2182.421999999704, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1826.42999999996, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 270, + "self_cpu_us": 1732.066999999999, + "self_device_us": 1221.016000000127, + "self_device_memory_bytes": 37748736 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1720.8809999998857, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 1615.1079999998499, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1522.9489999999714, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "vllm_rlt/worker/sampling.py(6): probabilities", + "count": 181, + "self_cpu_us": 1390.739999999816, + "self_device_us": 0.0, + "self_device_memory_bytes": -35586048 + } + ] + }, + "tokens_unchanged": true + } +} From 08d1b8d53ed1c25fabb1ae0c1b973facab9ec999 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 1 Oct 2026 12:40:35 +0000 Subject: [PATCH 4/9] Keep training contract review focused on the public API Signed-off-by: 0z5a --- benchmarks/logprob_overhead.py | 125 - benchmarks/nixl_preflight.py | 73 - benchmarks/pd_training_contract.py | 172 -- benchmarks/training_contract.py | 94 - docs/training_contract.md | 28 - .../README.md | 134 - .../raw/minhao-cleanup-smoke-removed.json | 4 - .../raw/minhao-cleanup-smoke.json | 2669 ---------------- .../raw/minhao-cleanup-smoke.log | 40 - .../raw/minhao-cleanup.json | 7 - .../raw/minhao-cpu.log | 12 - .../raw/minhao-cuda-parallel.log | 11 - .../raw/minhao-cuda.log | 11 - .../raw/minhao-driver-stack-wait.log | 19 - .../raw/minhao-driver-wait.log | 42 - .../raw/minhao-gpu-before.csv | 2 - .../raw/minhao-logprob-overhead.json | 2741 ----------------- .../raw/minhao-logprob-overhead.log | 40 - .../raw/minhao-seed-before.log | 14 - .../raw/minhao-slow-cleanup.json | 4 - .../raw/minhao-tiny-overhead.json | 2669 ---------------- .../training-contract-a100-20261001/README.md | 77 - .../raw/contract-cpu-final.log | 10 - .../raw/cpu-regression-final.log | 9 - .../raw/lint-final.log | 4 - .../raw/nixl-preflight-ipc.log | 51 - .../raw/nixl-preflight-shm.log | 51 - .../raw/nixl-preflight-tcp.log | 6 - .../raw/nixl-preflight.log | 12 - .../raw/pd-full-graph-matched.json | 1189 ------- .../raw/pd-full-graph-matched.log | 11 - .../raw/pd-full-graph-tcp.log | 16 - .../raw/pd-tiny-eager-tcp.json | 1160 ------- .../raw/pd-tiny-eager-tcp.log | 11 - .../raw/pd-tiny-graph-tcp.json | 1163 ------- .../raw/pd-tiny-graph-tcp.log | 11 - .../raw/runtime.json | 13 - .../training-contract-thor-20261001/README.md | 35 - .../raw/shared-full-rollout-benchmark.json | 98 - tests/__init__.py | 1 - tests/test_pd.py | 1 - tests/test_pd_weight_control.py | 91 - tests/test_speculative.py | 2 +- tests/test_training_contract.py | 238 -- 44 files changed, 1 insertion(+), 13170 deletions(-) delete mode 100644 benchmarks/logprob_overhead.py delete mode 100644 benchmarks/nixl_preflight.py delete mode 100644 benchmarks/pd_training_contract.py delete mode 100644 benchmarks/training_contract.py delete mode 100644 docs/validation/minhao-coordination-thor-20261001/README.md delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json delete mode 100644 docs/validation/training-contract-a100-20261001/README.md delete mode 100644 docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/lint-final.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/runtime.json delete mode 100644 docs/validation/training-contract-thor-20261001/README.md delete mode 100644 docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json delete mode 100644 tests/__init__.py delete mode 100644 tests/test_pd_weight_control.py delete mode 100644 tests/test_training_contract.py diff --git a/benchmarks/logprob_overhead.py b/benchmarks/logprob_overhead.py deleted file mode 100644 index cdf5bdbd..00000000 --- a/benchmarks/logprob_overhead.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Alternate unprofiled score modes; report profiler operator snapshots separately.""" - -import argparse -import json -import statistics -import time -from dataclasses import replace -from pathlib import Path - -import torch -from transformers import AutoTokenizer - -from vllm_rlt import ( - LLM, - CacheConfig, - ExecutionConfig, - ExitConfig, - SamplingParams, - SchedulerConfig, - SpeculativeConfig, -) -from vllm_rlt.models import OuroForCausalLM - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("model", type=Path) - parser.add_argument("output", type=Path) - parser.add_argument("--repeats", type=int, default=5) - args = parser.parse_args() - torch.set_num_threads(4) - native = OuroForCausalLM.from_pretrained(args.model, device="cuda", dtype=torch.bfloat16) - tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False) - prompts = [tokenizer.encode(text) for text in ("Compute 17 + 25.\n", "Compute 9 * 8.\n")] - base = [ - SamplingParams(max_tokens=32, temperature=0.9, seed=31 + i, ignore_eos=True) - for i in range(2) - ] - modes = { - "none": base, - **{ - name: [replace(p, logprobs=0, logprobs_mode=name) for p in base] - for name in ("raw", "processed") - }, - } - report = {} - for path in ("sync", "async", "speculative"): - engine = LLM( - native, - attention_backend="triton", - cache_config=CacheConfig(num_blocks=512), - scheduler_config=SchedulerConfig(max_num_seqs=2), - exit_config=ExitConfig("ouro_delayed" if path == "async" else "ouro"), - execution_config=ExecutionConfig(cuda_graphs=True, async_scheduling=path == "async"), - speculative_config=SpeculativeConfig(3) if path == "speculative" else None, - ) - for params in modes.values(): - engine.generate(prompts, params) - times = {name: [] for name in modes} - memory = {name: [] for name in modes} - for index in range(args.repeats): - names = list(modes) - names = names[index % 3 :] + names[: index % 3] - outputs = {} - for name in names: - torch.cuda.synchronize() - torch.cuda.reset_peak_memory_stats() - baseline = torch.cuda.memory_allocated() - started = time.monotonic() - outputs[name] = engine.generate(prompts, modes[name]) - torch.cuda.synchronize() - times[name].append(time.monotonic() - started) - memory[name].append(torch.cuda.max_memory_allocated() - baseline) - assert all(len(o.token_ids) == 32 for o in outputs["none"]) - assert all(o.log_probs is None for o in outputs["none"]) - for name in ("raw", "processed"): - assert [o.token_ids for o in outputs[name]] == [ - o.token_ids for o in outputs["none"] - ] - assert all(len(o.log_probs) == len(o.token_ids) for o in outputs[name]) - medians = {name: statistics.median(values) for name, values in times.items()} - profiles = {} - for name in ("none", "raw", "processed"): - for detail in ("ops", "full"): - with torch.profiler.profile( - activities=[ - torch.profiler.ProfilerActivity.CPU, - torch.profiler.ProfilerActivity.CUDA, - ], - record_shapes=detail == "full", - with_stack=detail == "full", - profile_memory=detail == "full", - ) as profile: - engine.generate(prompts, modes[name]) - torch.cuda.synchronize() - events = sorted( - profile.key_averages(), key=lambda e: e.self_cpu_time_total, reverse=True - ) - profiles[f"{name}-{detail}"] = [ - { - "operator": e.key, - "count": e.count, - "self_cpu_us": e.self_cpu_time_total, - "self_device_us": e.self_device_time_total, - "self_device_memory_bytes": e.self_device_memory_usage, - } - for e in events[:20] - ] - report[path] = { - "seconds": times, - "median_seconds": medians, - "tokens_per_second": {name: 64 / value for name, value in medians.items()}, - "peak_extra_allocated_bytes": memory, - "profiles": profiles, - "tokens_unchanged": True, - } - engine.close() - print(path, medians, flush=True) - args.output.write_text(json.dumps(report, indent=2) + "\n") - del engine - torch.cuda.empty_cache() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/nixl_preflight.py b/benchmarks/nixl_preflight.py deleted file mode 100644 index a56156b9..00000000 --- a/benchmarks/nixl_preflight.py +++ /dev/null @@ -1,73 +0,0 @@ -import multiprocessing as mp -import time - -import torch -from nixl._api import nixl_agent - - -def worker(device, pipe): - torch.cuda.set_device(device) - tensor = ( - torch.arange(32, dtype=torch.float32, device=f"cuda:{device}") - if device == 0 - else torch.zeros(32, device=f"cuda:{device}") - ) - torch.cuda.synchronize() - name = f"ouro-nixl-{device}" - agent = nixl_agent(name) - registration = agent.register_memory(tensor) - pipe.send( - dict(name=name, metadata=agent.get_agent_metadata(), ptr=tensor.data_ptr(), device=device) - ) - peer = pipe.recv() - agent.add_remote_agent(peer["metadata"]) - pipe.send("ready") - pipe.recv() - if device == 0: - handle = agent.initialize_xfer( - "WRITE", - agent.get_xfer_descs(tensor), - agent.get_xfer_descs([(peer["ptr"], 128, peer["device"])], "VRAM"), - peer["name"], - notif_msg=b"done", - backends=["UCX"], - ) - agent.transfer(handle) - while agent.check_xfer_state(handle) != "DONE": - time.sleep(0.001) - agent.release_xfer_handle(handle) - else: - while not agent.get_new_notifs(): - time.sleep(0.001) - pipe.send(tensor.cpu().tolist()) - pipe.recv() - agent.remove_remote_agent(peer["name"]) - agent.deregister_memory(registration) - pipe.close() - - -if __name__ == "__main__": - context = mp.get_context("spawn") - peers = [] - for device in range(2): - parent, child = context.Pipe() - process = context.Process(target=worker, args=(device, child)) - process.start() - child.close() - peers.append((process, parent)) - infos = [p.recv() for _, p in peers] - for i, (_, p) in enumerate(peers): - p.send(infos[1 - i]) - assert all(p.recv() == "ready" for _, p in peers) - for _, p in peers: - p.send("go") - values = [p.recv() for _, p in peers] - print("VALUES", values, flush=True) - for _, p in peers: - p.send("stop") - for process, p in peers: - process.join() - p.close() - assert process.exitcode == 0 - assert values[0] == values[1] - print("MULTIPROCESS_NIXL_PASSED", flush=True) diff --git a/benchmarks/pd_training_contract.py b/benchmarks/pd_training_contract.py deleted file mode 100644 index 38830d64..00000000 --- a/benchmarks/pd_training_contract.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Two spawned CUDA owners: NIXL KV, policy publication, and local-engine parity.""" - -import argparse -import json -import time -from dataclasses import asdict, replace -from pathlib import Path - -import torch - -from vllm_rlt import LLM, CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig -from vllm_rlt.models import OuroConfig, OuroForCausalLM -from vllm_rlt.pd.config import PDConfig -from vllm_rlt.pd.engine import PDEngine - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("output", type=Path) - parser.add_argument("--model", type=Path) - parser.add_argument("--cuda-graphs", action="store_true") - parser.add_argument("--temperature", type=float, default=0.9) - parser.add_argument("--max-num-seqs", type=int, default=4) - args = parser.parse_args() - torch.set_num_threads(4) - torch.manual_seed(42) - dtype = torch.bfloat16 if args.model else torch.float32 - source = args.model or OuroConfig( - vocab_size=64, - hidden_size=32, - intermediate_size=64, - num_hidden_layers=2, - num_attention_heads=4, - num_key_value_heads=2, - head_dim=8, - max_position_embeddings=128, - total_ut_steps=4, - bos_token_id=0, - eos_token_id=0, - ) - native = ( - OuroForCausalLM.from_pretrained(source, device="cuda:0", dtype=dtype) - if args.model - else OuroForCausalLM(source).to(device="cuda:0", dtype=dtype) - ) - execution = ExecutionConfig(cuda_graphs=args.cuda_graphs, async_scheduling=True) - cache = CacheConfig(num_blocks=512, enable_prefix_caching=True) - scheduler = SchedulerConfig( - max_num_seqs=args.max_num_seqs, max_num_batched_tokens=64, prefill_chunk_size=16 - ) - common = dict( - execution_config=execution, - exit_config=ExitConfig("ouro_delayed"), - attention_backend="triton", - ) - local = LLM(native, cache_config=cache, scheduler_config=scheduler, **common) - pd = PDEngine( - source, - dtype="bfloat16" if args.model else "float32", - pd_config=PDConfig( - max_control_messages=1, startup_timeout=600, request_timeout=600, shutdown_timeout=120 - ), - prefill_cache_config=cache, - decode_cache_config=cache, - prefill_scheduler_config=scheduler, - decode_scheduler_config=scheduler, - terminate_workers_on_failure=False, - **common, - ) - disaggregated = LLM(pd) - prompts = [[3, 4, 5] * 9 + [6], [3, 4, 5] * 9 + [7]] - report = dict(model=asdict(native.config), cuda_graphs=args.cuda_graphs, iterations=[]) - report["max_num_seqs"] = args.max_num_seqs - try: - for version, loops in enumerate((2, 3, 4), 1): - with torch.no_grad(): - native.lm_head.weight.mul_(0.98) - for arm in (local, disaggregated): - arm.start_weight_update(version) - weights = list(native.named_parameters()) - for offset in range(0, len(weights), 8): - arm.update_weights(weights[offset : offset + 8]) - arm.finish_weight_update() - assert arm.get_weight_version() == version - for mode in ("raw", "processed"): - params = [ - SamplingParams( - max_tokens=6, - max_loops=loops, - seed=11 + i, - temperature=args.temperature, - top_k=8, - top_p=0.8, - ignore_eos=True, - logprobs=0, - logprobs_mode=mode, - ) - for i in range(len(prompts)) - ] - outputs, timings = {}, {} - for label, arm in (("local", local), ("pd", disaggregated)): - started = time.monotonic() - outputs[label] = arm.generate(prompts, params) - timings[label] = time.monotonic() - started - errors = [] - for expected, result in zip(outputs["local"], outputs["pd"], strict=True): - assert expected.token_ids == result.token_ids - assert expected.exit_depths == result.exit_depths - assert result.weight_version == version - assert result.sampling_params == expected.sampling_params - delta = ( - torch.tensor(expected.log_probs) - torch.tensor(result.log_probs) - ).abs() - errors.append(delta.max().item()) - torch.testing.assert_close( - torch.tensor(expected.log_probs), - torch.tensor(result.log_probs), - atol=2e-5, - rtol=2e-5, - ) - row = dict( - version=version, - loops=loops, - mode=mode, - seconds=timings, - max_logprob_error=max(errors), - outputs=[asdict(o) for o in outputs["pd"]], - ) - report["iterations"].append(row) - print("PARITY", version, loops, mode, timings, max(errors), flush=True) - # A failed chunk cannot expose a mixed policy; a newer full epoch recovers. - disaggregated.start_weight_update(4) - try: - disaggregated.update_weights([("missing_parameter", torch.zeros(1))]) - except RuntimeError as error: - report["rejected_chunk"] = str(error) - else: - raise AssertionError("Invalid publication unexpectedly succeeded") - disaggregated.start_weight_update(5) - disaggregated.update_weights(native.named_parameters()) - disaggregated.finish_weight_update() - recovered = disaggregated.generate(prompts[:1], params[:1])[0] - assert recovered.weight_version == 5 - stopped = disaggregated.generate( - prompts[:1], [replace(params[0], stop_token_ids=(recovered.token_ids[0],))] - )[0] - assert len(stopped.token_ids) == len(stopped.log_probs) == 1 - assert stopped.finish_reason == "stop" - pd.add_request("abort", prompts[0], params[0]) - while not pd.transfers[next(iter(pd.transfers))].latest_output: - pd.step() - aborted = pd.abort_request("abort") - assert aborted.finish_reason == "abort" - assert len(aborted.log_probs) == len(aborted.token_ids) - while pd.has_unfinished_requests(): - pd.step() - disaggregated.reset_prefix_cache() - report.update(recovered_version=5, stop=asdict(stopped), abort=asdict(aborted)) - finally: - disaggregated.close() - local.close() - report["workers"] = pd.worker_metrics - assert all( - m["used_blocks"] == 0 and m["weight_version"] == 5 for m in pd.worker_metrics.values() - ) - assert sum(m["bytes_sent"] for m in pd.worker_metrics.values()) > 0 - args.output.write_text(json.dumps(report, indent=2)) - print("PD_CONTRACT_PASSED", args.output, flush=True) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/training_contract.py b/benchmarks/training_contract.py deleted file mode 100644 index 0bd6fbe0..00000000 --- a/benchmarks/training_contract.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Alternating-order warmed rollout timings, including returned log probabilities.""" - -import argparse -import json -import statistics -import time -from pathlib import Path - -import torch -from transformers import AutoTokenizer - -from vllm_rlt import ( - LLM, - CacheConfig, - ExecutionConfig, - SamplingParams, - SchedulerConfig, - SpeculativeConfig, -) -from vllm_rlt.models import OuroForCausalLM - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("model", type=Path) - parser.add_argument("output", type=Path) - parser.add_argument("--repeats", type=int, default=5) - args = parser.parse_args() - torch.set_num_threads(4) - tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False) - prompts = [tokenizer.encode(text) for text in ("Compute 17 + 25.\n", "Compute 9 * 8.\n")] - native = OuroForCausalLM.from_pretrained(args.model, device="cuda", dtype=torch.bfloat16) - report = {} - for speculative in (False, True): - arms = { - mode: LLM( - native, - attention_backend="triton", - cache_config=CacheConfig(num_blocks=512), - scheduler_config=SchedulerConfig(max_num_seqs=8, max_num_batched_tokens=128), - execution_config=ExecutionConfig(cuda_graphs=mode == "graph"), - speculative_config=SpeculativeConfig(3) if speculative else None, - ) - for mode in ("eager", "graph") - } - for loops in (4,) if speculative else (2, 3, 4): - params = [ - SamplingParams( - max_tokens=32, - max_loops=loops, - temperature=0.9, - seed=31 + i, - logprobs=0, - ignore_eos=True, - ) - for i in range(len(prompts)) - ] - for arm in arms.values(): - arm.generate(prompts, params) - arm.start_weight_update() - arm.update_weights(native.named_parameters()) - arm.finish_weight_update() - version = arms["eager"].get_weight_version() - times = {mode: [] for mode in arms} - for index in range(args.repeats): - outputs = {} - for mode in ("eager", "graph") if index % 2 == 0 else ("graph", "eager"): - torch.cuda.synchronize() - started = time.monotonic() - outputs[mode] = arms[mode].generate(prompts, params) - torch.cuda.synchronize() - times[mode].append(time.monotonic() - started) - for eager, graph in zip(outputs["eager"], outputs["graph"], strict=True): - assert eager.token_ids == graph.token_ids - assert eager.exit_depths == graph.exit_depths - assert eager.log_probs == graph.log_probs - assert eager.weight_version == graph.weight_version == version - medians = {mode: statistics.median(values) for mode, values in times.items()} - name = f"speculative-{loops}" if speculative else f"native-{loops}" - report[name] = { - "seconds": times, - "medians": medians, - "speedup": medians["eager"] / medians["graph"], - "tokens_and_logprobs_exact": True, - } - print(name, report[name], flush=True) - for arm in arms.values(): - arm.close() - del arms - args.output.write_text(json.dumps(report, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/docs/training_contract.md b/docs/training_contract.md index fa1d2b84..e5a73cfd 100644 --- a/docs/training_contract.md +++ b/docs/training_contract.md @@ -55,31 +55,3 @@ upstream self-speculation uses a full-depth target and synchronous execution; upstream PD does not implement self-speculation. Publication adds no such restriction to ordinary early-exit or async rollouts. Trainers must replay full-depth prompt prefill and the returned decode depths rather than assume the entire sequence used K. - -## Validation - -```bash -python -m unittest tests.test_training_contract tests.test_pd_weight_control -v -CONTRACT_DEVICE=cuda python -m unittest tests.test_training_contract -v -python benchmarks/training_contract.py /models/Ouro-1.4B timings.json -UCX_TLS=tcp,cuda_copy python benchmarks/nixl_preflight.py -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py pd.json --cuda-graphs -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py full-pd.json \ - --model /models/Ouro-1.4B --cuda-graphs --max-num-seqs 1 -``` - -CPU and Thor CUDA tests cover raw/processed probabilities against an independent -dense oracle, speculative correction/bonus alignment, publication, pointer stability, -prefix invalidation, early exit, async stop/abort and entropy seeds. The PD control -test drives two real CPU engines and checks recovery after one peer commits and -another fails. It uses in-process control delivery; it does **not** validate IPC or -NIXL transport. The separate two-A100 run uses real spawned workers and NIXL: -FP32 multi-request and full BF16 matched-batch checks cover three publications, -raw/processed scores, graph replay, prefix reuse, explicit stop/abort and recovery -to version five. See [PD evidence](validation/training-contract-a100-20261001/README.md). -No PD workers were launched on single-GPU Thor, where NIXL is absent. - -This is a draft implementation of the -[Phase 1 contract proposed in issue 70](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5888101273) -on upstream `ecb1f8b505b7e831815b40aec3b4598619cca23a`; formal upstream agreement -remains outside the scope of this contributor draft. diff --git a/docs/validation/minhao-coordination-thor-20261001/README.md b/docs/validation/minhao-coordination-thor-20261001/README.md deleted file mode 100644 index 59c4dce1..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/README.md +++ /dev/null @@ -1,134 +0,0 @@ -# Phase 1 / VIME coordination checks — 2026-10-01 - -These checks exercise the semantics in -[MinhaoLi0318's PR 1 proposal](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5927860064) -on the existing Thor runtime. They test the contributor implementation, not an -unpublished Phase 1 branch. The -[coordination reply](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5930413915) -records the interface differences and division of upstream work. - -The replay requirement exposed a real omission: an entropy-seeded request -returned `sampling_params.seed=None`. The -[unfixed regression](raw/minhao-seed-before.log) fails on that missing effective -seed. `Request` now resolves an independent 63-bit seed when it is created and -retains it through output, preemption, abort and existing PD parameter transport. -Default `seed=0` is unchanged. An abort before any sampling also reports the seed; -the caller's immutable `SamplingParams(seed=None)` is not modified. - -| Thor CPU check | Result | -|---|---| -| Entropy seed reporting and exact token/score/depth replay | Ordinary, async and speculative paths pass | -| Abort before sampling | Effective seed retained; zero selected scores | -| Preemption | Token IDs, selected scores, depths and RNG continuity match uninterrupted execution | -| Existing probability/publication/cache/stop/abort checks | All pass | -| Full focused suite including in-process PD publication | **7 tests pass, 1.736 s** | - -See [CPU log](raw/minhao-cpu.log). Independent VIME FP32 mixed-depth replay, -recomputed gradients and generic physical-parameter conversion are checked -separately. No runtime packages are installed or updated. - -The user authorized concurrent short CUDA validation while an existing -Qwen-Image training reservation remained active. **All six CUDA contract tests -pass in 135.474 s**, including entropy-seed replay with graphs, stop/abort, -preemption, speculative target scores and publication recovery; see -[CUDA log](raw/minhao-cuda-parallel.log). Startup is included in the suite time. The originally queued CUDA suite also -completed naturally after the reservation was released: **6/6 in 3.989 s**, -[log](raw/minhao-cuda.log). -Independent VIME FP32 mixed-depth replay/recomputed gradients and generic -physical-parameter conversion pass **two CPU tests in 0.302 s**. No existing -training process or reservation was interrupted. - -Earlier full-model training/save/resume and real-PD evidence is retained in the -[Thor](../training-contract-thor-20261001/README.md) and -[A100](../training-contract-a100-20261001/README.md) reports. - -## Selected-score cost - -`benchmarks/logprob_overhead.py` warms each mode and rotates the measurement -order over five repeats. Each unprofiled window generates two prompts with 32 -response tokens each, T=0.9, fixed per-request seeds, no filtering and CUDA -graphs enabled. Sync and speculative paths use Ouro exit; async uses delayed -Ouro exit. Token sequences must match across score modes; selected scores must -align with every committed token. Initialization and capture are excluded. - -Separately, all three modes receive an ops-only and a full-detail CPU/CUDA -profiler window (`record_shapes`, stacks and memory enabled for full detail). -The raw JSON retains the 20 operators with largest self CPU time per window, -with counts, self device time and memory. These are operator summaries, not -exported timeline traces. Profiled runs are excluded from throughput medians. -The [tiny smoke JSON](raw/minhao-tiny-overhead.json) checks all three paths and -18 profiler windows with one repeat; its timings are not full-model results. - -### Full Ouro-1.4B shared-GPU observations - -The full run overlaps another Qwen-Image training phase. GPU utilization -before this run was **98%**; [snapshot](raw/minhao-gpu-before.csv). -These measurements include resource contention and are not isolated throughput -results. Each mode has five unprofiled repeats. All three paths complete; -token IDs match across modes and selected scores align with committed tokens. - -| Path | None median (s) | Raw median (s) | Processed median (s) | Raw time overhead | Processed time overhead | -|---|---:|---:|---:|---:|---:| -| sync | 1.8017 | 1.8280 | 1.8281 | +1.46% | +1.47% | -| async | 1.6974 | 1.7849 | 1.7894 | +5.15% | +5.42% | -| speculative | 1.4366 | 1.4462 | 1.4486 | +0.67% | +0.83% | - -| Path / score mode | Tokens/s | Speed vs None | Median extra PyTorch allocation (KiB) | -|---|---:|---:|---:| -| sync / none | 35.52 | 1.000× | 933.0 | -| sync / raw | 35.01 | 0.986× | 965.0 | -| sync / processed | 35.01 | 0.986× | 933.0 | -| async / none | 37.70 | 1.000× | 965.0 | -| async / raw | 35.86 | 0.951× | 963.5 | -| async / processed | 35.77 | 0.949× | 965.0 | -| speculative / none | 44.55 | 1.000× | 2733.5 | -| speculative / raw | 44.25 | 0.993× | 2793.5 | -| speculative / processed | 44.18 | 0.992× | 2761.5 | - -[Raw timings / all 18 profiler operator summaries](raw/minhao-logprob-overhead.json), -[benchmark log](raw/minhao-logprob-overhead.log). Extra allocation is -per-window PyTorch peak above its starting allocation; it excludes resident -model/cache/graph storage and other processes. - -Between completed paths, the process temporarily waited in NVIDIA UVM external -allocation mapping (`os_acquire_rwlock_read/write`); an independent GPU status -query also waited. The [captured stack](raw/minhao-driver-stack-wait.log) and -[kernel log](raw/minhao-driver-wait.log) record concurrent `NV_ERR_NO_MEMORY` -allocation errors. Both waits resolved naturally; all paths completed without -process termination, driver reset or environment changes. Initialization waits -are outside the warmed measurement windows. The harness now drops the previous -engine and clears its allocator cache between paths to avoid retaining that -engine while constructing the next one. The recorded full run predates this -inter-path cleanup; its timed body is unchanged. -The updated cleanup harness then passes a separate tiny-model smoke across -all three paths and 18 profiler windows ([JSON](raw/minhao-cleanup-smoke.json), -[log](raw/minhao-cleanup-smoke.log)); its [57,039,421-byte model is removed](raw/minhao-cleanup-smoke-removed.json). - -All five full-model depth/effective-seed replay cases also complete, including -delayed async. The [VIME score-error table](https://github.com/0z5a/vime/blob/feat/ouro-rollout-cuda-graphs/docs/validation/minhao-coordination-thor-20261001/README.md) -records the remaining training/inference recomputation differences; full-depth -K=4 also has a nonzero residual. Exact seed replay and score-recomputation -agreement are separate checks. - -The [completed full-model copy and replacement shard](raw/minhao-cleanup.json) -are removed (3,232,344,846 bytes). The 358,667,054-byte local transfer copy -was also removed. The original slower download completed naturally; its -[2,873,677,792-byte model copy was removed](raw/minhao-slow-cleanup.json) by the -existing queue, without repeating GPU tests. All four task-owned model folders -and both replacement transfer copies are absent after validation. - -## Interface coordination - -| Proposed Phase 1 interface | Current reference draft | VIME consumer plan | -|---|---|---| -| `RequestOutput.logprobs` | `RequestOutput.log_probs` | Use the upstream spelling once its PR 1 branch is available | -| Engine-level `raw_logprobs` / `processed_logprobs` | Request-level `raw` / `processed` | Configure processed scores for the trainer; avoid a second compatibility API | -| Effective seed reported in resolved parameters | Fixed by this follow-up | Keep seeded reproduction and opt-in entropy groups | -| Full-depth first token and executed decode depths | Already present | Full four-loop prefill and differentiable `last_exited` trace replay | -| PR 2 versioned full publication | Validated reference implementation | Reuse one agreed upstream contract | - -The old `RLEngine.set_loop_budget` mutated `total_ut_steps`; the current VIME -adapter supersedes it and does not need reduced-depth prefill. No new transport -layer is needed for the pilot: the consumer is in-process `LLM`, and existing PD -worker IPC remains inside the engine. MinhaoLi0318 and IDEA-V retain the proposed -Phase 1 upstream roles; this draft supplies reusable implementation and checks. diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json deleted file mode 100644 index 1f9931bd..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "removed": "models/Ouro-minhao-cleanup-smoke", - "bytes": 57039421 -} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json deleted file mode 100644 index ed52a20b..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json +++ /dev/null @@ -1,2669 +0,0 @@ -{ - "sync": { - "seconds": { - "none": [ - 0.08066848499993284 - ], - "raw": [ - 0.0914349139998194 - ], - "processed": [ - 0.08466345700026068 - ] - }, - "median_seconds": { - "none": 0.08066848499993284, - "raw": 0.0914349139998194, - "processed": 0.08466345700026068 - }, - "tokens_per_second": { - "none": 793.3705461315318, - "raw": 699.9514430573688, - "processed": 755.9341688563809 - }, - "peak_extra_allocated_bytes": { - "none": [ - 758784 - ], - "raw": [ - 823808 - ], - "processed": [ - 791040 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 24741.71000000011, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 14785.07000000005, - "self_device_us": 6.143999999963853, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2004, - "self_cpu_us": 10337.619999999994, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2020, - "self_cpu_us": 6443.9159999996045, - "self_device_us": 2353.745999999865, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1348, - "self_cpu_us": 4180.128999999671, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Activity Buffer Request", - "count": 3, - "self_cpu_us": 3175.8589999999667, - "self_device_us": 15.329000000011547, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 1981.3640000000255, - "self_device_us": 966.4389999999644, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 208, - "self_cpu_us": 1895.6409999999596, - "self_device_us": 293.95500000013635, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1556, - "self_cpu_us": 1581.7819999997378, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 716, - "self_cpu_us": 1457.2760000001927, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1335.993999999857, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1012, - "self_cpu_us": 1159.8320000000067, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 962.9140000004063, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 879.5000000000755, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 867.1529999999693, - "self_device_us": 227.0119999999879, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 830.8970000000882, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 824.2710000000934, - "self_device_us": 553.6069999999727, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2415, - "self_cpu_us": 754.1939999999772, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 744.7460000000592, - "self_device_us": 198.56199999995624, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 116, - "self_cpu_us": 645.1380000000013, - "self_device_us": 236.03399999999692, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 51961.9490000005, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 16149.892999999873, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2004, - "self_cpu_us": 10752.365000000542, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9578.167000000489, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 7612.398999999911, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2020, - "self_cpu_us": 7304.089999999687, - "self_device_us": 5244.179000000046, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1348, - "self_cpu_us": 4891.489000000262, - "self_device_us": 0, - "self_device_memory_bytes": 26818560 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 4097.280999999177, - "self_device_us": 0.0, - "self_device_memory_bytes": 162304 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 3075.5700000002166, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1556, - "self_cpu_us": 3032.1670000008203, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 716, - "self_cpu_us": 2489.1329999999634, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 2429.843999999812, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2326.7890000000816, - "self_device_us": 1660.2469999999175, - "self_device_memory_bytes": 6881280 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 2264.487000000183, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "aten::cat", - "count": 208, - "self_cpu_us": 2253.055999999902, - "self_device_us": 608.0070000000824, - "self_device_memory_bytes": 305152 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 2143.693999999712, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 2037.7939999994514, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1728.5120000003517, - "self_device_us": 0.0, - "self_device_memory_bytes": -12845056 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 1563.099999999995, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 1466.244999999848, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 119556.1379999991, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 23820.948999999957, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2194, - "self_cpu_us": 14371.875999999655, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2212, - "self_cpu_us": 11208.853000000732, - "self_device_us": 9572.278999999877, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 9940.423999999859, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1540, - "self_cpu_us": 7332.918000000681, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 3733.2030000000573, - "self_device_us": 1393.6519999997668, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 2742.5369999992313, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 2586.1810000003697, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2453.102000000058, - "self_device_us": 2902.2410000000755, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2360.366000000333, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2117.0190000001458, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1898.4860000001345, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 624, - "self_cpu_us": 1705.4810000000034, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3537, - "self_cpu_us": 1688.5239999999105, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1140, - "self_cpu_us": 1436.9140000001375, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1312.8229999998948, - "self_device_us": 794.6019999998643, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1280.3379999999306, - "self_device_us": 2046.0730000000694, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1207.4139999993881, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 1164.5660000002226, - "self_device_us": 719.3059999997731, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 115092.6299999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 29600.60599999951, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 25946.991000000064, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 17345.223999999165, - "self_device_us": 0.0, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cudaLaunchKernel", - "count": 2194, - "self_cpu_us": 15940.768999999744, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 13333.437000000138, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2212, - "self_cpu_us": 13191.647000002034, - "self_device_us": 8704.526000000209, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1540, - "self_cpu_us": 8662.159000000243, - "self_device_us": 0, - "self_device_memory_bytes": 39434240 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 7366.031999998762, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 4815.7890000007, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 4814.504000000853, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 4703.688000000431, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 4365.908999999834, - "self_device_us": 1169.2249999997575, - "self_device_memory_bytes": 337920 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 4116.904000000264, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 4072.4490000006735, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 3742.0259999994387, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 3527.933999998534, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 3314.13099999951, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 624, - "self_cpu_us": 3090.420999999369, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 3038.0740000001883, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 105856.23600000067, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 23095.000000000022, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2322, - "self_cpu_us": 15003.759999999633, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2148, - "self_cpu_us": 10396.17099999906, - "self_device_us": 9188.07700000014, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 8986.292999999905, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1476, - "self_cpu_us": 6711.9949999997025, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 3563.392000000484, - "self_device_us": 1348.502999999796, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 2623.3650000003126, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2498.1059999999925, - "self_device_us": 2908.510000000058, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 2357.0779999990714, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2048.535, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2041.7819999998937, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3537, - "self_cpu_us": 1853.5020000003638, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1701.4389999999712, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 192, - "self_cpu_us": 1573.1770000001234, - "self_device_us": 1020.2990000001446, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 1523.2590000001437, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1140, - "self_cpu_us": 1365.1570000002835, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1229.6910000001153, - "self_device_us": 799.3670000000275, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1220.0819999998166, - "self_device_us": 2031.6709999999912, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1216.561000000036, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 155410.87200000056, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 26514.28400000018, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 22682.278999999973, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2322, - "self_cpu_us": 17120.490999999194, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 16900.313999997426, - "self_device_us": 0.0, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 13033.013999998899, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2148, - "self_cpu_us": 12944.766999999709, - "self_device_us": 12005.580999999922, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1476, - "self_cpu_us": 8355.110999999586, - "self_device_us": 0, - "self_device_memory_bytes": 39401472 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 7274.417999999696, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 4893.769999999835, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 4797.883999998665, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 4710.996000000819, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 4480.034000000242, - "self_device_us": 1730.5530000000113, - "self_device_memory_bytes": 337920 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 4113.781999999035, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 3937.3690000012284, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 3691.073999999741, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 3555.606999998805, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 3436.052999998901, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 3407.9150000012155, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2940.6499999998923, - "self_device_us": 0.0, - "self_device_memory_bytes": -12845056 - } - ] - }, - "tokens_unchanged": true - }, - "async": { - "seconds": { - "none": [ - 0.1108772420002424 - ], - "raw": [ - 0.1287477310002032 - ], - "processed": [ - 0.12239944100019784 - ] - }, - "median_seconds": { - "none": 0.1108772420002424, - "raw": 0.1287477310002032, - "processed": 0.12239944100019784 - }, - "tokens_per_second": { - "none": 577.214934692009, - "raw": 497.0961391148632, - "processed": 522.8782049739635 - }, - "peak_extra_allocated_bytes": { - "none": [ - 790016 - ], - "raw": [ - 822272 - ], - "processed": [ - 822272 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 15284.012000000086, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 1850, - "self_cpu_us": 8500.083000000152, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 6639.298000000122, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1439, - "self_cpu_us": 5531.486999999959, - "self_device_us": 1906.9919999998415, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 4616.57799999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2769, - "self_cpu_us": 4578.959000000186, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 601, - "self_cpu_us": 2438.2479999999, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2059.4019999999846, - "self_device_us": 778.1539999999927, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3851, - "self_cpu_us": 1576.196000000137, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 629, - "self_cpu_us": 1306.6099999999087, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1285.6129999997502, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 1154.325999999974, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 996.3809999998794, - "self_device_us": 225.15500000009706, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 798.8650000000216, - "self_device_us": 569.2259999999969, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 751.5409999999792, - "self_device_us": 210.21199999991222, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::record_stream", - "count": 436, - "self_cpu_us": 730.9929999999513, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 116, - "self_cpu_us": 650.3899999999953, - "self_device_us": 229.37999999999693, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventQuery", - "count": 513, - "self_cpu_us": 642.4939999999581, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventRecordWithFlags", - "count": 698, - "self_cpu_us": 624.0679999998281, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 281, - "self_cpu_us": 616.0749999999405, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 16433.368999999864, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10018.040999999977, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9557.15300000049, - "self_device_us": 0.0, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cudaLaunchKernel", - "count": 1850, - "self_cpu_us": 9201.422999999402, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8294.379000000605, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 7561.7619999999515, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2769, - "self_cpu_us": 7053.931000000085, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6636.100999999369, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1439, - "self_cpu_us": 6178.882000000486, - "self_device_us": 6708.658999999896, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5287.671999999828, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 4708.536999999357, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4653.8080000003965, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4520.836999999436, - "self_device_us": 0.0, - "self_device_memory_bytes": -79872 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4280.746000000392, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 3811.028999999038, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3031.376999999363, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 601, - "self_cpu_us": 2754.339000000004, - "self_device_us": 0, - "self_device_memory_bytes": 26515968 - }, - { - "operator": "", - "count": 904, - "self_cpu_us": 2668.428999999229, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/__init__.py(1235): current_stream", - "count": 1126, - "self_cpu_us": 2653.2650000008134, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/_utils.py(869): _get_device_index", - "count": 1504, - "self_cpu_us": 2575.899000002003, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 42234.66600000008, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 17010.12299999994, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 11070.726999999686, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2040, - "self_cpu_us": 10325.780999999826, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1631, - "self_cpu_us": 6525.653000000375, - "self_device_us": 8328.703999999532, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 4947.713000000557, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 4864.119999999892, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 793, - "self_cpu_us": 3114.437000000318, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2199.381999999973, - "self_device_us": 3114.2829999999713, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2073.5459999992754, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1458.7159999996475, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 693, - "self_cpu_us": 1455.3750000004352, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 116, - "self_cpu_us": 1195.895999999781, - "self_device_us": 692.2980000000898, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1053.418000000198, - "self_device_us": 860.6630000000077, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 1023.9610000000209, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1004.951999999932, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 409, - "self_cpu_us": 958.3949999998126, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 96, - "self_cpu_us": 888.4549999998671, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 859.6940000000086, - "self_device_us": 2180.226000000088, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 837.9099999999289, - "self_device_us": 781.5500000001175, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 22104.463000000447, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 17926.967000000168, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 14532.418000000032, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2040, - "self_cpu_us": 10800.322000000502, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10283.571000000464, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9719.242999999962, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8455.242999999638, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 7985.128999999445, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1631, - "self_cpu_us": 7096.114999998659, - "self_device_us": 7530.1170000006005, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6655.402000001546, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5625.204000000713, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 4933.312999999274, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4771.646000000193, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4725.08799999866, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4402.655000000255, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 3951.7359999998625, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "aten::empty_strided", - "count": 793, - "self_cpu_us": 3523.918000000247, - "self_device_us": 0, - "self_device_memory_bytes": 39131648 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3151.4589999996992, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2998.801000000943, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 904, - "self_cpu_us": 2673.5620000006493, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 43409.94599999997, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 17950.804999999928, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 15988.948999999806, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2168, - "self_cpu_us": 11112.410000000078, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1567, - "self_cpu_us": 6483.044000000588, - "self_device_us": 8276.219999999763, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 4952.348999999742, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 4921.737000000263, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 729, - "self_cpu_us": 2952.603999999926, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2189.6210000000665, - "self_device_us": 3186.7999999999442, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2120.180999999159, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 693, - "self_cpu_us": 1494.2490000004798, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1457.0889999997416, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 116, - "self_cpu_us": 1239.0429999999242, - "self_device_us": 708.8769999999126, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 192, - "self_cpu_us": 1220.1639999996369, - "self_device_us": 1123.057999999999, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1135.0230000010429, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 1087.3439999998745, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1070.1700000001292, - "self_device_us": 874.7320000000436, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 409, - "self_cpu_us": 998.1089999999881, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 96, - "self_cpu_us": 941.3419999995094, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 902.430999999955, - "self_device_us": 2233.6359999999077, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 23466.177000000756, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 18052.96899999999, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 12057.866999999911, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2168, - "self_cpu_us": 11316.609999999708, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10142.197000000597, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9814.517999998712, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8444.050000000905, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 7757.864000000339, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1567, - "self_cpu_us": 6859.583000000366, - "self_device_us": 7261.939000000045, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6667.205999998861, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5536.804000000327, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 4946.131999999503, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4856.363000000383, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4753.402000000731, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4567.222000000331, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 3955.984000000677, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "aten::empty_strided", - "count": 729, - "self_cpu_us": 3271.6339999996685, - "self_device_us": 0, - "self_device_memory_bytes": 39098880 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3136.034000000277, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 3133.5739999998086, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 904, - "self_cpu_us": 2658.5939999993807, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - }, - "speculative": { - "seconds": { - "none": [ - 0.06240307000007306 - ], - "raw": [ - 0.0662938270002087 - ], - "processed": [ - 0.06682750300024054 - ] - }, - "median_seconds": { - "none": 0.06240307000007306, - "raw": 0.0662938270002087, - "processed": 0.06682750300024054 - }, - "tokens_per_second": { - "none": 1025.590567898744, - "raw": 965.3990860988991, - "processed": 957.6895309072021 - }, - "peak_extra_allocated_bytes": { - "none": [ - 2750976 - ], - "raw": [ - 2790912 - ], - "processed": [ - 2774528 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 14082.55900000005, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2245, - "self_cpu_us": 9969.27999999969, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1128, - "self_cpu_us": 8384.83099999996, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1277, - "self_cpu_us": 3738.911000000112, - "self_device_us": 1777.0809999999603, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 143, - "self_cpu_us": 2681.666000000013, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 592, - "self_cpu_us": 2131.4490000001315, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1464.813999999862, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1219, - "self_cpu_us": 1242.6320000000528, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1164.5480000000407, - "self_device_us": 348.58399999995163, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 977.9220000000182, - "self_device_us": 302.44099999999275, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 969.8469999999941, - "self_device_us": 148.32200000000057, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 893.5579999999991, - "self_device_us": 672.4589999999498, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 814.1800000000003, - "self_device_us": 175.17400000007547, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 797.685999999996, - "self_device_us": 602.407000000032, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2113, - "self_cpu_us": 754.8929999999518, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 77, - "self_cpu_us": 751.3600000000063, - "self_device_us": 112.54500000000348, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 743.7420000000911, - "self_device_us": 1537.3079999999263, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 742.4579999999497, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 348, - "self_cpu_us": 714.7729999999433, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_softmax", - "count": 119, - "self_cpu_us": 704.4650000000765, - "self_device_us": 1838.0370000000748, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaStreamSynchronize", - "count": 143, - "self_cpu_us": 19864.37699999997, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 14970.81799999996, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2245, - "self_cpu_us": 10701.993000000055, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1128, - "self_cpu_us": 10585.129000000112, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9116.944000000163, - "self_device_us": 0.0, - "self_device_memory_bytes": -1536 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6067.3340000002245, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 4490.072999999706, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1277, - "self_cpu_us": 4200.721000000049, - "self_device_us": 2472.8799999996913, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 592, - "self_cpu_us": 2475.1309999999894, - "self_device_us": 0, - "self_device_memory_bytes": 52925952 - }, - { - "operator": "aten::slice", - "count": 1219, - "self_cpu_us": 2209.2680000001774, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2199.3709999999483, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1887.626000000164, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1604.1319999999032, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1586.2180000000506, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1252.9259999999376, - "self_device_us": 522.2780000000967, - "self_device_memory_bytes": 25165824 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1165.056000000348, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - }, - { - "operator": "aten::_to_copy", - "count": 348, - "self_cpu_us": 1149.622000000063, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1149.0220000000054, - "self_device_us": 234.2420000000002, - "self_device_memory_bytes": 786432 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1108.9360000000352, - "self_device_us": 466.69400000002315, - "self_device_memory_bytes": 1404416 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 1083.3829999999834, - "self_device_us": 1011.7250000000568, - "self_device_memory_bytes": 79872 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1219, - "self_cpu_us": 32065.39100000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 15680.841000000019, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2394, - "self_cpu_us": 11380.852999999855, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 4496.739999999833, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1428, - "self_cpu_us": 4368.5809999999665, - "self_device_us": 4428.808999999856, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 743, - "self_cpu_us": 2724.119000000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1590.5000000001273, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 1249.5310000003237, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1218.5759999999027, - "self_device_us": 777.1980000000331, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 100, - "self_cpu_us": 1051.9999999999136, - "self_device_us": 428.0050000000215, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1017.5759999999709, - "self_device_us": 1260.8519999999708, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1012.5349999999967, - "self_device_us": 733.8019999999997, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 959.7169999995767, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 930.4999999999745, - "self_device_us": 1327.9919999999292, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 913.5249999998941, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 857.5730000000585, - "self_device_us": 328.99800000001414, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 831.2279999999759, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 829.5409999999483, - "self_device_us": 1179.376999999913, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 778.833999999988, - "self_device_us": 2838.9859999999917, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 736.6149999998129, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1219, - "self_cpu_us": 17222.49800000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 15855.267000000127, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 11819.789999999986, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2394, - "self_cpu_us": 11519.752999999928, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9310.95399999983, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6072.020999999813, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1428, - "self_cpu_us": 4727.487999999833, - "self_device_us": 4655.986000000061, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 4696.012000000468, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::empty_strided", - "count": 743, - "self_cpu_us": 3005.7680000001283, - "self_device_us": 0, - "self_device_memory_bytes": 65541632 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 2301.55100000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2113.589000000433, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1948.2030000000268, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 1625.992999999985, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1563.8129999999483, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1353.5050000000588, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1338.518000000211, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1312.731999999898, - "self_device_us": 807.4049999999152, - "self_device_memory_bytes": 25165824 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 1273.8249999999016, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1184.5459999999466, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1142.3969999999777, - "self_device_us": 341.6040000000071, - "self_device_memory_bytes": 786432 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1155, - "self_cpu_us": 30886.555999999768, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 15259.450999999935, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2522, - "self_cpu_us": 11694.530000000093, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 4923.203000000066, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1364, - "self_cpu_us": 4200.7139999999345, - "self_device_us": 4325.325999999722, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 679, - "self_cpu_us": 2475.386999999806, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1579.0149999997666, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 270, - "self_cpu_us": 1526.4960000000283, - "self_device_us": 958.5179999999909, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 1211.8220000002698, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1046.327999999994, - "self_device_us": 727.1810000000069, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 100, - "self_cpu_us": 1044.046000000038, - "self_device_us": 422.95099999992226, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_softmax", - "count": 183, - "self_cpu_us": 1043.5719999998073, - "self_device_us": 4847.890000000012, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1012.0560000000614, - "self_device_us": 1260.6269999999504, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 924.9700000000994, - "self_device_us": 1308.5010000000257, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 921.9240000003183, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 911.6380000003298, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 842.8640000000414, - "self_device_us": 329.63399999997273, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 816.9300000000476, - "self_device_us": 1178.6429999999946, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 807.9510000000835, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 778.4430000000539, - "self_device_us": 2829.236000000021, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1155, - "self_cpu_us": 42451.45200000021, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 16473.516999999978, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2522, - "self_cpu_us": 12428.987999999705, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9528.780999999806, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 7053.55099999987, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6385.314000000195, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 4901.061999999998, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1364, - "self_cpu_us": 4642.906999999959, - "self_device_us": 6678.909000000535, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 679, - "self_cpu_us": 2844.3409999998976, - "self_device_us": 0, - "self_device_memory_bytes": 65508864 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 2370.445000000463, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2189.775000000467, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 2057.2630000004756, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "aten::div", - "count": 270, - "self_cpu_us": 1714.8240000003643, - "self_device_us": 1486.07199999988, - "self_device_memory_bytes": 37748736 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 1671.7550000000952, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1589.7839999998428, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1412.5379999998395, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1349.7980000001553, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "vllm_rlt/worker/sampling.py(6): probabilities", - "count": 181, - "self_cpu_us": 1349.6579999995265, - "self_device_us": 0.0, - "self_device_memory_bytes": -35586048 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 1343.5690000002296, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1245.2050000000427, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - } - ] - }, - "tokens_unchanged": true - } -} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log deleted file mode 100644 index 17965cd3..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log +++ /dev/null @@ -1,40 +0,0 @@ -[transformers] You are using a model of type `ouro` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating. -USDT:2026-10-01 20:16:37 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:37 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:16:39 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:39 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:16:46 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:47 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:16:50 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:50 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:16:59 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:59 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:02 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:02 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -sync {'none': 0.08066848499993284, 'raw': 0.0914349139998194, 'processed': 0.08466345700026068} -USDT:2026-10-01 20:17:12 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:12 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:16 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:16 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:31 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:31 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:34 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:35 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:52 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:52 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:56 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:56 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -async {'none': 0.1108772420002424, 'raw': 0.1287477310002032, 'processed': 0.12239944100019784} -USDT:2026-10-01 20:18:14 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:14 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:16 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:16 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:22 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:22 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:25 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:25 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:32 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:32 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:35 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:35 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -speculative {'none': 0.06240307000007306, 'raw': 0.0662938270002087, 'processed': 0.06682750300024054} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json deleted file mode 100644 index 8d18ff49..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "removed": [ - "models/Ouro-minhao-full", - "minhao-fast-part-2" - ], - "bytes": 3232344846 -} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log deleted file mode 100644 index 665bd66e..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log +++ /dev/null @@ -1,12 +0,0 @@ -test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok -test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok -test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok -test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok -test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok -test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok -test_all_workers_commit_and_partial_commit_requires_new_epoch (tests.test_pd_weight_control.PDWeightControlTest.test_all_workers_commit_and_partial_commit_requires_new_epoch) ... ok - ----------------------------------------------------------------------- -Ran 7 tests in 1.736s - -OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log deleted file mode 100644 index 8eef87ad..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log +++ /dev/null @@ -1,11 +0,0 @@ -test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok -test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok -test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok -test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok -test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok -test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok - ----------------------------------------------------------------------- -Ran 6 tests in 135.474s - -OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log deleted file mode 100644 index ae3542fd..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log +++ /dev/null @@ -1,11 +0,0 @@ -test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok -test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok -test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok -test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok -test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok -test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok - ----------------------------------------------------------------------- -Ran 6 tests in 3.989s - -OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log deleted file mode 100644 index 855a3b10..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log +++ /dev/null @@ -1,19 +0,0 @@ -Kernel stack captured from PID 54942 during the observed driver wait on 2026-10-01: -[<0>] os_acquire_rwlock_write+0x38/0x64 [nvidia] -[<0>] portSyncRwLockAcquireWrite+0x10/0x30 [nvidia] -[<0>] rmapiLockAcquire+0x23c/0x340 [nvidia] -[<0>] _nvGpuOpsLocksAcquireWithClientLockFlags+0x98/0x1f0 [nvidia] -[<0>] dupMemory+0x88/0x690 [nvidia] -[<0>] nvUvmInterfaceDupMemory+0x34/0x40 [nvidia] -[<0>] uvm_map_external_allocation_on_gpu+0x1f0/0x3fc [nvidia_uvm] -[<0>] uvm_api_map_external_allocation+0x474/0x4e4 [nvidia_uvm] -[<0>] uvm_ioctl+0x117c/0x16a0 [nvidia_uvm] -[<0>] uvm_unlocked_ioctl_entry.part.0+0xdc/0x148 [nvidia_uvm] -[<0>] uvm_unlocked_ioctl_entry+0x78/0x9c [nvidia_uvm] -[<0>] __arm64_sys_ioctl+0xac/0xf0 -[<0>] invoke_syscall+0x48/0x114 -[<0>] el0_svc_common.constprop.0+0xc0/0xe0 -[<0>] do_el0_svc+0x1c/0x28 -[<0>] el0_svc+0x30/0xd8 -[<0>] el0t_64_sync_handler+0x120/0x12c -[<0>] el0t_64_sync+0x194/0x198 diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log deleted file mode 100644 index b59b66e8..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log +++ /dev/null @@ -1,42 +0,0 @@ -Thu Oct 1 12:03:16 UTC 2026 -[<0>] futex_wait_queue+0x6c/0x98 -[<0>] __futex_wait+0xcc/0x160 -[<0>] futex_wait+0x88/0x110 -[<0>] do_futex+0xf8/0x1a0 -[<0>] __arm64_sys_futex+0xe0/0x13c -[<0>] invoke_syscall+0x48/0x114 -[<0>] el0_svc_common.constprop.0+0xc0/0xe0 -[<0>] do_el0_svc+0x1c/0x28 -[<0>] el0_svc+0x30/0xd8 -[<0>] el0t_64_sync_handler+0x120/0x12c -[<0>] el0t_64_sync+0x194/0x198 -[ 1630.645566] NVRM: GPU0 refcntRequestReference_IMPL: Failed to enter state 1 (current state: 0, status: 0x00000056) -[ 2474.164009] NVRM: GPU0 refcntRequestReference_IMPL: Failed to enter state 1 (current state: 0, status: 0x00000056) -[ 2478.973583] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2479.334052] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2479.459146] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2480.205368] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2480.300412] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2480.606304] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2480.823594] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2481.003826] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2481.261606] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2481.325949] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2482.079631] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2482.152349] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2482.562933] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2482.686092] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2483.701671] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2483.788894] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2484.193315] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2484.306663] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2484.419459] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.152876] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.231614] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.661010] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.775288] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.971025] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2486.389956] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2486.451899] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2486.841379] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2486.904428] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv deleted file mode 100644 index 9c1a4867..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv +++ /dev/null @@ -1,2 +0,0 @@ -name, driver_version, memory.used [MiB], utilization.gpu [%] -NVIDIA Thor, 595.78, [N/A], 98 % diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json deleted file mode 100644 index 9ca1eca6..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json +++ /dev/null @@ -1,2741 +0,0 @@ -{ - "sync": { - "seconds": { - "none": [ - 1.797056486999736, - 1.8016931980000663, - 1.7812917080000261, - 1.8116626850001012, - 1.8313193210001373 - ], - "raw": [ - 1.8279988220001542, - 1.8234451930002251, - 1.810924204999992, - 1.8776579790001051, - 1.8599433469998985 - ], - "processed": [ - 1.8894610560000729, - 1.7770136750000347, - 1.8161140520001027, - 1.8662560849998044, - 1.8281205210000735 - ] - }, - "median_seconds": { - "none": 1.8016931980000663, - "raw": 1.8279988220001542, - "processed": 1.8281205210000735 - }, - "tokens_per_second": { - "none": 35.52214110096099, - "raw": 35.01096348080393, - "processed": 35.00863278149123 - }, - "peak_extra_allocated_bytes": { - "none": [ - 955392, - 955392, - 955392, - 955392, - 955392 - ], - "raw": [ - 988160, - 955392, - 988160, - 988160, - 955392 - ], - "processed": [ - 955392, - 955392, - 988160, - 955392, - 955392 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 3115292.1020000014, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 297542.58400000003, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 47216.64400000275, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6232, - "self_cpu_us": 46928.53299999721, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2724, - "self_cpu_us": 25592.631000006077, - "self_device_us": 7924.548000013914, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 10, - "self_cpu_us": 19917.72599999947, - "self_device_us": 2.1440000000875443, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16707.02599999922, - "self_device_us": 69237.73299999899, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2140, - "self_cpu_us": 16282.546999996724, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Activity Buffer Request", - "count": 11, - "self_cpu_us": 11960.367999999446, - "self_device_us": 3.072000000017397, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 9020.897999999828, - "self_device_us": 3486.1089999998353, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 384, - "self_cpu_us": 8604.2179999959, - "self_device_us": 1033.1409999988641, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1420, - "self_cpu_us": 7515.799999997356, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 800, - "self_cpu_us": 5476.198000001276, - "self_device_us": 164.86600000051476, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 5247.854999999913, - "self_device_us": 1053.979999999935, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2084, - "self_cpu_us": 5008.964999994238, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3880.791000000514, - "self_device_us": 2027.154000000004, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3850.2629999997826, - "self_device_us": 547.1479999999751, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 2068, - "self_cpu_us": 3678.7860000009327, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3572.775999995909, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4087, - "self_cpu_us": 3312.0390000030416, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 1714448.007000006, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 312503.2759999983, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 88477.1809999905, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaLaunchKernel", - "count": 6232, - "self_cpu_us": 45877.57399999528, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2724, - "self_cpu_us": 30653.4359999993, - "self_device_us": 8406.053999994654, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 17, - "self_cpu_us": 21553.933999999936, - "self_device_us": 113.12300000005052, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 21004.616999993712, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2140, - "self_cpu_us": 18041.15100000197, - "self_device_us": 0, - "self_device_memory_bytes": 120670208 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 15299.454999999656, - "self_device_us": 186853.4359999999, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 13051.557000005414, - "self_device_us": 0.0, - "self_device_memory_bytes": -1185792 - }, - { - "operator": "aten::_to_copy", - "count": 1420, - "self_cpu_us": 10733.110999994191, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 10510.103999998966, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 384, - "self_cpu_us": 9402.348000001257, - "self_device_us": 4816.147000000074, - "self_device_memory_bytes": 15486976 - }, - { - "operator": "aten::slice", - "count": 2084, - "self_cpu_us": 8692.440000006642, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 566, - "self_cpu_us": 8643.991000002294, - "self_device_us": 0.0, - "self_device_memory_bytes": -8029696 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 7680, - "self_cpu_us": 8545.6210000067, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 8124.672999999564, - "self_device_us": 15076.379999999674, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "Activity Buffer Request", - "count": 10, - "self_cpu_us": 7915.628000000057, - "self_device_us": 0.47999999999956344, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 7900.098000011858, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 7111.301000002073, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 1687563.1620000014, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 297471.37799999845, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6422, - "self_cpu_us": 38591.12499999725, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2916, - "self_cpu_us": 19720.34099999547, - "self_device_us": 4969.288999992985, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 11, - "self_cpu_us": 13895.465999999578, - "self_device_us": 2.6239999998360872, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2332, - "self_cpu_us": 13841.695999991633, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 12845.257000000787, - "self_device_us": 85966.08499999985, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 11226.992999998267, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 448, - "self_cpu_us": 7124.344999999742, - "self_device_us": 1127.6500000008473, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 6689.6719999999805, - "self_device_us": 3976.72400000012, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1548, - "self_cpu_us": 5447.389000004618, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 800, - "self_cpu_us": 4433.354999998611, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2086, - "self_cpu_us": 4181.640999996726, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 3946.4330000000346, - "self_device_us": 1249.5749999998784, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 5209, - "self_cpu_us": 3355.9950000011663, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 3335.78000000483, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3175.459000005998, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 2957.2929999998587, - "self_device_us": 2255.626000000015, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 2879.41300000012, - "self_device_us": 626.137999999959, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 2196, - "self_cpu_us": 2788.2449999992214, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 1712470.881999998, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 303259.42099999957, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 84949.75999997946, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 58914.68900000312, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 38, - "self_cpu_us": 55984.85400000001, - "self_device_us": 141.47600000002421, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6422, - "self_cpu_us": 39227.52699999942, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2916, - "self_cpu_us": 20689.506999994737, - "self_device_us": 8596.109999999157, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 17150.899999998324, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2332, - "self_cpu_us": 15443.806000003227, - "self_device_us": 0, - "self_device_memory_bytes": 133285888 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 14933.654000000508, - "self_device_us": 216397.0109999997, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 10250.292999994825, - "self_device_us": 0.0, - "self_device_memory_bytes": -1185792 - }, - { - "operator": "aten::_to_copy", - "count": 1548, - "self_cpu_us": 8689.791999999334, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 7680, - "self_cpu_us": 7914.010000004513, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2086, - "self_cpu_us": 7845.527000000181, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 7781.584000000366, - "self_device_us": 14088.113000000061, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "aten::cat", - "count": 448, - "self_cpu_us": 7187.911999997445, - "self_device_us": 1075.74999999817, - "self_device_memory_bytes": 15519744 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 6351.897999999905, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 566, - "self_cpu_us": 6237.029000000981, - "self_device_us": 0.0, - "self_device_memory_bytes": -8029696 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 6197.680000002206, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 7680, - "self_cpu_us": 5868.171000009923, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 1785390.3010000014, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 287422.5859999989, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 54, - "self_cpu_us": 132692.09599999993, - "self_device_us": 137.35000000000582, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 63230.2860000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6550, - "self_cpu_us": 41654.006999997655, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2852, - "self_cpu_us": 20811.043999990427, - "self_device_us": 5011.1320000056, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2268, - "self_cpu_us": 14158.613000006142, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 13688.341999998862, - "self_device_us": 237601.12799999924, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 448, - "self_cpu_us": 7512.926999999517, - "self_device_us": 1107.5330000010308, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 6855.801000000043, - "self_device_us": 3760.478000000774, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1548, - "self_cpu_us": 5959.337000000479, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 800, - "self_cpu_us": 4528.025000001053, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2086, - "self_cpu_us": 4388.27499998413, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4043.9099999990876, - "self_device_us": 6436.210999999603, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 5209, - "self_cpu_us": 3740.4920000093116, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3374.305000002234, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 3299.6420000069775, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3031.1940000002096, - "self_device_us": 5960.095999999843, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3006.850000000014, - "self_device_us": 629.8799999999992, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 2196, - "self_cpu_us": 3004.803000003427, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 1681418.688000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 313165.76199999935, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 89656.83900000324, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 55077.46799999963, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 46, - "self_cpu_us": 49129.840000000055, - "self_device_us": 177.34899999998743, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6550, - "self_cpu_us": 45109.648999988836, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2852, - "self_cpu_us": 23547.901000007907, - "self_device_us": 22245.57399999522, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 18967.310000011475, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2268, - "self_cpu_us": 16894.335999999996, - "self_device_us": 0, - "self_device_memory_bytes": 133253120 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16466.671999998278, - "self_device_us": 191312.69199999896, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 11592.744999999559, - "self_device_us": 0.0, - "self_device_memory_bytes": -1185792 - }, - { - "operator": "aten::_to_copy", - "count": 1548, - "self_cpu_us": 9718.056999999331, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 7680, - "self_cpu_us": 8876.85000001891, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 448, - "self_cpu_us": 8632.131000004554, - "self_device_us": 1104.7309999994868, - "self_device_memory_bytes": 15519744 - }, - { - "operator": "aten::slice", - "count": 2086, - "self_cpu_us": 8531.177000011521, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 7987.785000000618, - "self_device_us": 16808.661000000444, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 566, - "self_cpu_us": 7880.714999992513, - "self_device_us": 0.0, - "self_device_memory_bytes": -8029696 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 7180.626999999522, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 6991.26000000355, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 7680, - "self_cpu_us": 6713.435000007084, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - }, - "async": { - "seconds": { - "none": [ - 1.695322036000107, - 1.6974444380002751, - 1.7046040849995734, - 1.6946781169999667, - 1.7272834199998215 - ], - "raw": [ - 1.7682591719999436, - 1.7628239639998355, - 1.7849022200002764, - 1.7954341879999447, - 1.8388650159999997 - ], - "processed": [ - 1.7741621370000757, - 1.770364053999856, - 1.7893874629999118, - 1.8360151079996285, - 1.8289056349999555 - ] - }, - "median_seconds": { - "none": 1.6974444380002751, - "raw": 1.7849022200002764, - "processed": 1.7893874629999118 - }, - "tokens_per_second": { - "none": 37.70373778796383, - "raw": 35.856305898924866, - "processed": 35.766429196225545 - }, - "peak_extra_allocated_bytes": { - "none": [ - 988160, - 988160, - 988160, - 988160, - 988160 - ], - "raw": [ - 986624, - 988160, - 986624, - 986624, - 988160 - ], - "processed": [ - 988160, - 988160, - 986624, - 988160, - 988160 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaEventSynchronize", - "count": 502, - "self_cpu_us": 1432410.3530000027, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 191290.37999999942, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 44, - "self_cpu_us": 34835.79299999999, - "self_device_us": 351.92999999994936, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6078, - "self_cpu_us": 33813.03999999954, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16364.334999999637, - "self_device_us": 183683.25199999977, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 11429.879000001887, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2143, - "self_cpu_us": 11020.502999999995, - "self_device_us": 4541.596999996408, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 10356.752000000413, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 8018.803000000195, - "self_device_us": 3866.8100000002287, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3297, - "self_cpu_us": 6736.454999988346, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1393, - "self_cpu_us": 6702.344000001056, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4626.323999999812, - "self_device_us": 9519.472000000073, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 5523, - "self_cpu_us": 3716.362000009337, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3477.173000000108, - "self_device_us": 619.724000000142, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3425.1280000001398, - "self_device_us": 12465.012999999697, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 1, - "self_cpu_us": 3281.159, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 985, - "self_cpu_us": 2562.969999997975, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 981, - "self_cpu_us": 2548.3720000009953, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2241.14700000608, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 228, - "self_cpu_us": 2224.8430000000544, - "self_device_us": 774.4480000006351, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaEventSynchronize", - "count": 502, - "self_cpu_us": 1334775.7140000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 213300.00299999965, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 37214.26000000528, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6078, - "self_cpu_us": 31441.57300000226, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16664.894000000244, - "self_device_us": 184634.85100000046, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "Command Buffer Full", - "count": 34, - "self_cpu_us": 16638.358999999953, - "self_device_us": 83.08099999994738, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 658, - "self_cpu_us": 14962.963000003772, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3297, - "self_cpu_us": 12126.546999992546, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 11609.655999997276, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 11161.335999998177, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2143, - "self_cpu_us": 10901.848999998805, - "self_device_us": 8184.701999998565, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 10622.565999999239, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 8326.869000005598, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 8074.22300000026, - "self_device_us": 0.0, - "self_device_memory_bytes": -79872 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 7691.808999999961, - "self_device_us": 7839.375999999642, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "", - "count": 658, - "self_cpu_us": 7110.883000002672, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1393, - "self_cpu_us": 7029.449000000028, - "self_device_us": 0, - "self_device_memory_bytes": 119249408 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 6454.799000000135, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 7680, - "self_cpu_us": 5952.708999990169, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 5775.1769999995595, - "self_device_us": 0.0, - "self_device_memory_bytes": -169984 - } - ], - "raw-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 865738.5560000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 440, - "self_cpu_us": 513537.4239999985, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 269982.26400000206, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6268, - "self_cpu_us": 37557.50699999495, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 19496.538000000193, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 15068.10300000109, - "self_device_us": 89085.99599999975, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2335, - "self_cpu_us": 15040.314000005079, - "self_device_us": 4926.857000000852, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 11644.150999997, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3299, - "self_cpu_us": 9157.908999995387, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1585, - "self_cpu_us": 9157.521999997385, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 7889.426999999985, - "self_device_us": 3726.571000000061, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 6645, - "self_cpu_us": 4706.0270000029495, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4578.1169999999665, - "self_device_us": 1194.1009999999906, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 1045, - "self_cpu_us": 3393.951000005082, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1113, - "self_cpu_us": 3387.474999996231, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3383.8789999999763, - "self_device_us": 2079.0089999998263, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 292, - "self_cpu_us": 3383.5519999986764, - "self_device_us": 885.724000001198, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3352.952999999814, - "self_device_us": 585.2520000001759, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3030.2580000000307, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::rsqrt", - "count": 388, - "self_cpu_us": 2112.1670000002123, - "self_device_us": 558.6339999998054, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 894514.4109999985, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 438, - "self_cpu_us": 356624.633000003, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 282386.8210000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 61673.15100000108, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 57640.45999999429, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6268, - "self_cpu_us": 37907.111000003744, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 658, - "self_cpu_us": 20129.23500000021, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 17286.161000000488, - "self_device_us": 209501.60500000042, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "Command Buffer Full", - "count": 24, - "self_cpu_us": 15990.03599999995, - "self_device_us": 94.47799999994459, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2335, - "self_cpu_us": 15931.423000005114, - "self_device_us": 5127.309000006906, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 15294.889999997744, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3299, - "self_cpu_us": 15165.398000015914, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 12728.342999995075, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 11771.589000000844, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1585, - "self_cpu_us": 10317.796999996372, - "self_device_us": 0, - "self_device_memory_bytes": 131865088 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 9568.225999996474, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "", - "count": 658, - "self_cpu_us": 8962.633000000906, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 8289.213999999814, - "self_device_us": 9734.966999999577, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 7695.529999998893, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 7675.931000003035, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 920932.4780000017, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 439, - "self_cpu_us": 482884.0049999996, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 305185.86299999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 73606.65199999898, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 58, - "self_cpu_us": 70443.25700000004, - "self_device_us": 161.2559999999503, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6396, - "self_cpu_us": 39018.222000004345, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2271, - "self_cpu_us": 15692.35799999762, - "self_device_us": 4989.233000002182, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 14275.767999999229, - "self_device_us": 203367.5870000003, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 12591.590999998225, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3299, - "self_cpu_us": 10243.788999999848, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1521, - "self_cpu_us": 8787.994000000632, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 6978.011999999464, - "self_device_us": 7834.872999999812, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 6645, - "self_cpu_us": 5005.443999995054, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4159.8260000002265, - "self_device_us": 1241.8309999997346, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 1045, - "self_cpu_us": 3796.858999999785, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1113, - "self_cpu_us": 3539.445999995534, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 292, - "self_cpu_us": 3381.941999998534, - "self_device_us": 891.852000000601, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3376.4410000027856, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3101.857999999801, - "self_device_us": 2076.3349999998336, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3023.192999999952, - "self_device_us": 613.0609999998796, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 860792.1709999986, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 439, - "self_cpu_us": 413039.0830000005, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 255455.0850000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 49905.24699999363, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6396, - "self_cpu_us": 37927.29899999841, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 658, - "self_cpu_us": 17880.09100000289, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16554.555999998465, - "self_device_us": 89435.19800000006, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 15447.242000002167, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2271, - "self_cpu_us": 13614.053999995802, - "self_device_us": 4926.6889999996565, - "self_device_memory_bytes": -1024 - }, - { - "operator": "aten::slice", - "count": 3299, - "self_cpu_us": 13604.380000005436, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 13420.60000000347, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 12200.938999997265, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 10094.503999997636, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1521, - "self_cpu_us": 8711.722999999149, - "self_device_us": 0, - "self_device_memory_bytes": 131832320 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 8518.236999999934, - "self_device_us": 3812.4549999999153, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 8272.147000002355, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "", - "count": 658, - "self_cpu_us": 8045.141999997961, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 6979.725999999505, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 6839.281000003541, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 6694.447999993816, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - }, - "speculative": { - "seconds": { - "none": [ - 1.4405976819998614, - 1.4354998989997512, - 1.4359760199999982, - 1.43785706799963, - 1.436613651999778 - ], - "raw": [ - 1.446159684000122, - 1.4462237470002037, - 1.4429731870000069, - 1.447812623000118, - 1.447334557999966 - ], - "processed": [ - 1.4485182270000223, - 1.4616416380004011, - 1.445917856000051, - 1.449624680999932, - 1.4485653670003558 - ] - }, - "median_seconds": { - "none": 1.436613651999778, - "raw": 1.4462237470002037, - "processed": 1.4485653670003558 - }, - "tokens_per_second": { - "none": 44.54920772255747, - "raw": 44.25318014086722, - "processed": 44.181644444895994 - }, - "peak_extra_allocated_bytes": { - "none": [ - 2799104, - 2799104, - 2799104, - 2799104, - 2799104 - ], - "raw": [ - 2860544, - 2827776, - 2860544, - 2860544, - 2827776 - ], - "processed": [ - 2827776, - 2827776, - 2860544, - 2827776, - 2827776 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 980, - "self_cpu_us": 1013133.5880000024, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 329889.5450000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6440, - "self_cpu_us": 44799.10400000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 14830.692999999728, - "self_device_us": 57957.05499999999, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1858, - "self_cpu_us": 11904.329999998092, - "self_device_us": 3719.296999995492, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 10062.069999999801, - "self_device_us": 3642.09899999938, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1349, - "self_cpu_us": 8686.187999998147, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 5437.313999999343, - "self_device_us": 1142.796999999984, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 5328.228999999939, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 4195.546000000275, - "self_device_us": 2040.650999999802, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 4130.804000000057, - "self_device_us": 541.9709999999882, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1036, - "self_cpu_us": 3498.193000000953, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 3478.147999996989, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1598, - "self_cpu_us": 3344.8489999989542, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 247, - "self_cpu_us": 3218.2849999998366, - "self_device_us": 773.1339999996012, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3629, - "self_cpu_us": 2757.4049999993445, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 204, - "self_cpu_us": 2495.4719999990484, - "self_device_us": 344.9210000009334, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::rsqrt", - "count": 388, - "self_cpu_us": 2447.126000000311, - "self_device_us": 568.7279999999582, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_index_put_impl_", - "count": 192, - "self_cpu_us": 2249.8519999999753, - "self_device_us": 1397.437999999971, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 134, - "self_cpu_us": 2245.1280000006864, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 980, - "self_cpu_us": 1004743.099000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 341942.8800000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 98, - "self_cpu_us": 65974.60500000956, - "self_device_us": 0.0, - "self_device_memory_bytes": -122880 - }, - { - "operator": "cudaLaunchKernel", - "count": 6440, - "self_cpu_us": 44354.85499999292, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 16, - "self_cpu_us": 43468.00100000008, - "self_device_us": 2872.282999999996, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 9, - "self_cpu_us": 17052.98900000006, - "self_device_us": 2.4319999997969717, - "self_device_memory_bytes": 122880 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 16737.56800000057, - "self_device_us": 133078.352, - "self_device_memory_bytes": 74514432 - }, - { - "operator": "aten::copy_", - "count": 1858, - "self_cpu_us": 12988.228999999212, - "self_device_us": 7612.869000000108, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 53, - "self_cpu_us": 12837.078999994766, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 10, - "self_cpu_us": 11424.281000001734, - "self_device_us": 0.0, - "self_device_memory_bytes": -41080832 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 10870.521999998786, - "self_device_us": 14274.18699999967, - "self_device_memory_bytes": 133641216 - }, - { - "operator": "aten::empty_strided", - "count": 1349, - "self_cpu_us": 9551.200000001178, - "self_device_us": 0, - "self_device_memory_bytes": 146153984 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 8481.077000000107, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 7766.478999998973, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "aten::slice", - "count": 1598, - "self_cpu_us": 7758.273000003454, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 9024, - "self_cpu_us": 6788.078000001582, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 6336, - "self_cpu_us": 6656.0759999922775, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", - "count": 96, - "self_cpu_us": 6457.112000000272, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 134, - "self_cpu_us": 6235.574999999869, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1036, - "self_cpu_us": 6220.435999998934, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1070, - "self_cpu_us": 1064068.1040000021, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 361931.5290000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 48, - "self_cpu_us": 104276.37, - "self_device_us": 164.09299999989162, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 158, - "self_cpu_us": 50917.7819999979, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6588, - "self_cpu_us": 44249.33099999894, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 9, - "self_cpu_us": 15985.770999999775, - "self_device_us": 3.1359999999403954, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2008, - "self_cpu_us": 13097.013999997875, - "self_device_us": 14456.707000001214, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 11822.664999999748, - "self_device_us": 173903.50500000032, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1499, - "self_cpu_us": 9125.855000001111, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 8022.662999998281, - "self_device_us": 14237.222000000518, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 4792.803000000362, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 4564.072000000597, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4263.941999999985, - "self_device_us": 1244.1820000001771, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1122, - "self_cpu_us": 3775.3999999982043, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 269, - "self_cpu_us": 3679.534000000115, - "self_device_us": 4876.952999999159, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1600, - "self_cpu_us": 3253.3939999918493, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3200.0180000000037, - "self_device_us": 5880.0210000000625, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3172.9669999997113, - "self_device_us": 606.8459999999995, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4139, - "self_cpu_us": 3093.79700000462, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 204, - "self_cpu_us": 2634.8199999991048, - "self_device_us": 346.6839999984659, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1070, - "self_cpu_us": 921827.2790000035, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 348470.8909999982, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 98, - "self_cpu_us": 82054.98900000838, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaLaunchKernel", - "count": 6588, - "self_cpu_us": 43856.03299999783, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 14546.498999999756, - "self_device_us": 58689.897, - "self_device_memory_bytes": 74514432 - }, - { - "operator": "aten::copy_", - "count": 2008, - "self_cpu_us": 13351.863999997171, - "self_device_us": 4054.520999999536, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 53, - "self_cpu_us": 13121.481000001226, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 10, - "self_cpu_us": 11731.000000006607, - "self_device_us": 0.0, - "self_device_memory_bytes": -41080832 - }, - { - "operator": "aten::empty_strided", - "count": 1499, - "self_cpu_us": 10030.803999998225, - "self_device_us": 0, - "self_device_memory_bytes": 158769664 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 9924.678000001335, - "self_device_us": 3753.6830000002865, - "self_device_memory_bytes": 133641216 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 7820.23400000037, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "aten::slice", - "count": 1600, - "self_cpu_us": 6671.56400000199, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 6336, - "self_cpu_us": 6460.941999995901, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1122, - "self_cpu_us": 6371.330999999439, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 9024, - "self_cpu_us": 6215.4499999884265, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", - "count": 96, - "self_cpu_us": 6019.520000000246, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/nn/modules/module.py(1782): _call_impl", - "count": 1397, - "self_cpu_us": 5992.312000000209, - "self_device_us": 0.0, - "self_device_memory_bytes": -99717120 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 5270.968999999644, - "self_device_us": 1170.8770000002514, - "self_device_memory_bytes": 28510208 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 5207.893999999975, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 4842.231000002561, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1006, - "self_cpu_us": 1086286.7079999999, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 366847.06800000044, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 48, - "self_cpu_us": 81405.13199999991, - "self_device_us": 132.25700000009965, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 158, - "self_cpu_us": 49747.239000002395, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6716, - "self_cpu_us": 44376.60800000187, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 9, - "self_cpu_us": 15859.352999999945, - "self_device_us": 2.464000000152737, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1944, - "self_cpu_us": 13164.344999999565, - "self_device_us": 7839.070999998783, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 12993.734999999626, - "self_device_us": 152600.5239999997, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1435, - "self_cpu_us": 8864.449999998058, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 8686.781999999766, - "self_device_us": 25866.910000000396, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 6254.039000003366, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4607.503999999629, - "self_device_us": 1228.5170000003036, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 4396.7750000001415, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1122, - "self_cpu_us": 4098.707000003469, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 269, - "self_cpu_us": 3794.4410000006155, - "self_device_us": 858.2720000006775, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3510.004999999982, - "self_device_us": 5943.891000000016, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1600, - "self_cpu_us": 3501.2590000032596, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3423.5350000000344, - "self_device_us": 600.6419999997925, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 268, - "self_cpu_us": 3386.080000000744, - "self_device_us": 443.71500000028755, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4139, - "self_cpu_us": 3143.4790000002977, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1006, - "self_cpu_us": 923970.0769999987, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 349496.5669999999, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 98, - "self_cpu_us": 79725.7660000025, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaLaunchKernel", - "count": 6716, - "self_cpu_us": 45678.692999995874, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 15432.809000000572, - "self_device_us": 59057.668000000325, - "self_device_memory_bytes": 74514432 - }, - { - "operator": "aten::copy_", - "count": 1944, - "self_cpu_us": 13113.157000004554, - "self_device_us": 3976.40699999536, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 53, - "self_cpu_us": 12922.643000007964, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 10, - "self_cpu_us": 11522.10300000626, - "self_device_us": 0.0, - "self_device_memory_bytes": -41080832 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 10494.68399999901, - "self_device_us": 3770.354000000312, - "self_device_memory_bytes": 133641216 - }, - { - "operator": "aten::empty_strided", - "count": 1435, - "self_cpu_us": 9768.620999997758, - "self_device_us": 0, - "self_device_memory_bytes": 158736896 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 8493.670999999964, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", - "count": 96, - "self_cpu_us": 6569.988999999679, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1600, - "self_cpu_us": 6520.582000002687, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1122, - "self_cpu_us": 6445.759999999112, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 117, - "self_cpu_us": 6329.600000002482, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 6336, - "self_cpu_us": 6263.172000000794, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/nn/modules/module.py(1782): _call_impl", - "count": 1397, - "self_cpu_us": 6239.434000000041, - "self_device_us": 0.0, - "self_device_memory_bytes": -99717120 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 9024, - "self_cpu_us": 6183.9280000038, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 5625.210000000348, - "self_device_us": 1181.320999999678, - "self_device_memory_bytes": 28510208 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 5410.53999999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - } -} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log deleted file mode 100644 index b108ece5..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log +++ /dev/null @@ -1,40 +0,0 @@ -[transformers] You are using a model of type `ouro` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating. -USDT:2026-10-01 19:52:11 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:52:14 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:52:32 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:52:45 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:53:16 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:53:18 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:53:32 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:53:36 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:54:03 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:54:20 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:54:40 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:54:45 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -sync {'none': 1.8016931980000663, 'raw': 1.8279988220001542, 'processed': 1.8281205210000735} -USDT:2026-10-01 20:02:51 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:03:17 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:03:43 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:03:49 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:04:25 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:04:51 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:05:18 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:05:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:06:14 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:06:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:06:52 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:07:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -async {'none': 1.6974444380002751, 'raw': 1.7849022200002764, 'processed': 1.7893874629999118} -USDT:2026-10-01 20:11:40 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:12:07 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:12:35 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:12:37 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:13:07 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:13:09 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:13:50 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:14:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:14:49 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:14:51 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:15:33 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:01 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -speculative {'none': 1.436613651999778, 'raw': 1.4462237470002037, 'processed': 1.4485653670003558} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log deleted file mode 100644 index 1a90a902..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log +++ /dev/null @@ -1,14 +0,0 @@ -test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... FAIL - -====================================================================== -FAIL: test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/home/jwipc/experiments/ouro-vime-20260930/rlt-training-contract-20261001/tests/test_training_contract.py", line 190, in test_entropy_seed_is_reported_and_replays_the_same_policy - self.assertIsInstance(output.sampling_params.seed, int) -AssertionError: None is not an instance of - ----------------------------------------------------------------------- -Ran 1 test in 0.067s - -FAILED (failures=1) diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json deleted file mode 100644 index b0c05fe2..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "removed": "models/Ouro-1.4B-parallel", - "bytes": 2873677792 -} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json deleted file mode 100644 index 96acb36b..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json +++ /dev/null @@ -1,2669 +0,0 @@ -{ - "sync": { - "seconds": { - "none": [ - 0.09519798499991339 - ], - "raw": [ - 0.11991658300007657 - ], - "processed": [ - 0.11633255899982942 - ] - }, - "median_seconds": { - "none": 0.09519798499991339, - "raw": 0.11991658300007657, - "processed": 0.11633255899982942 - }, - "tokens_per_second": { - "none": 672.2831370859186, - "raw": 533.7043334528564, - "processed": 550.1469283426821 - }, - "peak_extra_allocated_bytes": { - "none": [ - 758784 - ], - "raw": [ - 823808 - ], - "processed": [ - 791040 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 33297.56899999986, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 19800.59699999992, - "self_device_us": 11.039000000033411, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2004, - "self_cpu_us": 19130.309000000445, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2020, - "self_cpu_us": 9735.057999999846, - "self_device_us": 3466.580000000014, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1348, - "self_cpu_us": 6787.842999999959, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Activity Buffer Request", - "count": 3, - "self_cpu_us": 4397.798999999917, - "self_device_us": 17.24599999999691, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1556, - "self_cpu_us": 2589.577999999135, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2440.773999999882, - "self_device_us": 1127.5460000000476, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 208, - "self_cpu_us": 2438.501999999814, - "self_device_us": 416.11600000010685, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 2014.048999999999, - "self_device_us": 681.3790000000336, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 716, - "self_cpu_us": 2006.413000000081, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1787.3960000001898, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 1310.3229999998512, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1012, - "self_cpu_us": 1157.6350000001703, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 1153.9850000001802, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1078.1530000000166, - "self_device_us": 288.4610000001521, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1031.721999999896, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2415, - "self_cpu_us": 989.9190000005906, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 935.0519999999015, - "self_device_us": 286.7849999999162, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 64, - "self_cpu_us": 901.8880000001336, - "self_device_us": 541.9650000000875, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 96740.21499999995, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 29828.6660000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 19172.10899999903, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2004, - "self_cpu_us": 16537.040000000692, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 15321.057999998613, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2020, - "self_cpu_us": 13304.200000000319, - "self_device_us": 9814.611000000457, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 9246.910000000378, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1348, - "self_cpu_us": 8824.270000000583, - "self_device_us": 0, - "self_device_memory_bytes": 26818560 - }, - { - "operator": "(2): __init__", - "count": 780, - "self_cpu_us": 7972.586999999223, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 7952.3179999991335, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "torch/autograd/grad_mode.py(290): __enter__", - "count": 312, - "self_cpu_us": 5820.944000000607, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1556, - "self_cpu_us": 5617.165999999186, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 156, - "self_cpu_us": 5495.474999999722, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", - "count": 8, - "self_cpu_us": 4696.361000000001, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 716, - "self_cpu_us": 4452.799999999923, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 4376.564999998955, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 4297.467999999419, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(287): _make_scheduled_item", - "count": 376, - "self_cpu_us": 4069.740999999527, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Activity Buffer Request", - "count": 2, - "self_cpu_us": 4064.9590000000353, - "self_device_us": 8.286999999976615, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 3937.9390000010226, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 108719.8060000003, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 24011.792999999805, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2194, - "self_cpu_us": 15252.708000000432, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 13507.98999999966, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2212, - "self_cpu_us": 11690.644999998814, - "self_device_us": 8416.399999999961, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1540, - "self_cpu_us": 7301.129000000332, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 4101.939999999672, - "self_device_us": 1255.3850000000248, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 2997.545000000003, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2510.9329999999395, - "self_device_us": 2639.002999999942, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 2464.008999998726, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2282.966000000388, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2134.255999999961, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1791.3630000006428, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3537, - "self_cpu_us": 1679.8659999990111, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 624, - "self_cpu_us": 1648.1559999993642, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1140, - "self_cpu_us": 1481.7340000004865, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1324.1299999998773, - "self_device_us": 683.1499999999578, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1257.5680000000211, - "self_device_us": 1779.3249999999916, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1212.7460000002857, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 1187.963000000027, - "self_device_us": 634.3850000001239, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 101858.63600000048, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 19925.880999999914, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2194, - "self_cpu_us": 13766.511999999813, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 13020.369000001338, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "aten::copy_", - "count": 2212, - "self_cpu_us": 10548.945999998652, - "self_device_us": 9764.56400000026, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 10036.862999999756, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 8070.916999999871, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1540, - "self_cpu_us": 6907.167000000017, - "self_device_us": 0, - "self_device_memory_bytes": 39434240 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 5716.320000000458, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 3870.769999999612, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 3773.934000000846, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 3767.767999999496, - "self_device_us": 1359.2739999999485, - "self_device_memory_bytes": 337920 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 3737.9029999998384, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 3170.4539999998087, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 3001.5509999997157, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 2973.905000000177, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 2791.1890000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2571.306999999335, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2479.8980000000774, - "self_device_us": 2988.58499999993, - "self_device_memory_bytes": 6881280 - }, - { - "operator": "aten::clone", - "count": 624, - "self_cpu_us": 2234.690999999897, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 97750.7149999996, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 22683.500000000073, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2322, - "self_cpu_us": 20361.571000000113, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 13013.803999999898, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2148, - "self_cpu_us": 11927.50900000012, - "self_device_us": 8301.904999999215, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1476, - "self_cpu_us": 6909.296999999731, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 4398.034999999721, - "self_device_us": 1229.2659999998846, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 3254.1110000007093, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2565.4240000000195, - "self_device_us": 2622.8850000000243, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 2342.3889999999733, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2110.396999999586, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2086.5779999999377, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3537, - "self_cpu_us": 1777.3179999996833, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 192, - "self_cpu_us": 1635.8050000000076, - "self_device_us": 899.7680000000564, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1635.431000000337, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1140, - "self_cpu_us": 1513.9589999994594, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 1491.8950000000768, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1314.6709999997838, - "self_device_us": 686.7049999998162, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1258.967999999892, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1243.792000000154, - "self_device_us": 1745.1790000000492, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 103497.714000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 20846.46299999978, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2322, - "self_cpu_us": 15129.085999998966, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 13175.480000000473, - "self_device_us": 0.0, - "self_device_memory_bytes": -1024 - }, - { - "operator": "aten::copy_", - "count": 2148, - "self_cpu_us": 10448.986999999888, - "self_device_us": 10015.051999999127, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 10255.892999999633, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 8545.498999999993, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1476, - "self_cpu_us": 6776.009000000365, - "self_device_us": 0, - "self_device_memory_bytes": 39401472 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 5816.4070000001375, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 3927.1879999986068, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 3912.425999998626, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 3884.787000000045, - "self_device_us": 1454.8620000000992, - "self_device_memory_bytes": 337920 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 3826.4770000002713, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 3287.5009999989925, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 3132.696000000161, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 3023.1649999991896, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2872.8629999997174, - "self_device_us": 3133.3969999999745, - "self_device_memory_bytes": 6881280 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 2871.759000000642, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2640.3929999992906, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 2423.540000000653, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - }, - "async": { - "seconds": { - "none": [ - 0.11112063099994884 - ], - "raw": [ - 0.12377583400007097 - ], - "processed": [ - 0.12324414399995476 - ] - }, - "median_seconds": { - "none": 0.11112063099994884, - "raw": 0.12377583400007097, - "processed": 0.12324414399995476 - }, - "tokens_per_second": { - "none": 575.9506531242562, - "raw": 517.0637751466356, - "processed": 519.2944502095247 - }, - "peak_extra_allocated_bytes": { - "none": [ - 790016 - ], - "raw": [ - 822272 - ], - "processed": [ - 822272 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 16279.262000000004, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 1850, - "self_cpu_us": 8762.216999999695, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 6619.0009999997255, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1439, - "self_cpu_us": 5515.489000000218, - "self_device_us": 2130.588000000116, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2769, - "self_cpu_us": 4635.025999999778, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 4584.934999999944, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 601, - "self_cpu_us": 2461.7410000001278, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2147.010999999959, - "self_device_us": 809.0290000000505, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3851, - "self_cpu_us": 1570.322000000206, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1314.6100000000988, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 629, - "self_cpu_us": 1276.088999999649, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 990.0759999999482, - "self_device_us": 233.5900000000729, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 940.6050000001692, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 864.0409999999702, - "self_device_us": 601.2849999999398, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 782.1960000001072, - "self_device_us": 233.108000000002, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::record_stream", - "count": 436, - "self_cpu_us": 745.8910000001142, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 116, - "self_cpu_us": 660.8229999999921, - "self_device_us": 266.3260000000107, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 281, - "self_cpu_us": 619.7590000002324, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventRecordWithFlags", - "count": 698, - "self_cpu_us": 613.4480000000897, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 64, - "self_cpu_us": 609.95099999999, - "self_device_us": 486.4459999999799, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 17218.51800000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 9886.790999998972, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9667.236999999815, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 1850, - "self_cpu_us": 9417.455000000573, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8362.155000000257, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 7480.238000000069, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2769, - "self_cpu_us": 7326.50600000022, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6628.762999999737, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1439, - "self_cpu_us": 6172.138999999667, - "self_device_us": 7529.890999999854, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5232.246999999894, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 4782.417000000076, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4636.270000000101, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4530.803999999982, - "self_device_us": 0.0, - "self_device_memory_bytes": -79872 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4478.493999999459, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 3908.4060000003046, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3038.3940000000453, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 601, - "self_cpu_us": 2837.559999999785, - "self_device_us": 0, - "self_device_memory_bytes": 26515968 - }, - { - "operator": "", - "count": 904, - "self_cpu_us": 2774.022999999871, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/__init__.py(1235): current_stream", - "count": 1126, - "self_cpu_us": 2606.9919999999374, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2567.98200000016, - "self_device_us": 3231.842000000015, - "self_device_memory_bytes": 6881280 - } - ], - "raw-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 35043.07600000008, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 27261.345000000074, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 26212.683000000343, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2040, - "self_cpu_us": 17226.433999998942, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 9567.590000000226, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1631, - "self_cpu_us": 7822.165999999748, - "self_device_us": 7616.087999999582, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 5809.952999998892, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 793, - "self_cpu_us": 3797.753000000337, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2782.3519999998025, - "self_device_us": 2836.7000000000608, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2576.1720000005644, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1843.8050000004368, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 693, - "self_cpu_us": 1734.2529999999033, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 116, - "self_cpu_us": 1597.180999999936, - "self_device_us": 645.6439999999984, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 409, - "self_cpu_us": 1295.4429999995746, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1207.970999999644, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1207.7140000000436, - "self_device_us": 744.3319999997984, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 1078.0610000000052, - "self_device_us": 684.6229999998395, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 96, - "self_cpu_us": 1074.36400000082, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1062.3739999999489, - "self_device_us": 1878.7700000000477, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 1000.1569999997646, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 20903.29400000006, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 18533.75499999998, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 11627.489999999798, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2040, - "self_cpu_us": 11486.406999999133, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10235.82400000188, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9755.430000000004, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 8791.236000000776, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 8558.017999999998, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8461.214000000378, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1631, - "self_cpu_us": 7077.14399999875, - "self_device_us": 7515.524999999724, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6816.739999999478, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5592.861000000013, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4809.259999999384, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4741.460000002217, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4577.247000000707, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 4044.835999999861, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "aten::empty_strided", - "count": 793, - "self_cpu_us": 3699.4469999996372, - "self_device_us": 0, - "self_device_memory_bytes": 39131648 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 3092.7339999997967, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3059.046000000157, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2879.6480000001075, - "self_device_us": 2809.507000000074, - "self_device_memory_bytes": 6881280 - } - ], - "processed-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 40731.565000000046, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 18052.8380000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2168, - "self_cpu_us": 11742.620000000232, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 10820.819999999676, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1567, - "self_cpu_us": 6499.077000000206, - "self_device_us": 8344.468000000634, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5100.179999999666, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 4953.838000000167, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 729, - "self_cpu_us": 2996.4449999999683, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2412.3269999999034, - "self_device_us": 3132.7200000000075, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2234.7580000002854, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1542.5300000003408, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 693, - "self_cpu_us": 1528.0179999995985, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 192, - "self_cpu_us": 1305.0059999999976, - "self_device_us": 1111.3880000001209, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 116, - "self_cpu_us": 1274.8779999999783, - "self_device_us": 721.4030000000266, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1105.1659999998597, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1061.8390000001273, - "self_device_us": 851.0689999999304, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 409, - "self_cpu_us": 1038.5240000001413, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 956.9540000000325, - "self_device_us": 2173.455000000007, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 96, - "self_cpu_us": 929.1470000001209, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 908.5779999999498, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 19562.78000000015, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 12463.96400000043, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2168, - "self_cpu_us": 11787.197999999762, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10282.645999998535, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9901.121999998875, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 9359.13800000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8651.15399999867, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 7960.596999999834, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1567, - "self_cpu_us": 6933.717000000461, - "self_device_us": 7324.400999999833, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6819.291999999174, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5430.379000000365, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 5054.977999999375, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4980.245000000294, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4827.424999998944, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4674.382999999931, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 4062.3459999991246, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "aten::empty_strided", - "count": 729, - "self_cpu_us": 3426.792000000013, - "self_device_us": 0, - "self_device_memory_bytes": 39098880 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3211.9410000003777, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 3122.0590000002553, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2800.17300000012, - "self_device_us": 2851.515000000054, - "self_device_memory_bytes": 6881280 - } - ] - }, - "tokens_unchanged": true - }, - "speculative": { - "seconds": { - "none": [ - 0.0662261480001689 - ], - "raw": [ - 0.06795481500012102 - ], - "processed": [ - 0.06800570499990499 - ] - }, - "median_seconds": { - "none": 0.0662261480001689, - "raw": 0.06795481500012102, - "processed": 0.06800570499990499 - }, - "tokens_per_second": { - "none": 966.385663859489, - "raw": 941.8022843544792, - "processed": 941.0975152759523 - }, - "peak_extra_allocated_bytes": { - "none": [ - 2750976 - ], - "raw": [ - 2790912 - ], - "processed": [ - 2774528 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 14250.309000000101, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2245, - "self_cpu_us": 10531.197000000051, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1128, - "self_cpu_us": 8562.952999999969, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1277, - "self_cpu_us": 3761.6589999999446, - "self_device_us": 1806.0290000000139, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 143, - "self_cpu_us": 2708.568000000006, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 592, - "self_cpu_us": 2201.785999999958, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1515.5650000001278, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1219, - "self_cpu_us": 1244.1550000002997, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1168.2059999999747, - "self_device_us": 362.9779999999464, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1026.8629999999969, - "self_device_us": 156.73599999999442, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 995.331999999973, - "self_device_us": 317.34799999996153, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 917.5149999999649, - "self_device_us": 683.0130000000008, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 843.7139999999272, - "self_device_us": 172.83400000001166, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 790.9519999999757, - "self_device_us": 621.252999999977, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 77, - "self_cpu_us": 775.0059999999964, - "self_device_us": 124.67199999998161, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2113, - "self_cpu_us": 760.4689999999889, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 759.0639999997917, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 756.3700000000226, - "self_device_us": 1531.8530000000064, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 348, - "self_cpu_us": 708.1709999999481, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_softmax", - "count": 119, - "self_cpu_us": 698.9639999999526, - "self_device_us": 1821.2669999999434, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1128, - "self_cpu_us": 50386.84900000042, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 18524.925000000065, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2245, - "self_cpu_us": 12642.707000000097, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 10299.16400000074, - "self_device_us": 0.0, - "self_device_memory_bytes": -1536 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6868.614000000468, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 143, - "self_cpu_us": 6569.547999999943, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 5634.887000000228, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1277, - "self_cpu_us": 4802.597999998776, - "self_device_us": 6307.480999999992, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 592, - "self_cpu_us": 3066.6099999999415, - "self_device_us": 0, - "self_device_memory_bytes": 52925952 - }, - { - "operator": "aten::slice", - "count": 1219, - "self_cpu_us": 2758.779000000186, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2436.3480000002855, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 2412.122000000076, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 2321.6109999999353, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1901.2210000000298, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2113, - "self_cpu_us": 1567.9659999997698, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1554.9709999999468, - "self_device_us": 1259.2209999998413, - "self_device_memory_bytes": 25165824 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1469.638999999961, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - }, - { - "operator": "aten::_to_copy", - "count": 348, - "self_cpu_us": 1457.5210000002992, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1289.5729999998493, - "self_device_us": 1402.8940000000239, - "self_device_memory_bytes": 1404416 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 1265.5860000001667, - "self_device_us": 2400.320000000047, - "self_device_memory_bytes": 79872 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1219, - "self_cpu_us": 32791.54100000013, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 17470.00599999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2394, - "self_cpu_us": 12090.505000000023, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 4819.1389999999865, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1428, - "self_cpu_us": 4432.517000000222, - "self_device_us": 4649.904999999926, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 743, - "self_cpu_us": 2811.108999999853, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1714.3729999998868, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 1327.7300000001896, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1264.6509999999944, - "self_device_us": 774.6999999999425, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 100, - "self_cpu_us": 1105.010999999929, - "self_device_us": 438.56399999991436, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1051.4090000000483, - "self_device_us": 1278.6749999999424, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1050.9589999995637, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1045.2189999999996, - "self_device_us": 741.7150000000001, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 980.1500000000906, - "self_device_us": 1307.7370000000465, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 938.0559999998939, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 902.338000000078, - "self_device_us": 333.67199999995864, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 879.014999999983, - "self_device_us": 1192.532999999974, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 833.2999999999356, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 790.916999999994, - "self_device_us": 2837.7279999999555, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 769.0820000001117, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 17014.585000000065, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1219, - "self_cpu_us": 15616.989000000358, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2394, - "self_cpu_us": 12305.728000000006, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9651.556000000543, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6259.108999999342, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 4917.951999999987, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1428, - "self_cpu_us": 4794.027999999778, - "self_device_us": 4887.427999999511, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 4222.749999999923, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 743, - "self_cpu_us": 3183.7439999999515, - "self_device_us": 0, - "self_device_memory_bytes": 65541632 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 2523.371999999762, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2202.935999999663, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 2086.883999999831, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 1783.7669999996692, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1756.5959999997158, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1639.7709999998492, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1436.0750000000262, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 1346.087000000095, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1336.519999999975, - "self_device_us": 813.8999999999978, - "self_device_memory_bytes": 25165824 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1293.4269999999624, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1222.6059999999607, - "self_device_us": 738.8599999998873, - "self_device_memory_bytes": 1404416 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1155, - "self_cpu_us": 30564.47400000023, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 16297.44999999991, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2522, - "self_cpu_us": 12829.769999999564, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 6427.685000000046, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1364, - "self_cpu_us": 4445.575999999577, - "self_device_us": 4287.241000000021, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 679, - "self_cpu_us": 2640.5459999999166, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1695.7159999999421, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 270, - "self_cpu_us": 1634.4419999999154, - "self_device_us": 895.1859999999397, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 1283.0529999999708, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 100, - "self_cpu_us": 1112.1259999999893, - "self_device_us": 597.6139999999978, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_softmax", - "count": 183, - "self_cpu_us": 1092.33499999993, - "self_device_us": 4761.238999999965, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1076.8700000000263, - "self_device_us": 1272.6010000000606, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1041.7010000000118, - "self_device_us": 734.4430000000102, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1010.8409999999494, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 1003.2569999999541, - "self_device_us": 1286.9970000000249, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 993.8319999999167, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 919.8220000000947, - "self_device_us": 318.9200000000674, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 879.4340000000484, - "self_device_us": 1155.8990000000413, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 858.7320000000709, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 807.774000000074, - "self_device_us": 2808.9659999999603, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1155, - "self_cpu_us": 20902.823999999953, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 16891.75100000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2522, - "self_cpu_us": 15932.245000000374, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9363.272999999954, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6491.891000000185, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 5420.91799999993, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 5071.345999999881, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1364, - "self_cpu_us": 4659.201999999755, - "self_device_us": 5583.503000000237, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 679, - "self_cpu_us": 3039.1240000001817, - "self_device_us": 0, - "self_device_memory_bytes": 65508864 - }, - { - "operator": "aten::bitwise_and", - "count": 87, - "self_cpu_us": 2845.459999999992, - "self_device_us": 366.18600000001607, - "self_device_memory_bytes": 44544 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 2434.940000000655, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2247.8800000001356, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::eq", - "count": 87, - "self_cpu_us": 2212.9000000000815, - "self_device_us": 339.5000000001128, - "self_device_memory_bytes": 44544 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 2182.421999999704, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1826.42999999996, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 270, - "self_cpu_us": 1732.066999999999, - "self_device_us": 1221.016000000127, - "self_device_memory_bytes": 37748736 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1720.8809999998857, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 1615.1079999998499, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1522.9489999999714, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "vllm_rlt/worker/sampling.py(6): probabilities", - "count": 181, - "self_cpu_us": 1390.739999999816, - "self_device_us": 0.0, - "self_device_memory_bytes": -35586048 - } - ] - }, - "tokens_unchanged": true - } -} diff --git a/docs/validation/training-contract-a100-20261001/README.md b/docs/validation/training-contract-a100-20261001/README.md deleted file mode 100644 index 40473595..00000000 --- a/docs/validation/training-contract-a100-20261001/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Training contract with two A100 workers — 2026-10-01 - -Two A100-SXM4-40GB GPUs, driver 580.105.08, the existing `/venv/main` runtime, -PyTorch 2.13.0+cu130, Triton 3.7.1 and NIXL 1.4.1. Separate spawned P/D owners -register their GPU KV/hidden pools and exchange real NIXL WRITE transfers. -The public `LLM` interface stages physical policy chunks through CPU IPC and waits -for every worker to commit. No package, driver or service changes are made. - -| Case | Active slots | Maximum logprob error vs local | Published/recovered version | D graph captures / replays | P bytes transferred | -|---|---:|---:|---:|---:|---:| -| Random FP32, eager, K=2/3/4 | 4 | 4.77e-7 | 5 | Disabled | 300,928 | -| Random FP32, graphs, K=2/3/4 | 4 | 4.77e-7 | 5 | 2 / 116 | 300,928 | -| Full BF16 Ouro-1.4B, graphs, K=2/3/4 | 1 | 0 | 5 | 1 / 206 | 229,699,584 | - -Each case compares raw and processed scores at temperature 0.9, top-k 8 and -nucleus 0.8, with two prompts and six response tokens. Token IDs, physical exit -depths, effective parameters and policy versions agree. Prompt prefill is full -K=4; decode budgets change after each complete publication. Prefix reuse, -chunked updates, a rejected update, recovery into version five, explicit stop -and active abort are exercised. Both workers finish with zero used blocks and -version five. Graph captures remain valid through publication and recovery. - -The full-model source is `ByteDance/Ouro-1.4B` at -`574fa66cb8bf5abdc979642d01cf2b79b16bfab1`, safetensors SHA256 -`58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af`. -See [full outputs/counters](raw/pd-full-graph-matched.json), -[tiny eager](raw/pd-tiny-eager-tcp.json), [tiny graphs](raw/pd-tiny-graph-tcp.json) -and [runtime](raw/runtime.json). - -## Transport and numerical limits - -Default UCX GPU IPC reports completion but leaves the destination zero in an -independent two-process, 128-byte probe. Explicit IPC and shared-memory-only -configurations also fail on this container. `UCX_TLS=tcp,cuda_copy` passes the -probe and every case above with the installed library. Transfers are host staged; -this report does not establish direct GPU IPC/RDMA performance. The workaround -is scoped to each command. [Probe source](../../../benchmarks/nixl_preflight.py), -[default failure](raw/nixl-preflight.log) and [passing probe](raw/nixl-preflight-tcp.log) -are retained. No processes are forcibly terminated; successful workers use the -normal stop/acknowledgment protocol. - -A full BF16 comparison with four slots has a maximum observed selected-score -difference of 0.11199 when local and PD prefill batches differ. The matched -one-slot run above is exact; the FP32 four-slot run is within 4.77e-7. -Different BF16 batching is therefore not claimed to be bitwise equivalent. See -[the rejected strict comparison](raw/pd-full-graph-tcp.log). -Other jobs were already running on both GPUs. Timings in these parity probes -include startup/capture/prefix-state differences and contention, and are not a -controlled throughput comparison. The [Thor table](../training-contract-thor-20261001/README.md) -contains alternating-order warmed eager/graph timings. - -## Reproduction - -From the repository root, in an existing CUDA/NIXL environment: - -```bash -UCX_TLS=tcp,cuda_copy python benchmarks/nixl_preflight.py -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py tiny.json -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py tiny-graph.json --cuda-graphs -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py full.json \ - --model /models/Ouro-1.4B --cuda-graphs --max-num-seqs 1 -``` - -The VIME training/save/resume and general HF export checks are recorded in -[0z5a/vime#5](https://github.com/0z5a/vime/pull/5). - -## Repository checks - -The complete default CPU regression suite passes: **421 passed, 141 skipped** -in 19.41 s; GPU tests, including the upstream forced-peer-failure test, are skipped -by the repository's ordinary collection rules. See -[regression log](raw/cpu-regression-final.log) and -[focused contract tests](raw/contract-cpu-final.log). -Ruff lint and format checks pass for all 88 Python files. -The existing runtimes lack `pre_commit`, so its three `language: system` hooks -are run directly without installing it: Python-file lint, format check and the -full default CPU suite. See [lint log](raw/lint-final.log). diff --git a/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log b/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log deleted file mode 100644 index 885048c7..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log +++ /dev/null @@ -1,10 +0,0 @@ -test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok -test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok -test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok -test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok -test_all_workers_commit_and_partial_commit_requires_new_epoch (tests.test_pd_weight_control.PDWeightControlTest.test_all_workers_commit_and_partial_commit_requires_new_epoch) ... ok - ----------------------------------------------------------------------- -Ran 5 tests in 2.095s - -OK diff --git a/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log b/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log deleted file mode 100644 index d10641bc..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log +++ /dev/null @@ -1,9 +0,0 @@ -.......s.s.s.sssssssssssssssssss....sssssssssssssss.s................... [ 12%] -............................................ssssss......s..............s [ 25%] -ssssssss.sssssssssssssssssssssssss...................................... [ 38%] -....ssssssss.ssss....sssssss...............................s............ [ 51%] -..sssssssssssssssss.................sss................................. [ 64%] -..ssssss..sss.....ss...ss............................................... [ 76%] -........................................................................ [ 89%] -.............ss.ss...............................sssss.... [100%] -421 passed, 141 skipped in 19.41s diff --git a/docs/validation/training-contract-a100-20261001/raw/lint-final.log b/docs/validation/training-contract-a100-20261001/raw/lint-final.log deleted file mode 100644 index 3d3e3c6d..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/lint-final.log +++ /dev/null @@ -1,4 +0,0 @@ -Required pre-commit hooks use language: system. pre_commit is absent; no installation performed. -ruff 0.16.8 -All checks passed! -88 files already formatted diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log deleted file mode 100644 index 11287164..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log +++ /dev/null @@ -1,51 +0,0 @@ -[1790822017.515227] [25f216012e8e:21943:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy, cuda_ipc/cuda - no am bcopy -W1001 02:33:37.515292 21943 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable -E1001 02:33:37.515377 21943 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep -E1001 02:33:37.515387 21943 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND -Process SpawnProcess-1: -Traceback (most recent call last): - File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap - self.run() - File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run - self._target(*self._args, **self._kwargs) - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker - agent = nixl_agent(name) - ^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ - self.create_backend(bknd, init) - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend - self.backends[backend] = self.agent.createBackend(backend, initParams) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND -[1790822017.583102] [25f216012e8e:21944:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy, cuda_ipc/cuda - no am bcopy -W1001 02:33:37.583151 21944 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable -E1001 02:33:37.583303 21944 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep -E1001 02:33:37.583311 21944 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND -Process SpawnProcess-2: -Traceback (most recent call last): - File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap - self.run() - File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run - self._target(*self._args, **self._kwargs) - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker - agent = nixl_agent(name) - ^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ - self.create_backend(bknd, init) - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend - self.backends[backend] = self.agent.createBackend(backend, initParams) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND -Traceback (most recent call last): - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 42, in - infos=[p.recv() for _,p in peers] - ^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 250, in recv - buf = self._recv_bytes() - ^^^^^^^^^^^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes - buf = self._recv(4) - ^^^^^^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 399, in _recv - raise EOFError -EOFError diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log deleted file mode 100644 index d2072821..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log +++ /dev/null @@ -1,51 +0,0 @@ -[1790822145.345936] [25f216012e8e:23279:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy -W1001 02:35:45.345994 23279 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable -E1001 02:35:45.346127 23279 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep -E1001 02:35:45.346146 23279 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND -Process SpawnProcess-1: -Traceback (most recent call last): - File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap - self.run() - File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run - self._target(*self._args, **self._kwargs) - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker - agent = nixl_agent(name) - ^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ - self.create_backend(bknd, init) - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend - self.backends[backend] = self.agent.createBackend(backend, initParams) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND -[1790822145.362990] [25f216012e8e:23280:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy -W1001 02:35:45.363052 23280 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable -E1001 02:35:45.363210 23280 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep -E1001 02:35:45.363225 23280 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND -Process SpawnProcess-2: -Traceback (most recent call last): - File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap - self.run() - File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run - self._target(*self._args, **self._kwargs) - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker - agent = nixl_agent(name) - ^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ - self.create_backend(bknd, init) - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend - self.backends[backend] = self.agent.createBackend(backend, initParams) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND -Traceback (most recent call last): - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 42, in - infos=[p.recv() for _,p in peers] - ^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 250, in recv - buf = self._recv_bytes() - ^^^^^^^^^^^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes - buf = self._recv(4) - ^^^^^^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 399, in _recv - raise EOFError -EOFError diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log deleted file mode 100644 index c5bfffe1..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log +++ /dev/null @@ -1,6 +0,0 @@ -2026-10-01 02:12:24 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:12:24 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-1 -2026-10-01 02:12:24 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:12:24 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-0 -VALUES [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0]] -MULTIPROCESS_NIXL_PASSED diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log deleted file mode 100644 index ce2d6594..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log +++ /dev/null @@ -1,12 +0,0 @@ -2026-10-01 02:11:20 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:11:20 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-0 -2026-10-01 02:11:20 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:11:20 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-1 -VALUES [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]] -[1790820681.044462] [25f216012e8e:15986:0] cuda_ctx.c:23 UCX ERROR cuDevicePrimaryCtxGetState(cuda_device, &flags, &active) failed: unrecognized error code 4 -[1790820681.060647] [25f216012e8e:15985:0] cuda_ctx.c:23 UCX ERROR cuDevicePrimaryCtxGetState(cuda_device, &flags, &active) failed: unrecognized error code 4 -Traceback (most recent call last): - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 50, in - assert values[0]==values[1] - ^^^^^^^^^^^^^^^^^^^^ -AssertionError diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json deleted file mode 100644 index 03da93de..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json +++ /dev/null @@ -1,1189 +0,0 @@ -{ - "model": { - "vocab_size": 49152, - "hidden_size": 2048, - "intermediate_size": 5632, - "num_hidden_layers": 24, - "num_attention_heads": 16, - "num_key_value_heads": 16, - "head_dim": 128, - "hidden_act": "silu", - "max_position_embeddings": 65536, - "initializer_range": 0.02, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "total_ut_steps": 4, - "early_exit_threshold": 1.0, - "bos_token_id": 0, - "eos_token_id": 0, - "pad_token_id": null, - "tie_word_embeddings": false, - "attention_dropout": 0.0, - "rope_scaling": null, - "use_sliding_window": false, - "sliding_window": null, - "layer_types": [ - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention" - ] - }, - "cuda_graphs": true, - "iterations": [ - { - "version": 1, - "loops": 2, - "mode": "raw", - "seconds": { - "local": 1.501969379838556, - "pd": 15.95300066191703 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "0", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.4585607051849365, - -1.3445959091186523, - -0.6607269048690796, - -0.4440094828605652, - -0.288143515586853, - -0.2284386157989502 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "1", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 4, - 4, - 4, - 198 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.4905070662498474, - -1.852401614189148, - -1.8474926948547363, - -4.226284027099609, - -2.1031107902526855, - -3.4267778396606445 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 1, - "loops": 2, - "mode": "processed", - "seconds": { - "local": 0.544591479934752, - "pd": 6.968707168009132 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "2", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.5379835367202759, - -0.22262510657310486, - 0.0, - 0.0, - 0.0 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "3", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 4, - 4, - 4, - 198 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.5063599348068237, - -1.3956828117370605, - -1.77802312374115, - -1.5563204288482666, - -0.19239495694637299 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "raw", - "seconds": { - "local": 0.7185850627720356, - "pd": 14.518414614256471 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "4", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.5055561065673828, - -0.5541938543319702, - -0.48413512110710144, - -0.3641066551208496, - -0.22259598970413208, - -0.19062571227550507 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "5", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 4, - 3, - 4, - 3 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.54900062084198, - -0.6617550849914551, - -1.9724218845367432, - -0.34641382098197937, - -1.458582878112793, - -0.42156851291656494 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "processed", - "seconds": { - "local": 0.66817781701684, - "pd": 7.080546109005809 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "6", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.1622927039861679, - -0.15219146013259888, - 0.0, - 0.0, - 0.0 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "7", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 4, - 3, - 4, - 3 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.2678171992301941, - -1.781590223312378, - 0.0, - -1.1942179203033447, - 0.0 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "raw", - "seconds": { - "local": 0.849096744786948, - "pd": 14.640745197888464 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "8", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.5625201463699341, - -0.37419480085372925, - -0.2975218594074249, - -0.2469334453344345, - -0.20516522228717804, - -0.1743372529745102 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "9", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.616585373878479, - -0.5554823875427246, - -0.43136879801750183, - -0.33480215072631836, - -0.2979239821434021, - -0.46127814054489136 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "processed", - "seconds": { - "local": 0.7937315646559, - "pd": 7.206241376232356 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "10", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "11", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.1622924953699112, - 0.0, - 0.0, - 0.0, - 0.0 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - } - ], - "max_num_seqs": 1, - "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", - "recovered_version": 5, - "stop": { - "request_id": "13", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "stop", - "text": "", - "log_probs": [ - 0.0 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [ - 3 - ], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "abort": { - "request_id": "abort", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "abort", - "text": "", - "log_probs": [ - 0.0 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "workers": { - "cae75e7035744e8795024750bd65b22e-decode-1": { - "kind": "stopped", - "prefill_tokens": 0, - "bytes_sent": 0, - "transfers": 0, - "transfer_seconds": 0.0, - "used_blocks": 0, - "weight_version": 5, - "graph_captures": 1, - "graph_replays": 206, - "graph_fallbacks": 0 - }, - "cae75e7035744e8795024750bd65b22e-prefill-0": { - "kind": "stopped", - "prefill_tokens": 244, - "bytes_sent": 229699584, - "transfers": 22, - "transfer_seconds": 207.36763043235987, - "used_blocks": 0, - "weight_version": 5 - } - } -} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log deleted file mode 100644 index d37a3c92..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log +++ /dev/null @@ -1,11 +0,0 @@ -2026-10-01 02:30:31 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:30:31 NIXL INFO _api.py:285 Initialized NIXL agent: cae75e7035744e8795024750bd65b22e-decode-1 -2026-10-01 02:30:31 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:30:31 NIXL INFO _api.py:285 Initialized NIXL agent: cae75e7035744e8795024750bd65b22e-prefill-0 -PARITY 1 2 raw {'local': 1.501969379838556, 'pd': 15.95300066191703} 0.0 -PARITY 1 2 processed {'local': 0.544591479934752, 'pd': 6.968707168009132} 0.0 -PARITY 2 3 raw {'local': 0.7185850627720356, 'pd': 14.518414614256471} 0.0 -PARITY 2 3 processed {'local': 0.66817781701684, 'pd': 7.080546109005809} 0.0 -PARITY 3 4 raw {'local': 0.849096744786948, 'pd': 14.640745197888464} 0.0 -PARITY 3 4 processed {'local': 0.7937315646559, 'pd': 7.206241376232356} 0.0 -PD_CONTRACT_PASSED pd-full-graph-matched.json diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log deleted file mode 100644 index 03b98612..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log +++ /dev/null @@ -1,16 +0,0 @@ -2026-10-01 02:19:01 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:19:01 NIXL INFO _api.py:285 Initialized NIXL agent: 35f3775d875246919e95de82111ac017-prefill-0 -2026-10-01 02:19:01 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:19:01 NIXL INFO _api.py:285 Initialized NIXL agent: 35f3775d875246919e95de82111ac017-decode-1 -Traceback (most recent call last): - File "/workspace/0z5a/work/ouro-vime-contract-20261001/rlt/benchmarks/pd_training_contract.py", line 168, in - main() - File "/workspace/0z5a/work/ouro-vime-contract-20261001/rlt/benchmarks/pd_training_contract.py", line 111, in main - torch.testing.assert_close( - File "/usr/local/lib/python3.12/dist-packages/torch/testing/_comparison.py", line 1631, in assert_close - raise error_metas[0].to_error(msg) -AssertionError: Tensor-likes are not close! - -Mismatched elements: 6 / 6 (100.0%) -Greatest absolute difference: 0.11199069023132324 at index (1,) (up to 2e-05 allowed) -Greatest relative difference: 0.07553135603666306 at index (1,) (up to 2e-05 allowed) diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json deleted file mode 100644 index c5c64fd2..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json +++ /dev/null @@ -1,1160 +0,0 @@ -{ - "model": { - "vocab_size": 64, - "hidden_size": 32, - "intermediate_size": 64, - "num_hidden_layers": 2, - "num_attention_heads": 4, - "num_key_value_heads": 2, - "head_dim": 8, - "hidden_act": "silu", - "max_position_embeddings": 128, - "initializer_range": 0.02, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "total_ut_steps": 4, - "early_exit_threshold": 1.0, - "bos_token_id": 0, - "eos_token_id": 0, - "pad_token_id": null, - "tie_word_embeddings": false, - "attention_dropout": 0.0, - "rope_scaling": null, - "use_sliding_window": false, - "sliding_window": null, - "layer_types": null - }, - "cuda_graphs": false, - "iterations": [ - { - "version": 1, - "loops": 2, - "mode": "raw", - "seconds": { - "local": 1.1838269280269742, - "pd": 2.0369991376064718 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "0", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 42, - 42, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.992061138153076, - -3.8517885208129883, - -3.85848069190979, - -3.9884941577911377, - -3.8614566326141357, - -3.8722712993621826 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "1", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 51, - 15, - 55 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.861236810684204, - -3.9610671997070312, - -3.9557578563690186, - -4.008734226226807, - -3.9762279987335205, - -4.012908458709717 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 1, - "loops": 2, - "mode": "processed", - "seconds": { - "local": 0.23476281203329563, - "pd": 0.41898630978539586 - }, - "max_logprob_error": 1.1920928955078125e-07, - "outputs": [ - { - "request_id": "2", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 42, - 42, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.969618320465088, - -1.8275089263916016, - -1.835660457611084, - -1.980717658996582, - -1.8372858762741089, - -1.8491119146347046 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "3", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 51, - 15, - 55 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8221769332885742, - -1.9443289041519165, - -1.942601203918457, - -2.004945993423462, - -1.9664429426193237, - -2.008206367492676 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "raw", - "seconds": { - "local": 0.21088168676942587, - "pd": 0.45077179512009025 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "4", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 12, - 61, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.9952800273895264, - -4.035950660705566, - -3.7910518646240234, - -3.8775575160980225, - -3.849231719970703, - -3.8591208457946777 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "5", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 56, - 15, - 56 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.8670716285705566, - -3.976059913635254, - -3.975245237350464, - -4.008485317230225, - -3.941457986831665, - -4.011442184448242 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "processed", - "seconds": { - "local": 0.24939796887338161, - "pd": 0.38997333077713847 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "6", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 12, - 61, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.9691096544265747, - -2.072470188140869, - -1.8017675876617432, - -1.8991870880126953, - -1.8665740489959717, - -1.8762420415878296 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "7", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 56, - 15, - 56 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8246119022369385, - -2.0073468685150146, - -2.00469970703125, - -2.0388402938842773, - -1.96330726146698, - -2.036628007888794 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "raw", - "seconds": { - "local": 0.22700949804857373, - "pd": 0.45369746908545494 - }, - "max_logprob_error": 4.76837158203125e-07, - "outputs": [ - { - "request_id": "8", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 60, - 9, - 25, - 60, - 42 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.9984395503997803, - -4.025251865386963, - -4.024652004241943, - -3.9303245544433594, - -4.02800178527832, - -4.010705947875977 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "9", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 60, - 60, - 9, - 15, - 56 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.8727941513061523, - -4.021127223968506, - -4.020061016082764, - -4.025237083435059, - -3.993036985397339, - -4.017432689666748 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "processed", - "seconds": { - "local": 0.25585177494212985, - "pd": 0.42146871984004974 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "10", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 60, - 9, - 25, - 60, - 42 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.9686124324798584, - -1.9942327737808228, - -1.9923925399780273, - -1.8824831247329712, - -1.9928045272827148, - -1.9732145071029663 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "11", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 60, - 60, - 9, - 15, - 56 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8269999027252197, - -1.9946293830871582, - -1.9918233156204224, - -1.995558738708496, - -1.9558247327804565, - -1.983024001121521 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - } - ], - "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", - "recovered_version": 5, - "stop": { - "request_id": "13", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "stop", - "text": "", - "log_probs": [ - -1.9686124324798584 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [ - 42 - ], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "abort": { - "request_id": "abort", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "abort", - "text": "", - "log_probs": [ - -1.9686124324798584 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "workers": { - "e7b52c471c5a4a7383476001ca2853cc-decode-1": { - "kind": "stopped", - "prefill_tokens": 0, - "bytes_sent": 0, - "transfers": 0, - "transfer_seconds": 0.0, - "used_blocks": 0, - "weight_version": 5 - }, - "e7b52c471c5a4a7383476001ca2853cc-prefill-0": { - "kind": "stopped", - "prefill_tokens": 292, - "bytes_sent": 300928, - "transfers": 22, - "transfer_seconds": 2.774511670228094, - "used_blocks": 0, - "weight_version": 5 - } - } -} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log deleted file mode 100644 index 9ae443a5..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log +++ /dev/null @@ -1,11 +0,0 @@ -2026-10-01 02:14:34 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:14:34 NIXL INFO _api.py:285 Initialized NIXL agent: e7b52c471c5a4a7383476001ca2853cc-decode-1 -2026-10-01 02:14:34 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:14:34 NIXL INFO _api.py:285 Initialized NIXL agent: e7b52c471c5a4a7383476001ca2853cc-prefill-0 -PARITY 1 2 raw {'local': 1.1838269280269742, 'pd': 2.0369991376064718} 2.384185791015625e-07 -PARITY 1 2 processed {'local': 0.23476281203329563, 'pd': 0.41898630978539586} 1.1920928955078125e-07 -PARITY 2 3 raw {'local': 0.21088168676942587, 'pd': 0.45077179512009025} 2.384185791015625e-07 -PARITY 2 3 processed {'local': 0.24939796887338161, 'pd': 0.38997333077713847} 2.384185791015625e-07 -PARITY 3 4 raw {'local': 0.22700949804857373, 'pd': 0.45369746908545494} 4.76837158203125e-07 -PARITY 3 4 processed {'local': 0.25585177494212985, 'pd': 0.42146871984004974} 2.384185791015625e-07 -PD_CONTRACT_PASSED pd-tiny-eager-tcp.json diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json deleted file mode 100644 index e95c674c..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json +++ /dev/null @@ -1,1163 +0,0 @@ -{ - "model": { - "vocab_size": 64, - "hidden_size": 32, - "intermediate_size": 64, - "num_hidden_layers": 2, - "num_attention_heads": 4, - "num_key_value_heads": 2, - "head_dim": 8, - "hidden_act": "silu", - "max_position_embeddings": 128, - "initializer_range": 0.02, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "total_ut_steps": 4, - "early_exit_threshold": 1.0, - "bos_token_id": 0, - "eos_token_id": 0, - "pad_token_id": null, - "tie_word_embeddings": false, - "attention_dropout": 0.0, - "rope_scaling": null, - "use_sliding_window": false, - "sliding_window": null, - "layer_types": null - }, - "cuda_graphs": true, - "iterations": [ - { - "version": 1, - "loops": 2, - "mode": "raw", - "seconds": { - "local": 1.2744349539279938, - "pd": 2.329219114035368 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "0", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 42, - 42, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.992061138153076, - -3.8517885208129883, - -3.85848069190979, - -3.9884941577911377, - -3.8614566326141357, - -3.8722712993621826 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "1", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 51, - 15, - 55 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.861236810684204, - -3.9610671997070312, - -3.9557578563690186, - -4.008734226226807, - -3.9762279987335205, - -4.012908458709717 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 1, - "loops": 2, - "mode": "processed", - "seconds": { - "local": 0.23766471119597554, - "pd": 0.436434387229383 - }, - "max_logprob_error": 1.1920928955078125e-07, - "outputs": [ - { - "request_id": "2", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 42, - 42, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.969618320465088, - -1.8275089263916016, - -1.835660457611084, - -1.980717658996582, - -1.8372858762741089, - -1.8491119146347046 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "3", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 51, - 15, - 55 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8221769332885742, - -1.9443289041519165, - -1.942601203918457, - -2.004945993423462, - -1.9664429426193237, - -2.008206367492676 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "raw", - "seconds": { - "local": 0.22012542001903057, - "pd": 0.4495909558609128 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "4", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 12, - 61, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.9952800273895264, - -4.035950660705566, - -3.7910518646240234, - -3.8775575160980225, - -3.849231719970703, - -3.8591208457946777 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "5", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 56, - 15, - 56 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.8670716285705566, - -3.976059913635254, - -3.975245237350464, - -4.008485317230225, - -3.941457986831665, - -4.011442184448242 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "processed", - "seconds": { - "local": 0.24550614319741726, - "pd": 0.40186773892492056 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "6", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 12, - 61, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.9691096544265747, - -2.072470188140869, - -1.8017675876617432, - -1.8991870880126953, - -1.8665740489959717, - -1.8762420415878296 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "7", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 56, - 15, - 56 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8246119022369385, - -2.0073468685150146, - -2.00469970703125, - -2.0388402938842773, - -1.96330726146698, - -2.036628007888794 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "raw", - "seconds": { - "local": 0.23163969395682216, - "pd": 0.4639628282748163 - }, - "max_logprob_error": 4.76837158203125e-07, - "outputs": [ - { - "request_id": "8", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 60, - 9, - 25, - 60, - 42 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.9984395503997803, - -4.025251865386963, - -4.024652004241943, - -3.9303245544433594, - -4.02800178527832, - -4.010705947875977 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "9", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 60, - 60, - 9, - 15, - 56 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.8727941513061523, - -4.021127223968506, - -4.020061016082764, - -4.025237083435059, - -3.993036985397339, - -4.017432689666748 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "processed", - "seconds": { - "local": 0.2783969510346651, - "pd": 0.46485115215182304 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "10", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 60, - 9, - 25, - 60, - 42 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.9686124324798584, - -1.9942327737808228, - -1.9923925399780273, - -1.8824831247329712, - -1.9928045272827148, - -1.9732145071029663 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "11", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 60, - 60, - 9, - 15, - 56 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8269999027252197, - -1.9946293830871582, - -1.9918233156204224, - -1.995558738708496, - -1.9558247327804565, - -1.983024001121521 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - } - ], - "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", - "recovered_version": 5, - "stop": { - "request_id": "13", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "stop", - "text": "", - "log_probs": [ - -1.9686124324798584 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [ - 42 - ], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "abort": { - "request_id": "abort", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "abort", - "text": "", - "log_probs": [ - -1.9686124324798584 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "workers": { - "0759283ec0764cee8d1bd7369fa54644-prefill-0": { - "kind": "stopped", - "prefill_tokens": 292, - "bytes_sent": 300928, - "transfers": 22, - "transfer_seconds": 2.949465547222644, - "used_blocks": 0, - "weight_version": 5 - }, - "0759283ec0764cee8d1bd7369fa54644-decode-1": { - "kind": "stopped", - "prefill_tokens": 0, - "bytes_sent": 0, - "transfers": 0, - "transfer_seconds": 0.0, - "used_blocks": 0, - "weight_version": 5, - "graph_captures": 2, - "graph_replays": 116, - "graph_fallbacks": 0 - } - } -} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log deleted file mode 100644 index e391a5ae..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log +++ /dev/null @@ -1,11 +0,0 @@ -2026-10-01 02:16:37 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:16:37 NIXL INFO _api.py:285 Initialized NIXL agent: 0759283ec0764cee8d1bd7369fa54644-prefill-0 -2026-10-01 02:16:37 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:16:37 NIXL INFO _api.py:285 Initialized NIXL agent: 0759283ec0764cee8d1bd7369fa54644-decode-1 -PARITY 1 2 raw {'local': 1.2744349539279938, 'pd': 2.329219114035368} 2.384185791015625e-07 -PARITY 1 2 processed {'local': 0.23766471119597554, 'pd': 0.436434387229383} 1.1920928955078125e-07 -PARITY 2 3 raw {'local': 0.22012542001903057, 'pd': 0.4495909558609128} 2.384185791015625e-07 -PARITY 2 3 processed {'local': 0.24550614319741726, 'pd': 0.40186773892492056} 2.384185791015625e-07 -PARITY 3 4 raw {'local': 0.23163969395682216, 'pd': 0.4639628282748163} 4.76837158203125e-07 -PARITY 3 4 processed {'local': 0.2783969510346651, 'pd': 0.46485115215182304} 2.384185791015625e-07 -PD_CONTRACT_PASSED pd-tiny-graph-tcp.json diff --git a/docs/validation/training-contract-a100-20261001/raw/runtime.json b/docs/validation/training-contract-a100-20261001/raw/runtime.json deleted file mode 100644 index f4813a5e..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/runtime.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "torch": "2.13.0+cu130", - "triton": "3.7.1", - "nixl": "1.4.1", - "transformers": "5.17.0", - "numpy": "2.3.5", - "safetensors": "0.8.0", - "gpus": [ - "NVIDIA A100-SXM4-40GB, 580.105.08, 40960 MiB", - "NVIDIA A100-SXM4-40GB, 580.105.08, 40960 MiB" - ], - "UCX_TLS": "tcp,cuda_copy" -} \ No newline at end of file diff --git a/docs/validation/training-contract-thor-20261001/README.md b/docs/validation/training-contract-thor-20261001/README.md deleted file mode 100644 index 52920253..00000000 --- a/docs/validation/training-contract-thor-20261001/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Training contract on Thor — 2026-10-01 - -These timings use full BF16 `ByteDance/Ouro-1.4B` at revision -`574fa66cb8bf5abdc979642d01cf2b79b16bfab1`, with the public `LLM` rollout/update -API on upstream `ecb1f8b505b7e831815b40aec3b4598619cca23a`. - -| Rollout | Eager (s) | CUDA graphs (s) | Speedup | Time saved | -|---|---:|---:|---:|---:| -| Native K=2 decode | 1.0689 | 0.9939 | 1.076× | 7.02% | -| Native K=3 decode | 1.5190 | 1.4207 | 1.069× | 6.47% | -| Native K=4 decode | 1.9414 | 1.8234 | 1.065× | 6.08% | -| Self-speculative target K=4, draft K=2, 3 candidates | 1.4952 | 1.3741 | 1.088× | 8.11% | - -Five alternating-order pairs after per-arm warmup and full-weight publication; -two prompts, 32 response tokens each, temperature=0.9 and fixed per-request seeds. -Returned selected-token raw logprobs, tokens, exit depths and weight versions -match exactly between eager and graph arms in every pair. Each native prompt -prefill uses full K=4. Measurements include logprob delivery and exclude model -loading and initial capture. Each speculative comparison uses the same speculative -algorithm in both arms. See [raw timing pairs](raw/shared-full-rollout-benchmark.json). - -CPU tests pass all five checks; CUDA/Triton tests pass all four applicable checks. -The independent FP32 oracle covers processed probabilities with temperature and -top-k/top-p, speculative target scores after publication, early exit, async -stop/abort, prefix invalidation, stable parameter addresses and failed publication -recovery. The PD control test confirms a newer complete epoch recovers from a -partial peer commit; delivery is in-process, so IPC and NIXL are not covered. - -Full-model VIME training and fresh-process checkpoint/HF export parity are -reported in [0z5a/vime#5](https://github.com/0z5a/vime/pull/5). -Real multi-GPU PD publication and rollout checks are recorded in the -[A100 report](../training-contract-a100-20261001/README.md); Thor has one GPU and no NIXL. -No packages, CUDA drivers or environments were updated. GPU runs use the existing -performance lock and finish naturally. Completed experiment weights/checkpoints -are removed after preserving logs and tensor hashes. diff --git a/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json b/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json deleted file mode 100644 index 22ae874c..00000000 --- a/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "native-2": { - "seconds": { - "eager": [ - 1.068922233011108, - 1.0962036999990232, - 1.0150359050021507, - 1.0776073549932335, - 1.0171921259898227 - ], - "graph": [ - 0.99387969498639, - 1.031959738000296, - 0.9904317960026674, - 1.0318653039867058, - 0.9868912409874611 - ] - }, - "medians": { - "eager": 1.068922233011108, - "graph": 0.99387969498639 - }, - "speedup": 1.0755046495096627, - "tokens_and_logprobs_exact": true - }, - "native-3": { - "seconds": { - "eager": [ - 1.5189770860015415, - 1.5267343849991448, - 1.4750143490091432, - 1.5399230089969933, - 1.4762638669926673 - ], - "graph": [ - 1.4206840230035596, - 1.4595294760074466, - 1.3996001509949565, - 1.4518495189840905, - 1.4113349709950853 - ] - }, - "medians": { - "eager": 1.5189770860015415, - "graph": 1.4206840230035596 - }, - "speedup": 1.0691871390164396, - "tokens_and_logprobs_exact": true - }, - "native-4": { - "seconds": { - "eager": [ - 1.9413879640051164, - 1.9600694489781745, - 1.8969329710234888, - 1.9612911650037859, - 1.8895282850135118 - ], - "graph": [ - 1.7855259570060298, - 1.8588317520043347, - 1.8060117769928183, - 1.8618553779961076, - 1.8234225899796002 - ] - }, - "medians": { - "eager": 1.9413879640051164, - "graph": 1.8234225899796002 - }, - "speedup": 1.0646944787641552, - "tokens_and_logprobs_exact": true - }, - "speculative-4": { - "seconds": { - "eager": [ - 1.4952338639996015, - 1.523701586003881, - 1.4740307310130447, - 1.5373682600038592, - 1.4766599040012807 - ], - "graph": [ - 1.3729902119957842, - 1.415411785012111, - 1.373799616994802, - 1.416570646018954, - 1.3740531750081573 - ] - }, - "medians": { - "eager": 1.4952338639996015, - "graph": 1.3740531750081573 - }, - "speedup": 1.0881921392821823, - "tokens_and_logprobs_exact": true - } -} \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index f4a55800..00000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Repository tests; keep imports separate from installed test packages.""" diff --git a/tests/test_pd.py b/tests/test_pd.py index 71999078..9b07ff14 100644 --- a/tests/test_pd.py +++ b/tests/test_pd.py @@ -239,7 +239,6 @@ def test_engine_yields_while_waiting_for_remote_kv(async_scheduling): from vllm_rlt.request import Request, Stage engine = object.__new__(LLMEngine) - engine._update_version = None engine.profiling = Profiler("cpu") engine.preemption = PreemptionManager(engine) engine.execution_config = ExecutionConfig(async_scheduling=async_scheduling) diff --git a/tests/test_pd_weight_control.py b/tests/test_pd_weight_control.py deleted file mode 100644 index 523efbbb..00000000 --- a/tests/test_pd_weight_control.py +++ /dev/null @@ -1,91 +0,0 @@ -"""PD publication control with two real CPU engines; no NIXL transport claim.""" - -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -import torch - -from tests.helpers import tiny_ouro_config -from vllm_rlt import LLM, SamplingParams -from vllm_rlt.engine.llm_engine import LLMEngine -from vllm_rlt.models import OuroForCausalLM -from vllm_rlt.pd.config import PDConfig -from vllm_rlt.pd.engine import PDEngine -from vllm_rlt.pd.worker import PDWorker - - -def control_pair(): - coordinator = PDEngine.__new__(PDEngine) - coordinator.config = PDConfig() - coordinator.closed, coordinator.failure = False, None - coordinator.requests, coordinator.transfers = {}, {} - coordinator._weight_version, coordinator._update_version = 0, None - coordinator._update_failed, coordinator._paused = False, False - coordinator._weight_replies = {} - coordinator.peers = { - role: SimpleNamespace(name=role, role=role) for role in ("prefill", "decode") - } - workers = {} - for role, peer in coordinator.peers.items(): - worker = PDWorker.__new__(PDWorker) - worker.engine = LLMEngine(OuroForCausalLM(tiny_ouro_config())) - worker.work = {} - worker.send = lambda kind, peer=peer, **fields: coordinator._message( - peer, dict(kind=kind, **fields) - ) - workers[role] = worker - coordinator._send = lambda name, kind, **fields: workers[name].command( - dict(kind=kind, **fields) - ) - return coordinator, workers - - -class PDWeightControlTest(unittest.TestCase): - def test_all_workers_commit_and_partial_commit_requires_new_epoch(self): - coordinator, workers = control_pair() - facade = LLM(coordinator) - weights = list(workers["prefill"].engine.model.named_parameters()) - pointers = { - role: [p.data_ptr() for p in worker.engine.model.parameters()] - for role, worker in workers.items() - } - facade.start_weight_update() - facade.update_weights(weights) - facade.finish_weight_update() - self.assertEqual(facade.get_weight_version(), 1) - coordinator.start_weight_update() - coordinator.update_weights(weights) - with patch.object( - workers["decode"].engine, - "finish_weight_update", - side_effect=RuntimeError("copy failure"), - ): - with self.assertRaisesRegex(RuntimeError, "copy failure"): - coordinator.finish_weight_update() - self.assertEqual(workers["prefill"].engine.get_weight_version(), 2) - self.assertEqual(workers["decode"].engine.get_weight_version(), 1) - with self.assertRaises(RuntimeError): - coordinator.step() - with self.assertRaises(RuntimeError): - coordinator.add_request("blocked", [3]) - coordinator.start_weight_update() - coordinator.update_weights(weights) - coordinator.finish_weight_update() - self.assertEqual(coordinator.get_weight_version(), 3) - for role, worker in workers.items(): - self.assertEqual(worker.engine.get_weight_version(), 3) - self.assertEqual( - pointers[role], [p.data_ptr() for p in worker.engine.model.parameters()] - ) - output = LLM(worker.engine).generate( - [[3, 4]], SamplingParams(max_tokens=2, logprobs=0) - )[0] - self.assertEqual(output.weight_version, 3) - self.assertEqual(len(output.log_probs), len(output.token_ids)) - coordinator.reset_prefix_cache() - - -if __name__ == "__main__": - torch.set_num_threads(1) - unittest.main() diff --git a/tests/test_speculative.py b/tests/test_speculative.py index 162f54ec..412a8b13 100644 --- a/tests/test_speculative.py +++ b/tests/test_speculative.py @@ -395,7 +395,7 @@ def test_greedy_round_reads_back_once_with_ragged_requests(): # Draft IDs stay on the device; every speculative round, including ragged and # budget-shortened ones, synchronizes with the host exactly once. torch.manual_seed(123) - m = OuroForCausalLM(tiny_ouro_config(hidden_size=256, head_dim=64)).to( + m = OuroForCausalLM(OuroConfig.tiny(hidden_size=256, head_dim=64)).to( device="cuda", dtype=torch.bfloat16 ) e = engine(m, k=3, cache_config=CacheConfig(128, 16), attention_backend="triton") diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py deleted file mode 100644 index b536eaa5..00000000 --- a/tests/test_training_contract.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Training contract checks runnable with unittest on CPU or CUDA/Triton.""" - -import os -import unittest -from dataclasses import replace - -import torch - -from tests.reference import dense_reference -from vllm_rlt import ( - LLM, - CacheConfig, - ExecutionConfig, - ExitConfig, - SamplingParams, - SchedulerConfig, - SpeculativeConfig, -) -from vllm_rlt.engine.llm_engine import LLMEngine -from vllm_rlt.models import OuroConfig, OuroForCausalLM -from vllm_rlt.request import Request -from vllm_rlt.worker.sampling import generator_for, probabilities - -DEVICE = os.environ.get("CONTRACT_DEVICE", "cpu") - - -def model(): - torch.manual_seed(17) - config = OuroConfig( - vocab_size=64, - hidden_size=256 if DEVICE == "cuda" else 32, - intermediate_size=512 if DEVICE == "cuda" else 64, - num_hidden_layers=2, - num_attention_heads=4, - num_key_value_heads=2, - head_dim=64 if DEVICE == "cuda" else 8, - max_position_embeddings=128, - total_ut_steps=4, - eos_token_id=0, - ) - return OuroForCausalLM(config).to(DEVICE) - - -def llm(native, *, graphs=False, asynchronous=False, speculative=False, prefix=False): - return LLM( - native, - attention_backend="triton" if DEVICE == "cuda" else "torch", - cache_config=CacheConfig(num_blocks=128, block_size=2, enable_prefix_caching=prefix), - scheduler_config=SchedulerConfig(max_num_seqs=4, max_num_batched_tokens=32), - exit_config=ExitConfig("ouro_delayed" if asynchronous else "ouro"), - execution_config=ExecutionConfig(cuda_graphs=graphs, async_scheduling=asynchronous), - speculative_config=SpeculativeConfig(3) if speculative else None, - ) - - -class TrainingContractTest(unittest.TestCase): - def test_selected_probabilities_speculative_target_and_post_update(self): - native = model() - prompt = [3, 4, 5] - for graphs in [False, True] if DEVICE == "cuda" else [False]: - for speculative in (False, True): - rollout = llm(native, graphs=graphs, speculative=speculative) - pointers = [p.data_ptr() for p in native.parameters()] - for version in (0, 1): - if version: - weights = [(n, p.detach() * 0.97) for n, p in native.named_parameters()] - rollout.start_weight_update(version) - rollout.update_weights(weights[:5]) - rollout.update_weights(weights[5:]) - rollout.finish_weight_update() - for mode in ("raw", "processed"): - params = SamplingParams( - max_tokens=8, - temperature=0.8, - top_k=13, - top_p=0.9, - logprobs=0, - logprobs_mode=mode, - seed=29, - ignore_eos=True, - ) - output = rollout.generate([prompt], params)[0] - logits = dense_reference(native, prompt + output.token_ids[:-1], 4)[-1][2] - rows = logits[len(prompt) - 1 :] - expected = torch.stack( - [ - probabilities(row, params)[token].log() - if mode == "processed" - else row.float().log_softmax(-1)[token] - for row, token in zip(rows, output.token_ids, strict=True) - ] - ) - torch.testing.assert_close( - torch.tensor(output.log_probs, device=DEVICE), - expected, - atol=4e-5, - rtol=4e-5, - ) - self.assertEqual(output.weight_version, version) - self.assertEqual(output.sampling_params, replace(params, max_loops=4)) - self.assertEqual(pointers, [p.data_ptr() for p in native.parameters()]) - if graphs: - bank = ( - rollout.engine.speculative_runner.graphs - if speculative - else rollout.engine.model_runner.graphs - ) - self.assertGreater(bank.replays, 0) - - def test_early_exit_async_stop_and_abort(self): - for graphs in [False, True] if DEVICE == "cuda" else [False]: - for asynchronous in (False, True): - rollout = llm(model(), graphs=graphs, asynchronous=asynchronous) - rollout.start_weight_update(2) - rollout.update_weights(rollout.engine.model.named_parameters()) - rollout.finish_weight_update() - params = SamplingParams(max_tokens=5, exit_threshold=0, logprobs=0, ignore_eos=True) - output = rollout.generate([[3, 4]], params)[0] - self.assertEqual(output.weight_version, 2) - self.assertEqual(output.exit_depths, [4] + [3 if asynchronous else 2] * 4) - self.assertEqual(len(output.log_probs), 5) - stopped = rollout.generate( - [[3, 4]], replace(params, stop_token_ids=(output.token_ids[0],)) - )[0] - self.assertEqual(stopped.token_ids, output.token_ids[:1]) - self.assertEqual(len(stopped.log_probs), 1) - self.assertEqual(stopped.finish_reason, "stop") - rollout.engine.add_request("abort", [3, 4], params) - while not rollout.engine.step(): - pass - aborted = rollout.engine.abort_request("abort") - self.assertEqual(len(aborted.log_probs), len(aborted.token_ids)) - self.assertEqual(aborted.finish_reason, "abort") - - def test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy(self): - rollout = llm(model(), prefix=True) - native = rollout.engine.model - params = SamplingParams(max_tokens=3, logprobs=0, ignore_eos=True) - rollout.generate([[3, 4, 5, 6]], params) - self.assertTrue(rollout.engine.cache_manager.lookup_prefix([3, 4, 5, 6])) - rollout.pause_generation() - with self.assertRaises(RuntimeError): - rollout.generate([[3]], params) - rollout.resume_generation() - rollout.engine.add_request("active", [3], params) - with self.assertRaises(RuntimeError): - rollout.start_weight_update() - rollout.engine.abort_request("active") - rollout.start_weight_update(7) - self.assertEqual(rollout.engine.cache_manager.num_used_blocks, 0) - self.assertFalse(rollout.engine.cache_manager.lookup_prefix([3, 4, 5, 6])) - names = list(native.named_parameters()) - before = names[0][1].detach().clone() - with self.assertRaises(ValueError): - rollout.update_weights( - [(names[0][0], torch.zeros_like(before)), ("invalid", torch.zeros(1))] - ) - torch.testing.assert_close(before, names[0][1], rtol=0, atol=0) - with self.assertRaises(RuntimeError): - rollout.finish_weight_update() - with self.assertRaises(RuntimeError): - rollout.generate([[3]], params) - rollout.start_weight_update(7) - rollout.update_weights(names[:1]) - with self.assertRaises(ValueError): - rollout.finish_weight_update() - rollout.update_weights(names[1:]) - rollout.finish_weight_update() - self.assertEqual(rollout.generate([[3]], params)[0].weight_version, 7) - rollout.reset_prefix_cache() - self.assertEqual(rollout.engine.cache_manager.num_used_blocks, 0) - - def test_seed_none_and_validation(self): - params = SamplingParams(seed=None, temperature=1) - requests = [Request(str(i), [3], params) for i in range(2)] - seeds = [generator_for(r, DEVICE).initial_seed() for r in requests] - self.assertNotEqual(*seeds) - for values in ({"logprobs": 1}, {"logprobs_mode": "invalid"}, {"stop_token_ids": (-1,)}): - with self.assertRaises(ValueError): - SamplingParams(**values) - with self.assertRaises(ValueError): - llm(model()).generate([[3]], SamplingParams(stop_token_ids=(64,))) - - def test_entropy_seed_is_reported_and_replays_the_same_policy(self): - params = SamplingParams( - max_tokens=8, temperature=0.9, seed=None, logprobs=0, ignore_eos=True - ) - for options in ({}, {"asynchronous": True}, {"speculative": True}): - rollout = llm(model(), graphs=DEVICE == "cuda", **options) - output = rollout.generate([[3, 4, 5]], params)[0] - self.assertIsInstance(output.sampling_params.seed, int) - self.assertLess(output.sampling_params.seed, 2**63) - replay = rollout.generate([[3, 4, 5]], output.sampling_params)[0] - self.assertEqual(output.token_ids, replay.token_ids) - self.assertEqual(output.log_probs, replay.log_probs) - self.assertEqual(output.exit_depths, replay.exit_depths) - rollout.engine.add_request("unstarted", [3], replace(params, temperature=0)) - aborted = rollout.engine.abort_request("unstarted") - self.assertIsInstance(aborted.sampling_params.seed, int) - self.assertEqual(aborted.log_probs, []) - rollout.close() - self.assertIsNone(params.seed) - - def test_preemption_preserves_selected_scores_and_rng(self): - native = model() - params = SamplingParams( - max_tokens=8, temperature=0.8, seed=45, logprobs=0, exit_threshold=0, ignore_eos=True - ) - results = [] - for suspend in (False, True): - engine = LLMEngine( - native, - cache_config=CacheConfig(64, 2), - scheduler_config=SchedulerConfig(enable_preemption=True), - attention_backend="triton" if DEVICE == "cuda" else "torch", - ) - engine.add_request("a", [1, 2, 3], params) - while len(engine.scheduler.requests["a"].generated_token_ids) < 3: - engine.step() - if suspend: - engine.add_request("b", [5], SamplingParams(max_tokens=1)) - engine.scheduler.selected_request_ids.clear() - self.assertTrue(engine.preemption.preempt(engine.scheduler.requests["b"])) - while engine.has_unfinished_requests(): - for output in engine.step(): - if output.request_id == "a" and output.finished: - results.append(output) - if suspend: - self.assertEqual(engine.preemption.resumptions, 1) - engine.close() - self.assertEqual(results[0].token_ids, results[1].token_ids) - self.assertEqual(results[0].log_probs, results[1].log_probs) - self.assertEqual(results[0].exit_depths, results[1].exit_depths) - - -if __name__ == "__main__": - torch.set_num_threads(1) - unittest.main() From f941596599fbe8bf0c2fc1c6ef61709bd7b1c240 Mon Sep 17 00:00:00 2001 From: MinhaoLi0318 Date: Thu, 22 Oct 2026 11:22:23 +0800 Subject: [PATCH 5/9] feat: add rollout sampling fields and logprob scoring primitives SamplingParams gains the vLLM-named rollout fields of RFC #70: logprobs (None or 0; top-k alternatives are rejected for now), seed=None (resolved by the engine, see resolve_seed) and stop_token_ids. Sampler.processed_logits factors out the temperature/top-k/top-p arithmetic that probabilities() normalizes, so processed logprobs score exactly the distribution a token is drawn from. raw_logprobs and sample_with_logprob consume no RNG. worker/logprobs.py holds the mode check and the grouped processed scoring used by the runner. --- tests/test_sampler.py | 192 +++++++++++++++++++++++++++++++++++- vllm_rlt/sampling_params.py | 43 ++++++-- vllm_rlt/worker/logprobs.py | 86 ++++++++++++++++ vllm_rlt/worker/sampler.py | 64 ++++++++++-- 4 files changed, 366 insertions(+), 19 deletions(-) create mode 100644 vllm_rlt/worker/logprobs.py diff --git a/tests/test_sampler.py b/tests/test_sampler.py index a9595f67..58e8261e 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -1,18 +1,24 @@ -"""Contract tests for the extracted Sampler: greedy, top-k and top-p. +"""Contract tests for sampling: the extracted Sampler, per-token logprobs and seeds. ``Sampler`` only consumes a generator the caller resolved, so these tests create one themselves (mirroring what ``RngRegistry.acquire`` does in production) and keep ``RngRegistry``'s own creation and lifetime contracts separate. The distribution tests state the sampling contract itself, including strict top-k: exactly ``k`` candidates survive by index, so a tie at the k-th logit does not -widen the support. +widen the support. The rest cover ``token_logprob`` (``worker/logprobs.py``) in raw +and processed modes, the batched ``processed_logprobs`` and +``Sampler.sample_with_logprob`` against it, and the registry's rejection of an +unresolved ``seed=None``. """ +import math + import pytest import torch import vllm_rlt.worker.sampler as sampler_module from vllm_rlt.sampling_params import SamplingParams +from vllm_rlt.worker.logprobs import processed_logprobs, token_logprob, truncates from vllm_rlt.worker.sampler import RngRegistry, Sampler @@ -250,3 +256,185 @@ def test_rng_registry_drop_removes_only_that_requests_slot(): registry.drop("dropped") assert registry.peek("kept") is kept assert dropped is not kept + + +def test_unresolved_seed_is_rejected_before_any_sampling(): + # The registry is the only generator creator, so it rejects an unresolved seed, + # and without a generator no non-greedy token can be drawn. Engines resolve + # seed=None at admission, greedy requests included (tests/test_rollout_outputs.py). + params = SamplingParams(seed=None, temperature=1.0) + registry = RngRegistry() + with pytest.raises(ValueError, match="seed=None"): + registry.acquire("r", params, torch.device("cpu")) + assert registry.peek("r") is None + with pytest.raises(ValueError, match="requires a generator"): + Sampler().sample(torch.zeros(4), params, None) + + +LOGPROB_LOGITS = torch.tensor([1.5, 0.2, -0.7, 2.1, 0.0, -3.0]) + + +@pytest.mark.parametrize("token", range(len(LOGPROB_LOGITS)), ids=lambda t: f"token={t}") +def test_raw_token_logprob_is_the_model_log_softmax_for_any_token(token): + for dtype in (torch.float32, torch.bfloat16): + logits = LOGPROB_LOGITS.to(dtype) + params = SamplingParams(temperature=0.6, top_k=2, top_p=0.5, seed=1) + value = token_logprob(logits, params, torch.tensor(token), "raw_logprobs") + assert value.dim() == 0 and value.dtype == torch.float32 + expected = torch.log_softmax(logits.float(), -1)[token] + assert torch.equal(value, expected) + + +PROCESSED_PARAMS = [ + SamplingParams(temperature=0.6, seed=1), + SamplingParams(temperature=1.3, top_k=3, seed=1), + SamplingParams(temperature=0.9, top_p=0.7, seed=1), + SamplingParams(temperature=0.8, top_k=4, top_p=0.9, seed=1), +] +PROCESSED_IDS = ["temperature", "temperature-top_k", "temperature-top_p", "temperature-top_k-top_p"] + + +@pytest.mark.parametrize("params", PROCESSED_PARAMS, ids=PROCESSED_IDS) +def test_processed_token_logprob_scores_the_sampling_distribution(params): + probs = Sampler.probabilities(LOGPROB_LOGITS, params) + for token in range(len(LOGPROB_LOGITS)): + value = token_logprob(LOGPROB_LOGITS, params, torch.tensor(token), "processed_logprobs") + if probs[token] == 0: + assert value.item() == -math.inf # Filtered tokens cannot be sampled. + else: + assert value.item() == pytest.approx(probs[token].log().item(), abs=1e-6) + + +def test_processed_token_logprob_edge_cases(): + argmax = torch.tensor(int(LOGPROB_LOGITS.argmax())) + greedy = SamplingParams(temperature=0.0, top_k=1) + # Greedy requests report the unscaled model distribution, as vLLM does. + assert torch.equal( + token_logprob(LOGPROB_LOGITS, greedy, argmax, "processed_logprobs"), + token_logprob(LOGPROB_LOGITS, greedy, argmax, "raw_logprobs"), + ) + # Greedy has no temperature-scaled logits to score, so building them is an error. + for build in (Sampler.processed_logits, Sampler.probabilities): + with pytest.raises(ValueError, match="greedy"): + build(LOGPROB_LOGITS, greedy) + # One token survives top_k=1, or top_p=0.6 after top_k=2: top-p sees the renormalized + # top-2 mass (argmax 0.65), not the full distribution (0.53, which would keep two). + for single in (dict(temperature=0.7, top_k=1), dict(temperature=1.0, top_k=2, top_p=0.6)): + params = SamplingParams(seed=1, **single) + assert token_logprob(LOGPROB_LOGITS, params, argmax, "processed_logprobs").item() == 0.0 + + +def test_processed_token_logprob_scores_the_strict_top_k_support(): + # A tie at the k-th logit: strict top-k keeps exactly k tokens, so the processed + # logprob of the surviving tied token is log(e^4 / (e^5 + e^4)), and the other + # tied token cannot be sampled. + logits = torch.tensor([5.0, 4.0, 4.0, 1.0]) + params = SamplingParams(temperature=1.0, top_k=2, seed=1) + probs = Sampler.probabilities(logits, params) + kept = next(index for index in (1, 2) if probs[index] > 0) + dropped = 3 - kept + value = token_logprob(logits, params, torch.tensor(kept), "processed_logprobs") + assert value.item() == pytest.approx(torch.log_softmax(torch.tensor([5.0, 4.0]), -1)[1].item()) + assert token_logprob(logits, params, torch.tensor(dropped), "processed_logprobs") == -math.inf + + +def test_token_logprob_consumes_no_rng(): + params = SamplingParams(temperature=0.8, top_k=4, top_p=0.9, seed=42) + generator = seeded_generator(params) + token = Sampler().sample(LOGPROB_LOGITS, params, generator) + state, global_state = generator.get_state().clone(), torch.get_rng_state() + for logprobs_mode in ("raw_logprobs", "processed_logprobs"): + token_logprob(LOGPROB_LOGITS, params, token, logprobs_mode) + assert torch.equal(generator.get_state(), state) + assert torch.equal(torch.get_rng_state(), global_state) + + +@pytest.mark.parametrize( + "params", [SamplingParams(top_k=1), *PROCESSED_PARAMS], ids=["greedy", *PROCESSED_IDS] +) +def test_sample_with_logprob_draws_like_sample_and_scores_the_drawn_logits(params): + sampler, plain, scored = Sampler(), seeded_generator(params), seeded_generator(params) + initial = plain.get_state().clone() + for _ in range(4): + token = sampler.sample(LOGPROB_LOGITS, params, plain) + same, logprob = sampler.sample_with_logprob(LOGPROB_LOGITS, params, scored) + assert torch.equal(same, token) + expected = token_logprob(LOGPROB_LOGITS, params, token, "processed_logprobs") + assert torch.equal(logprob, expected) + # Same RNG consumption as sample(): none for greedy rows, which ignore the generator. + assert torch.equal(plain.get_state(), scored.get_state()) + assert torch.equal(scored.get_state(), initial) == (params.temperature == 0) + if params.temperature != 0: # Like sample(), never a silent global-RNG fallback. + with pytest.raises(ValueError, match="requires a generator"): + sampler.sample_with_logprob(LOGPROB_LOGITS, params, None) + + +# One row per processed-logprob path: batched raw (greedy, even with top-k, and T == 1), batched +# per temperature, the sampler's top-k/top-p logits, and (last) a row without logprobs. +BATCH_PARAMS = [ + SamplingParams(temperature=0.0, top_k=3, logprobs=0), + SamplingParams(temperature=1.0, logprobs=0, seed=1), + SamplingParams(temperature=0.7, logprobs=0, seed=2), + SamplingParams(temperature=1.3, logprobs=0, seed=3), + SamplingParams(temperature=0.9, top_k=8, logprobs=0, seed=4), + SamplingParams(temperature=1.0, top_p=0.8, logprobs=0, seed=5), + SamplingParams(temperature=0.7, top_k=16, top_p=0.9, logprobs=0, seed=6), + SamplingParams(temperature=0.8, seed=7), +] + + +ROWS = 48 # Rows laid out by a BATCH_LAYOUTS entry; a row without logprobs follows them. + + +def at(temperature): + return SamplingParams(temperature=temperature, logprobs=0, seed=1) + + +# Layout -> (row -> params, log_softmax passes of processed_logprobs or None: only bounded). +BATCH_LAYOUTS = { + "mixed": (lambda row: BATCH_PARAMS[row % 8], None), + "t1": (lambda row: BATCH_PARAMS[1], 1), # The only group: one pass over the whole batch. + "t0.7": (lambda row: BATCH_PARAMS[2], 1), + "top_k-top_p": (lambda row: BATCH_PARAMS[4 + row % 3], 0), + # Many temperatures: one per row; rows r and ROWS - 1 - r sharing one (nested spans, the + # outer ones scored row by row); two alternating (one pass per overlapping span). + "distinct_t": (lambda row: at(0.5 + 0.01 * row), ROWS), + "nested_t": (lambda row: at(0.5 + 0.01 * min(row, ROWS - 1 - row)), None), + "alternating_t": (lambda row: at((0.7, 1.3)[row % 2]), 2), +} + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("layout", BATCH_LAYOUTS.values(), ids=BATCH_LAYOUTS.keys()) +def test_batched_processed_logprobs_equal_per_row_token_logprob(monkeypatch, layout, dtype): + params = [layout[0](row) for row in range(ROWS)] + [BATCH_PARAMS[-1]] + logits = 4 * torch.randn(len(params), 256, generator=torch.Generator().manual_seed(0)) + logits, sampler, tokens, truncated = logits.to(dtype), Sampler(), [], {} + for row, row_params in enumerate(params): # As ModelRunner._sample_with_logprobs. + # A fresh generator per row, as RngRegistry.acquire creates on a first draw. + generator = None if row_params.temperature == 0 else seeded_generator(row_params) + if row_params.logprobs is not None and truncates(row_params): + token, truncated[row] = sampler.sample_with_logprob(logits[row], row_params, generator) + else: + token = sampler.sample(logits[row], row_params, generator) + tokens.append(token) + token_ids, passes = torch.stack(tokens), [] # passes: rows scored by each log_softmax. + + raw_logprobs = Sampler.raw_logprobs + + def counted(logits, token_ids): + passes.append(len(logits)) + return raw_logprobs(logits, token_ids) + + monkeypatch.setattr(Sampler, "raw_logprobs", staticmethod(counted)) + values = processed_logprobs(logits, params, token_ids, truncated) + monkeypatch.undo() + # However many temperatures mix, at most twice the batched rows are scored (57d2308 scored + # each requesting row once, in a pass of its own). + assert sum(passes) <= 2 * (sum(p.logprobs is not None for p in params) - len(truncated)) + assert layout[1] is None or len(passes) == layout[1] + assert values.shape == (len(params),) and values.dtype == torch.float32 + for row, row_params in enumerate(params): + if row_params.logprobs is not None: + expected = token_logprob(logits[row], row_params, token_ids[row], "processed_logprobs") + assert torch.equal(values[row], expected) diff --git a/vllm_rlt/sampling_params.py b/vllm_rlt/sampling_params.py index c5f75096..f6016701 100644 --- a/vllm_rlt/sampling_params.py +++ b/vllm_rlt/sampling_params.py @@ -1,5 +1,6 @@ import math -from dataclasses import dataclass +import secrets +from dataclasses import dataclass, replace @dataclass(frozen=True) @@ -8,15 +9,18 @@ class SamplingParams: temperature: float = 0.0 top_p: float = 1.0 top_k: int = -1 + # None lets the engine draw a per-request seed; outputs report the seed used. seed: int | None = 0 - logprobs: int | None = None - logprobs_mode: str = "raw" - stop_token_ids: tuple[int, ...] = () min_loops: int = 2 max_loops: int | None = None exit_threshold: float = 1.0 ignore_eos: bool = False priority: int = 0 + # vLLM convention: 0 returns only the sampled token's log-probability. + logprobs: int | None = None + logprobs_mode: str = "raw" + # Stop after emitting any of these IDs, even with ignore_eos (vLLM semantics). + stop_token_ids: tuple[int, ...] = () def __post_init__(self): if type(self.priority) is not int: @@ -41,10 +45,31 @@ def __post_init__(self): if not 0 <= self.exit_threshold <= 1: raise ValueError("exit_threshold must be in [0, 1]") if self.seed is not None and (type(self.seed) is not int or not 0 <= self.seed < 2**63): - raise ValueError("seed must be an integer in [0, 2**63)") - if self.logprobs is not None and (type(self.logprobs) is not int or self.logprobs != 0): - raise ValueError("logprobs=0 returns the selected-token probability") + raise ValueError("seed must be None or an integer in [0, 2**63)") + if self.logprobs is not None: + if type(self.logprobs) is not int or self.logprobs < 0: + raise ValueError("logprobs must be None or 0") + if self.logprobs > 0: + raise ValueError( + "logprobs > 0 (top-k alternatives) is not supported yet; " + "use 0 for the sampled token's log-probability" + ) if self.logprobs_mode not in ("raw", "processed"): raise ValueError("logprobs_mode must be raw or processed") - if any(type(token) is not int or token < 0 for token in self.stop_token_ids): - raise ValueError("stop_token_ids must contain nonnegative integer IDs") + if not isinstance(self.stop_token_ids, (list, tuple)) or any( + type(token) is not int or token < 0 for token in self.stop_token_ids + ): + raise ValueError("stop_token_ids must be a list of nonnegative integers") + object.__setattr__(self, "stop_token_ids", tuple(self.stop_token_ids)) + + +def resolve_seed(params: SamplingParams) -> SamplingParams: + """Return ``params`` with ``seed=None`` replaced by a fresh 63-bit seed. + + The seed comes from OS entropy, so torch and Python global RNG streams are + not consumed. Engines call this before creating a Request, so outputs and + every PD worker observe the same effective seed. + """ + if params.seed is not None: + return params + return replace(params, seed=secrets.randbits(63)) diff --git a/vllm_rlt/worker/logprobs.py b/vllm_rlt/worker/logprobs.py new file mode 100644 index 00000000..90331ed1 --- /dev/null +++ b/vllm_rlt/worker/logprobs.py @@ -0,0 +1,86 @@ +"""Rollout logprobs: the FP32 log-probability of each sampled token. + +Scoring consumes no RNG. Raw logprobs score the model distribution. Processed +logprobs score ``Sampler.processed_logits``, the logits ``Sampler.probabilities`` +normalizes, so they score the distribution the token was actually drawn from. +This module imports ``worker/sampler.py``, never the reverse. +""" + +from collections import defaultdict + +import torch + +from vllm_rlt.worker.sampler import Sampler + +LOGPROBS_MODES = ("raw_logprobs", "processed_logprobs") + + +def check_logprobs_mode(mode): + if mode not in LOGPROBS_MODES: + raise ValueError( + f"logprobs_mode must be one of {LOGPROBS_MODES}, got {mode!r}; " + "raw_logits and processed_logits are not supported" + ) + + +def token_logprob(logits, params, token_id, mode): + """0-dim FP32 log-probability of ``token_id`` (0-dim long tensor) for one logits row. + + raw_logprobs uses the model distribution. processed_logprobs uses the + distribution actually sampled from; greedy rows have no temperature (vLLM). + Consumes no RNG and keeps the result on the logits device. + """ + if mode == "processed_logprobs" and params.temperature != 0: + logits = Sampler.processed_logits(logits, params) + return Sampler.raw_logprobs(logits[None], token_id.view(1))[0] + + +def truncates(params): + """Whether this row is drawn from top-k/top-p truncated logits (see ``processed_logits``). + + Greedy rows never are: ``argmax`` ignores top-k and top-p. + """ + return params.temperature != 0 and (params.top_k > 0 or params.top_p < 1) + + +def scaled_logprobs(logits, token_ids, temperature): + """``raw_logprobs`` of ``logits / temperature``, divided as ``processed_logits`` divides.""" + scaled = logits if temperature == 1 else logits.float() / temperature + return Sampler.raw_logprobs(scaled, token_ids) + + +def processed_logprobs(logits, params, token_ids, truncated): + """FP32 [rows] processed logprobs of ``token_ids``, equal to per-row ``token_logprob``. + + ``logits`` [rows, vocab] are the CODA rows of ``params`` (one per row), at least one of + which requests logprobs. ``truncated`` maps every requesting row that ``truncates`` to + the 0-dim logprob ``Sampler.sample_with_logprob`` computed from the processed logits it + drew from. Other requesting rows are grouped by temperature: greedy and T == 1 rows are + scored by a raw ``log_softmax`` (``x / 1.0 == x``), each other temperature by + ``log_softmax(logits.float() / T)`` with the Python scalar T, as ``processed_logits`` + divides. A group is scored in one pass over its row span ``[first, last]`` when that span + is at most twice its rows, else row by row, so the scored rows never exceed twice the + requesting rows, however many temperatures the batch mixes. The only group, holding at + least half the rows, is scored over the whole batch and returned as is; otherwise one + stack of 0-dim views assembles the rows. No host sync, per-row device write or per-row + device metadata (spans are host-int slices). Rows that did not request logprobs hold + arbitrary values. + """ + groups = defaultdict(list) # Temperature (greedy: 1) -> requesting untruncated rows. + for row, row_params in enumerate(params): + if row_params.logprobs is not None and row not in truncated: + groups[1 if row_params.temperature == 0 else row_params.temperature].append(row) + values = dict(truncated) + for temperature, rows in groups.items(): + if not truncated and len(groups) == 1 and len(params) <= 2 * len(rows): + return scaled_logprobs(logits, token_ids, temperature) # One pass, no assembly. + first, end = rows[0], rows[-1] + 1 + if end - first > 2 * len(rows): # Sparse group: a span pass would mostly score others. + for row in rows: + one = slice(row, row + 1) + values[row] = scaled_logprobs(logits[one], token_ids[one], temperature)[0] + continue + scored = scaled_logprobs(logits[first:end], token_ids[first:end], temperature) + values.update((row, scored[row - first]) for row in rows) + filler = next(iter(values.values())) + return torch.stack([values.get(row, filler) for row in range(len(params))]) diff --git a/vllm_rlt/worker/sampler.py b/vllm_rlt/worker/sampler.py index d737ae82..0c4feea9 100644 --- a/vllm_rlt/worker/sampler.py +++ b/vllm_rlt/worker/sampler.py @@ -14,6 +14,10 @@ caller obtained from ``RngRegistry``. The registry is the single owner of RNG creation, lookup and lifetime, and the engine shares one instance between its runners. + +Rollout logprobs (``worker/logprobs.py``) score sampled tokens with +``Sampler.raw_logprobs``, on ``Sampler.processed_logits`` in processed mode. That +module imports this one, never the reverse. """ import torch @@ -25,14 +29,15 @@ class Sampler: """Greedy / top-k / top-p sampling for one logits row.""" @staticmethod - def probabilities(logits: torch.Tensor, params: SamplingParams) -> torch.Tensor: - """Return the temperature/top-k/top-p distribution for one logits row. + def processed_logits(logits: torch.Tensor, params: SamplingParams) -> torch.Tensor: + """Return the FP32 temperature/top-k/top-p logits for one logits row. ``params`` is assumed to be validated by ``SamplingParams.__post_init__``. Greedy decoding has no temperature-scaled distribution, so it is a caller - error rather than a silent argmax. Top-k keeps exactly ``k`` candidates - (or every token when ``k`` reaches the vocabulary); top-p may trim that - support further. + error rather than a silent argmax. Excluded tokens are ``-inf``. Top-k + keeps exactly ``k`` candidates (or every token when ``k`` reaches the + vocabulary); top-p may trim that support further. ``probabilities`` (CODA + and speculative sampling) and processed logprobs share this arithmetic. """ if params.temperature == 0: raise ValueError("greedy decoding has no temperature-scaled distribution") @@ -52,7 +57,15 @@ def probabilities(logits: torch.Tensor, params: SamplingParams) -> torch.Tensor: remove[1:] = remove[:-1].clone() remove[0] = False logits = logits.scatter(0, indices, sorted_logits.masked_fill(remove, -torch.inf)) - return logits.softmax(-1) + return logits + + @staticmethod + def probabilities(logits: torch.Tensor, params: SamplingParams) -> torch.Tensor: + """Return the temperature/top-k/top-p distribution for one logits row. + + It is the softmax of ``processed_logits``, which rejects greedy decoding. + """ + return Sampler.processed_logits(logits, params).softmax(-1) @staticmethod def selected_logprob( @@ -69,6 +82,15 @@ def draw(probs: torch.Tensor, generator: torch.Generator) -> torch.Tensor: """Draw one token from a normalized distribution as a 0-dim long tensor.""" return torch.multinomial(probs, 1, generator=generator).squeeze(0) + @staticmethod + def raw_logprobs(logits: torch.Tensor, token_ids: torch.Tensor) -> torch.Tensor: + """FP32 [rows] log-probabilities of ``token_ids`` [rows] under ``logits`` [rows, vocab]. + + Scoring consumes no RNG and keeps the result on the logits device. + """ + logprobs = torch.log_softmax(logits, -1, dtype=torch.float32) + return logprobs.gather(-1, token_ids[:, None]).squeeze(-1) + def sample( self, logits: torch.Tensor, @@ -90,6 +112,28 @@ def sample( raise ValueError("non-greedy sampling requires a generator") return self.draw(self.probabilities(logits, params), generator) + def sample_with_logprob( + self, + logits: torch.Tensor, + params: SamplingParams, + generator: torch.Generator | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """``sample()`` plus the 0-dim FP32 processed logprob of the drawn token. + + The logprob is ``log_softmax`` of the same processed logits the token is + drawn from, so it equals ``token_logprob(..., "processed_logprobs")`` + without rebuilding them; greedy rows report the unscaled distribution. + Tokens, generator requirements and RNG consumption are those of ``sample()``. + """ + if params.temperature == 0: + token = logits.argmax() + return token, self.raw_logprobs(logits[None], token.view(1))[0] + if generator is None: + raise ValueError("non-greedy sampling requires a generator") + processed = self.processed_logits(logits, params) + token = self.draw(processed.softmax(-1), generator) + return token, self.raw_logprobs(processed[None], token.view(1))[0] + class RngRegistry: """Per-request sampling generators for one engine. @@ -115,11 +159,15 @@ def acquire( """Return the request's generator, creating it on ``device`` if needed. The caller passes the device of the distribution it will sample from, so - ``multinomial`` and its generator agree. An existing generator is returned - as-is and never migrated. + ``multinomial`` and its generator agree. A new generator is seeded with + ``params.seed``, which must be engine-resolved, so its ``initial_seed()`` + is the seed the request's outputs report. An existing generator is + returned as-is and never reseeded or migrated. """ generator = self.generators.get(request_id) if generator is None: + if params.seed is None: # Engines resolve seed=None at admission. + raise ValueError("seed=None must be resolved by the engine before sampling") generator = torch.Generator(device=device).manual_seed(params.seed) self.generators[request_id] = generator return generator From 855668ed4eb8eb7ab7cd8d828efc8965c3fc40de Mon Sep 17 00:00:00 2001 From: MinhaoLi0318 Date: Thu, 22 Oct 2026 11:24:53 +0800 Subject: [PATCH 6/9] feat: expose selected-token logprobs, effective seeds and stop token IDs Port of #83's runtime contract onto the stage/loop engine (#118) and the Scheduler-owned result application (#107): - Scheduler._record_token is the single append point. It records the token's logprob and stops on stop_token_ids even with ignore_eos, as in vLLM, for synchronous and delivered asynchronous CODA outputs alike. - ModelRunner scores CODA rows under logprobs_mode (raw_logprobs or processed_logprobs) next to sampling, with the generators resolved before execution, and returns them in ModelRunnerOutput.logprobs. CUDA readback uses a dedicated pinned pool, so exit-score leases of a request with an undelivered CODA output cannot starve its logprobs. - StageEngine resolves seed=None at admission; RequestOutput carries logprobs and the effective sampling_params across the StageClient pipe, aborts included. StageClient validates logprobs_mode before starting a worker. PDEngine resolves one seed for P and D and mirrors D's logprobs. - LLM.generate omits the text of a stop-terminating ID; HTTP rejects the new fields until the token-in/token-out API. - Speculative decoding rejects logprobs at admission for now. --- README.md | 4 +- benchmarks/cdb_runtime.py | 19 +- docs/async-speculative.md | 5 +- docs/launching.md | 49 ++ docs/training_contract.md | 38 +- tests/test_cdb_runtime.py | 3 + tests/test_engine.py | 2 +- tests/test_mixed_execution.py | 2 +- tests/test_rollout_outputs.py | 974 ++++++++++++++++++++++++++++++++ tests/test_serving.py | 23 +- tests/test_split_prefill.py | 3 +- tests/test_training_contract.py | 45 +- vllm_rlt/core/scheduler.py | 22 +- vllm_rlt/engine/llm_engine.py | 3 + vllm_rlt/engine/stage_engine.py | 16 +- vllm_rlt/entrypoints/llm.py | 7 +- vllm_rlt/pd/engine.py | 18 +- vllm_rlt/request.py | 20 +- vllm_rlt/sampling_params.py | 3 - vllm_rlt/serving/protocol.py | 6 +- vllm_rlt/worker/model_runner.py | 160 ++++-- vllm_rlt/worker/output.py | 7 +- vllm_rlt/worker/sampler.py | 10 - vllm_rlt/worker/speculative.py | 10 - 24 files changed, 1273 insertions(+), 176 deletions(-) create mode 100644 tests/test_rollout_outputs.py diff --git a/README.md b/README.md index 4a128539..99c4caa7 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,9 @@ for supported combinations and limitations. and [execution scope and constraints](docs/async-speculative.md#cuda-graph-scope-and-ownership). - **Offline and online inference.** A Python API, a command-line interface, and an OpenAI-compatible completions endpoint with streaming, greedy decoding, - and seeded top-k/top-p sampling. + and seeded top-k/top-p sampling. RL rollouts can request per-token + log-probabilities aligned with exit depths, engine-chosen seeds, and stop + token IDs. - **Prefill/decode disaggregation.** Separate prefill and decode worker pools across GPUs on one host, with NIXL KV transfer and overlap between chunked prefill computation and transfer. diff --git a/benchmarks/cdb_runtime.py b/benchmarks/cdb_runtime.py index 0b75e849..399c05c0 100644 --- a/benchmarks/cdb_runtime.py +++ b/benchmarks/cdb_runtime.py @@ -63,13 +63,18 @@ def benchmark(device="cpu", layout="last_exited", trace=None, seed=123): rid: SamplingParams(max_tokens=6, exit_threshold=q, ignore_eos=True) for rid, q in zip(prompts, [0.0, 1.0, 0.5]) } - fingerprint = dict( - model_config=model.config.to_dict(), - seed=seed, - dtype=str(dtype), - layout=layout, - prompts=prompts, - sampling={rid: asdict(p) for rid, p in parameters.items()}, + # JSON-canonical (tuples become lists) to match a --trace-out/--trace-in round trip. + fingerprint = json.loads( + json.dumps( + dict( + model_config=model.config.to_dict(), + seed=seed, + dtype=str(dtype), + layout=layout, + prompts=prompts, + sampling={rid: asdict(p) for rid, p in parameters.items()}, + ) + ) ) backend = "triton" if device == "cuda" else "torch" diff --git a/docs/async-speculative.md b/docs/async-speculative.md index 0c61088c..ae8e773a 100644 --- a/docs/async-speculative.md +++ b/docs/async-speculative.md @@ -244,8 +244,9 @@ use the reserved allocation and stream-ordered writes instead. Only completed full prompt pages are eligible for prefix caching. Every ticket owns its result buffer and page-table snapshot until compute and -copy events complete. EOS/length stopping is applied in output order on the CPU; -one already-submitted later round can finish but its outputs are discarded. +copy events complete. EOS, `stop_token_ids` and length stopping +(`Scheduler._record_token`) is applied in output order on the CPU; one +already-submitted later round can finish but its outputs are discarded. Cancellation removes scheduling eligibility immediately. KV allocations and device state survive until their final GPU user completes. Request object identity isolates stale tickets when a request ID is reused. diff --git a/docs/launching.md b/docs/launching.md index 6fbc0de5..a1aedda1 100644 --- a/docs/launching.md +++ b/docs/launching.md @@ -260,6 +260,55 @@ execution. The first generated token comes from full-depth prefill, reflected in `exit_depths`. For dynamic arrivals, stepwise output, and cancellation, see the [architecture overview](unified_execution.md). +### Rollout outputs + +RL trainers can request, for each generated token, its log-probability under the +policy that sampled it (RFC [#70](https://github.com/InferMatrixLab/vllm-rlt/issues/70)). +Names follow vLLM. + +| Option | Default | Contract | +| --- | --- | --- | +| `SamplingParams.logprobs` | `None` | `0` adds `output.logprobs`, the FP32 log-probability of each sampled token. Positive values (top-k alternatives) raise `ValueError` for now. | +| `SamplingParams.seed` | `0` | `None` asks the engine for a fresh 63-bit seed from OS entropy at `add_request`, consuming neither the torch nor the Python global RNG. `output.sampling_params` reports the effective parameters, including the seed the request's generator is created with. | +| `SamplingParams.stop_token_ids` | `()` | Emitting one finishes the request with `stop`, even with `ignore_eos=True`; `finish_reason` does not say which stop ID or EOS matched. The token stays in `token_ids`; `LLM.generate` omits its text. IDs outside the vocabulary are rejected. | +| `logprobs_mode` of `LLM`, `StageEngine` (`LLMEngine`), `PDEngine` | `"raw_logprobs"` | `"raw_logprobs"` is `log_softmax` of the CODA logits. `"processed_logprobs"` is `log_softmax` of the logits sampling draws from (temperature, then top-k, then top-p); greedy rows report the raw value, as vLLM does. No CLI flag yet. | + +Every output of a request with `logprobs=0`, including streamed, `stop`, `length` +and `abort` outputs, has `len(logprobs) == len(token_ids) == len(exit_depths)`. +Requesting logprobs consumes no RNG and changes no token or exit depth. Replaying +the reported `sampling_params` reproduces tokens and logprobs bitwise only with the +same batch composition, configuration and hardware. A resumed preempted request or +a prefix-cache hit matches the uninterrupted cold run numerically, not necessarily +bitwise, because the batch composition (and, for a prefix hit, the prefill +chunking) changes. Huginn is the exception: it draws its random recurrent-state +initialization from the request's generator, greedy requests included, so its +`seed` matters even at `temperature=0`, and the draws depend on how the prompt is +split into prefill chunks. When the chunking changes, for example under memory +pressure, a Huginn request's tokens and logprobs can differ, not just numerically. + +`logprobs[i]` is conditional on `exit_depths[i]`: the policy is the model truncated +at that loop depth. A teacher-forced recompute therefore has to replay the depths: + +- Prompt positions run at the model depth and the input `token_ids[i]` (position + `len(prompt) + i`) runs `exit_depths[i + 1]` loops, so the input depths are + `[depth] * len(prompt) + exit_depths[1:]`. The last generated token is never an + input. +- Under `last_exited` KV, a token that exited at depth `d` has its depth-`d` KV + copied into the deeper loop planes, and later tokens attend to those copies. An + ordinary full-depth forward matches only rollouts whose `exit_depths` all equal + the model depth: `exit_threshold=1` and `max_loops` equal to the model depth, in + a non-`trace` exit mode (`trace` takes its depths from `depths_by_request`). +- Under `shared` KV, positions read earlier tokens' final retained KV at every + loop depth (see [KV layout examples](kv_layout_computation.md)), so no ordinary + forward reproduces the logprobs, even at full depth. +- Huginn's logprobs also depend on its random recurrent-state initialization, which + shares the request's generator with sampling, so no ordinary forward reproduces + them either. + +Speculative decoding does not accept `logprobs` yet. + +The HTTP API does not accept `logprobs`, `stop_token_ids` or `seed: null` yet. + ## Troubleshooting | Symptom | What to check | diff --git a/docs/training_contract.md b/docs/training_contract.md index bcab9fb4..07c1eb10 100644 --- a/docs/training_contract.md +++ b/docs/training_contract.md @@ -14,22 +14,22 @@ outputs = llm.generate(prompts, SamplingParams( )) ``` -Each output contains aligned selected-token `log_probs`, `exit_depths`, the +Each output contains aligned selected-token `logprobs`, `exit_depths`, the committed `weight_version`, and its effective `sampling_params`. `logprobs=None` -keeps the ordinary inference path. Reported parameters resolve the model's -default `max_loops` and cap `top_k` at its vocabulary size. `logprobs=0` requests only the emitted token's -probability; top-N alternatives are not implemented. +keeps the ordinary inference path. `logprobs=0` requests only the emitted token's +log-probability; top-N alternatives are not implemented. -`logprobs_mode="raw"` is the default: FP32 log-softmax of unscaled model logits. -`"processed"` returns probabilities after positive temperature and top-k/top-p -filtering. Greedy requests return the raw score in either mode. Speculative -outputs use the verified **target** policy's score, including correction/bonus -tokens; proposal or rejection-residual probabilities are never exposed as the -policy score. Explicit `stop_token_ids` apply even with `ignore_eos=True`. -`seed=None` resolves an independent seed in `[0, 2**63)` when the request is -created. The effective seed is returned in `sampling_params`, including for an -abort before sampling. Reusing those parameters replays the same request on the -same policy and execution configuration. The default remains `seed=0`. +`logprobs_mode` is an engine option of `LLM`, `StageClient`, `LLMEngine` and +`PDEngine`, not a request field. `"raw_logprobs"` is the default: FP32 log-softmax +of unscaled model logits. `"processed_logprobs"` scores the distribution the token +was drawn from, after positive temperature and top-k/top-p filtering. Greedy +requests return the raw score in either mode. Speculative decoding rejects +`logprobs` requests at admission for now. Explicit `stop_token_ids` apply even +with `ignore_eos=True`. `seed=None` resolves an independent seed in `[0, 2**63)` +when the engine admits the request. The effective seed is returned in +`sampling_params`, including for an abort before sampling. The default remains +`seed=0`. [Rollout outputs](launching.md#rollout-outputs) states when replaying +those parameters reproduces tokens and logprobs. Publication requires an idle engine. Each chunk is checked for duplicate names, unknown names and mismatched shapes before that chunk is copied. `finish_weight_update` @@ -51,11 +51,9 @@ does not replace registered KV storage. PD worker control errors return replies without terminating the worker. Existing exit, async and speculative configuration rules still apply. Self-speculation -uses a full-depth target. Selected-token scores are supported by synchronous -self-speculation and ordinary asynchronous decoding; cross-round asynchronous -self-speculation rejects logprobs requests at admission because its compact result -buffers contain token IDs only. PD follows the same synchronous and asynchronous -self-speculation limits on its decode workers. Publication -adds no such restriction to ordinary early-exit or async rollouts. Trainers must +uses a full-depth target. Selected-token scores are supported by synchronous and +asynchronous decoding; synchronous and asynchronous self-speculation reject +logprobs requests at admission for now. PD follows the same limits on its decode +workers. Publication adds no such restriction to ordinary early-exit or async rollouts. Trainers must replay full-depth prompt prefill and the returned decode depths rather than assume the entire sequence used K. diff --git a/tests/test_cdb_runtime.py b/tests/test_cdb_runtime.py index 8b345dcb..ab955c7f 100644 --- a/tests/test_cdb_runtime.py +++ b/tests/test_cdb_runtime.py @@ -1,5 +1,6 @@ """End-to-end checks of cache semantics, delayed exits and scheduling progress.""" +import json from dataclasses import replace import pytest @@ -455,6 +456,8 @@ def test_synthetic_replay_variants_preserve_outputs(): assert trace["depths_by_request"]["A"] == [4, 2, 2, 2, 2, 2] assert len(results) == 8 assert all(r["seconds"] > 0 for r in results) + # --trace-out/--trace-in round-trip the trace through JSON. + assert len(benchmark(trace=json.loads(json.dumps(trace)))[1]) == 8 with pytest.raises(ValueError, match="does not match"): benchmark(layout="shared", trace=trace) diff --git a/tests/test_engine.py b/tests/test_engine.py index 1ad00979..b07976ca 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -266,7 +266,7 @@ def unexpected(*args, **kwargs): pytest.fail("sampling generators must be resolved before execution") monkeypatch.setattr(engine.rng, "acquire", unexpected) - token = runner._execute(None, prepared) + token = runner._execute(None, prepared).values assert token.shape == (1,) and token.device == runner.device engine.close() diff --git a/tests/test_mixed_execution.py b/tests/test_mixed_execution.py index 542b8e96..be537cbc 100644 --- a/tests/test_mixed_execution.py +++ b/tests/test_mixed_execution.py @@ -316,7 +316,7 @@ def test_prepared_batch_survives_progress_changes_and_contains_no_policy_label() scheduled_request.request.generated_token_ids.append(17) # The prepared batch owns its addresses and routing; later request progress cannot # retarget the submitted operation, even for mixed multi-row state bindings. - actual = engine.model_runner._execute(batch, prepared) + actual = engine.model_runner._execute(batch, prepared).values torch.testing.assert_close(actual, torch.tensor(expected.gate_scores)) assert not hasattr(prepared.batch, "stage") for left, right in zip(prepared.batch.spans, build_execution_batch(reference_batch, 4).spans): diff --git a/tests/test_rollout_outputs.py b/tests/test_rollout_outputs.py new file mode 100644 index 00000000..f8a4f4d6 --- /dev/null +++ b/tests/test_rollout_outputs.py @@ -0,0 +1,974 @@ +"""Rollout outputs: per-token logprobs, effective seeds and stop token IDs.""" + +import gc +import math +import random +import time +from collections import defaultdict, deque +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch + +from tests.helpers import tiny_ouro_config +from tests.reference import dense_reference +from tests.test_huginn import tiny_huginn_config +from vllm_rlt import LLM, CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig +from vllm_rlt.config import SpeculativeConfig +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.models import HuginnForCausalLM, OuroForCausalLM +from vllm_rlt.pd.config import PDConfig +from vllm_rlt.pd.engine import PDEngine, Peer +from vllm_rlt.request import Operation, RequestOutput, RequestPhase +from vllm_rlt.serving.protocol import CompletionRequest +from vllm_rlt.worker.logprobs import token_logprob +from vllm_rlt.worker.model_runner import ReadbackSlot, Submission +from vllm_rlt.worker.output import ModelRunnerOutput, SpeculativeResult +from vllm_rlt.worker.sampler import Sampler + +TRACES = {"a": [4, 1, 3, 2, 4, 1, 2, 3], "b": [4, 2, 1, 4, 3, 2, 1, 4]} # Keyed by request ID. +EXIT_MODES = [ # (async scheduling, exit config, exit_threshold) for every CPU exit policy. + pytest.param(False, ExitConfig("ouro"), 1.0, id="ouro-full-depth"), + pytest.param(False, ExitConfig("ouro"), 0.5, id="ouro-early-exit"), + pytest.param(False, ExitConfig("trace", depths_by_request=TRACES), 1.0, id="trace"), + pytest.param(True, ExitConfig("ouro_delayed"), 0.3, id="async-ouro_delayed"), + pytest.param(True, ExitConfig("random_lookahead"), 0.3, id="async-random_lookahead"), + pytest.param(True, ExitConfig("trace", depths_by_request=TRACES), 1.0, id="async-trace"), +] +ASYNC_EXIT_MODES = [pytest.param(*p.values[1:], id=p.id) for p in EXIT_MODES if p.values[0]] +LOGPROBS_MODES = ["raw_logprobs", "processed_logprobs"] +# How request "a" ends: max_tokens, EOS (honored or ignored), stop IDs, both, abort. +FINISHES = "length eos stop_id ignored_eos eos_with_stop_ids stop_id_at_max eos_at_max abort" + + +def model(seed=123, device="cpu", dtype=torch.float32, **overrides): + torch.manual_seed(seed) + return OuroForCausalLM(tiny_ouro_config(**overrides)).to(device, dtype) + + +def rollout(**overrides): + return SamplingParams(**{"ignore_eos": True, "logprobs": 0, **overrides}) + + +def make_engine(m, *, asynchronous=False, exit_config=None, execution=None, **kwargs): + """Small CPU engine by default; asynchronous engines default to ouro_delayed exits.""" + kwargs.setdefault("cache_config", CacheConfig(128, 2)) + kwargs.setdefault("scheduler_config", SchedulerConfig(max_num_seqs=3, max_num_batched_tokens=3)) + execution = ExecutionConfig(async_scheduling=asynchronous, **(execution or {})) + exit_config = exit_config or ExitConfig("ouro_delayed" if asynchronous else "ouro") + return LLMEngine(m, execution_config=execution, exit_config=exit_config, **kwargs) + + +def check_aligned(output): + n, logprobs = len(output.token_ids), output.logprobs + assert len(output.exit_depths) == n + assert (logprobs is None) == (output.sampling_params.logprobs is None) + assert logprobs is None or len(logprobs) == n + assert all(math.isfinite(value) and value <= 0 for value in logprobs or ()) + + +def generate(engine, prompts, params): + """``LLM.generate`` on an in-process engine, whose runners tests can patch (``LLM`` + runs its engine in a stage worker process).""" + params = params if isinstance(params, list) else [params] * len(prompts) + for i, (prompt, request_params) in enumerate(zip(prompts, params)): + engine.add_request(str(i), prompt, request_params) + outputs = final(stream(engine)) + return [outputs[str(i)] for i in range(len(prompts))] + + +def stream(engine, until=None, streamed=None): + """Step until ``until()`` holds, else drain, checking every output's alignment. A drained + LLMEngine (1000-step limit) holds zero used KV blocks; PD gets a 90 s wall-clock limit.""" + streamed = defaultdict(list) if streamed is None else streamed + done = until or (lambda: not engine.has_unfinished_requests()) + local, deadline, steps = isinstance(engine, LLMEngine), time.monotonic() + 90, 0 + while not done(): + assert engine.has_unfinished_requests() + assert (steps < 1000) if local else (time.monotonic() < deadline) + steps += 1 + for output in engine.step(): + check_aligned(output) + streamed[output.request_id].append(output) + if until is None and local: + assert engine.cache_manager.num_used_blocks == 0 + return streamed + + +def final(streamed): + return {rid: outputs[-1] for rid, outputs in streamed.items()} + + +def assert_same(actual, expected, atol=None): + """Same tokens and exit depths; logprobs equal, or within ``atol`` when given.""" + assert (actual.token_ids, actual.exit_depths) == (expected.token_ids, expected.exit_depths) + logprobs = expected.logprobs + if atol is not None and logprobs is not None: + logprobs = pytest.approx(logprobs, abs=atol) + assert actual.logprobs == logprobs + + +def hold_coda_delivery(monkeypatch): + """Report CODA tickets unfinished, so only the one-output bound delivers them.""" + ready = Submission.ready + monkeypatch.setattr(Submission, "ready", lambda t: t.operation != Operation.CODA and ready(t)) + return ready + + +def lease_cpu_logprob_slots(monkeypatch, engine, count=8): + """Return the logprob readback pool. On CPU, which has none, install ``count`` emulated + slots and monkeypatch ``runner.submit`` to lease one per logprob ticket (CUDA: unchanged).""" + runner, submit = engine.model_runner, engine.model_runner.submit + if runner.device.type == "cuda": + return runner.logprob_slots + runner.logprob_slots = [ReadbackSlot(torch.empty(1)) for _ in range(count)] + + def leasing_submit(*args, **kwargs): + ticket = submit(*args, **kwargs) + if ticket.logprobs is not None: + ticket.logprob_slot = runner._readback_slot(ticket.logprobs, runner.logprob_slots) + return ticket + + monkeypatch.setattr(runner, "submit", leasing_submit) + return runner.logprob_slots + + +def test_valid_rollout_params_are_normalized_and_hashable(): + params = SamplingParams(seed=None, logprobs=0, stop_token_ids=[5, 3, 5]) + assert (params.seed, params.logprobs, params.stop_token_ids) == (None, 0, (5, 3, 5)) + assert {params} == {replace(params, stop_token_ids=(5, 3, 5))} + assert SamplingParams().logprobs is None and SamplingParams(seed=2**63 - 1).seed == 2**63 - 1 + + +def rejected(field, match, *values): + return [pytest.param(field, value, match, id=f"{field}={value!r}") for value in values] + + +@pytest.mark.parametrize( + "field,value,match", + rejected("logprobs", "not supported yet", 1, 5) + + rejected("logprobs", "None or 0", -1, True, 0.0, "0") + + rejected("seed", "seed", True, -1, 1.5, "1") + + [pytest.param("seed", 2**63, "seed", id="seed=2**63")] + + rejected("stop_token_ids", "stop_token_ids", None, 3, "3", [True], [1.0], [-1], {3}), +) +def test_invalid_rollout_params_are_rejected(field, value, match): + with pytest.raises(ValueError, match=match): + SamplingParams(**{field: value}) + + +def test_logprobs_mode_is_validated_and_forwarded_to_stage_workers(monkeypatch): + # LLM and PDEngine run StageEngine in worker processes; capture what they would start. + start = Mock(side_effect=RuntimeError("stop before workers")) + monkeypatch.setattr("vllm_rlt.engine.llm_engine.start_stage_worker", start) + for logprobs_mode in LOGPROBS_MODES: + assert LLMEngine(model(), logprobs_mode=logprobs_mode).model_runner.logprobs_mode == ( + logprobs_mode + ) + for build, source in ((LLM, model), (PDEngine, tiny_ouro_config)): + with pytest.raises(RuntimeError, match="stop before workers"): + build(source(), logprobs_mode=logprobs_mode) + for logprobs_mode in ("raw_logits", "processed_logits", "logprobs", None): + for build, source in ((LLMEngine, model), (LLM, model), (PDEngine, tiny_ouro_config)): + with pytest.raises(ValueError, match="not supported"): + build(source(), logprobs_mode=logprobs_mode) + # Valid modes reach the worker engines (LLM, then PD); invalid ones start no worker. + forwarded = [c.args[3]["engine"]["logprobs_mode"] for c in start.call_args_list] + assert forwarded == [mode for mode in LOGPROBS_MODES for _ in range(2)] + + +def test_admission_rejects_out_of_vocabulary_stop_ids(): + engine = LLMEngine(model()) + for stop in ([64], [3, 1000]): # The tiny vocabulary is IDs 0..63. + with pytest.raises(ValueError, match="stop_token_ids"): + engine.add_request("bad", [1], SamplingParams(stop_token_ids=stop)) + assert not engine.has_unfinished_requests() + engine.add_request("ok", [1], SamplingParams(max_tokens=2, stop_token_ids=[63], logprobs=0)) + assert final(stream(engine))["ok"].finished + + +def test_engine_rejects_missing_logprob_for_requesting_row(): + engine = LLMEngine(model()) + engine.add_request("r", [2], SamplingParams(logprobs=0)) + with pytest.raises(RuntimeError, match="missing logprob"): + engine.scheduler._record_token(engine.scheduler.requests["r"], 3, 4, None) + + +def test_stage_client_round_trips_rollout_outputs(): + """LLM runs StageEngine in a worker process: outputs cross the pipe with their logprobs + and effective params, an abort before the first sample included, and replay exactly.""" + params = rollout(max_tokens=6, temperature=0.9, top_k=8, seed=None) + with LLM(model(), logprobs_mode="processed_logprobs") as llm: + client = llm.engine + client.add_request("early", [2, 3], params) + aborted = client.abort_request("early") + check_aligned(aborted) + assert aborted.finish_reason == "abort" and type(aborted.sampling_params.seed) is int + outputs = llm.generate([[2, 3], [4]], params) + for output in outputs: + check_aligned(output) + assert output.sampling_params == replace(params, seed=output.sampling_params.seed) + assert outputs[0].sampling_params.seed != outputs[1].sampling_params.seed + replay = llm.generate([[2, 3], [4]], [output.sampling_params for output in outputs]) + for actual, expected in zip(replay, outputs): + assert_same(actual, expected) + # A stop ID ends the request in the worker; the stop token keeps its logprob. + stop = outputs[0].token_ids[0] + params = replace(outputs[0].sampling_params, stop_token_ids=[stop]) + (stopped,) = llm.generate([[2, 3]], params) + assert (stopped.finish_reason, stopped.token_ids) == ("stop", [stop]) + assert stopped.logprobs == outputs[0].logprobs[:1] + + +def test_http_request_fields_are_unchanged(): + # tests/test_serving.py covers the other rollout fields. + for value in (0, True): + with pytest.raises(ValueError, match="logprobs"): + CompletionRequest.parse({"model": "m", "prompt": "a", "logprobs": value}, "m") + assert CompletionRequest.parse({"model": "m", "prompt": "a", "seed": 7}, "m").params.seed == 7 + + +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +@pytest.mark.parametrize("finish", FINISHES.split()) +@pytest.mark.parametrize("asynchronous,exit_config,threshold", EXIT_MODES) +def test_logprobs_are_defined_and_aligned_through_each_finish( + monkeypatch, asynchronous, exit_config, threshold, finish, logprobs_mode +): + rows = defaultdict(list) # CODA logits rows in token order; the baseline runs first. + common = dict(min_loops=1, exit_threshold=threshold) + options = dict(asynchronous=asynchronous, exit_config=exit_config, logprobs_mode=logprobs_mode) + # The unrequested row exercises mixed batches (ignored values). Admitted first, it puts + # "a" behind row 0 in shared batches, so each row must read its own logprob. + unrequested = rollout(max_tokens=6, temperature=0.9, seed=5, logprobs=None, **common) + + def run(params, *, eos=0, abort=False): + engine = make_engine(model(eos_token_id=eos), **options) + runner = engine.model_runner + if params.logprobs is None: # No logprob work when no request asks. + unexpected = AssertionError("logprobs computed without a request") + monkeypatch.setattr(runner, "_sample_with_logprobs", Mock(side_effect=unexpected)) + + def recording(sample): + def record(row, row_params, generator): + if row_params.seed == params.seed: # Request "a"; "b" has seed 5. + rows["a"].append(row.detach().clone()) + return sample(row, row_params, generator) + + return record + + # Processed top-k/top-p rows that request logprobs draw through sample_with_logprob. + for name in ("sample", "sample_with_logprob"): + setattr(runner.sampler, name, recording(getattr(runner.sampler, name))) + engine.add_request("b", [4], unrequested) + engine.add_request("a", [2, 3], replace(params, **common)) + if not abort: + return stream(engine) + request = engine.scheduler.requests["a"] + streamed = stream(engine, lambda: len(request.generated_token_ids) >= 2) + streamed["a"].append(engine.abort_request("a")) + return stream(engine, streamed=streamed) + + params = rollout(max_tokens=6, temperature=1.0, top_k=20, top_p=0.9, seed=7) + baseline = final(run(params))["a"] + tokens = baseline.token_ids + assert baseline.finish_reason == "length" and len(rows["a"]) == len(tokens) == 6 + # Raw scores the token's CODA row; processed, the distribution the sampler draws from. + processed = logprobs_mode == "processed_logprobs" + expected = [ + (Sampler.probabilities(row, params).log() if processed else row.float().log_softmax(-1))[t] + for row, t in zip(rows["a"], tokens) + ] + assert baseline.logprobs == pytest.approx([value.item() for value in expected], abs=1e-6) + # Every policy except fixed full depth must exercise shallow exits here. + shallow = threshold < 1 or exit_config.mode == "trace" + assert any(depth < 4 for depth in baseline.exit_depths) == shallow + # Asking for logprobs changes no token or exit depth. + quiet = final(run(replace(params, logprobs=None)))["a"] + assert (quiet.token_ids, quiet.exit_depths) == (tokens, baseline.exit_depths) + # Stop on the first token that is new at index >= 2, so a stop must cut the stream. + index = next(i for i, t in enumerate(tokens) if i >= 2 and t not in tokens[:i]) + stop, other = tokens[index], next(t for t in range(64) if t not in tokens) + # finish -> (EOS ID, overrides, reason, length); stop IDs, unlike EOS, override ignore_eos. + honor, cut = dict(ignore_eos=False), dict(max_tokens=index + 1) + cases = { + "length": (0, {}, "length", 6), + "eos": (stop, honor, "stop", index + 1), + "stop_id": (0, dict(stop_token_ids=[other, stop]), "stop", index + 1), + "ignored_eos": (stop, {}, "length", 6), + "eos_with_stop_ids": (stop, dict(honor, stop_token_ids=[other]), "stop", index + 1), + "stop_id_at_max": (0, dict(cut, stop_token_ids=[stop]), "stop", index + 1), + "eos_at_max": (stop, dict(honor, **cut), "stop", index + 1), + "abort": (0, {}, "abort", None), # After two delivered tokens. + } + eos, overrides, reason, length = cases[finish] + outputs = run(replace(params, **overrides), eos=eos, abort=finish == "abort")["a"] + n = len(outputs[-1].token_ids) + assert outputs[-1].finish_reason == reason and (2 <= n < 6 if length is None else n == length) + if finish != "abort": # One token per output; a stop token keeps its depth and logprob. + assert [len(o.token_ids) for o in outputs] == list(range(1, n + 1)) + for output in outputs: + # Re-checked after draining: each output must own its lists, not the request's. + check_aligned(output) + k = len(output.token_ids) + assert output.token_ids == tokens[:k] and output.exit_depths == baseline.exit_depths[:k] + assert output.logprobs == baseline.logprobs[:k] + + +# Request ID -> overrides covering each processed-logprob path: one batched raw log_softmax +# (greedy, even with top-k, and T == 1), one per other temperature (0.7 twice, 1.3), the +# sampler's top-k/top-p logits, and a row that asks for no logprobs. +MIXED_ROWS = { + "greedy": dict(temperature=0.0, top_k=3), + "t1": dict(temperature=1.0), + "t07": dict(temperature=0.7), + "t07b": dict(temperature=0.7), + "t13": dict(temperature=1.3), + "top_k": dict(temperature=0.9, top_k=8), + "top_p": dict(temperature=1.0, top_p=0.8), + "top_k_top_p": dict(temperature=0.7, top_k=16, top_p=0.9), + "quiet": dict(temperature=0.8, logprobs=None), +} + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_grouped_processed_logprobs_equal_the_per_row_reference(monkeypatch, asynchronous): + mode, built = "processed_logprobs", [] # built: one entry per processed_logits call. + processed_logits = Sampler.processed_logits + + def counted(logits, params): + built.append(params) + return processed_logits(logits, params) + + monkeypatch.setattr(Sampler, "processed_logits", staticmethod(counted)) + batch_sizes = [] + + def run(logprobs): + scheduler = SchedulerConfig(max_num_seqs=9, max_num_batched_tokens=16) + options = dict(logprobs_mode=mode, scheduler_config=scheduler) + engine = make_engine(model(), asynchronous=asynchronous, **options) + runner = engine.model_runner + batched = runner._sample_with_logprobs + + def checked(logits, params, generators): + before = len(built) + token_ids, values = batched(logits, params, generators) + # Sampling builds each sampled row's processed logits once; logprobs reuse them. + assert len(built) - before == sum(p.temperature > 0 for p in params) + for row, row_params in enumerate(params): # 57d2308 ran token_logprob per row. + if row_params.logprobs is not None: + reference = token_logprob(logits[row], row_params, token_ids[row], mode) + assert torch.equal(values[row], reference) + batch_sizes.append(len(params)) + return token_ids, values + + monkeypatch.setattr(runner, "_sample_with_logprobs", checked) + for i, (rid, overrides) in enumerate(MIXED_ROWS.items()): + params = rollout(max_tokens=6, seed=20 + i, exit_threshold=1.0, **overrides) + engine.add_request(rid, [1 + i], params if logprobs else replace(params, logprobs=None)) + return final(stream(engine)) + + grouped, quiet = run(logprobs=True), run(logprobs=False) + assert max(batch_sizes) == len(MIXED_ROWS) # Every path shared one CODA batch. + for rid, output in quiet.items(): # Logprobs change no token, depth or RNG draw. + assert grouped[rid].token_ids == output.token_ids + assert grouped[rid].exit_depths == output.exit_depths + assert (grouped[rid].logprobs is None) == (rid == "quiet") + + +@pytest.mark.parametrize("layout", ["last_exited", "shared"]) +def test_full_depth_raw_logprobs_vs_teacher_forced_reference(layout): + m = model() + engine = make_engine(m, cache_config=CacheConfig(128, 2, layout)) + prompts = {"a": [2, 3, 5], "s": [7]} + engine.add_request("a", prompts["a"], rollout(max_tokens=6)) + engine.add_request("s", prompts["s"], rollout(max_tokens=6, temperature=0.8, seed=3)) + for rid, output in final(stream(engine)).items(): + assert output.exit_depths == [4] * 6 + tokens = prompts[rid] + output.token_ids + rows = dense_reference(m, tokens, 4)[-1][2][len(prompts[rid]) - 1 : -1] + expected = rows.float().log_softmax(-1)[torch.arange(6), torch.tensor(output.token_ids)] + difference = (torch.tensor(output.logprobs) - expected).abs().max().item() + # Shared KV reads earlier tokens' final-loop KV at every depth: no plain forward matches. + assert difference < 2e-5 if layout == "last_exited" else difference > 1e-2 + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_seed_none_is_resolved_at_admission_and_replays_exactly(asynchronous): + params = rollout(max_tokens=8, temperature=1.0, seed=None) + engine = make_engine(model(), asynchronous=asynchronous) + torch_state, python_state = torch.get_rng_state(), random.getstate() + for rid in ("x", "y", "unsampled"): + engine.add_request(rid, [2, 3], params) + # Seeds come from OS entropy, not from the torch or Python global RNG. + assert torch.equal(torch.get_rng_state(), torch_state) and random.getstate() == python_state + seeds = {rid: r.sampling_params.seed for rid, r in engine.scheduler.requests.items()} + assert all(type(seed) is int and 0 <= seed < 2**63 for seed in seeds.values()) + # Every output, even an abort before the first sample, reports the effective params. + outputs = {"unsampled": engine.abort_request("unsampled")} + assert engine.rng.peek("unsampled") is None # Aborted before its first draw. + sampled = [engine.scheduler.requests[rid] for rid in ("x", "y")] + streamed = stream(engine, lambda: all(r.generated_token_ids for r in sampled)) + # The engine's registry seeded each generator with exactly the seed outputs report. + assert all(engine.rng.peek(r).initial_seed() == seeds[r] for r in ("x", "y")) + outputs |= final(stream(engine, streamed=streamed)) + assert engine.rng.peek("x") is None and engine.rng.peek("y") is None # Dropped at finish. + for rid, output in outputs.items(): + check_aligned(output) + assert output.sampling_params == replace(params, seed=seeds[rid]) + assert len(set(seeds.values())) == 3 and outputs["x"].token_ids != outputs["y"].token_ids + # Replaying the reported params reproduces the rollout exactly. + replay = make_engine(model(), asynchronous=asynchronous) + for rid in ("x", "y"): + replay.add_request(rid, [2, 3], outputs[rid].sampling_params) + for rid, output in final(stream(replay)).items(): + assert_same(output, outputs[rid]) + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_greedy_seed_none_is_resolved_at_admission(asynchronous): + # Greedy Ouro rows never reach the RNG registry, so admission alone resolves their seed + # (Huginn's greedy rows do reach it: see the Huginn test below). + engine = make_engine(model(), asynchronous=asynchronous) + engine.rng.acquire = Mock(side_effect=AssertionError("greedy Ouro rows draw no RNG")) + engine.add_request("g", [2, 3], rollout(max_tokens=3, seed=None)) + seed = engine.scheduler.requests["g"].sampling_params.seed + assert type(seed) is int and 0 <= seed < 2**63 + (output,) = final(stream(engine)).values() + assert output.sampling_params == rollout(max_tokens=3, seed=seed) + + +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +@pytest.mark.parametrize("temperature", [0.0, 0.8], ids=["greedy", "top_k-top_p"]) +def test_huginn_reports_the_seed_of_its_recurrent_state_generator(temperature, logprobs_mode): + # Huginn draws its random recurrent-state initialization from the request's generator + # at prefill and in every PRELUDE, greedy requests included, so its generator is + # created before any CODA sample. That generator must still carry the reported seed. + torch.manual_seed(42) + m = HuginnForCausalLM(tiny_huginn_config()).eval() + prompts = {"a": [4, 7, 3], "b": [9, 2, 5, 6]} + scheduler = SchedulerConfig(max_num_seqs=3, max_num_batched_tokens=8, prefill_chunk_size=3) + options = dict(cache_config=CacheConfig(128, 2), scheduler_config=scheduler) + + def run(params): + engine = LLMEngine(m, logprobs_mode=logprobs_mode, **options) + acquire, created = engine.rng.acquire, {} + + def recording_acquire(request_id, request_params, device): + generator = acquire(request_id, request_params, device) + created.setdefault(request_id, generator.initial_seed()) + return generator + + engine.rng.acquire = recording_acquire + for rid, prompt in prompts.items(): + engine.add_request(rid, prompt, params[rid]) + return final(stream(engine)), created + + sampling = dict(temperature=temperature, top_k=5, top_p=0.9) if temperature else {} + params = rollout(max_tokens=5, min_loops=3, max_loops=3, seed=None, **sampling) + outputs, created = run(dict.fromkeys(prompts, params)) + assert created == {rid: output.sampling_params.seed for rid, output in outputs.items()} + # Requesting logprobs changes no token, and replaying the reported params is bitwise. + quiet, _ = run({rid: replace(o.sampling_params, logprobs=None) for rid, o in outputs.items()}) + replay, _ = run({rid: output.sampling_params for rid, output in outputs.items()}) + for rid, output in outputs.items(): + assert quiet[rid].token_ids == output.token_ids + assert quiet[rid].exit_depths == output.exit_depths + assert_same(replay[rid], output) + + +PREEMPTED_PHASES = [ + pytest.param(False, RequestPhase.CODA, id="sync-coda"), + pytest.param(False, RequestPhase.PRELUDE, id="sync-prelude"), + pytest.param(False, RequestPhase.RECURRENT, id="sync-recurrent"), + # Asynchronous preemption skips requests with an undelivered output. With a single + # request, one waiting in PRELUDE still has its previous output undelivered, so this + # construction reaches no async PRELUDE victim; the pressure test below does. + pytest.param(True, RequestPhase.CODA, id="async-coda"), + pytest.param(True, RequestPhase.RECURRENT, id="async-recurrent"), +] + + +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +@pytest.mark.parametrize("asynchronous,phase", PREEMPTED_PHASES) +def test_preemption_at_each_phase_keeps_the_rng_slot(asynchronous, phase, logprobs_mode): + # Top-k/top-p rows: processed mode samples them through _sample_with_logprob. + params = rollout(max_tokens=7, temperature=0.8, top_k=3, top_p=0.9, seed=45, exit_threshold=0) + options = dict(asynchronous=asynchronous, logprobs_mode=logprobs_mode) + scheduler = SchedulerConfig(enable_preemption=True) + options.update(cache_config=CacheConfig(64, 2), scheduler_config=scheduler) + results = [] + for preempt in (False, True): + engine = make_engine(model(), **options) + engine.add_request("a", [1, 2, 3], params) + request = engine.scheduler.requests["a"] + if preempt: + stream( + engine, + lambda: ( + len(request.generated_token_ids) >= 2 + and request.phase == phase + and not request.num_output_placeholders + ), + ) + generator = engine.rng.peek("a") + state = generator.get_state().clone() + engine.add_request("b", [5], SamplingParams(max_tokens=1)) + assert engine.scheduler.preempt(engine.scheduler.requests["b"]) + # Suspension keeps the request's generator, at the same point of its stream. + assert engine.rng.peek("a") is generator + assert torch.equal(generator.get_state(), state) + results.append(final(stream(engine))["a"]) + assert engine.preemption.resumptions == int(preempt) + # A CODA victim resumes into "b"'s CODA batch, so its LM head runs two rows instead of + # one. CPU GEMMs are not batch invariant, so its logprobs match numerically (up to ~5e-7). + # PRELUDE and RECURRENT victims share no batch with "b": theirs stay bitwise. + assert_same(results[1], results[0], atol=1e-6 if phase == RequestPhase.CODA else None) + + +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +def test_async_prelude_preemption_keeps_the_rng_slot(logprobs_mode): + """Memory pressure preempts asynchronous requests waiting in PRELUDE with no undelivered + output (the construction of tests/test_prefix_growth.py's GPU test for #109). On CPU no + AsyncState exists, so this covers the RNG slot and logprobs, not #109's device token.""" + trace = ExitConfig("trace", depths_by_request={"t": [4, 1, 2, 3, 4, 1, 2, 3]}) + params = rollout(max_tokens=8, min_loops=1, temperature=0.9, top_k=5, top_p=0.95) + scheduler = SchedulerConfig( + max_num_seqs=2, max_num_batched_tokens=2, prefill_chunk_size=2, enable_preemption=True + ) + options = dict(asynchronous=True, exit_config=trace, logprobs_mode=logprobs_mode) + results, victims = [], [] + for blocks in (128, 24): + cache = CacheConfig(blocks, 2, incremental_allocation=True) + engine = make_engine(model(72), cache_config=cache, scheduler_config=scheduler, **options) + runner, suspended = engine.model_runner, {} + suspend, resume = runner.suspend, engine.scheduler.resume_callback + + def checked_suspend(request_id, engine=engine, suspend=suspend, suspended=suspended): + request, generator = engine.scheduler.requests[request_id], engine.rng.peek(request_id) + victims.append((request.phase, generator is not None)) + state = None if generator is None else generator.get_state().clone() + suspended[request_id] = generator, state + suspend(request_id) + + def checked_resume(request, engine=engine, resume=resume, suspended=suspended): + if request.request_id in suspended: # Its stream is untouched while suspended. + generator, state = suspended[request.request_id] + assert engine.rng.peek(request.request_id) is generator + assert generator is None or torch.equal(generator.get_state(), state) + return resume(request) + + runner.suspend, engine.scheduler.resume_callback = checked_suspend, checked_resume + for i in range(4): + prompt = [2 * i + 1, 2 * i + 2] + engine.add_request(str(i), prompt, replace(params, seed=10 + i), trace_id="t") + results.append(final(stream(engine))) + assert (engine.preemption.preemptions > 0) == (blocks == 24) + assert engine.preemption.resumptions == engine.preemption.preemptions + assert (RequestPhase.PRELUDE, True) in victims # A PRELUDE victim that had already sampled. + for rid, output in results[0].items(): + # Pressure changes the batch composition, so logprobs match numerically. + assert_same(results[1][rid], output, atol=1e-6) + + +def test_preemption_and_prefix_hits_keep_logprobs(): + params = rollout(max_tokens=7, temperature=0.8, seed=45, exit_threshold=0) + results = [] + for suspend in (False, True): + scheduler = SchedulerConfig(enable_preemption=True) + engine = make_engine(model(), cache_config=CacheConfig(64, 2), scheduler_config=scheduler) + engine.add_request("a", [1, 2, 3], params) + request = engine.scheduler.requests["a"] + stream(engine, lambda: len(request.generated_token_ids) >= 3) + if suspend: + before = list(request.logprobs) + engine.add_request("b", [5], SamplingParams(max_tokens=1)) + assert engine.scheduler.preempt(engine.scheduler.requests["b"]) + assert request.logprobs == before and request.phase == RequestPhase.WAITING + results.append(final(stream(engine))["a"]) + assert engine.preemption.resumptions == int(suspend) + assert_same(results[1], results[0]) + # Memory-pressure preemption matches an unpressured run. + params, results = rollout(max_tokens=8, exit_threshold=0), [] + for blocks, preempt in ((128, False), (24, True)): + cache = CacheConfig(blocks, 2, incremental_allocation=True) + scheduler = SchedulerConfig(max_num_seqs=3, prefill_chunk_size=2, enable_preemption=preempt) + engine = make_engine(model(15), cache_config=cache, scheduler_config=scheduler) + for i in range(3): + engine.add_request(str(i), [1, 2], params) + results.append(final(stream(engine))) + assert (engine.preemption.preemptions > 0) == preempt + for rid, output in results[0].items(): + assert_same(results[1][rid], output, atol=1e-6) + # A prefix-cache hit matches the cold run. + cache = CacheConfig(64, 2, enable_prefix_caching=True) + scheduler = SchedulerConfig(prefill_chunk_size=2) + engine = make_engine(model(15), cache_config=cache, scheduler_config=scheduler) + engine.add_request("cold", [1, 2, 3, 4, 5], replace(params, max_tokens=4)) + cold = final(stream(engine))["cold"] + engine.add_request("warm", [1, 2, 3, 4, 5], replace(params, max_tokens=4)) + engine.step() + assert engine.last_schedule.requests[0].token_start == 4 + assert_same(final(stream(engine))["warm"], cold, atol=1e-6) + + +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +@pytest.mark.parametrize( + "old,new", [(0, None), (None, 0)], ids=["logprobs-then-none", "none-then-logprobs"] +) +@pytest.mark.parametrize("exit_config,threshold", ASYNC_EXIT_MODES) +def test_lagging_delivery_abort_and_id_reuse_keep_logprobs( + monkeypatch, exit_config, threshold, old, new, logprobs_mode +): + sampling = dict(temperature=0.9, top_p=0.95, seed=7, min_loops=1, exit_threshold=threshold) + params = rollout(max_tokens=6, **sampling) # T != 1 and top-p: processed differs from raw. + options = dict(exit_config=exit_config, logprobs_mode=logprobs_mode) + + def fresh(*, asynchronous): + engine = make_engine(model(), asynchronous=asynchronous, **options) + engine.add_request("a", [5, 6], replace(params, logprobs=new)) + return final(stream(engine))["a"] + + expected = fresh(asynchronous=True) + if exit_config.mode == "trace": # Asynchronous delivery matches the synchronous path. + assert_same(fresh(asynchronous=False), expected, atol=1e-6) + hold_coda_delivery(monkeypatch) + engine = make_engine(model(), asynchronous=True, **options) + slots = lease_cpu_logprob_slots(monkeypatch, engine) + engine.add_request("a", [2, 3], replace(params, logprobs=old)) + engine.add_request("b", [4], replace(params, seed=5)) + request = engine.scheduler.requests["a"] + # Step until the request has delivered a token and has one undelivered CODA output. + stream(engine, lambda: request.generated_token_ids and request.num_output_placeholders == 1) + delivered = len(request.generated_token_ids) + aborted = engine.abort_request("a") + check_aligned(aborted) + # The undelivered token is not part of the abort output. + assert aborted.finish_reason == "abort" and len(aborted.token_ids) == delivered + # Reuse the ID with the opposite setting while the stale ticket (and lease) is pending. + assert engine._pending_coda and (old is None or any(slot.leased for slot in slots)) + engine.add_request("a", [5, 6], replace(params, logprobs=new)) + assert_same(final(stream(engine))["a"], expected, atol=1e-6) + assert not engine._pending_coda and not any(slot.leased for slot in slots) + + +def test_speculative_admission_rejects_logprobs(): + engine = LLMEngine(model(), speculative_config=SpeculativeConfig(3)) + with pytest.raises(ValueError, match="speculative"): + engine.add_request("r", [2, 3], SamplingParams(logprobs=0)) + assert not engine.has_unfinished_requests() + + +@pytest.mark.parametrize( + "finish,max_tokens,emitted", + [ + ("round", 9, [10, 50, 0, 13]), + ("stop_token_ids", 9, [10, 50]), + ("eos", 9, [10, 50, 0]), + ("length", 3, [10, 50]), + ], + ids=["continue", "stop-id-mid-round", "eos-mid-round", "length-at-round-end"], +) +def test_async_speculative_collect_uses_shared_finish_rules(finish, max_tokens, emitted): + """Deliver one async speculative round on CPU. The runner needs CUDA, so a stub ticket + stands in for its readback; _collect_speculative itself is the code under test.""" + engine = LLMEngine(model(), speculative_config=SpeculativeConfig(3)) + stops = [50] if finish == "stop_token_ids" else [] + params = SamplingParams(max_tokens=max_tokens, ignore_eos=finish != "eos", stop_token_ids=stops) + engine.add_request("r", [2, 3], params) + request = engine.scheduler.requests["r"] + stream(engine, lambda: request.generated_token_ids) # The bootstrap CODA token. + batch = engine.scheduler.schedule() + [scheduled_request] = batch.requests + # Apply _step_async_speculative's submission bookkeeping, then deliver the round. + engine.scheduler.update_after_submission(batch) + assert request.num_output_placeholders == scheduled_request.token_count + tokens = [10, 50, 0, 13][: scheduled_request.token_count] + frontier = len(request.prompt_token_ids) + len(request.generated_token_ids) - 1 + len(tokens) + result = ModelRunnerOutput( + speculative=[SpeculativeResult(tokens, len(tokens) - 1)], frontiers=[frontier] + ) + ticket = SimpleNamespace( + batch=batch, + states=(SimpleNamespace(delivered=0),), + rounds=(0,), + ready=lambda: True, + collect=lambda: result, + retire=lambda: None, + ) + stats = engine.speculative_runner.stats + engine.speculative_runner = SimpleNamespace( + stats=stats, + readback_waits=0, + discarded_rounds=0, + request_ready=lambda r: True, + release=lambda r: None, + ) + engine.async_speculative, engine._pending_speculative = True, [ticket] + [output] = engine._collect_speculative() + reason = {"round": None, "length": "length"}.get(finish, "stop") + assert (output.token_ids[1:], output.finish_reason) == (emitted, reason) + assert output.exit_depths == [4] * len(output.token_ids) and output.logprobs is None + assert stats.committed_tokens == len(emitted) and request.num_output_placeholders == 0 + if reason is None: + assert list(engine.scheduler.queues[RequestPhase.SPECULATIVE]) == ["r"] + else: + assert not engine.has_unfinished_requests() + assert engine.cache_manager.num_used_blocks == 0 + + +def test_pd_coordinator_resolves_seeds_and_mirrors_logprobs(): + engine = object.__new__(PDEngine) # Coordinator only: no worker processes. + engine.closed, engine.failure = False, None + engine.requests, engine.transfers, engine.outputs = {}, {}, deque() + engine.config, engine.exit_config = PDConfig(), ExitConfig("ouro_delayed") + engine.speculative_config, engine._async_speculative = None, False + engine.model_config = tiny_ouro_config() + engine._weight_version, engine._update_version = 0, None + engine._update_failed, engine._paused = False, False + info, roles = dict(block_size=4, depths=4, num_blocks=1024), ("prefill", "decode") + engine.peers = { + role: Peer(role, role, SimpleNamespace(info=info, send=Mock())) for role in roles + } + with pytest.raises(ValueError, match="stop_token_ids"): + engine.add_request("bad", [1], SamplingParams(stop_token_ids=[64])) + # P and D get these effective params (one resolved seed), which an early abort reports. + for rid in ("unsampled", "r"): + engine.add_request(rid, [1, 2], rollout(max_tokens=8, temperature=1.0, seed=None)) + unsampled = engine.requests["unsampled"].sampling_params + assert engine.abort_request("unsampled").sampling_params == unsampled + request = engine.requests["r"] + assert type(unsampled.seed) is int and type(request.sampling_params.seed) is int + assert request.logprobs == [] + w = next(iter(engine.transfers.values())) + w.p, w.d, w.phase = "prefill", "decode", "decode" + for n in (1, 2): + mirrored = dict(logprobs=[-0.5] * n, sampling_params=request.sampling_params) + output = RequestOutput(w.tid, [1, 2], [5] * n, [4] * n, False, **mirrored) + message = dict(kind="output", request_id=w.tid, output=output) + engine._message(engine.peers["decode"], message) + assert [o.request_id for o in engine.outputs] == ["r", "r"] + assert engine.outputs[-1].logprobs == [-0.5, -0.5] + aborted = engine.abort_request("r") + assert aborted.finish_reason == "abort" and aborted.sampling_params == request.sampling_params + assert (aborted.token_ids, aborted.exit_depths) == ([5, 5], [4, 4]) + assert aborted.logprobs == [-0.5, -0.5] + + +def gpu_engine( + m, *, asynchronous, exit_config=None, seqs=3, logprobs_mode="raw_logprobs", **execution +): + scheduler = SchedulerConfig(max_num_seqs=seqs, max_num_batched_tokens=8, prefill_chunk_size=4) + options = dict(cache_config=CacheConfig(128, 16), scheduler_config=scheduler) + options |= dict(logprobs_mode=logprobs_mode, attention_backend="triton", execution=execution) + exit_config = exit_config or ExitConfig("ouro_delayed") + return make_engine(m, asynchronous=asynchronous, exit_config=exit_config, **options) + + +def gpu_params(rid, **overrides): + values = dict(max_tokens=6 + int(rid), temperature=0.8, top_k=16, seed=40 + int(rid)) + return rollout(**(values | dict(min_loops=2, exit_threshold=0.2) | overrides)) + + +@pytest.mark.gpu +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_gpu_cuda_graph_logprobs_match_eager(asynchronous, logprobs_mode): + # Static eager pads RECURRENT rows to a power of two while graph replay runs len(rows), + # so GEMM reductions may round; FP32 keeps tokens and exits exact and bounds logprobs. + m = model(321, "cuda", torch.float32, head_dim=64) + results = [] + for graphs in (False, True): + # One stream (no prefer_recurrent overlap): batches cannot depend on graph timing. + static = dict(static_buffers=True, pad_to_power_of_two=True, multi_stream=False) + options = dict(asynchronous=asynchronous, logprobs_mode=logprobs_mode, cuda_graphs=graphs) + engine = gpu_engine(m, **options, **static) + for i in range(3): + engine.add_request(str(i), [2 + i, 3, 4] * (i + 1), gpu_params(str(i))) + results.append(final(stream(engine))) + assert not graphs or engine.model_runner.graphs.replays > 0 + eager, graphed = results + assert graphed.keys() == eager.keys() + for rid, output in eager.items(): + assert_same(graphed[rid], output, atol=1e-4) + + +@pytest.mark.gpu +@pytest.mark.parametrize("multi_stream", [False, True], ids=["one-stream", "multi-stream"]) +def test_gpu_async_readback_matches_sync_and_returns_leases(monkeypatch, multi_stream): + # FP32 keeps sampled tokens stable across sync/async batch compositions. + m = model(321, "cuda", torch.float32, head_dim=64) + traces = {str(i): [4, 1, 3, 2, 4, 1, 2, 3, 4, 2, 1, 3] for i in range(4)} + options = dict(exit_config=ExitConfig("trace", depths_by_request=traces), seqs=4) + + def run(*, asynchronous): + engine = gpu_engine(m, asynchronous=asynchronous, multi_stream=multi_stream, **options) + for i in range(4): + params = gpu_params(str(i), max_tokens=12, min_loops=1, exit_threshold=1.0) + engine.add_request(str(i), [2, 3 + i], params) + return engine, final(stream(engine)) + + _, expected = run(asynchronous=False) + # Delivery lags to the one-output bound: logprob leases overlap live exit-score tickets. + hold_coda_delivery(monkeypatch) + engine, actual = run(asynchronous=True) + assert not any(slot.leased for slot in engine.model_runner.logprob_slots) + for rid, output in expected.items(): + assert_same(actual[rid], output, atol=1e-4) + + +@pytest.mark.gpu +def test_gpu_delayed_exit_scores_and_logprobs_do_not_exhaust_readback_pools(monkeypatch): + engine = gpu_engine(model(5, "cuda", torch.bfloat16, head_dim=64), asynchronous=True, seqs=4) + assert len(engine.model_runner.logprob_slots) == 4 + 4 + # Lagging delivery holds a logprob lease per request while delayed exit scores hold theirs. + hold_coda_delivery(monkeypatch) + for round_index in range(3): + for i in range(4): + engine.add_request(f"{round_index}-{i}", [2, 3, i], gpu_params(str(i), max_tokens=16)) + assert len(final(stream(engine))) == 4 + assert not any(slot.leased for slot in engine.model_runner.logprob_slots) + + +@pytest.mark.gpu +def test_gpu_abort_with_pending_coda_returns_lease_and_reuses_id(monkeypatch): + engine = gpu_engine(model(5, "cuda", torch.bfloat16, head_dim=64), asynchronous=True) + slots = engine.model_runner.logprob_slots + ready = hold_coda_delivery(monkeypatch) + for i in range(2): + engine.add_request(str(i), [2, 3, i], gpu_params(str(i), max_tokens=12)) + request = engine.scheduler.requests["0"] + # Step until the request has delivered a token and has one undelivered CODA output. + stream(engine, lambda: request.generated_token_ids and request.num_output_placeholders == 1) + delivered = len(request.generated_token_ids) + aborted = engine.abort_request("0") + check_aligned(aborted) + assert len(aborted.token_ids) == delivered and any(slot.leased for slot in slots) + stale = [t for t in engine._pending_coda if any(i.request is request for i in t.batch.requests)] + assert stale + # release() synchronized the aborted request, so the next step collects its ticket. + monkeypatch.setattr(Submission, "ready", ready) + engine.add_request("0", [2, 3], gpu_params("0", logprobs=None)) + for output in engine.step(): + check_aligned(output) + assert not any(t is s for t in engine._pending_coda for s in stale) + outputs = final(stream(engine)) + assert outputs["0"].logprobs is None and len(outputs["0"].token_ids) == 6 + assert len(outputs["1"].logprobs) == 12 + assert not engine._pending_coda and not any(slot.leased for slot in slots) + + +@pytest.mark.parametrize("device", ["cpu", pytest.param("cuda", marks=pytest.mark.gpu)]) +def test_execution_error_returns_undelivered_logprob_leases(monkeypatch, device): + cuda = device == "cuda" + m = model(5, "cuda", torch.bfloat16, head_dim=64) if cuda else model(5) + engine = gpu_engine(m, asynchronous=True, seqs=4) if cuda else make_engine(m, asynchronous=True) + slots = lease_cpu_logprob_slots(monkeypatch, engine) + runner = engine.model_runner + ready, execute = hold_coda_delivery(monkeypatch), runner._execute + held = [] + + def fail(batch, prepared): + result = execute(batch, prepared) + if prepared.batch.operation == Operation.RECURRENT and engine._pending_coda: + held.append(sum(slot.leased for slot in slots)) + raise RuntimeError("injected failure after CODA submission") + return result + + monkeypatch.setattr(runner, "_execute", fail) + for i in range(3): + engine.add_request(str(i), [2, 3, i], gpu_params(str(i), max_tokens=8)) + with pytest.raises(RuntimeError, match="injected"): + stream(engine) + # The failure path drops undelivered tickets without collect(); __del__ returns leases. + assert held and held[0] > 0 and not engine._pending_coda + gc.collect() + assert not any(slot.leased for slot in slots) + monkeypatch.setattr(runner, "_execute", execute) + monkeypatch.setattr(Submission, "ready", ready) + engine.add_request("3", [2, 3], gpu_params("3", max_tokens=4)) + check_aligned(final(stream(engine))["3"]) + + +@pytest.mark.gpu +def test_gpu_async_speculative_stop_token_ids_match_sync(): + """Cross-round async speculation (CUDA, Triton, greedy) stops on stop IDs like sync.""" + m = model(device="cuda") + prompts = {"a": [2, 3, 4, 5], "b": [9], "c": [2, 3, 4, 7]} + + def run(asynchronous, **overrides): + engine = LLMEngine( + m, + attention_backend="triton", + cache_config=CacheConfig(256, 2, incremental_allocation=True), + scheduler_config=SchedulerConfig( + max_num_seqs=4, max_num_batched_tokens=16, prefill_chunk_size=2 + ), + execution_config=ExecutionConfig(async_scheduling=asynchronous), + speculative_config=SpeculativeConfig(3), + ) + for rid, prompt in prompts.items(): + params = SamplingParams(max_tokens=16, ignore_eos=True, **overrides) + engine.add_request(rid, prompt, params) + outputs = final(stream(engine)) + engine.close() + return outputs + + free = run(False) + # Stop on the latest first occurrence after the bootstrap token, so a round is cut. + first = { + (rid, i): t + for rid, o in free.items() + for i, t in enumerate(o.token_ids) + if i > 0 and t not in o.token_ids[:i] + } + assert first, "greedy outputs repeat their bootstrap token; pick other prompts" + stop = first[max(first, key=lambda key: key[1])] + expected, actual = run(False, stop_token_ids=[stop]), run(True, stop_token_ids=[stop]) + assert any(o.finish_reason == "stop" for o in expected.values()) + for rid, output in expected.items(): + tokens = free[rid].token_ids + cut = tokens[: tokens.index(stop) + 1] if stop in tokens else tokens + assert output.token_ids == cut + assert (actual[rid].token_ids, actual[rid].finish_reason) == ( + output.token_ids, + output.finish_reason, + ) + + +@pytest.mark.gpu +def test_gpu_pd_forwards_logprobs_and_abort_stays_aligned(): + pytest.importorskip("nixl") + if torch.cuda.device_count() < 2: + pytest.skip("requires 2 visible GPUs") + exit_config = ExitConfig("trace", depths_by_request={"t": [4, 2, 3, 1] * 16}) + prefill = SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=5, prefill_chunk_size=3) + transfer = dict(transfer_chunk_bytes=4096, max_inflight_bytes=16384) + caches = dict(prefill_cache_config=CacheConfig(128, 4), decode_cache_config=CacheConfig(128, 4)) + with PDEngine( + tiny_ouro_config(head_dim=64), + pd_config=PDConfig(**transfer, request_timeout=90, startup_timeout=90), # GPUs 0, 1. + prefill_scheduler_config=prefill, + decode_scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=2), + exit_config=exit_config, + execution_config=ExecutionConfig(async_scheduling=True), + attention_backend="triton", + seed=123, + **caches, + ) as e: + m = model(123, "cuda:0", torch.bfloat16, head_dim=64) + options = dict(cache_config=CacheConfig(128, 4), scheduler_config=prefill) + reference = make_engine(m, exit_config=exit_config, attention_backend="triton", **options) + params = rollout(max_tokens=6, min_loops=1, temperature=0.7, top_k=32, seed=947) + for target in (e, reference): + target.add_request("r", [1, 2, 3, 4, 5], params, trace_id="t") + assert_same(final(stream(e))["r"], final(stream(reference))["r"], atol=5e-2) + e.add_request("abort", [1, 2, 3, 4, 5], replace(params, max_tokens=64), trace_id="t") + delivered = defaultdict(list) + stream(e, lambda: len(delivered["abort"]) >= 2, delivered) + output = e.abort_request("abort") + check_aligned(output) + assert output.token_ids == delivered["abort"][-1].token_ids + assert output.logprobs == delivered["abort"][-1].logprobs + stream(e) diff --git a/tests/test_serving.py b/tests/test_serving.py index 223ab911..f28a6ab0 100644 --- a/tests/test_serving.py +++ b/tests/test_serving.py @@ -14,7 +14,7 @@ from aiohttp.test_utils import TestClient, TestServer from tests.helpers import tiny_ouro_config -from vllm_rlt import CacheConfig, SamplingParams, SchedulerConfig +from vllm_rlt import LLM, CacheConfig, SamplingParams, SchedulerConfig from vllm_rlt.engine.llm_engine import LLMEngine from vllm_rlt.models import OuroForCausalLM from vllm_rlt.request import RequestPhase @@ -485,6 +485,27 @@ async def run(): asyncio.run(run()) +def test_offline_stop_token_text_is_omitted_and_http_keeps_its_fields(): + engine, tokenizer = factory() + engine.add_request("direct", tokenizer.encode("abc"), SamplingParams(max_tokens=4)) + outputs = [] + while engine.has_unfinished_requests(): + outputs.extend(engine.step()) + tokens = outputs[-1].token_ids + index = next(i for i, t in enumerate(tokens) if i and t not in tokens[:i]) + stop, expected = tokens[index], tokenizer.decode(tokens[:index]) + # The stop ID stays in token_ids; only its text is omitted, as in vLLM. + llm = LLM(engine.model, tokenizer=tokenizer) + output = llm.generate("abc", SamplingParams(max_tokens=4, stop_token_ids=[stop]))[0] + assert (output.text, output.token_ids) == (expected, tokens[: index + 1]) + output = llm.generate("abc", SamplingParams(max_tokens=4))[0] # Other finishes keep it. + assert (output.text, output.finish_reason) == (tokenizer.decode(tokens), "length") + # HTTP request fields are unchanged until the token-in/token-out API. + for fields in ({"stop_token_ids": [stop]}, {"seed": None}): + with pytest.raises(ValueError): + CompletionRequest.parse(body(**fields), "test-model") + + def test_owner_thread_dynamic_batching_cancellation_and_overload(): async def run(): blocked, release = threading.Event(), threading.Event() diff --git a/tests/test_split_prefill.py b/tests/test_split_prefill.py index 66e5f6bc..8dfb6fb1 100644 --- a/tests/test_split_prefill.py +++ b/tests/test_split_prefill.py @@ -27,6 +27,7 @@ from vllm_rlt.models import OuroForCausalLM from vllm_rlt.request import Operation, Request, RequestPhase from vllm_rlt.worker.executor import build_execution_batch +from vllm_rlt.worker.model_runner import ExecutionOutput TOTAL_DEPTHS = 4 @@ -587,7 +588,7 @@ def test_cuda_submit_readback_failure_releases_batch_input_buffers(monkeypatch): def execute_with_result(batch, prepared): original_execute(batch, prepared) - return torch.zeros(1, device=runner.device, dtype=torch.float32) + return ExecutionOutput(torch.zeros(1, device=runner.device, dtype=torch.float32)) monkeypatch.setattr(runner, "_execute", execute_with_result) diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py index e6b03c1d..3c12d7f8 100644 --- a/tests/test_training_contract.py +++ b/tests/test_training_contract.py @@ -22,8 +22,8 @@ from vllm_rlt.pd.engine import PDEngine -@pytest.mark.parametrize("asynchronous,speculative", [(False, False), (True, False), (False, True)]) -@pytest.mark.parametrize("mode", ["raw", "processed"]) +@pytest.mark.parametrize("asynchronous,speculative", [(False, False), (True, False)]) +@pytest.mark.parametrize("mode", ["raw_logprobs", "processed_logprobs"]) @pytest.mark.parametrize("temperature", [0, 0.7]) def test_selected_scores_match_dense_target_and_keep_tokens( asynchronous, speculative, mode, temperature @@ -35,25 +35,20 @@ def test_selected_scores_match_dense_target_and_keep_tokens( execution_config=ExecutionConfig(async_scheduling=asynchronous), exit_config=ExitConfig("ouro_delayed" if asynchronous else "ouro"), speculative_config=SpeculativeConfig(2) if speculative else None, - ) - params = SamplingParams( - max_tokens=7, - ignore_eos=True, - temperature=temperature, - logprobs=0, logprobs_mode=mode, ) + params = SamplingParams(max_tokens=7, ignore_eos=True, temperature=temperature, logprobs=0) prompts = [[2, 3], [4, 5, 6]] try: outputs = llm.generate(prompts, params) ordinary = llm.generate(prompts, replace(params, logprobs=None)) assert [o.token_ids for o in outputs] == [o.token_ids for o in ordinary] for prompt, output in zip(prompts, outputs): - assert len(output.log_probs) == len(output.token_ids) == 7 + assert len(output.logprobs) == len(output.token_ids) == 7 prefix = list(prompt) - for token, actual in zip(output.token_ids, output.log_probs): + for token, actual in zip(output.token_ids, output.logprobs): logits = dense_reference(model, prefix, model.config.total_ut_steps)[-1][2][-1] - if mode == "processed" and temperature > 0: + if mode == "processed_logprobs" and temperature > 0: logits = logits / temperature expected = logits.float().log_softmax(-1)[token].item() assert actual == pytest.approx(expected, abs=2e-5) @@ -89,7 +84,7 @@ def test_full_publication_invalidates_prefix_and_preserves_parameter_addresses() torch.testing.assert_close(value, expected) output = llm.generate([[2, 3]], params)[0] assert output.weight_version == 3 - assert output.sampling_params.max_loops == model.config.total_ut_steps + assert output.sampling_params == params finally: llm.close() @@ -147,7 +142,7 @@ def test_publication_waits_for_requests_and_remote_transfer_leases(): engine.close() -@pytest.mark.parametrize("speculative", [False, True]) +@pytest.mark.parametrize("speculative", [False]) def test_explicit_stop_applies_with_ignore_eos_and_scores_align(speculative): llm = LocalLLM( tiny_ouro_model(), @@ -161,7 +156,7 @@ def test_explicit_stop_applies_with_ignore_eos_and_scores_align(speculative): stopped = llm.generate([[2, 3]], replace(params, stop_token_ids=(stop,)))[0] count = full.token_ids.index(stop) + 1 assert stopped.token_ids == full.token_ids[:count] - assert stopped.log_probs == pytest.approx(full.log_probs[:count], abs=2e-5) + assert stopped.logprobs == pytest.approx(full.logprobs[:count], abs=2e-5) assert stopped.finish_reason == "stop" finally: llm.close() @@ -205,7 +200,7 @@ def test_cpu_stage_clients_publish_to_all_pd_peers_and_recover_control_errors(): llm = LLM(clients[0]) output = llm.generate([[2, 3]], SamplingParams(max_tokens=2, logprobs=0))[0] assert output.weight_version == 2 - assert len(output.log_probs) == len(output.token_ids) + assert len(output.logprobs) == len(output.token_ids) finally: for client in clients: client.close() @@ -230,25 +225,13 @@ def test_mixed_score_admission_and_unspecified_seed_replay(asynchronous): ignore_eos=True, ) outputs = llm.generate([[2, 3], [4, 5]], [replace(scored, logprobs=None), scored]) - assert outputs[0].log_probs is None - assert len(outputs[1].log_probs) == 4 + assert outputs[0].logprobs is None + assert len(outputs[1].logprobs) == 4 effective = outputs[1].sampling_params - assert effective.top_k == model.config.vocab_size + assert effective == replace(scored, seed=effective.seed) assert 0 <= effective.seed < 2**63 replay = llm.generate([[4, 5]], effective)[0] assert replay.token_ids == outputs[1].token_ids - assert replay.log_probs == pytest.approx(outputs[1].log_probs, abs=2e-5) + assert replay.logprobs == pytest.approx(outputs[1].logprobs, abs=2e-5) finally: llm.close() - - -def test_async_speculative_logprob_rejected_before_any_request_or_transfer(): - engine = StageEngine(tiny_ouro_model(), cache_config=CacheConfig(64, 2)) - engine.async_speculative = True # Admission only: no CUDA speculative runner is constructed. - try: - with pytest.raises(ValueError, match="logprobs"): - engine.add_request("unsupported", [1], SamplingParams(logprobs=0)) - assert not engine.has_unfinished_requests() - finally: - engine.async_speculative = False - engine.close() diff --git a/vllm_rlt/core/scheduler.py b/vllm_rlt/core/scheduler.py index 821e8a3b..abb599b1 100644 --- a/vllm_rlt/core/scheduler.py +++ b/vllm_rlt/core/scheduler.py @@ -1190,9 +1190,20 @@ def needs_exit_signal(self, request: Request) -> bool: maximum = request.sampling_params.max_loops or self.total_ut_steps return self.exit_mode != "trace" and request.loops_done + 1 < maximum - def _record_token(self, request: Request, token: int, depth: int) -> bool: + def _record_token( + self, request: Request, token: int, depth: int, logprob: float | None = None + ) -> bool: + """Record one sampled token; return whether it finished the request. + + EOS stops unless ignore_eos; stop_token_ids stop even with ignore_eos, as + in vLLM. Either stop token stays in the output. Then max_tokens is LENGTH. + """ request.generated_token_ids.append(token) request.exit_depths.append(depth) + if request.logprobs is not None: + if logprob is None: + raise RuntimeError("missing logprob for a request that asked for logprobs") + request.logprobs.append(logprob) params = request.sampling_params if token in params.stop_token_ids or ( token in self.eos_token_ids and not params.ignore_eos @@ -1296,9 +1307,8 @@ def update_from_output( raise RuntimeError("invalid pending output count") request.num_output_placeholders -= 1 depth = result.depths[row] if delivered else request.loops_done - if request.sampling_params.logprobs is not None: - request.log_probs.append(result.log_probs[row]) - if self._record_token(request, result.token_ids[row], depth): + logprob = result.logprobs[row] if result.logprobs is not None else None + if self._record_token(request, result.token_ids[row], depth, logprob): update.finished.append(request) elif not delivered: self.enqueue( @@ -1321,9 +1331,7 @@ def update_from_output( "speculative GPU frontier disagrees with committed prefix" ) emitted = 0 - for index, token in enumerate(spec.token_ids): - if request.sampling_params.logprobs is not None: - request.log_probs.append(spec.log_probs[index]) + for token in spec.token_ids: emitted += 1 if self._record_token(request, token, self.speculative_config.target_loops): update.finished.append(request) diff --git a/vllm_rlt/engine/llm_engine.py b/vllm_rlt/engine/llm_engine.py index a9b9534e..ee58735b 100644 --- a/vllm_rlt/engine/llm_engine.py +++ b/vllm_rlt/engine/llm_engine.py @@ -12,6 +12,7 @@ from vllm_rlt.engine.stage_engine import StageEngine from vllm_rlt.engine.worker import start_stage_worker from vllm_rlt.models import prepare_model +from vllm_rlt.worker.logprobs import check_logprobs_mode LLMEngine = StageEngine @@ -40,6 +41,8 @@ def __init__( wait_for_ready=True, **engine_options, ): + if "logprobs_mode" in engine_options: # Fail before starting a worker. + check_logprobs_mode(engine_options["logprobs_mode"]) model, revision, self.model_config = prepare_model( model, revision=revision, model_config=model_config ) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 303c733e..22e921f3 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -15,8 +15,9 @@ from vllm_rlt.engine.stage_config import StageConfig from vllm_rlt.profiling import Profiler from vllm_rlt.request import FinishReason, Request, RequestOutput, RequestPhase -from vllm_rlt.sampling_params import SamplingParams +from vllm_rlt.sampling_params import SamplingParams, resolve_seed from vllm_rlt.worker.executor import Executor +from vllm_rlt.worker.logprobs import check_logprobs_mode from vllm_rlt.worker.model_runner import ModelRunner from vllm_rlt.worker.output import ExitSignal, ModelRunnerOutput from vllm_rlt.worker.sampler import RngRegistry @@ -47,7 +48,9 @@ def __init__( speculative_config=None, stage_config=None, connector: StageConnector | None = None, + logprobs_mode="raw_logprobs", ): + check_logprobs_mode(logprobs_mode) self.model = model self._weight_version = 0 self._update_version = None @@ -192,6 +195,7 @@ def __init__( scheduler_config=scheduler_config, staged_prefill=self.scheduler.staged_prefill, rng=self.rng, + logprobs_mode=logprobs_mode, ) self.speculative_runner = ( SpeculativeRunner( @@ -285,11 +289,10 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i raise RuntimeError("Generation admission is paused") if not isinstance(request_id, str) or not request_id: raise ValueError("request_id must be a nonempty string") - params = sampling_params or SamplingParams() + # Every output, including an abort before the first sample, reports this seed. + params = resolve_seed(sampling_params or SamplingParams()) if self.async_speculative and params.temperature != 0: raise ValueError("async speculative decoding currently requires greedy sampling") - if self.async_speculative and params.logprobs is not None: - raise ValueError("async speculative decoding does not yet support logprobs") if request_id in self._retiring_speculative: self._retire_speculative(wait=True, request_id=request_id) config = self.model.config @@ -298,15 +301,16 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i if any(type(t) is not int or not 0 <= t < config.vocab_size for t in prompt_token_ids): raise ValueError("prompt token IDs must be integers within the model vocabulary") if any(t >= config.vocab_size for t in params.stop_token_ids): - raise ValueError("stop token IDs must be within the model vocabulary") + raise ValueError("stop_token_ids must be within the model vocabulary") max_loops = params.max_loops or config.total_ut_steps if max_loops > config.total_ut_steps or params.min_loops > max_loops: raise ValueError("requested loop bounds exceed the model's supported depth") - params = replace(params, max_loops=max_loops, top_k=min(params.top_k, config.vocab_size)) if self.speculative_config is not None and ( max_loops != self.speculative_config.target_loops or params.exit_threshold != 1.0 ): raise ValueError("speculative requests require fixed target depth and exit_threshold=1") + if self.speculative_config is not None and params.logprobs is not None: + raise ValueError("logprobs are not yet supported with speculative decoding") if trace_id is not None: if not isinstance(trace_id, str) or not trace_id: raise ValueError("trace_id must be a nonempty string") diff --git a/vllm_rlt/entrypoints/llm.py b/vllm_rlt/entrypoints/llm.py index 3b11a233..996d5a24 100644 --- a/vllm_rlt/entrypoints/llm.py +++ b/vllm_rlt/entrypoints/llm.py @@ -30,6 +30,7 @@ def __init__( exit_config=None, execution_config=None, speculative_config=None, + logprobs_mode="raw_logprobs", ): self._tokenizer_source = None if isinstance(model, (LLMEngine, StageClient, PDEngine)): @@ -55,6 +56,7 @@ def __init__( exit_config=exit_config, execution_config=execution_config, speculative_config=speculative_config, + logprobs_mode=logprobs_mode, ) self._next_request_id = 0 @@ -91,10 +93,13 @@ def generate(self, prompts, sampling_params=None): for output in self.engine.step(): if output.finished: if self.tokenizer is not None: + # As in vLLM, the stop-terminating ID adds no text. + stop = output.finish_reason == "stop" output = replace( output, text=self.tokenizer.decode( - output.token_ids, skip_special_tokens=True + output.token_ids[:-1] if stop else output.token_ids, + skip_special_tokens=True, ), ) results[output.request_id] = output diff --git a/vllm_rlt/pd/engine.py b/vllm_rlt/pd/engine.py index 1823d516..d6319a92 100644 --- a/vllm_rlt/pd/engine.py +++ b/vllm_rlt/pd/engine.py @@ -21,7 +21,8 @@ from vllm_rlt.profiling import ProfileConfig from vllm_rlt.profiling_artifacts import _write_json_atomic, profile_timestamp from vllm_rlt.request import FinishReason, Request, RequestOutput, RequestPhase -from vllm_rlt.sampling_params import SamplingParams +from vllm_rlt.sampling_params import SamplingParams, resolve_seed +from vllm_rlt.worker.logprobs import check_logprobs_mode from .config import PDConfig from .connector import PDStageConnector @@ -74,7 +75,9 @@ def __init__( seed=0, speculative_config=None, terminate_workers_on_failure: bool = True, + logprobs_mode="raw_logprobs", ): + check_logprobs_mode(logprobs_mode) self.speculative_config = speculative_config model, revision, self.model_config = prepare_model( model, revision=revision, model_config=model_config @@ -150,6 +153,7 @@ def __init__( speculative_config=speculative_config if role == "decode" else None, execution_config=execution, attention_backend=attention_backend, + logprobs_mode=logprobs_mode, ) self.peers[name] = Peer(name, role, client) deadline = time.monotonic() + self.config.startup_timeout @@ -240,6 +244,8 @@ def _message(self, peer, m): w.latest_output = output w.request.generated_token_ids = list(output.token_ids) w.request.exit_depths = list(output.exit_depths) + if output.logprobs is not None: + w.request.logprobs = list(output.logprobs) w.timings.setdefault("first_token", now) self.outputs.append(output) if output.finished: @@ -289,7 +295,8 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra raise ValueError("request ID must be nonempty and unique among active requests") if len(self.transfers) >= self.config.max_pending_requests: raise ValueError("PD pending-request limit reached") - params = sampling_params or SamplingParams() + # P and D receive the same resolved params; only D samples. + params = resolve_seed(sampling_params or SamplingParams()) cfg = self.model_config if self.speculative_config is not None: if ( @@ -299,8 +306,8 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra raise ValueError( "speculative requests require fixed target depth and exit_threshold=1" ) - if self._async_speculative and params.logprobs is not None: - raise ValueError("async speculative decoding does not yet support logprobs") + if params.logprobs is not None: + raise ValueError("logprobs are not yet supported with speculative decoding") if self._async_speculative and params.temperature != 0: raise ValueError("async speculative decoding currently requires greedy sampling") if not prompt_token_ids or any( @@ -308,11 +315,10 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra ): raise ValueError("prompt must contain valid model token IDs") if any(t >= cfg.vocab_size for t in params.stop_token_ids): - raise ValueError("stop token IDs must be within the model vocabulary") + raise ValueError("stop_token_ids must be within the model vocabulary") maximum = params.max_loops or cfg.total_ut_steps if maximum > cfg.total_ut_steps or params.min_loops > maximum: raise ValueError("requested loop bounds exceed model depth") - params = replace(params, max_loops=maximum, top_k=min(params.top_k, cfg.vocab_size)) if len(prompt_token_ids) + params.max_tokens - 1 > cfg.max_position_embeddings: raise ValueError("prompt plus decode exceeds context capacity") if trace_id is not None and (not isinstance(trace_id, str) or not trace_id): diff --git a/vllm_rlt/request.py b/vllm_rlt/request.py index 075c747b..cc6c3d1c 100644 --- a/vllm_rlt/request.py +++ b/vllm_rlt/request.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field, replace from enum import Enum -from secrets import randbits import torch @@ -43,6 +42,8 @@ class Request: phase: RequestPhase = RequestPhase.WAITING generated_token_ids: list[int] = field(default_factory=list) exit_depths: list[int] = field(default_factory=list) + # Aligned with generated_token_ids when sampling_params.logprobs is set, else None. + logprobs: list[float] | None = None # Scheduling advances before coda's CPU output delivery. num_output_placeholders: int = 0 num_speculative_rounds: int = 0 @@ -69,11 +70,10 @@ class Request: finish_reason: FinishReason | None = None exit_trace: tuple[int, ...] = field(default=(), repr=False) weight_version: int = 0 - log_probs: list[torch.Tensor] = field(default_factory=list, repr=False) def __post_init__(self): - if self.sampling_params.seed is None: - self.sampling_params = replace(self.sampling_params, seed=randbits(63)) + if self.logprobs is None and self.sampling_params.logprobs is not None: + self.logprobs = [] @staticmethod def prefill_boundaries( @@ -211,8 +211,10 @@ class RequestOutput: finished: bool finish_reason: str | None = None text: str = "" - log_probs: list[float] | None = None + # Log-probability of each token_ids entry, conditional on its exit_depths entry. + logprobs: list[float] | None = None weight_version: int = 0 + # Effective parameters, including the engine-chosen seed for seed=None. sampling_params: SamplingParams | None = None @classmethod @@ -226,13 +228,7 @@ def from_request(cls, request: Request): finish_reason=request.finish_reason.value if request.finish_reason is not None else None, - log_probs=( - torch.stack(request.log_probs[: len(request.generated_token_ids)]).tolist() - if request.generated_token_ids - else [] - ) - if request.sampling_params.logprobs is not None - else None, + logprobs=list(request.logprobs) if request.logprobs is not None else None, weight_version=request.weight_version, sampling_params=request.sampling_params, ) diff --git a/vllm_rlt/sampling_params.py b/vllm_rlt/sampling_params.py index f6016701..ad71b906 100644 --- a/vllm_rlt/sampling_params.py +++ b/vllm_rlt/sampling_params.py @@ -18,7 +18,6 @@ class SamplingParams: priority: int = 0 # vLLM convention: 0 returns only the sampled token's log-probability. logprobs: int | None = None - logprobs_mode: str = "raw" # Stop after emitting any of these IDs, even with ignore_eos (vLLM semantics). stop_token_ids: tuple[int, ...] = () @@ -54,8 +53,6 @@ def __post_init__(self): "logprobs > 0 (top-k alternatives) is not supported yet; " "use 0 for the sampled token's log-probability" ) - if self.logprobs_mode not in ("raw", "processed"): - raise ValueError("logprobs_mode must be raw or processed") if not isinstance(self.stop_token_ids, (list, tuple)) or any( type(token) is not int or token < 0 for token in self.stop_token_ids ): diff --git a/vllm_rlt/serving/protocol.py b/vllm_rlt/serving/protocol.py index c08243cd..83d5f9f4 100644 --- a/vllm_rlt/serving/protocol.py +++ b/vllm_rlt/serving/protocol.py @@ -61,7 +61,6 @@ class CompletionRequest: def parse(cls, body, model): if not isinstance(body, dict): raise ValueError("request body must be a JSON object") - params_keys = {field.name for field in fields(SamplingParams)} neutral = { "n": 1, "best_of": 1, @@ -74,6 +73,9 @@ def parse(cls, body, model): "suffix": None, "logit_bias": None, } + # Rollout fields stay offline-only until the token-in/token-out API (#70). + params_keys = {field.name for field in fields(SamplingParams)} - neutral.keys() + params_keys -= {"stop_token_ids"} allowed = ( params_keys | neutral.keys() @@ -119,6 +121,8 @@ def parse(cls, body, model): raise ValueError(f"{key} must be a finite number") if "ignore_eos" in params and type(params["ignore_eos"]) is not bool: raise ValueError("ignore_eos must be a boolean") + if "seed" in params and params["seed"] is None: + raise ValueError("seed must be an integer in [0, 2**63)") trace_id = body.get("trace_id") if "trace_id" in body and (not isinstance(trace_id, str) or not trace_id): raise ValueError("trace_id must be a nonempty string") diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index b31b2939..c4b4cae8 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -18,12 +18,24 @@ ) from vllm_rlt.worker.cuda_graph import CodaGraphs, RecurrentGraphs from vllm_rlt.worker.executor import ForwardBatch, ModelRuntime +from vllm_rlt.worker.logprobs import processed_logprobs, truncates from vllm_rlt.worker.output import ModelRunnerOutput from vllm_rlt.worker.sampler import RngRegistry, Sampler from vllm_rlt.worker.submission_queue import MAX_CONCURRENT_SUBMISSIONS, SubmissionQueue from vllm_rlt.worker.tensors import StagedInputTensor, TensorTransfer, UvaBuffer, pinned_tensor +class ExecutionOutput(NamedTuple): + """One submission's device outputs. + + ``values`` holds RECURRENT exit scores or CODA sampled IDs, else None. + ``logprobs`` holds CODA FP32 logprobs of those IDs when a row asked, else None. + """ + + values: torch.Tensor | None + logprobs: torch.Tensor | None = None + + @dataclass class ReadbackSlot: storage: torch.Tensor @@ -101,7 +113,6 @@ class PreparedExecution: packed_metadata: PackedMetadataBuffer | None = None sampling_params: tuple = () sampling_generators: tuple = () - log_probs: list | None = None @dataclass @@ -116,11 +127,14 @@ class Submission: positions: tuple[int, ...] = () output_indices: tuple[int, ...] = () operation: Operation | None = None - log_probs: tuple = () + # CODA: FP32 logprobs of the sampled IDs, read back on the same event. + logprobs: torch.Tensor | None = None + logprob_slot: ReadbackSlot | None = None def __del__(self): - if self.slot is not None: - self.slot.leased = False + for slot in (self.slot, self.logprob_slot): + if slot is not None: + slot.leased = False @property def has_result(self): @@ -135,24 +149,23 @@ def collect(self) -> ModelRunnerOutput: if self.event is not None: self.event.synchronize() values = self.values.tolist() if self.values is not None else None + logprobs = self.logprobs.tolist() if self.logprobs is not None else None is_coda = self.operation == Operation.CODA is_recurrent = self.operation == Operation.RECURRENT self.cached = ModelRunnerOutput( gate_scores=values if is_recurrent else None, token_ids=values if is_coda else None, - log_probs=[ - score.detach().cpu() if score is not None else None for score in self.log_probs - ] - if self.log_probs - else None, + logprobs=logprobs if is_coda else None, completion=self.event, depths=self.depths, output_indices=self.output_indices, ) - self.values = None - if self.slot is not None: - self.slot.leased = False - self.slot = None + self.values = self.logprobs = None + for name in ("slot", "logprob_slot"): + slot = getattr(self, name) + if slot is not None: + slot.leased = False + setattr(self, name, None) return self.cached @@ -167,6 +180,7 @@ def __init__( scheduler_config=None, staged_prefill=None, rng: RngRegistry | None = None, + logprobs_mode="raw_logprobs", ): self.model = model.eval() self.cache_manager = cache_manager @@ -178,6 +192,7 @@ def __init__( # Sampling RNG state lives with the engine, which shares this registry # between the CODA runner and the speculative runner. self.rng = rng if rng is not None else RngRegistry() + self.logprobs_mode = logprobs_mode self.exit_config = exit_config or ExitConfig() self.execution_config = execution_config or ExecutionConfig() scheduler = scheduler_config or SchedulerConfig() @@ -244,6 +259,7 @@ def __init__( ): stream.wait_stream(torch.cuda.current_stream(self.device)) self.readback_slots = [] + self.logprob_slots = [] if self.device.type == "cuda": # cudaHostAlloc during submission can serialize streams. Allocate all # readback storage before work starts, retaining leases until consumed. @@ -254,6 +270,19 @@ def __init__( pinned_tensor(scheduler.max_num_batched_tokens, dtype, self.device) ) ) + # A separate pool keeps exit-score leases from starving CODA logprobs: a + # request whose CODA output is undelivered can already run its next + # token's recurrent loops, so it can hold both kinds of lease at once. + # A CODA ticket keeps its logprob lease until it is delivered or dropped. + # Each pending ticket holds a request with an undelivered output (at most + # one per request), so with the new ticket at most max_num_seqs leases are + # live; a stale ticket of an aborted request is still delivered (its + # results are ignored) or dropped on failure. The +4 is headroom, as for + # the exit-score pool. + for _ in range(scheduler.max_num_seqs + 4): + self.logprob_slots.append( + ReadbackSlot(pinned_tensor(scheduler.max_num_seqs, torch.float32, self.device)) + ) # Retire the oldest consumer BEFORE preparing another batch's inputs. # All rotating prefill pools share this maximum in-flight depth. self.submissions = SubmissionQueue() @@ -889,13 +918,7 @@ def _prepare_cpu_inputs(self, batch: ExecutionBatch): prepared, gate_indices=self.input_buffers[torch.int64].append(gate_rows) ) if batch.operation == Operation.CODA: - prepared = replace( - prepared, - sampling_params=tuple(r.sampling_params for r in requests), - log_probs=[] - if any(r.sampling_params.logprobs is not None for r in requests) - else None, - ) + prepared = replace(prepared, sampling_params=tuple(r.sampling_params for r in requests)) if batch.operation == Operation.PRELUDE and batch.loop_count: packed_metadata = ( self.packed_metadata_buffers[self.submissions.slot] @@ -1106,12 +1129,12 @@ def _prepare_tokens(self, prepared, size): return tokens @torch.inference_mode() - def _execute(self, scheduled_batch, prepared): + def _execute(self, scheduled_batch, prepared) -> ExecutionOutput: return self._execute_batch(prepared.batch, prepared) - def _execute_batch(self, batch: ExecutionBatch, prepared): + def _execute_batch(self, batch: ExecutionBatch, prepared) -> ExecutionOutput: hidden = None - values = None + values = logprobs = None for entry in prepared.forwards: forward = entry.forward if entry.hidden_rows is not None: @@ -1136,30 +1159,49 @@ def _execute_batch(self, batch: ExecutionBatch, prepared): ) values = selected.float().sigmoid() elif forward.compute_logits: - values = torch.stack( - [ - self.sampler.sample(row, params, generator) - for row, params, generator in zip( - result.logits, prepared.sampling_params, prepared.sampling_generators - ) - ] - ) - if prepared.log_probs is not None: - prepared.log_probs.extend( - self.sampler.selected_logprob(row, token, params) - if params.logprobs is not None - else None - for row, token, params in zip( - result.logits, values, prepared.sampling_params - ) + params, generators = prepared.sampling_params, prepared.sampling_generators + if any(p.logprobs is not None for p in params): + values, logprobs = self._sample_with_logprobs(result.logits, params, generators) + else: + values = torch.stack( + [ + self.sampler.sample(row, p, generator) + for row, p, generator in zip(result.logits, params, generators) + ] ) if prepared.routing is not None: prepared.routing.scatter(values, tokens=True) - return values + return ExecutionOutput(values, logprobs) + + def _sample_with_logprobs(self, logits, params, generators): + """Sampled IDs [rows] and FP32 logprobs [rows] of them under ``self.logprobs_mode``. - def _readback_slot(self, result): + Raw mode scores every row in one batched op. Processed mode samples a requesting + top-k/top-p row through ``Sampler.sample_with_logprob``, which scores the processed + logits it drew from, and scores the other requesting rows in temperature batches + (``processed_logprobs``); no row rebuilds its processed logits. All other rows + sample through ``Sampler.sample``. Both draw identically, so tokens and RNG use do + not depend on logprobs. Values of rows that did not request logprobs are arbitrary. + """ + rows = len(params) + if self.logprobs_mode == "raw_logprobs": + token_ids = torch.stack( + [self.sampler.sample(row, p, g) for row, p, g in zip(logits, params, generators)] + ) + return token_ids, Sampler.raw_logprobs(logits[:rows], token_ids) + tokens, truncated = [], {} + for index, (row, p, generator) in enumerate(zip(logits, params, generators)): + if p.logprobs is not None and truncates(p): + token, truncated[index] = self.sampler.sample_with_logprob(row, p, generator) + else: + token = self.sampler.sample(row, p, generator) + tokens.append(token) + token_ids = torch.stack(tokens) + return token_ids, processed_logprobs(logits[:rows], params, token_ids, truncated) + + def _readback_slot(self, result, slots=None): pending = None - for slot in self.readback_slots: + for slot in self.readback_slots if slots is None else slots: if slot.storage.dtype != result.dtype or slot.leased: continue if slot.event is None or slot.event.query(): @@ -1173,6 +1215,13 @@ def _readback_slot(self, result): return pending raise RuntimeError("readback buffers exhausted; retire completed submissions") + @staticmethod + def _copy_to_host(values, slot, stream): + values.record_stream(stream) + host = slot.storage[: values.numel()].view(values.shape) + host.copy_(values, non_blocking=True) + return host + def _synchronize_submission(self, stream, prepared): if self.device.type != "cuda": return @@ -1208,7 +1257,7 @@ def submit(self, scheduled_batch: SchedulerOutput | None = None, *, batch: Execu stream = self.core_stream if batch.loop_count else self.boundary_stream prepared = None routing_finish_attempted = False - slot = None + slot = logprob_slot = None try: if self.submissions.begin() and split_prefill: self.split_prefill_submission_waits += 1 @@ -1228,7 +1277,7 @@ def submit(self, scheduled_batch: SchedulerOutput | None = None, *, batch: Execu self._wait_for_requests(batch, stream) prepared = self.prepare_on_gpu(prepared) try: - result = self._execute(scheduled_batch, prepared) + result, logprobs = self._execute(scheduled_batch, prepared) finally: if prepared.routing is not None: routing_finish_attempted = True @@ -1262,17 +1311,22 @@ def submit(self, scheduled_batch: SchedulerOutput | None = None, *, batch: Execu event = compute_done if result is not None: slot = self._readback_slot(result) + if logprobs is not None: + logprob_slot = self._readback_slot(logprobs, self.logprob_slots) readback_stream = self.readback_stream or compute_stream with torch.cuda.stream(readback_stream): if readback_stream != compute_stream: readback_stream.wait_event(compute_done) - result.record_stream(readback_stream) - host = slot.storage[: result.numel()].view(result.shape) - host.copy_(result, non_blocking=True) + result = self._copy_to_host(result, slot, readback_stream) + if logprobs is not None: + logprobs = self._copy_to_host( + logprobs, logprob_slot, readback_stream + ) event = torch.cuda.Event() event.record(readback_stream) - slot.event = event - result = host + for leased in (slot, logprob_slot): + if leased is not None: + leased.event = event for span in batch.spans: self.events[span.request.request_id] = compute_done self.submissions.complete(compute_done) @@ -1288,7 +1342,8 @@ def submit(self, scheduled_batch: SchedulerOutput | None = None, *, batch: Execu positions=prepared.positions, output_indices=prepared.output_indices, operation=batch.operation, - log_probs=tuple(prepared.log_probs or ()), + logprobs=logprobs, + logprob_slot=logprob_slot, ) except Exception: self._synchronize_submission(stream, prepared) @@ -1305,8 +1360,9 @@ def submit(self, scheduled_batch: SchedulerOutput | None = None, *, batch: Execu routing_finish_attempted = True prepared.routing.record_done() finally: - if slot is not None: - slot.leased = False + for leased in (slot, logprob_slot): + if leased is not None: + leased.leased = False raise finally: if prepared is not None and prepared.workspace is not None: diff --git a/vllm_rlt/worker/output.py b/vllm_rlt/worker/output.py index 3b8c6289..1948e8a6 100644 --- a/vllm_rlt/worker/output.py +++ b/vllm_rlt/worker/output.py @@ -4,7 +4,7 @@ in this single-process runtime. Only delayed delivery needs depth/index snapshots. """ -from dataclasses import dataclass, field +from dataclasses import dataclass from vllm_rlt.request import Request @@ -21,14 +21,15 @@ class ExitSignal: class SpeculativeResult: token_ids: list[int] accepted_count: int - log_probs: list = field(default_factory=list) @dataclass(frozen=True, kw_only=True) class ModelRunnerOutput: gate_scores: list[float] | None = None token_ids: list[int] | None = None - log_probs: list | None = None + # CODA: FP32 logprob of each token_ids entry; None when no row asked. Rows that did + # not ask hold arbitrary values. + logprobs: list[float] | None = None completion: object = None depths: tuple[int, ...] = () output_indices: tuple[int, ...] = () diff --git a/vllm_rlt/worker/sampler.py b/vllm_rlt/worker/sampler.py index 0c4feea9..9d968edb 100644 --- a/vllm_rlt/worker/sampler.py +++ b/vllm_rlt/worker/sampler.py @@ -67,16 +67,6 @@ def probabilities(logits: torch.Tensor, params: SamplingParams) -> torch.Tensor: """ return Sampler.processed_logits(logits, params).softmax(-1) - @staticmethod - def selected_logprob( - logits: torch.Tensor, token: int | torch.Tensor, params: SamplingParams - ) -> torch.Tensor: - """Score the selected token using the same filters as token sampling.""" - if params.logprobs_mode == "processed" and params.temperature > 0: - return Sampler.probabilities(logits, params)[token].log() - # Integer speculative tokens otherwise retain a view of the vocabulary row. - return logits.float().log_softmax(-1)[token].clone() - @staticmethod def draw(probs: torch.Tensor, generator: torch.Generator) -> torch.Tensor: """Draw one token from a normalized distribution as a 0-dim long tensor.""" diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index 68b681e0..c799e883 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -292,11 +292,6 @@ def _collect_round(self, submission): ) start += scheduled_request.token_count results.append(SpeculativeResult(tokens, accepted)) - if request.sampling_params.logprobs is not None: - results[-1].log_probs.extend( - Sampler.selected_logprob(row, token, request.sampling_params).detach().cpu() - for row, token in zip(rows, tokens) - ) continue start += scheduled_request.token_count # Sampling keeps sequential rejection; RNG consumption order is unchanged. @@ -320,9 +315,4 @@ def _collect_round(self, submission): ) tokens.append(int(token.item())) results.append(SpeculativeResult(tokens, accepted)) - if request.sampling_params.logprobs is not None: - results[-1].log_probs.extend( - Sampler.selected_logprob(row, token, request.sampling_params).detach().cpu() - for row, token in zip(rows, tokens) - ) return results From c9db33785f940faff62445b3a279ed6fcf4bc1cb Mon Sep 17 00:00:00 2001 From: MinhaoLi0318 Date: Thu, 22 Oct 2026 11:25:20 +0800 Subject: [PATCH 7/9] feat: score speculative tokens with target logprobs Port of ef93010 (IDEA-V) from the original #83 head onto the submitted speculative runners and the Scheduler-owned result application. Lift the admission rejection of logprobs with speculative_config. Every committed token (accepted draft, correction or bonus) is scored on its target verification row, never under the draft q: the round's output is distributed as p, including corrections drawn from (p - q)+. - Sync runner: _collect_round scores greedy rows raw; sampled rows in processed_logprobs mode reuse the processed logits p was normalized from. At most two batched scoring ops and one readback per round, and none when no request asks for logprobs. - Async runner (greedy-only): the raw logprob of each verification row's argmax is copied with the round's results on the same event. - SpeculativeResult.logprobs reaches Scheduler._record_token per token. Co-authored-by: IDEA-V --- docs/async-speculative.md | 7 +- docs/launching.md | 17 +- docs/training_contract.md | 22 +-- tests/test_rollout_outputs.py | 231 ++++++++++++++++++++++++++- tests/test_training_contract.py | 4 +- vllm_rlt/core/scheduler.py | 6 +- vllm_rlt/engine/stage_engine.py | 4 +- vllm_rlt/pd/engine.py | 2 - vllm_rlt/worker/async_speculative.py | 52 +++++- vllm_rlt/worker/output.py | 10 ++ vllm_rlt/worker/speculative.py | 142 ++++++++++++---- 11 files changed, 437 insertions(+), 60 deletions(-) diff --git a/docs/async-speculative.md b/docs/async-speculative.md index ae8e773a..5ad78a24 100644 --- a/docs/async-speculative.md +++ b/docs/async-speculative.md @@ -244,8 +244,11 @@ use the reserved allocation and stream-ordered writes instead. Only completed full prompt pages are eligible for prefix caching. Every ticket owns its result buffer and page-table snapshot until compute and -copy events complete. EOS, `stop_token_ids` and length stopping -(`Scheduler._record_token`) is applied in output order on the CPU; one +copy events complete. For rounds where a request asked for `logprobs`, each +verification row's FP32 raw logprob of its argmax is copied with the round's +results on the same event. Committed tokens are exactly those argmaxes. EOS, +`stop_token_ids` and length stopping (`Scheduler._record_token`) is applied in +output order on the CPU; one already-submitted later round can finish but its outputs are discarded. Cancellation removes scheduling eligibility immediately. KV allocations and device state survive until their final GPU user completes. Request object diff --git a/docs/launching.md b/docs/launching.md index a1aedda1..5b70ecea 100644 --- a/docs/launching.md +++ b/docs/launching.md @@ -305,7 +305,22 @@ at that loop depth. A teacher-forced recompute therefore has to replay the depth shares the request's generator with sampling, so no ordinary forward reproduces them either. -Speculative decoding does not accept `logprobs` yet. +Under [speculative decoding](async-speculative.md), every `exit_depths` entry is +`target_loops`, and each committed token is scored on its target verification row. +This covers an accepted draft token, the correction after a rejection, and the bonus +token. The draft distribution q never enters `logprobs`: + +- Verification resumes from the hidden state and KV that the draft loops computed + with the same weights. The scores therefore match a full-depth forward + numerically, not necessarily bitwise. The rollout replays like any other + full-depth rollout. +- A correction is drawn from the residual `(p - q)+`. The round's output is still + distributed as p, so its policy logprob is `log p[token]`. +- For a given seed, the sampled tokens depend on q, because draft draws and + rejection tests consume the request's generator. The logprob of a given token + does not depend on q. Greedy tokens do not depend on it either. +- Cross-round async speculation is greedy-only, so it reports raw values in both + modes. The HTTP API does not accept `logprobs`, `stop_token_ids` or `seed: null` yet. diff --git a/docs/training_contract.md b/docs/training_contract.md index 07c1eb10..23d0067f 100644 --- a/docs/training_contract.md +++ b/docs/training_contract.md @@ -23,13 +23,14 @@ log-probability; top-N alternatives are not implemented. `PDEngine`, not a request field. `"raw_logprobs"` is the default: FP32 log-softmax of unscaled model logits. `"processed_logprobs"` scores the distribution the token was drawn from, after positive temperature and top-k/top-p filtering. Greedy -requests return the raw score in either mode. Speculative decoding rejects -`logprobs` requests at admission for now. Explicit `stop_token_ids` apply even -with `ignore_eos=True`. `seed=None` resolves an independent seed in `[0, 2**63)` -when the engine admits the request. The effective seed is returned in -`sampling_params`, including for an abort before sampling. The default remains -`seed=0`. [Rollout outputs](launching.md#rollout-outputs) states when replaying -those parameters reproduces tokens and logprobs. +requests return the raw score in either mode. Speculative outputs use the +verified **target** policy's score, including correction/bonus tokens; proposal or +rejection-residual probabilities are never exposed as the policy score. Explicit +`stop_token_ids` apply even with `ignore_eos=True`. `seed=None` resolves an +independent seed in `[0, 2**63)` when the engine admits the request. The +effective seed is returned in `sampling_params`, including for an abort before +sampling. The default remains `seed=0`. [Rollout outputs](launching.md#rollout-outputs) +states when replaying those parameters reproduces tokens and logprobs. Publication requires an idle engine. Each chunk is checked for duplicate names, unknown names and mismatched shapes before that chunk is copied. `finish_weight_update` @@ -52,8 +53,9 @@ without terminating the worker. Existing exit, async and speculative configuration rules still apply. Self-speculation uses a full-depth target. Selected-token scores are supported by synchronous and -asynchronous decoding; synchronous and asynchronous self-speculation reject -logprobs requests at admission for now. PD follows the same limits on its decode -workers. Publication adds no such restriction to ordinary early-exit or async rollouts. Trainers must +asynchronous decoding and by synchronous and cross-round asynchronous +self-speculation; the latter is greedy-only, so it reports raw scores in both +modes. PD follows the same rules on its decode workers. Publication +adds no such restriction to ordinary early-exit or async rollouts. Trainers must replay full-depth prompt prefill and the returned decode depths rather than assume the entire sequence used K. diff --git a/tests/test_rollout_outputs.py b/tests/test_rollout_outputs.py index f8a4f4d6..35cdabef 100644 --- a/tests/test_rollout_outputs.py +++ b/tests/test_rollout_outputs.py @@ -23,10 +23,12 @@ from vllm_rlt.pd.engine import PDEngine, Peer from vllm_rlt.request import Operation, RequestOutput, RequestPhase from vllm_rlt.serving.protocol import CompletionRequest +from vllm_rlt.worker.async_speculative import SpeculativeRoundTicket from vllm_rlt.worker.logprobs import token_logprob from vllm_rlt.worker.model_runner import ReadbackSlot, Submission from vllm_rlt.worker.output import ModelRunnerOutput, SpeculativeResult from vllm_rlt.worker.sampler import Sampler +from vllm_rlt.worker.speculative import _CommittedScores TRACES = {"a": [4, 1, 3, 2, 4, 1, 2, 3], "b": [4, 2, 1, 4, 3, 2, 1, 4]} # Keyed by request ID. EXIT_MODES = [ # (async scheduling, exit config, exit_threshold) for every CPU exit policy. @@ -653,11 +655,155 @@ def fresh(*, asynchronous): assert not engine._pending_coda and not any(slot.leased for slot in slots) -def test_speculative_admission_rejects_logprobs(): +@pytest.mark.parametrize( + "finish,max_tokens,count", + [("round", 9, 8), ("stop_token_ids", 9, 4), ("eos", 9, 5), ("length", 4, 3)], + ids=["length-at-round-end", "stop-id-mid-round", "eos-mid-round", "length-mid-round"], +) +def test_speculative_update_slices_logprobs(monkeypatch, finish, max_tokens, count): engine = LLMEngine(model(), speculative_config=SpeculativeConfig(3)) - with pytest.raises(ValueError, match="speculative"): - engine.add_request("r", [2, 3], SamplingParams(logprobs=0)) - assert not engine.has_unfinished_requests() + stops = [50] if finish == "stop_token_ids" else [] + params = SamplingParams(max_tokens=max_tokens, ignore_eos=finish != "eos", stop_token_ids=stops) + engine.add_request("r", [2, 3], replace(params, logprobs=0)) + # Each round accepts all but its last draft token; token t has logprob -1 - t / 100. + rounds = ([10, 11], [12, 50, 0, 13], [14, 15], [16, 17]) + results = deque(SpeculativeResult(t, len(t) - 1, [-1 - i / 100 for i in t]) for t in rounds) + runner = engine.speculative_runner + monkeypatch.setattr( + runner, "_collect_round", lambda s: [results.popleft() for _ in s.batch.requests] + ) + result = stream(engine)["r"][-1] + emitted = [10, 11, 12, 50, 0, 13, 14, 15][:count] + reason = "length" if finish in ("round", "length") else "stop" + assert (result.token_ids[1:], result.finish_reason) == (emitted, reason) + assert result.exit_depths == [4] * len(result.token_ids) + assert result.logprobs[1:] == [-1 - t / 100 for t in emitted] + assert math.isfinite(result.logprobs[0]) + assert runner.stats.committed_tokens == len(emitted) + + +SPECULATIVE_SAMPLING = { # Greedy, truncated and unfiltered sampling. + "greedy": {}, + "top_k_top_p": dict(temperature=0.8, top_k=7, top_p=0.8, seed=42), + "unfiltered": dict(temperature=1.0, seed=5), +} + + +def speculative_engine(m, logprobs_mode="raw_logprobs", **kwargs): + return LLMEngine( + m, + speculative_config=SpeculativeConfig(3), + cache_config=CacheConfig(256, 2), + logprobs_mode=logprobs_mode, + **kwargs, + ) + + +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +def test_speculative_logprobs_match_full_depth_reference(monkeypatch, logprobs_mode): + """Every committed token (accepted, correction, bonus) is scored under the target p on + its verification row, which matches a teacher-forced forward at the model depth + numerically. The mixed batch puts greedy (raw-scored), sampled (processed-scored in + processed mode) and non-requesting rows in the same rounds.""" + m = model() + prompts = [[2, 3], [7, 8, 9], [5], [4, 6]] + runs = {name: [rollout(max_tokens=12, **o)] * 4 for name, o in SPECULATIVE_SAMPLING.items()} + runs["mixed"] = [rollout(max_tokens=12, **o) for o in SPECULATIVE_SAMPLING.values()] + runs["mixed"].append(SamplingParams(max_tokens=12, ignore_eos=True)) + kinds = set() # (sampled, accepted every draft) of each committed round. + mixed_rounds = 0 # Rounds scoring both greedy and sampled requests. + for name, params in runs.items(): + engine = speculative_engine(m, logprobs_mode) + runner = engine.speculative_runner + + def recording(submission, collect=runner._collect_round): + nonlocal mixed_rounds + results = collect(submission) + greedy = set() + for scheduled_request, result in zip(submission.batch.requests, results): + request_params = scheduled_request.request.sampling_params + if request_params.logprobs is None: + assert result.logprobs is None + continue + assert len(result.logprobs) == len(result.token_ids) + sampled = request_params.temperature != 0 + drafts = scheduled_request.token_count - 1 + kinds.add((sampled, result.accepted_count == drafts)) + greedy.add(not sampled) + mixed_rounds += len(greedy) == 2 + return results + + monkeypatch.setattr(runner, "_collect_round", recording) + outputs = generate(engine, prompts, params) + for prompt, output in zip(prompts, outputs): + check_aligned(output) + assert output.exit_depths == [4] * len(output.token_ids) + if output.logprobs is None: + continue + tokens = prompt + output.token_ids + rows = dense_reference(m, tokens, 4)[-1][2][len(prompt) - 1 : -1] + expected = [ + token_logprob(row, output.sampling_params, torch.tensor(token), logprobs_mode) + for row, token in zip(rows, output.token_ids) + ] + assert output.logprobs == pytest.approx(torch.stack(expected).tolist(), abs=2e-5) + if name == "greedy": # Greedy speculation equals the non-speculative engine. + native = generate(LLMEngine(m, logprobs_mode=logprobs_mode), prompts, params) + for actual, expected in zip(outputs, native): + assert_same(actual, expected, atol=2e-5) + # Both the correction (a rejected draft) and the bonus path were scored when sampling. + assert {(True, False), (True, True)} <= kinds + assert mixed_rounds > 0 + + +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +def test_speculative_logprobs_change_no_token_and_replay_exactly(logprobs_mode): + m = model() + prompts = [[2, 3], [7, 8, 9]] + params = rollout(max_tokens=10, **SPECULATIVE_SAMPLING["top_k_top_p"]) + + def run(params): + return generate(speculative_engine(m, logprobs_mode), prompts, params) + + first, second, quiet = run(params), run(params), run(replace(params, logprobs=None)) + for a, b, q in zip(first, second, quiet): + assert_same(a, b) # Same seed: bitwise tokens and logprobs. + # Scoring consumes no RNG, so the tokens do not depend on logprobs. + assert (a.token_ids, a.exit_depths) == (q.token_ids, q.exit_depths) + assert q.logprobs is None + + +def test_speculative_rows_without_logprobs_skip_scoring(monkeypatch): + """Without a requesting row nothing is scored; a mixed round scores only its requesting + request's committed tokens.""" + engine = speculative_engine(model()) + runner, resolve, raw = engine.speculative_runner, _CommittedScores.resolve, [] + collect, rounds = runner._collect_round, [] # Per round: IDs of the scored requests. + + def recording(submission): + rounds.append(None) + results = collect(submission) + indices = rounds[-1] + rounds[-1] = {submission.batch.requests[i].request.request_id for i in indices} + return results + + def resolving(scores, logits): + rounds[-1] = scores.raw_spans.keys() | scores.processed_spans.keys() + return resolve(scores, logits) + + monkeypatch.setattr(runner, "_collect_round", recording) + monkeypatch.setattr(_CommittedScores, "resolve", resolving) + scoring_op = Sampler.raw_logprobs + monkeypatch.setattr( + Sampler, "raw_logprobs", staticmethod(lambda *a: raw.append(1) or scoring_op(*a)) + ) + params = SamplingParams(max_tokens=8, ignore_eos=True) + assert generate(engine, [[2, 3]], params)[0].logprobs is None + assert rounds and not any(rounds) and not raw # No round scored or ran a scoring op. + rounds.clear() + quiet, loud = generate(engine, [[2, 3], [7, 8]], [params, replace(params, logprobs=0)]) + assert quiet.logprobs is None and len(loud.logprobs) == len(loud.token_ids) + assert set().union(*rounds) == {loud.request_id} @pytest.mark.parametrize( @@ -718,6 +864,45 @@ def test_async_speculative_collect_uses_shared_finish_rules(finish, max_tokens, assert engine.cache_manager.num_used_blocks == 0 +def test_async_speculative_ticket_scores_only_requesting_rows(): + """The async runner needs CUDA, so stub buffers drive SpeculativeRoundTicket.collect: in a + scored round each request that asked gets its committed rows' scores, the others None.""" + + class Done: + def query(self): + return True + + def synchronize(self): + pass + + def collect(logprobs_copied, wants): + # Two requests with 3 and 2 verification rows; 1 and 0 drafts accepted. + buffer = SimpleNamespace( + host_result=torch.tensor([7, 8, 9, 4, 5, 1, 0, 12, 20]), + host_logprobs=torch.tensor([-0.1, -0.2, -0.3, -0.4, -0.5]), + copy_done=Done(), + ) + params = [SamplingParams(logprobs=0 if w else None) for w in wants] + requests = [ + SimpleNamespace(request=SimpleNamespace(sampling_params=p), token_count=n) + for p, n in zip(params, (3, 2)) + ] + batch = SimpleNamespace(requests=requests) + ticket = SpeculativeRoundTicket( + batch, buffer, (), (0, 0), 5, logprobs_copied=logprobs_copied + ) + output = ticket.collect() + return output.speculative, output.frontiers + + results, frontiers = collect(True, (True, False)) + assert [r.token_ids for r in results] == [[7, 8], [4]] and frontiers == [12, 20] + assert results[0].logprobs == pytest.approx([-0.1, -0.2]) and results[1].logprobs is None + results, _ = collect(True, (False, True)) + assert results[0].logprobs is None and results[1].logprobs == pytest.approx([-0.4]) + results, _ = collect(False, (True, True)) + assert [r.logprobs for r in results] == [None, None] + + def test_pd_coordinator_resolves_seeds_and_mirrors_logprobs(): engine = object.__new__(PDEngine) # Coordinator only: no worker processes. engine.closed, engine.failure = False, None @@ -937,6 +1122,44 @@ def run(asynchronous, **overrides): ) +@pytest.mark.gpu +@pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) +def test_gpu_speculative_logprobs_match_across_graphs_async_and_native(logprobs_mode): + """CUDA/Triton speculation scores like the non-speculative engine, with CUDA Graphs and + with cross-round async speculation (greedy); sampled rounds match with graphs on/off.""" + m = model(device="cuda") + prompts = {"a": [2, 3, 4, 5], "b": [9], "c": [2, 3, 4, 7]} + + def run(speculative=True, asynchronous=False, graphs=False, **overrides): + engine = LLMEngine( + m, + attention_backend="triton", + cache_config=CacheConfig(256, 2, incremental_allocation=True), + scheduler_config=SchedulerConfig( + max_num_seqs=4, max_num_batched_tokens=16, prefill_chunk_size=2 + ), + execution_config=ExecutionConfig(async_scheduling=asynchronous, cuda_graphs=graphs), + speculative_config=SpeculativeConfig(3) if speculative else None, + logprobs_mode=logprobs_mode, + ) + for rid, prompt in prompts.items(): + engine.add_request(rid, prompt, rollout(max_tokens=16, **overrides)) + outputs = final(stream(engine)) + if speculative and graphs: + assert engine.speculative_runner.coda_graphs.captures > 0 + engine.close() + return outputs + + native = run(speculative=False) + for variant in (run(), run(graphs=True), run(asynchronous=True)): + for rid, expected in native.items(): + assert_same(variant[rid], expected, atol=1e-4) + sampled = SPECULATIVE_SAMPLING["top_k_top_p"] + eager, graphed = run(**sampled), run(graphs=True, **sampled) + for rid, expected in eager.items(): + assert_same(graphed[rid], expected, atol=1e-4) + + @pytest.mark.gpu def test_gpu_pd_forwards_logprobs_and_abort_stays_aligned(): pytest.importorskip("nixl") diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py index 3c12d7f8..2fcee540 100644 --- a/tests/test_training_contract.py +++ b/tests/test_training_contract.py @@ -22,7 +22,7 @@ from vllm_rlt.pd.engine import PDEngine -@pytest.mark.parametrize("asynchronous,speculative", [(False, False), (True, False)]) +@pytest.mark.parametrize("asynchronous,speculative", [(False, False), (True, False), (False, True)]) @pytest.mark.parametrize("mode", ["raw_logprobs", "processed_logprobs"]) @pytest.mark.parametrize("temperature", [0, 0.7]) def test_selected_scores_match_dense_target_and_keep_tokens( @@ -142,7 +142,7 @@ def test_publication_waits_for_requests_and_remote_transfer_leases(): engine.close() -@pytest.mark.parametrize("speculative", [False]) +@pytest.mark.parametrize("speculative", [False, True]) def test_explicit_stop_applies_with_ignore_eos_and_scores_align(speculative): llm = LocalLLM( tiny_ouro_model(), diff --git a/vllm_rlt/core/scheduler.py b/vllm_rlt/core/scheduler.py index abb599b1..74e0467b 100644 --- a/vllm_rlt/core/scheduler.py +++ b/vllm_rlt/core/scheduler.py @@ -1331,9 +1331,11 @@ def update_from_output( "speculative GPU frontier disagrees with committed prefix" ) emitted = 0 - for token in spec.token_ids: + for index, token in enumerate(spec.token_ids): emitted += 1 - if self._record_token(request, token, self.speculative_config.target_loops): + logprob = spec.logprobs[index] if spec.logprobs is not None else None + depth = self.speculative_config.target_loops + if self._record_token(request, token, depth, logprob): update.finished.append(request) break update.committed_tokens += emitted diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 22e921f3..6413ee7b 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -206,6 +206,7 @@ def __init__( rng=self.rng, runtime=self.model_runner.runtime, compute_stream=self.model_runner.core_stream, + logprobs_mode=logprobs_mode, ) if speculative_config is not None and not self.async_speculative else None @@ -223,6 +224,7 @@ def __init__( copy_stream=self.model_runner.copy_stream, rng=self.rng, runtime=self.model_runner.runtime, + logprobs_mode=logprobs_mode, ) self.executor = Executor(self.model_runner, speculation=self.speculative_runner) self._pending_speculative = [] @@ -309,8 +311,6 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i max_loops != self.speculative_config.target_loops or params.exit_threshold != 1.0 ): raise ValueError("speculative requests require fixed target depth and exit_threshold=1") - if self.speculative_config is not None and params.logprobs is not None: - raise ValueError("logprobs are not yet supported with speculative decoding") if trace_id is not None: if not isinstance(trace_id, str) or not trace_id: raise ValueError("trace_id must be a nonempty string") diff --git a/vllm_rlt/pd/engine.py b/vllm_rlt/pd/engine.py index d6319a92..86c36bbe 100644 --- a/vllm_rlt/pd/engine.py +++ b/vllm_rlt/pd/engine.py @@ -306,8 +306,6 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra raise ValueError( "speculative requests require fixed target depth and exit_threshold=1" ) - if params.logprobs is not None: - raise ValueError("logprobs are not yet supported with speculative decoding") if self._async_speculative and params.temperature != 0: raise ValueError("async speculative decoding currently requires greedy sampling") if not prompt_token_ids or any( diff --git a/vllm_rlt/worker/async_speculative.py b/vllm_rlt/worker/async_speculative.py index dd31ebae..cc2886f8 100644 --- a/vllm_rlt/worker/async_speculative.py +++ b/vllm_rlt/worker/async_speculative.py @@ -10,7 +10,7 @@ from vllm_rlt.request import Operation, Request from vllm_rlt.worker.executor import ForwardBatch from vllm_rlt.worker.output import ModelRunnerOutput, SpeculativeResult -from vllm_rlt.worker.sampler import RngRegistry +from vllm_rlt.worker.sampler import RngRegistry, Sampler from vllm_rlt.worker.speculative import SpeculativeRunner @@ -37,6 +37,11 @@ def __init__(self, runner, scheduler): size = scheduler.max_num_batched_tokens + 2 * scheduler.max_num_seqs self.host_result = torch.empty(size, dtype=torch.int64, pin_memory=True) self.result = torch.empty(size, dtype=torch.int64, device=cache.device) + # FP32 target logprob of each verification row's argmax, copied only for + # rounds with a request that asked for logprobs. + rows = scheduler.max_num_batched_tokens + self.host_logprobs = torch.empty(rows, dtype=torch.float32, pin_memory=True) + self.logprobs = torch.empty(rows, dtype=torch.float32, device=cache.device) self.leased = False self.copy_done: torch.cuda.Event | None = None self.compute_done: torch.cuda.Event | None = None @@ -50,6 +55,7 @@ class SpeculativeRoundTicket: states: tuple[SpeculativeRequestState, ...] rounds: tuple[int, ...] rows: int + logprobs_copied: bool = False cached: ModelRunnerOutput | None = None retired: bool = False @@ -69,13 +75,19 @@ def collect(self) -> ModelRunnerOutput: self.buffer.copy_done.synchronize() b = len(self.batch.requests) values = self.buffer.host_result[: self.rows + 2 * b].tolist() + scores = ( + self.buffer.host_logprobs[: self.rows].tolist() if self.logprobs_copied else None + ) results, start = [], 0 for i, scheduled_request in enumerate(self.batch.requests): accepted = values[self.rows + i] + params = scheduled_request.request.sampling_params + wants = scores is not None and params.logprobs is not None results.append( SpeculativeResult( values[start : start + accepted + 1], accepted, + scores[start : start + accepted + 1] if wants else None, ) ) start += scheduled_request.token_count @@ -117,8 +129,19 @@ def __init__( copy_stream, rng: RngRegistry | None = None, runtime=None, + logprobs_mode="raw_logprobs", ): - super().__init__(model, cache, config, execution, rng=rng, runtime=runtime) + # Async speculation is greedy-only, and greedy rows report raw logprobs in + # both modes, so the mode never changes a score here. + super().__init__( + model, + cache, + config, + execution, + rng=rng, + runtime=runtime, + logprobs_mode=logprobs_mode, + ) self.compute_stream = compute_stream self.copy_stream = copy_stream self.compute_stream.wait_stream(torch.cuda.current_stream(self.device)) @@ -233,7 +256,11 @@ def _draft(self, scheduled_requests, states, tables, allocations): candidates[i].append(proposed[row]) return starts, candidates, hidden_rows - def _verify(self, scheduled_requests, starts, hidden_rows, tables, allocations): + def _verify(self, scheduled_requests, starts, hidden_rows, tables, allocations, *, score=False): + """Return (target, scores): the target argmax of every verification row and, with + ``score``, each row's FP32 raw logprob of that argmax (else None). Committed tokens + are exactly the target argmax of their rows. + """ positions = torch.cat( [ start + torch.arange(scheduled_request.token_count, device=self.device) @@ -249,8 +276,9 @@ def _verify(self, scheduled_requests, starts, hidden_rows, tables, allocations): hidden = torch.stack([h for rows in hidden_rows for h in rows]) for depth in range(self.config.draft_loops, self.config.target_loops): hidden = self._execute_recurrent_step(hidden, positions, row_tables, depth, allocations) - target = self._coda(hidden).argmax(-1) - return target + logits = self._coda(hidden) + target = logits.argmax(-1) + return target, Sampler.raw_logprobs(logits, target) if score else None def _accept_and_update_state( self, scheduled_requests, states, starts, candidates, target, buffer @@ -295,7 +323,15 @@ def submit(self, batch): starts, candidates, hidden_rows = self._draft( scheduled_requests, states, tables, allocations ) - target = self._verify(scheduled_requests, starts, hidden_rows, tables, allocations) + scoring = any( + scheduled_request.request.sampling_params.logprobs is not None + for scheduled_request in scheduled_requests + ) + target, scores = self._verify( + scheduled_requests, starts, hidden_rows, tables, allocations, score=scoring + ) + if scoring: + buffer.logprobs[: len(target)].copy_(scores) count = self._accept_and_update_state( scheduled_requests, states, starts, candidates, target, buffer ) @@ -308,6 +344,8 @@ def submit(self, batch): self.copy_stream.wait_event(buffer.compute_done) n = count + 2 * len(scheduled_requests) buffer.host_result[:n].copy_(buffer.result[:n], non_blocking=True) + if scoring: + buffer.host_logprobs[:count].copy_(buffer.logprobs[:count], non_blocking=True) buffer.copy_done = torch.cuda.Event() buffer.copy_done.record(self.copy_stream) self.stats.rounds += len(scheduled_requests) @@ -315,7 +353,7 @@ def submit(self, batch): scheduled_request.token_count - 1 for scheduled_request in scheduled_requests ) self.stats.verified_rows += count - return SpeculativeRoundTicket(batch, buffer, states, rounds, count) + return SpeculativeRoundTicket(batch, buffer, states, rounds, count, logprobs_copied=scoring) def request_ready(self, request): state = self.states.get(request.request_id) diff --git a/vllm_rlt/worker/output.py b/vllm_rlt/worker/output.py index 1948e8a6..e9e552b4 100644 --- a/vllm_rlt/worker/output.py +++ b/vllm_rlt/worker/output.py @@ -19,8 +19,18 @@ class ExitSignal: @dataclass(frozen=True) class SpeculativeResult: + """One request's committed tokens from one round. + + ``token_ids[:accepted_count]`` are accepted draft tokens; ``token_ids[accepted_count]`` + is the correction, or the bonus when every draft was accepted. Token ``j`` comes from + verification row ``j``. + """ + token_ids: list[int] accepted_count: int + # Aligned with token_ids when the request asked for logprobs; else None. Each is the + # target's logprob of the committed token on its verification row, never the draft's. + logprobs: list[float] | None = None @dataclass(frozen=True, kw_only=True) diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index c799e883..18a03366 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -5,7 +5,7 @@ """ from contextlib import nullcontext -from dataclasses import dataclass +from dataclasses import dataclass, field import torch @@ -29,6 +29,56 @@ class SpeculativeStats: verified_rows: int = 0 +@dataclass +class _CommittedScores: + """Committed tokens of one round to score, read back together at the end. + + Raw-scored tokens name their verification row in the round's logits; processed-scored + tokens bring the FP32 processed logits their p was normalized from. Spans map a + result index to its tokens' contiguous slice within its group. + """ + + raw_rows: list[int] = field(default_factory=list) + raw_tokens: list[int] = field(default_factory=list) + raw_spans: dict[int, slice] = field(default_factory=dict) + processed_logits: list[torch.Tensor] = field(default_factory=list) + processed_tokens: list[int] = field(default_factory=list) + processed_spans: dict[int, slice] = field(default_factory=dict) + + def add_raw(self, index, first_row, tokens): + start = len(self.raw_tokens) + self.raw_spans[index] = slice(start, start + len(tokens)) + self.raw_rows.extend(range(first_row, first_row + len(tokens))) + self.raw_tokens.extend(tokens) + + def add_processed(self, index, processed_logits, tokens): + start = len(self.processed_tokens) + self.processed_spans[index] = slice(start, start + len(tokens)) + self.processed_logits.extend(processed_logits) + self.processed_tokens.extend(tokens) + + def resolve(self, logits) -> dict[int, list[float]]: + """Map each added result index to its tokens' FP32 logprobs. + + At most two batched scoring ops and one host readback; none if nothing was added. + """ + parts = [] + if self.raw_tokens: + rows = torch.tensor(self.raw_rows, device=logits.device) + tokens = torch.tensor(self.raw_tokens, device=logits.device) + parts.append(Sampler.raw_logprobs(logits[rows], tokens)) + if self.processed_tokens: + tokens = torch.tensor(self.processed_tokens, device=logits.device) + parts.append(Sampler.raw_logprobs(torch.stack(self.processed_logits), tokens)) + if not parts: + return {} + values = torch.cat(parts).tolist() + raw, processed = values[: len(self.raw_tokens)], values[len(self.raw_tokens) :] + scores = {index: raw[span] for index, span in self.raw_spans.items()} + scores.update((index, processed[span]) for index, span in self.processed_spans.items()) + return scores + + def greedy_accept(candidates, targets): """Return (tokens, accepted) for one request from host candidate and target IDs. @@ -107,6 +157,7 @@ def __init__( *, runtime=None, compute_stream=None, + logprobs_mode="raw_logprobs", ): self.model, self.cache, self.config = model, cache, config self.metadata = cache.metadata_builder @@ -121,6 +172,7 @@ def __init__( # The engine injects one registry per engine so both sampling paths share # each request's RNG slot. self.rng = rng if rng is not None else RngRegistry() + self.logprobs_mode = logprobs_mode self.stats = SpeculativeStats() self.graphs = ( RecurrentGraphs(model, cache, execution, False) if execution.cuda_graphs else None @@ -282,37 +334,71 @@ def _collect_round(self, submission): candidates[i].append(candidate) start += len(rows) targets = values[start:] - results, start = [], 0 + committed, scores, start = [], _CommittedScores(), 0 for i, scheduled_request in enumerate(scheduled_requests): rows = logits[start : start + scheduled_request.token_count] - request = scheduled_request.request - if request.sampling_params.temperature == 0: + params = scheduled_request.request.sampling_params + # Greedy rows report raw values in both modes (vLLM). Processed scores of + # sampled rows reuse the processed logits p was normalized from. + score_processed = ( + params.logprobs is not None + and params.temperature != 0 + and self.logprobs_mode == "processed_logprobs" + ) + if params.temperature == 0: tokens, accepted = greedy_accept( candidates[i], targets[start : start + scheduled_request.token_count] ) - start += scheduled_request.token_count - results.append(SpeculativeResult(tokens, accepted)) - continue - start += scheduled_request.token_count - # Sampling keeps sequential rejection; RNG consumption order is unchanged. - tokens, accepted = [], 0 - for offset, candidate in enumerate(candidates[i]): - p = Sampler.probabilities(rows[offset], request.sampling_params) - token, accept = rejection_sample( - candidate, - p, - proposals[i][offset], - self._acquire_generator(request, rows[offset]), - ) - tokens.append(token) - if not accept: - break - accepted += 1 else: - token = Sampler.draw( - Sampler.probabilities(rows[-1], request.sampling_params), - self._acquire_generator(request, rows[-1]), + tokens, accepted, processed = self._sample_round( + scheduled_request.request, + rows, + candidates[i], + proposals[i], + score_processed=score_processed, ) - tokens.append(int(token.item())) - results.append(SpeculativeResult(tokens, accepted)) - return results + # Token j was committed on verification row j. + if score_processed: + scores.add_processed(i, processed, tokens) + elif params.logprobs is not None: + scores.add_raw(i, start, tokens) + start += scheduled_request.token_count + committed.append((tokens, accepted)) + logprobs = scores.resolve(logits) + return [ + SpeculativeResult(tokens, accepted, logprobs.get(i)) + for i, (tokens, accepted) in enumerate(committed) + ] + + def _sample_round(self, request, rows, candidates, proposals, *, score_processed=False): + """Return (tokens, accepted, processed) by sequential rejection over ``rows``. + + RNG consumption order is unchanged. With ``score_processed``, ``processed`` holds + the FP32 processed logits of the row each committed token was drawn on, one per + token; otherwise it is None. A correction is drawn from the residual (p - q)+, + but the round's output is distributed as p, so every token is scored under p. + """ + params = request.sampling_params + processed = [] if score_processed else None + + def target(row): + if processed is None: + return Sampler.probabilities(row, params) + processed.append(Sampler.processed_logits(row, params)) + return processed[-1].softmax(-1) + + tokens, accepted = [], 0 + for offset, candidate in enumerate(candidates): + token, accept = rejection_sample( + candidate, + target(rows[offset]), + proposals[offset], + self._acquire_generator(request, rows[offset]), + ) + tokens.append(token) + if not accept: + return tokens, accepted, processed + accepted += 1 + token = Sampler.draw(target(rows[-1]), self._acquire_generator(request, rows[-1])) + tokens.append(int(token.item())) + return tokens, accepted, processed From d659914cbb64f9ea77e257d729e620c762ef9ecf Mon Sep 17 00:00:00 2001 From: MinhaoLi0318 Date: Sun, 25 Oct 2026 10:19:42 +0800 Subject: [PATCH 8/9] test: update GPU test fakes for the scoring and logprob-return interfaces fail_after_verify now forwards keyword arguments (score=), and the split-prefill consume() fake returns ExecutionOutput(None) to match the (result, logprobs) unpacking in submit. --- tests/test_async_speculative_graphs.py | 4 ++-- tests/test_split_prefill.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_async_speculative_graphs.py b/tests/test_async_speculative_graphs.py index f43b560e..bbd49f9a 100644 --- a/tests/test_async_speculative_graphs.py +++ b/tests/test_async_speculative_graphs.py @@ -190,8 +190,8 @@ def test_failed_graph_round_drains_readers_and_releases_buffers(backend, monkeyp runner = engine.speculative_runner verify = runner._verify - def fail_after_verify(*args): - verify(*args) + def fail_after_verify(*args, **kwargs): + verify(*args, **kwargs) raise RuntimeError("failed after graph submission") monkeypatch.setattr(runner, "_verify", fail_after_verify) diff --git a/tests/test_split_prefill.py b/tests/test_split_prefill.py index 8dfb6fb1..7f58283b 100644 --- a/tests/test_split_prefill.py +++ b/tests/test_split_prefill.py @@ -1956,7 +1956,7 @@ def consume(scheduled, prepared): hidden = prepared.batch_inputs.hidden captured.append(hidden.clone()) runner._store_hidden(batch, hidden + 1, prepared) - return None + return ExecutionOutput(None) runner._execute = consume try: From 205dcc6e5b1c5261708329f753656ce2b4e044cc Mon Sep 17 00:00:00 2001 From: MinhaoLi0318 Date: Sun, 25 Oct 2026 23:32:08 +0800 Subject: [PATCH 9/9] test: cover async speculation with CUDA Graphs in the GPU logprob test The GPU test that compares speculative logprobs with the non-speculative engine ran eager sync, graphed sync and eager async variants. Add the async + graphs variant, so cross-round speculation scores its committed tokens like native decode when the recurrent core and coda replay graphs. --- tests/test_rollout_outputs.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_rollout_outputs.py b/tests/test_rollout_outputs.py index 35cdabef..8a7d42a3 100644 --- a/tests/test_rollout_outputs.py +++ b/tests/test_rollout_outputs.py @@ -1125,8 +1125,9 @@ def run(asynchronous, **overrides): @pytest.mark.gpu @pytest.mark.parametrize("logprobs_mode", LOGPROBS_MODES) def test_gpu_speculative_logprobs_match_across_graphs_async_and_native(logprobs_mode): - """CUDA/Triton speculation scores like the non-speculative engine, with CUDA Graphs and - with cross-round async speculation (greedy); sampled rounds match with graphs on/off.""" + """CUDA/Triton speculation scores like the non-speculative engine, with CUDA Graphs, with + cross-round async speculation, and with both (greedy); sampled rounds match with graphs + on/off.""" m = model(device="cuda") prompts = {"a": [2, 3, 4, 5], "b": [9], "c": [2, 3, 4, 7]} @@ -1151,7 +1152,12 @@ def run(speculative=True, asynchronous=False, graphs=False, **overrides): return outputs native = run(speculative=False) - for variant in (run(), run(graphs=True), run(asynchronous=True)): + for variant in ( + run(), + run(graphs=True), + run(asynchronous=True), + run(asynchronous=True, graphs=True), + ): for rid, expected in native.items(): assert_same(variant[rid], expected, atol=1e-4) sampled = SPECULATIVE_SAMPLING["top_k_top_p"]