From f6cfefe5d7df8e31884b9396701245b934590bd6 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 1 Oct 2026 03:20:00 +0000 Subject: [PATCH 1/5] Expose training rollout scores and full policy publication Signed-off-by: 0z5a --- benchmarks/nixl_preflight.py | 73 + benchmarks/pd_training_contract.py | 172 +++ benchmarks/training_contract.py | 94 ++ docs/training_contract.md | 82 ++ .../training-contract-a100-20261001/README.md | 77 ++ .../raw/contract-cpu-final.log | 10 + .../raw/cpu-regression-final.log | 9 + .../raw/lint-final.log | 4 + .../raw/nixl-preflight-ipc.log | 51 + .../raw/nixl-preflight-shm.log | 51 + .../raw/nixl-preflight-tcp.log | 6 + .../raw/nixl-preflight.log | 12 + .../raw/pd-full-graph-matched.json | 1189 +++++++++++++++++ .../raw/pd-full-graph-matched.log | 11 + .../raw/pd-full-graph-tcp.log | 16 + .../raw/pd-tiny-eager-tcp.json | 1160 ++++++++++++++++ .../raw/pd-tiny-eager-tcp.log | 11 + .../raw/pd-tiny-graph-tcp.json | 1163 ++++++++++++++++ .../raw/pd-tiny-graph-tcp.log | 11 + .../raw/runtime.json | 13 + .../training-contract-thor-20261001/README.md | 35 + .../raw/shared-full-rollout-benchmark.json | 98 ++ tests/__init__.py | 1 + tests/test_pd.py | 1 + tests/test_pd_weight_control.py | 91 ++ tests/test_speculative.py | 2 +- tests/test_training_contract.py | 186 +++ vllm_rlt/core/kv_cache_manager.py | 6 + vllm_rlt/engine/llm_engine.py | 94 +- vllm_rlt/entrypoints/llm.py | 28 + vllm_rlt/pd/engine.py | 107 +- vllm_rlt/pd/worker.py | 33 +- vllm_rlt/request.py | 14 + vllm_rlt/sampling_params.py | 20 +- vllm_rlt/worker/model_runner.py | 3 + vllm_rlt/worker/sampler.py | 6 +- vllm_rlt/worker/sampling.py | 13 +- vllm_rlt/worker/speculative.py | 14 +- 38 files changed, 4952 insertions(+), 15 deletions(-) create mode 100644 benchmarks/nixl_preflight.py create mode 100644 benchmarks/pd_training_contract.py create mode 100644 benchmarks/training_contract.py create mode 100644 docs/training_contract.md create mode 100644 docs/validation/training-contract-a100-20261001/README.md create mode 100644 docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/lint-final.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json create mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log create mode 100644 docs/validation/training-contract-a100-20261001/raw/runtime.json create mode 100644 docs/validation/training-contract-thor-20261001/README.md create mode 100644 docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json create mode 100644 tests/__init__.py create mode 100644 tests/test_pd_weight_control.py create mode 100644 tests/test_training_contract.py diff --git a/benchmarks/nixl_preflight.py b/benchmarks/nixl_preflight.py new file mode 100644 index 00000000..a56156b9 --- /dev/null +++ b/benchmarks/nixl_preflight.py @@ -0,0 +1,73 @@ +import multiprocessing as mp +import time + +import torch +from nixl._api import nixl_agent + + +def worker(device, pipe): + torch.cuda.set_device(device) + tensor = ( + torch.arange(32, dtype=torch.float32, device=f"cuda:{device}") + if device == 0 + else torch.zeros(32, device=f"cuda:{device}") + ) + torch.cuda.synchronize() + name = f"ouro-nixl-{device}" + agent = nixl_agent(name) + registration = agent.register_memory(tensor) + pipe.send( + dict(name=name, metadata=agent.get_agent_metadata(), ptr=tensor.data_ptr(), device=device) + ) + peer = pipe.recv() + agent.add_remote_agent(peer["metadata"]) + pipe.send("ready") + pipe.recv() + if device == 0: + handle = agent.initialize_xfer( + "WRITE", + agent.get_xfer_descs(tensor), + agent.get_xfer_descs([(peer["ptr"], 128, peer["device"])], "VRAM"), + peer["name"], + notif_msg=b"done", + backends=["UCX"], + ) + agent.transfer(handle) + while agent.check_xfer_state(handle) != "DONE": + time.sleep(0.001) + agent.release_xfer_handle(handle) + else: + while not agent.get_new_notifs(): + time.sleep(0.001) + pipe.send(tensor.cpu().tolist()) + pipe.recv() + agent.remove_remote_agent(peer["name"]) + agent.deregister_memory(registration) + pipe.close() + + +if __name__ == "__main__": + context = mp.get_context("spawn") + peers = [] + for device in range(2): + parent, child = context.Pipe() + process = context.Process(target=worker, args=(device, child)) + process.start() + child.close() + peers.append((process, parent)) + infos = [p.recv() for _, p in peers] + for i, (_, p) in enumerate(peers): + p.send(infos[1 - i]) + assert all(p.recv() == "ready" for _, p in peers) + for _, p in peers: + p.send("go") + values = [p.recv() for _, p in peers] + print("VALUES", values, flush=True) + for _, p in peers: + p.send("stop") + for process, p in peers: + process.join() + p.close() + assert process.exitcode == 0 + assert values[0] == values[1] + print("MULTIPROCESS_NIXL_PASSED", flush=True) diff --git a/benchmarks/pd_training_contract.py b/benchmarks/pd_training_contract.py new file mode 100644 index 00000000..38830d64 --- /dev/null +++ b/benchmarks/pd_training_contract.py @@ -0,0 +1,172 @@ +"""Two spawned CUDA owners: NIXL KV, policy publication, and local-engine parity.""" + +import argparse +import json +import time +from dataclasses import asdict, replace +from pathlib import Path + +import torch + +from vllm_rlt import LLM, CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig +from vllm_rlt.models import OuroConfig, OuroForCausalLM +from vllm_rlt.pd.config import PDConfig +from vllm_rlt.pd.engine import PDEngine + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("output", type=Path) + parser.add_argument("--model", type=Path) + parser.add_argument("--cuda-graphs", action="store_true") + parser.add_argument("--temperature", type=float, default=0.9) + parser.add_argument("--max-num-seqs", type=int, default=4) + args = parser.parse_args() + torch.set_num_threads(4) + torch.manual_seed(42) + dtype = torch.bfloat16 if args.model else torch.float32 + source = args.model or OuroConfig( + vocab_size=64, + hidden_size=32, + intermediate_size=64, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=8, + max_position_embeddings=128, + total_ut_steps=4, + bos_token_id=0, + eos_token_id=0, + ) + native = ( + OuroForCausalLM.from_pretrained(source, device="cuda:0", dtype=dtype) + if args.model + else OuroForCausalLM(source).to(device="cuda:0", dtype=dtype) + ) + execution = ExecutionConfig(cuda_graphs=args.cuda_graphs, async_scheduling=True) + cache = CacheConfig(num_blocks=512, enable_prefix_caching=True) + scheduler = SchedulerConfig( + max_num_seqs=args.max_num_seqs, max_num_batched_tokens=64, prefill_chunk_size=16 + ) + common = dict( + execution_config=execution, + exit_config=ExitConfig("ouro_delayed"), + attention_backend="triton", + ) + local = LLM(native, cache_config=cache, scheduler_config=scheduler, **common) + pd = PDEngine( + source, + dtype="bfloat16" if args.model else "float32", + pd_config=PDConfig( + max_control_messages=1, startup_timeout=600, request_timeout=600, shutdown_timeout=120 + ), + prefill_cache_config=cache, + decode_cache_config=cache, + prefill_scheduler_config=scheduler, + decode_scheduler_config=scheduler, + terminate_workers_on_failure=False, + **common, + ) + disaggregated = LLM(pd) + prompts = [[3, 4, 5] * 9 + [6], [3, 4, 5] * 9 + [7]] + report = dict(model=asdict(native.config), cuda_graphs=args.cuda_graphs, iterations=[]) + report["max_num_seqs"] = args.max_num_seqs + try: + for version, loops in enumerate((2, 3, 4), 1): + with torch.no_grad(): + native.lm_head.weight.mul_(0.98) + for arm in (local, disaggregated): + arm.start_weight_update(version) + weights = list(native.named_parameters()) + for offset in range(0, len(weights), 8): + arm.update_weights(weights[offset : offset + 8]) + arm.finish_weight_update() + assert arm.get_weight_version() == version + for mode in ("raw", "processed"): + params = [ + SamplingParams( + max_tokens=6, + max_loops=loops, + seed=11 + i, + temperature=args.temperature, + top_k=8, + top_p=0.8, + ignore_eos=True, + logprobs=0, + logprobs_mode=mode, + ) + for i in range(len(prompts)) + ] + outputs, timings = {}, {} + for label, arm in (("local", local), ("pd", disaggregated)): + started = time.monotonic() + outputs[label] = arm.generate(prompts, params) + timings[label] = time.monotonic() - started + errors = [] + for expected, result in zip(outputs["local"], outputs["pd"], strict=True): + assert expected.token_ids == result.token_ids + assert expected.exit_depths == result.exit_depths + assert result.weight_version == version + assert result.sampling_params == expected.sampling_params + delta = ( + torch.tensor(expected.log_probs) - torch.tensor(result.log_probs) + ).abs() + errors.append(delta.max().item()) + torch.testing.assert_close( + torch.tensor(expected.log_probs), + torch.tensor(result.log_probs), + atol=2e-5, + rtol=2e-5, + ) + row = dict( + version=version, + loops=loops, + mode=mode, + seconds=timings, + max_logprob_error=max(errors), + outputs=[asdict(o) for o in outputs["pd"]], + ) + report["iterations"].append(row) + print("PARITY", version, loops, mode, timings, max(errors), flush=True) + # A failed chunk cannot expose a mixed policy; a newer full epoch recovers. + disaggregated.start_weight_update(4) + try: + disaggregated.update_weights([("missing_parameter", torch.zeros(1))]) + except RuntimeError as error: + report["rejected_chunk"] = str(error) + else: + raise AssertionError("Invalid publication unexpectedly succeeded") + disaggregated.start_weight_update(5) + disaggregated.update_weights(native.named_parameters()) + disaggregated.finish_weight_update() + recovered = disaggregated.generate(prompts[:1], params[:1])[0] + assert recovered.weight_version == 5 + stopped = disaggregated.generate( + prompts[:1], [replace(params[0], stop_token_ids=(recovered.token_ids[0],))] + )[0] + assert len(stopped.token_ids) == len(stopped.log_probs) == 1 + assert stopped.finish_reason == "stop" + pd.add_request("abort", prompts[0], params[0]) + while not pd.transfers[next(iter(pd.transfers))].latest_output: + pd.step() + aborted = pd.abort_request("abort") + assert aborted.finish_reason == "abort" + assert len(aborted.log_probs) == len(aborted.token_ids) + while pd.has_unfinished_requests(): + pd.step() + disaggregated.reset_prefix_cache() + report.update(recovered_version=5, stop=asdict(stopped), abort=asdict(aborted)) + finally: + disaggregated.close() + local.close() + report["workers"] = pd.worker_metrics + assert all( + m["used_blocks"] == 0 and m["weight_version"] == 5 for m in pd.worker_metrics.values() + ) + assert sum(m["bytes_sent"] for m in pd.worker_metrics.values()) > 0 + args.output.write_text(json.dumps(report, indent=2)) + print("PD_CONTRACT_PASSED", args.output, flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/training_contract.py b/benchmarks/training_contract.py new file mode 100644 index 00000000..0bd6fbe0 --- /dev/null +++ b/benchmarks/training_contract.py @@ -0,0 +1,94 @@ +"""Alternating-order warmed rollout timings, including returned log probabilities.""" + +import argparse +import json +import statistics +import time +from pathlib import Path + +import torch +from transformers import AutoTokenizer + +from vllm_rlt import ( + LLM, + CacheConfig, + ExecutionConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.models import OuroForCausalLM + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("model", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--repeats", type=int, default=5) + args = parser.parse_args() + torch.set_num_threads(4) + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False) + prompts = [tokenizer.encode(text) for text in ("Compute 17 + 25.\n", "Compute 9 * 8.\n")] + native = OuroForCausalLM.from_pretrained(args.model, device="cuda", dtype=torch.bfloat16) + report = {} + for speculative in (False, True): + arms = { + mode: LLM( + native, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=512), + scheduler_config=SchedulerConfig(max_num_seqs=8, max_num_batched_tokens=128), + execution_config=ExecutionConfig(cuda_graphs=mode == "graph"), + speculative_config=SpeculativeConfig(3) if speculative else None, + ) + for mode in ("eager", "graph") + } + for loops in (4,) if speculative else (2, 3, 4): + params = [ + SamplingParams( + max_tokens=32, + max_loops=loops, + temperature=0.9, + seed=31 + i, + logprobs=0, + ignore_eos=True, + ) + for i in range(len(prompts)) + ] + for arm in arms.values(): + arm.generate(prompts, params) + arm.start_weight_update() + arm.update_weights(native.named_parameters()) + arm.finish_weight_update() + version = arms["eager"].get_weight_version() + times = {mode: [] for mode in arms} + for index in range(args.repeats): + outputs = {} + for mode in ("eager", "graph") if index % 2 == 0 else ("graph", "eager"): + torch.cuda.synchronize() + started = time.monotonic() + outputs[mode] = arms[mode].generate(prompts, params) + torch.cuda.synchronize() + times[mode].append(time.monotonic() - started) + for eager, graph in zip(outputs["eager"], outputs["graph"], strict=True): + assert eager.token_ids == graph.token_ids + assert eager.exit_depths == graph.exit_depths + assert eager.log_probs == graph.log_probs + assert eager.weight_version == graph.weight_version == version + medians = {mode: statistics.median(values) for mode, values in times.items()} + name = f"speculative-{loops}" if speculative else f"native-{loops}" + report[name] = { + "seconds": times, + "medians": medians, + "speedup": medians["eager"] / medians["graph"], + "tokens_and_logprobs_exact": True, + } + print(name, report[name], flush=True) + for arm in arms.values(): + arm.close() + del arms + args.output.write_text(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/training_contract.md b/docs/training_contract.md new file mode 100644 index 00000000..9345cbf6 --- /dev/null +++ b/docs/training_contract.md @@ -0,0 +1,82 @@ +# Training rollout and full-weight publication + +`LLM` and `LLMEngine` expose the same full-update operations. `LLM` also accepts +an existing `LLMEngine` or `PDEngine`, so framework adapters use the same generation +and publication calls with either backend. + +```python +llm.start_weight_update(version=1) +for chunk in converted_hf_weights: + llm.update_weights(chunk) # Iterable[(physical_parameter_name, torch.Tensor)] +llm.finish_weight_update() +outputs = llm.generate(prompts, SamplingParams( + max_tokens=128, temperature=1.0, seed=42, logprobs=0, +)) +``` + +Each output contains aligned selected-token `log_probs`, `exit_depths`, the +committed `weight_version`, and its effective `sampling_params`. `logprobs=None` +keeps the ordinary inference path. Reported parameters resolve the model's +default `max_loops` and cap `top_k` at its vocabulary size. `logprobs=0` requests only the emitted token's +probability; top-N alternatives are not implemented. + +`logprobs_mode="raw"` is the default: FP32 log-softmax of unscaled model logits. +`"processed"` returns probabilities after positive temperature and top-k/top-p +filtering. Greedy requests return the raw score in either mode. Speculative +outputs use the verified **target** policy's score, including correction/bonus +tokens; proposal or rejection-residual probabilities are never exposed as the +policy score. Explicit `stop_token_ids` apply even with `ignore_eos=True`. +`seed=None` creates an independent entropy-seeded request generator. + +Publication requires an idle engine. Each chunk is checked for duplicate names, +unknown names and mismatched shapes before that chunk is copied. `finish_weight_update` +requires every physical parameter exactly once. Copies preserve parameter addresses +and captured graphs; a fresh transaction invalidates all cached prefix KV. If a +transfer fails, generation stays blocked until a fresh full transaction succeeds. +Starting again discards an unfinished transaction. `pause_generation` stops +admission while existing requests drain; `resume_generation` requires publication +to be complete. `abort_request` remains available to cancel requests before an update. + +PDEngine waits for all transfers to release, broadcasts CPU-staged weight chunks +to every P/D worker, and commits a version only after every peer acknowledges it. +Recovery uses a version newer than **all attempted** PD versions because some +workers may already have committed. With `terminate_workers_on_failure=False`, fatal workers retain their registered +memory rather than receive forced process signals; normal `close()` remains +cooperative. No CUDA tensors change ownership over IPC; +NIXL's existing GPU KV path and registrations remain intact. Prefix invalidation +does not replace registered KV storage. PD worker control errors return replies +without terminating the worker. + +Existing exit, async and speculative configuration rules still apply. In particular, +upstream self-speculation uses a full-depth target and synchronous execution; +upstream PD does not implement self-speculation. Publication adds no such restriction +to ordinary early-exit or async rollouts. Trainers must replay full-depth prompt +prefill and the returned decode depths rather than assume the entire sequence used K. + +## Validation + +```bash +python -m unittest tests.test_training_contract tests.test_pd_weight_control -v +CONTRACT_DEVICE=cuda python -m unittest tests.test_training_contract -v +python benchmarks/training_contract.py /models/Ouro-1.4B timings.json +UCX_TLS=tcp,cuda_copy python benchmarks/nixl_preflight.py +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py pd.json --cuda-graphs +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py full-pd.json \ + --model /models/Ouro-1.4B --cuda-graphs --max-num-seqs 1 +``` + +CPU and Thor CUDA tests cover raw/processed probabilities against an independent +dense oracle, speculative correction/bonus alignment, publication, pointer stability, +prefix invalidation, early exit, async stop/abort and entropy seeds. The PD control +test drives two real CPU engines and checks recovery after one peer commits and +another fails. It uses in-process control delivery; it does **not** validate IPC or +NIXL transport. The separate two-A100 run uses real spawned workers and NIXL: +FP32 multi-request and full BF16 matched-batch checks cover three publications, +raw/processed scores, graph replay, prefix reuse, explicit stop/abort and recovery +to version five. See [PD evidence](validation/training-contract-a100-20261001/README.md). +No PD workers were launched on single-GPU Thor, where NIXL is absent. + +This is a draft implementation of the +[Phase 1 contract proposed in issue 70](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5888101273) +on upstream `ecb1f8b505b7e831815b40aec3b4598619cca23a`; formal upstream agreement +remains outside the scope of this contributor draft. diff --git a/docs/validation/training-contract-a100-20261001/README.md b/docs/validation/training-contract-a100-20261001/README.md new file mode 100644 index 00000000..40473595 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/README.md @@ -0,0 +1,77 @@ +# Training contract with two A100 workers — 2026-10-01 + +Two A100-SXM4-40GB GPUs, driver 580.105.08, the existing `/venv/main` runtime, +PyTorch 2.13.0+cu130, Triton 3.7.1 and NIXL 1.4.1. Separate spawned P/D owners +register their GPU KV/hidden pools and exchange real NIXL WRITE transfers. +The public `LLM` interface stages physical policy chunks through CPU IPC and waits +for every worker to commit. No package, driver or service changes are made. + +| Case | Active slots | Maximum logprob error vs local | Published/recovered version | D graph captures / replays | P bytes transferred | +|---|---:|---:|---:|---:|---:| +| Random FP32, eager, K=2/3/4 | 4 | 4.77e-7 | 5 | Disabled | 300,928 | +| Random FP32, graphs, K=2/3/4 | 4 | 4.77e-7 | 5 | 2 / 116 | 300,928 | +| Full BF16 Ouro-1.4B, graphs, K=2/3/4 | 1 | 0 | 5 | 1 / 206 | 229,699,584 | + +Each case compares raw and processed scores at temperature 0.9, top-k 8 and +nucleus 0.8, with two prompts and six response tokens. Token IDs, physical exit +depths, effective parameters and policy versions agree. Prompt prefill is full +K=4; decode budgets change after each complete publication. Prefix reuse, +chunked updates, a rejected update, recovery into version five, explicit stop +and active abort are exercised. Both workers finish with zero used blocks and +version five. Graph captures remain valid through publication and recovery. + +The full-model source is `ByteDance/Ouro-1.4B` at +`574fa66cb8bf5abdc979642d01cf2b79b16bfab1`, safetensors SHA256 +`58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af`. +See [full outputs/counters](raw/pd-full-graph-matched.json), +[tiny eager](raw/pd-tiny-eager-tcp.json), [tiny graphs](raw/pd-tiny-graph-tcp.json) +and [runtime](raw/runtime.json). + +## Transport and numerical limits + +Default UCX GPU IPC reports completion but leaves the destination zero in an +independent two-process, 128-byte probe. Explicit IPC and shared-memory-only +configurations also fail on this container. `UCX_TLS=tcp,cuda_copy` passes the +probe and every case above with the installed library. Transfers are host staged; +this report does not establish direct GPU IPC/RDMA performance. The workaround +is scoped to each command. [Probe source](../../../benchmarks/nixl_preflight.py), +[default failure](raw/nixl-preflight.log) and [passing probe](raw/nixl-preflight-tcp.log) +are retained. No processes are forcibly terminated; successful workers use the +normal stop/acknowledgment protocol. + +A full BF16 comparison with four slots has a maximum observed selected-score +difference of 0.11199 when local and PD prefill batches differ. The matched +one-slot run above is exact; the FP32 four-slot run is within 4.77e-7. +Different BF16 batching is therefore not claimed to be bitwise equivalent. See +[the rejected strict comparison](raw/pd-full-graph-tcp.log). +Other jobs were already running on both GPUs. Timings in these parity probes +include startup/capture/prefix-state differences and contention, and are not a +controlled throughput comparison. The [Thor table](../training-contract-thor-20261001/README.md) +contains alternating-order warmed eager/graph timings. + +## Reproduction + +From the repository root, in an existing CUDA/NIXL environment: + +```bash +UCX_TLS=tcp,cuda_copy python benchmarks/nixl_preflight.py +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py tiny.json +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py tiny-graph.json --cuda-graphs +UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py full.json \ + --model /models/Ouro-1.4B --cuda-graphs --max-num-seqs 1 +``` + +The VIME training/save/resume and general HF export checks are recorded in +[0z5a/vime#5](https://github.com/0z5a/vime/pull/5). + +## Repository checks + +The complete default CPU regression suite passes: **421 passed, 141 skipped** +in 19.41 s; GPU tests, including the upstream forced-peer-failure test, are skipped +by the repository's ordinary collection rules. See +[regression log](raw/cpu-regression-final.log) and +[focused contract tests](raw/contract-cpu-final.log). +Ruff lint and format checks pass for all 88 Python files. +The existing runtimes lack `pre_commit`, so its three `language: system` hooks +are run directly without installing it: Python-file lint, format check and the +full default CPU suite. See [lint log](raw/lint-final.log). diff --git a/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log b/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log new file mode 100644 index 00000000..885048c7 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log @@ -0,0 +1,10 @@ +test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok +test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok +test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok +test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok +test_all_workers_commit_and_partial_commit_requires_new_epoch (tests.test_pd_weight_control.PDWeightControlTest.test_all_workers_commit_and_partial_commit_requires_new_epoch) ... ok + +---------------------------------------------------------------------- +Ran 5 tests in 2.095s + +OK diff --git a/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log b/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log new file mode 100644 index 00000000..d10641bc --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log @@ -0,0 +1,9 @@ +.......s.s.s.sssssssssssssssssss....sssssssssssssss.s................... [ 12%] +............................................ssssss......s..............s [ 25%] +ssssssss.sssssssssssssssssssssssss...................................... [ 38%] +....ssssssss.ssss....sssssss...............................s............ [ 51%] +..sssssssssssssssss.................sss................................. [ 64%] +..ssssss..sss.....ss...ss............................................... [ 76%] +........................................................................ [ 89%] +.............ss.ss...............................sssss.... [100%] +421 passed, 141 skipped in 19.41s diff --git a/docs/validation/training-contract-a100-20261001/raw/lint-final.log b/docs/validation/training-contract-a100-20261001/raw/lint-final.log new file mode 100644 index 00000000..3d3e3c6d --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/lint-final.log @@ -0,0 +1,4 @@ +Required pre-commit hooks use language: system. pre_commit is absent; no installation performed. +ruff 0.16.8 +All checks passed! +88 files already formatted diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log new file mode 100644 index 00000000..11287164 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log @@ -0,0 +1,51 @@ +[1790822017.515227] [25f216012e8e:21943:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy, cuda_ipc/cuda - no am bcopy +W1001 02:33:37.515292 21943 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable +E1001 02:33:37.515377 21943 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep +E1001 02:33:37.515387 21943 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND +Process SpawnProcess-1: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker + agent = nixl_agent(name) + ^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ + self.create_backend(bknd, init) + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend + self.backends[backend] = self.agent.createBackend(backend, initParams) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND +[1790822017.583102] [25f216012e8e:21944:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy, cuda_ipc/cuda - no am bcopy +W1001 02:33:37.583151 21944 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable +E1001 02:33:37.583303 21944 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep +E1001 02:33:37.583311 21944 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND +Process SpawnProcess-2: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker + agent = nixl_agent(name) + ^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ + self.create_backend(bknd, init) + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend + self.backends[backend] = self.agent.createBackend(backend, initParams) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND +Traceback (most recent call last): + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 42, in + infos=[p.recv() for _,p in peers] + ^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 250, in recv + buf = self._recv_bytes() + ^^^^^^^^^^^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes + buf = self._recv(4) + ^^^^^^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 399, in _recv + raise EOFError +EOFError diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log new file mode 100644 index 00000000..d2072821 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log @@ -0,0 +1,51 @@ +[1790822145.345936] [25f216012e8e:23279:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy +W1001 02:35:45.345994 23279 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable +E1001 02:35:45.346127 23279 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep +E1001 02:35:45.346146 23279 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND +Process SpawnProcess-1: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker + agent = nixl_agent(name) + ^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ + self.create_backend(bknd, init) + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend + self.backends[backend] = self.agent.createBackend(backend, initParams) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND +[1790822145.362990] [25f216012e8e:23280:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy +W1001 02:35:45.363052 23280 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable +E1001 02:35:45.363210 23280 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep +E1001 02:35:45.363225 23280 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND +Process SpawnProcess-2: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker + agent = nixl_agent(name) + ^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ + self.create_backend(bknd, init) + File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend + self.backends[backend] = self.agent.createBackend(backend, initParams) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND +Traceback (most recent call last): + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 42, in + infos=[p.recv() for _,p in peers] + ^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 250, in recv + buf = self._recv_bytes() + ^^^^^^^^^^^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes + buf = self._recv(4) + ^^^^^^^^^^^^^ + File "/usr/lib/python3.12/multiprocessing/connection.py", line 399, in _recv + raise EOFError +EOFError diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log new file mode 100644 index 00000000..c5bfffe1 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log @@ -0,0 +1,6 @@ +2026-10-01 02:12:24 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:12:24 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-1 +2026-10-01 02:12:24 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:12:24 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-0 +VALUES [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0]] +MULTIPROCESS_NIXL_PASSED diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log new file mode 100644 index 00000000..ce2d6594 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log @@ -0,0 +1,12 @@ +2026-10-01 02:11:20 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:11:20 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-0 +2026-10-01 02:11:20 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:11:20 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-1 +VALUES [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]] +[1790820681.044462] [25f216012e8e:15986:0] cuda_ctx.c:23 UCX ERROR cuDevicePrimaryCtxGetState(cuda_device, &flags, &active) failed: unrecognized error code 4 +[1790820681.060647] [25f216012e8e:15985:0] cuda_ctx.c:23 UCX ERROR cuDevicePrimaryCtxGetState(cuda_device, &flags, &active) failed: unrecognized error code 4 +Traceback (most recent call last): + File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 50, in + assert values[0]==values[1] + ^^^^^^^^^^^^^^^^^^^^ +AssertionError diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json new file mode 100644 index 00000000..03da93de --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json @@ -0,0 +1,1189 @@ +{ + "model": { + "vocab_size": 49152, + "hidden_size": 2048, + "intermediate_size": 5632, + "num_hidden_layers": 24, + "num_attention_heads": 16, + "num_key_value_heads": 16, + "head_dim": 128, + "hidden_act": "silu", + "max_position_embeddings": 65536, + "initializer_range": 0.02, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "total_ut_steps": 4, + "early_exit_threshold": 1.0, + "bos_token_id": 0, + "eos_token_id": 0, + "pad_token_id": null, + "tie_word_embeddings": false, + "attention_dropout": 0.0, + "rope_scaling": null, + "use_sliding_window": false, + "sliding_window": null, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ] + }, + "cuda_graphs": true, + "iterations": [ + { + "version": 1, + "loops": 2, + "mode": "raw", + "seconds": { + "local": 1.501969379838556, + "pd": 15.95300066191703 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "0", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.4585607051849365, + -1.3445959091186523, + -0.6607269048690796, + -0.4440094828605652, + -0.288143515586853, + -0.2284386157989502 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "1", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 4, + 4, + 4, + 198 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.4905070662498474, + -1.852401614189148, + -1.8474926948547363, + -4.226284027099609, + -2.1031107902526855, + -3.4267778396606445 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 1, + "loops": 2, + "mode": "processed", + "seconds": { + "local": 0.544591479934752, + "pd": 6.968707168009132 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "2", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.5379835367202759, + -0.22262510657310486, + 0.0, + 0.0, + 0.0 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "3", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 4, + 4, + 4, + 198 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.5063599348068237, + -1.3956828117370605, + -1.77802312374115, + -1.5563204288482666, + -0.19239495694637299 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "raw", + "seconds": { + "local": 0.7185850627720356, + "pd": 14.518414614256471 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "4", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.5055561065673828, + -0.5541938543319702, + -0.48413512110710144, + -0.3641066551208496, + -0.22259598970413208, + -0.19062571227550507 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "5", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 4, + 3, + 4, + 3 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.54900062084198, + -0.6617550849914551, + -1.9724218845367432, + -0.34641382098197937, + -1.458582878112793, + -0.42156851291656494 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "processed", + "seconds": { + "local": 0.66817781701684, + "pd": 7.080546109005809 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "6", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.1622927039861679, + -0.15219146013259888, + 0.0, + 0.0, + 0.0 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "7", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 4, + 3, + 4, + 3 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.2678171992301941, + -1.781590223312378, + 0.0, + -1.1942179203033447, + 0.0 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "raw", + "seconds": { + "local": 0.849096744786948, + "pd": 14.640745197888464 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "8", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.5625201463699341, + -0.37419480085372925, + -0.2975218594074249, + -0.2469334453344345, + -0.20516522228717804, + -0.1743372529745102 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "9", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -0.616585373878479, + -0.5554823875427246, + -0.43136879801750183, + -0.33480215072631836, + -0.2979239821434021, + -0.46127814054489136 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "processed", + "seconds": { + "local": 0.7937315646559, + "pd": 7.206241376232356 + }, + "max_logprob_error": 0.0, + "outputs": [ + { + "request_id": "10", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "11", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + 0.0, + -0.1622924953699112, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + } + ], + "max_num_seqs": 1, + "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", + "recovered_version": 5, + "stop": { + "request_id": "13", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "stop", + "text": "", + "log_probs": [ + 0.0 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [ + 3 + ], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "abort": { + "request_id": "abort", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 3 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "abort", + "text": "", + "log_probs": [ + 0.0 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "workers": { + "cae75e7035744e8795024750bd65b22e-decode-1": { + "kind": "stopped", + "prefill_tokens": 0, + "bytes_sent": 0, + "transfers": 0, + "transfer_seconds": 0.0, + "used_blocks": 0, + "weight_version": 5, + "graph_captures": 1, + "graph_replays": 206, + "graph_fallbacks": 0 + }, + "cae75e7035744e8795024750bd65b22e-prefill-0": { + "kind": "stopped", + "prefill_tokens": 244, + "bytes_sent": 229699584, + "transfers": 22, + "transfer_seconds": 207.36763043235987, + "used_blocks": 0, + "weight_version": 5 + } + } +} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log new file mode 100644 index 00000000..d37a3c92 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log @@ -0,0 +1,11 @@ +2026-10-01 02:30:31 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:30:31 NIXL INFO _api.py:285 Initialized NIXL agent: cae75e7035744e8795024750bd65b22e-decode-1 +2026-10-01 02:30:31 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:30:31 NIXL INFO _api.py:285 Initialized NIXL agent: cae75e7035744e8795024750bd65b22e-prefill-0 +PARITY 1 2 raw {'local': 1.501969379838556, 'pd': 15.95300066191703} 0.0 +PARITY 1 2 processed {'local': 0.544591479934752, 'pd': 6.968707168009132} 0.0 +PARITY 2 3 raw {'local': 0.7185850627720356, 'pd': 14.518414614256471} 0.0 +PARITY 2 3 processed {'local': 0.66817781701684, 'pd': 7.080546109005809} 0.0 +PARITY 3 4 raw {'local': 0.849096744786948, 'pd': 14.640745197888464} 0.0 +PARITY 3 4 processed {'local': 0.7937315646559, 'pd': 7.206241376232356} 0.0 +PD_CONTRACT_PASSED pd-full-graph-matched.json diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log new file mode 100644 index 00000000..03b98612 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log @@ -0,0 +1,16 @@ +2026-10-01 02:19:01 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:19:01 NIXL INFO _api.py:285 Initialized NIXL agent: 35f3775d875246919e95de82111ac017-prefill-0 +2026-10-01 02:19:01 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:19:01 NIXL INFO _api.py:285 Initialized NIXL agent: 35f3775d875246919e95de82111ac017-decode-1 +Traceback (most recent call last): + File "/workspace/0z5a/work/ouro-vime-contract-20261001/rlt/benchmarks/pd_training_contract.py", line 168, in + main() + File "/workspace/0z5a/work/ouro-vime-contract-20261001/rlt/benchmarks/pd_training_contract.py", line 111, in main + torch.testing.assert_close( + File "/usr/local/lib/python3.12/dist-packages/torch/testing/_comparison.py", line 1631, in assert_close + raise error_metas[0].to_error(msg) +AssertionError: Tensor-likes are not close! + +Mismatched elements: 6 / 6 (100.0%) +Greatest absolute difference: 0.11199069023132324 at index (1,) (up to 2e-05 allowed) +Greatest relative difference: 0.07553135603666306 at index (1,) (up to 2e-05 allowed) diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json new file mode 100644 index 00000000..c5c64fd2 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json @@ -0,0 +1,1160 @@ +{ + "model": { + "vocab_size": 64, + "hidden_size": 32, + "intermediate_size": 64, + "num_hidden_layers": 2, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 8, + "hidden_act": "silu", + "max_position_embeddings": 128, + "initializer_range": 0.02, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "total_ut_steps": 4, + "early_exit_threshold": 1.0, + "bos_token_id": 0, + "eos_token_id": 0, + "pad_token_id": null, + "tie_word_embeddings": false, + "attention_dropout": 0.0, + "rope_scaling": null, + "use_sliding_window": false, + "sliding_window": null, + "layer_types": null + }, + "cuda_graphs": false, + "iterations": [ + { + "version": 1, + "loops": 2, + "mode": "raw", + "seconds": { + "local": 1.1838269280269742, + "pd": 2.0369991376064718 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "0", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 42, + 42, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.992061138153076, + -3.8517885208129883, + -3.85848069190979, + -3.9884941577911377, + -3.8614566326141357, + -3.8722712993621826 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "1", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 51, + 15, + 55 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.861236810684204, + -3.9610671997070312, + -3.9557578563690186, + -4.008734226226807, + -3.9762279987335205, + -4.012908458709717 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 1, + "loops": 2, + "mode": "processed", + "seconds": { + "local": 0.23476281203329563, + "pd": 0.41898630978539586 + }, + "max_logprob_error": 1.1920928955078125e-07, + "outputs": [ + { + "request_id": "2", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 42, + 42, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.969618320465088, + -1.8275089263916016, + -1.835660457611084, + -1.980717658996582, + -1.8372858762741089, + -1.8491119146347046 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "3", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 51, + 15, + 55 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8221769332885742, + -1.9443289041519165, + -1.942601203918457, + -2.004945993423462, + -1.9664429426193237, + -2.008206367492676 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "raw", + "seconds": { + "local": 0.21088168676942587, + "pd": 0.45077179512009025 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "4", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 12, + 61, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.9952800273895264, + -4.035950660705566, + -3.7910518646240234, + -3.8775575160980225, + -3.849231719970703, + -3.8591208457946777 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "5", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 56, + 15, + 56 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.8670716285705566, + -3.976059913635254, + -3.975245237350464, + -4.008485317230225, + -3.941457986831665, + -4.011442184448242 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "processed", + "seconds": { + "local": 0.24939796887338161, + "pd": 0.38997333077713847 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "6", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 12, + 61, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.9691096544265747, + -2.072470188140869, + -1.8017675876617432, + -1.8991870880126953, + -1.8665740489959717, + -1.8762420415878296 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "7", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 56, + 15, + 56 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8246119022369385, + -2.0073468685150146, + -2.00469970703125, + -2.0388402938842773, + -1.96330726146698, + -2.036628007888794 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "raw", + "seconds": { + "local": 0.22700949804857373, + "pd": 0.45369746908545494 + }, + "max_logprob_error": 4.76837158203125e-07, + "outputs": [ + { + "request_id": "8", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 60, + 9, + 25, + 60, + 42 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.9984395503997803, + -4.025251865386963, + -4.024652004241943, + -3.9303245544433594, + -4.02800178527832, + -4.010705947875977 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "9", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 60, + 60, + 9, + 15, + 56 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.8727941513061523, + -4.021127223968506, + -4.020061016082764, + -4.025237083435059, + -3.993036985397339, + -4.017432689666748 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "processed", + "seconds": { + "local": 0.25585177494212985, + "pd": 0.42146871984004974 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "10", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 60, + 9, + 25, + 60, + 42 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.9686124324798584, + -1.9942327737808228, + -1.9923925399780273, + -1.8824831247329712, + -1.9928045272827148, + -1.9732145071029663 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "11", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 60, + 60, + 9, + 15, + 56 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8269999027252197, + -1.9946293830871582, + -1.9918233156204224, + -1.995558738708496, + -1.9558247327804565, + -1.983024001121521 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + } + ], + "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", + "recovered_version": 5, + "stop": { + "request_id": "13", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "stop", + "text": "", + "log_probs": [ + -1.9686124324798584 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [ + 42 + ], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "abort": { + "request_id": "abort", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "abort", + "text": "", + "log_probs": [ + -1.9686124324798584 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "workers": { + "e7b52c471c5a4a7383476001ca2853cc-decode-1": { + "kind": "stopped", + "prefill_tokens": 0, + "bytes_sent": 0, + "transfers": 0, + "transfer_seconds": 0.0, + "used_blocks": 0, + "weight_version": 5 + }, + "e7b52c471c5a4a7383476001ca2853cc-prefill-0": { + "kind": "stopped", + "prefill_tokens": 292, + "bytes_sent": 300928, + "transfers": 22, + "transfer_seconds": 2.774511670228094, + "used_blocks": 0, + "weight_version": 5 + } + } +} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log new file mode 100644 index 00000000..9ae443a5 --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log @@ -0,0 +1,11 @@ +2026-10-01 02:14:34 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:14:34 NIXL INFO _api.py:285 Initialized NIXL agent: e7b52c471c5a4a7383476001ca2853cc-decode-1 +2026-10-01 02:14:34 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:14:34 NIXL INFO _api.py:285 Initialized NIXL agent: e7b52c471c5a4a7383476001ca2853cc-prefill-0 +PARITY 1 2 raw {'local': 1.1838269280269742, 'pd': 2.0369991376064718} 2.384185791015625e-07 +PARITY 1 2 processed {'local': 0.23476281203329563, 'pd': 0.41898630978539586} 1.1920928955078125e-07 +PARITY 2 3 raw {'local': 0.21088168676942587, 'pd': 0.45077179512009025} 2.384185791015625e-07 +PARITY 2 3 processed {'local': 0.24939796887338161, 'pd': 0.38997333077713847} 2.384185791015625e-07 +PARITY 3 4 raw {'local': 0.22700949804857373, 'pd': 0.45369746908545494} 4.76837158203125e-07 +PARITY 3 4 processed {'local': 0.25585177494212985, 'pd': 0.42146871984004974} 2.384185791015625e-07 +PD_CONTRACT_PASSED pd-tiny-eager-tcp.json diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json new file mode 100644 index 00000000..e95c674c --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json @@ -0,0 +1,1163 @@ +{ + "model": { + "vocab_size": 64, + "hidden_size": 32, + "intermediate_size": 64, + "num_hidden_layers": 2, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 8, + "hidden_act": "silu", + "max_position_embeddings": 128, + "initializer_range": 0.02, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "total_ut_steps": 4, + "early_exit_threshold": 1.0, + "bos_token_id": 0, + "eos_token_id": 0, + "pad_token_id": null, + "tie_word_embeddings": false, + "attention_dropout": 0.0, + "rope_scaling": null, + "use_sliding_window": false, + "sliding_window": null, + "layer_types": null + }, + "cuda_graphs": true, + "iterations": [ + { + "version": 1, + "loops": 2, + "mode": "raw", + "seconds": { + "local": 1.2744349539279938, + "pd": 2.329219114035368 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "0", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 42, + 42, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.992061138153076, + -3.8517885208129883, + -3.85848069190979, + -3.9884941577911377, + -3.8614566326141357, + -3.8722712993621826 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "1", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 51, + 15, + 55 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.861236810684204, + -3.9610671997070312, + -3.9557578563690186, + -4.008734226226807, + -3.9762279987335205, + -4.012908458709717 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 1, + "loops": 2, + "mode": "processed", + "seconds": { + "local": 0.23766471119597554, + "pd": 0.436434387229383 + }, + "max_logprob_error": 1.1920928955078125e-07, + "outputs": [ + { + "request_id": "2", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 42, + 42, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.969618320465088, + -1.8275089263916016, + -1.835660457611084, + -1.980717658996582, + -1.8372858762741089, + -1.8491119146347046 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "3", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 51, + 15, + 55 + ], + "exit_depths": [ + 4, + 2, + 2, + 2, + 2, + 2 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8221769332885742, + -1.9443289041519165, + -1.942601203918457, + -2.004945993423462, + -1.9664429426193237, + -2.008206367492676 + ], + "weight_version": 1, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 2, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "raw", + "seconds": { + "local": 0.22012542001903057, + "pd": 0.4495909558609128 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "4", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 12, + 61, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.9952800273895264, + -4.035950660705566, + -3.7910518646240234, + -3.8775575160980225, + -3.849231719970703, + -3.8591208457946777 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "5", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 56, + 15, + 56 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.8670716285705566, + -3.976059913635254, + -3.975245237350464, + -4.008485317230225, + -3.941457986831665, + -4.011442184448242 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 2, + "loops": 3, + "mode": "processed", + "seconds": { + "local": 0.24550614319741726, + "pd": 0.40186773892492056 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "6", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 12, + 61, + 25, + 42, + 42 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.9691096544265747, + -2.072470188140869, + -1.8017675876617432, + -1.8991870880126953, + -1.8665740489959717, + -1.8762420415878296 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "7", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 47, + 47, + 56, + 15, + 56 + ], + "exit_depths": [ + 4, + 3, + 3, + 3, + 3, + 3 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8246119022369385, + -2.0073468685150146, + -2.00469970703125, + -2.0388402938842773, + -1.96330726146698, + -2.036628007888794 + ], + "weight_version": 2, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 3, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "raw", + "seconds": { + "local": 0.23163969395682216, + "pd": 0.4639628282748163 + }, + "max_logprob_error": 4.76837158203125e-07, + "outputs": [ + { + "request_id": "8", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 60, + 9, + 25, + 60, + 42 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.9984395503997803, + -4.025251865386963, + -4.024652004241943, + -3.9303245544433594, + -4.02800178527832, + -4.010705947875977 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "9", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 60, + 60, + 9, + 15, + 56 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -3.8727941513061523, + -4.021127223968506, + -4.020061016082764, + -4.025237083435059, + -3.993036985397339, + -4.017432689666748 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "raw", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + }, + { + "version": 3, + "loops": 4, + "mode": "processed", + "seconds": { + "local": 0.2783969510346651, + "pd": 0.46485115215182304 + }, + "max_logprob_error": 2.384185791015625e-07, + "outputs": [ + { + "request_id": "10", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42, + 60, + 9, + 25, + 60, + 42 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.9686124324798584, + -1.9942327737808228, + -1.9923925399780273, + -1.8824831247329712, + -1.9928045272827148, + -1.9732145071029663 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + { + "request_id": "11", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 7 + ], + "token_ids": [ + 61, + 60, + 60, + 9, + 15, + 56 + ], + "exit_depths": [ + 4, + 4, + 4, + 4, + 4, + 4 + ], + "finished": true, + "finish_reason": "length", + "text": "", + "log_probs": [ + -1.8269999027252197, + -1.9946293830871582, + -1.9918233156204224, + -1.995558738708496, + -1.9558247327804565, + -1.983024001121521 + ], + "weight_version": 3, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 12, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + } + ] + } + ], + "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", + "recovered_version": 5, + "stop": { + "request_id": "13", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "stop", + "text": "", + "log_probs": [ + -1.9686124324798584 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [ + 42 + ], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "abort": { + "request_id": "abort", + "prompt_token_ids": [ + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 3, + 4, + 5, + 6 + ], + "token_ids": [ + 42 + ], + "exit_depths": [ + 4 + ], + "finished": true, + "finish_reason": "abort", + "text": "", + "log_probs": [ + -1.9686124324798584 + ], + "weight_version": 5, + "sampling_params": { + "max_tokens": 6, + "temperature": 0.9, + "top_p": 0.8, + "top_k": 8, + "seed": 11, + "logprobs": 0, + "logprobs_mode": "processed", + "stop_token_ids": [], + "min_loops": 2, + "max_loops": 4, + "exit_threshold": 1.0, + "ignore_eos": true, + "priority": 0 + } + }, + "workers": { + "0759283ec0764cee8d1bd7369fa54644-prefill-0": { + "kind": "stopped", + "prefill_tokens": 292, + "bytes_sent": 300928, + "transfers": 22, + "transfer_seconds": 2.949465547222644, + "used_blocks": 0, + "weight_version": 5 + }, + "0759283ec0764cee8d1bd7369fa54644-decode-1": { + "kind": "stopped", + "prefill_tokens": 0, + "bytes_sent": 0, + "transfers": 0, + "transfer_seconds": 0.0, + "used_blocks": 0, + "weight_version": 5, + "graph_captures": 2, + "graph_replays": 116, + "graph_fallbacks": 0 + } + } +} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log new file mode 100644 index 00000000..e391a5ae --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log @@ -0,0 +1,11 @@ +2026-10-01 02:16:37 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:16:37 NIXL INFO _api.py:285 Initialized NIXL agent: 0759283ec0764cee8d1bd7369fa54644-prefill-0 +2026-10-01 02:16:37 NIXL INFO _api.py:407 Backend UCX was instantiated +2026-10-01 02:16:37 NIXL INFO _api.py:285 Initialized NIXL agent: 0759283ec0764cee8d1bd7369fa54644-decode-1 +PARITY 1 2 raw {'local': 1.2744349539279938, 'pd': 2.329219114035368} 2.384185791015625e-07 +PARITY 1 2 processed {'local': 0.23766471119597554, 'pd': 0.436434387229383} 1.1920928955078125e-07 +PARITY 2 3 raw {'local': 0.22012542001903057, 'pd': 0.4495909558609128} 2.384185791015625e-07 +PARITY 2 3 processed {'local': 0.24550614319741726, 'pd': 0.40186773892492056} 2.384185791015625e-07 +PARITY 3 4 raw {'local': 0.23163969395682216, 'pd': 0.4639628282748163} 4.76837158203125e-07 +PARITY 3 4 processed {'local': 0.2783969510346651, 'pd': 0.46485115215182304} 2.384185791015625e-07 +PD_CONTRACT_PASSED pd-tiny-graph-tcp.json diff --git a/docs/validation/training-contract-a100-20261001/raw/runtime.json b/docs/validation/training-contract-a100-20261001/raw/runtime.json new file mode 100644 index 00000000..f4813a5e --- /dev/null +++ b/docs/validation/training-contract-a100-20261001/raw/runtime.json @@ -0,0 +1,13 @@ +{ + "torch": "2.13.0+cu130", + "triton": "3.7.1", + "nixl": "1.4.1", + "transformers": "5.17.0", + "numpy": "2.3.5", + "safetensors": "0.8.0", + "gpus": [ + "NVIDIA A100-SXM4-40GB, 580.105.08, 40960 MiB", + "NVIDIA A100-SXM4-40GB, 580.105.08, 40960 MiB" + ], + "UCX_TLS": "tcp,cuda_copy" +} \ No newline at end of file diff --git a/docs/validation/training-contract-thor-20261001/README.md b/docs/validation/training-contract-thor-20261001/README.md new file mode 100644 index 00000000..52920253 --- /dev/null +++ b/docs/validation/training-contract-thor-20261001/README.md @@ -0,0 +1,35 @@ +# Training contract on Thor — 2026-10-01 + +These timings use full BF16 `ByteDance/Ouro-1.4B` at revision +`574fa66cb8bf5abdc979642d01cf2b79b16bfab1`, with the public `LLM` rollout/update +API on upstream `ecb1f8b505b7e831815b40aec3b4598619cca23a`. + +| Rollout | Eager (s) | CUDA graphs (s) | Speedup | Time saved | +|---|---:|---:|---:|---:| +| Native K=2 decode | 1.0689 | 0.9939 | 1.076× | 7.02% | +| Native K=3 decode | 1.5190 | 1.4207 | 1.069× | 6.47% | +| Native K=4 decode | 1.9414 | 1.8234 | 1.065× | 6.08% | +| Self-speculative target K=4, draft K=2, 3 candidates | 1.4952 | 1.3741 | 1.088× | 8.11% | + +Five alternating-order pairs after per-arm warmup and full-weight publication; +two prompts, 32 response tokens each, temperature=0.9 and fixed per-request seeds. +Returned selected-token raw logprobs, tokens, exit depths and weight versions +match exactly between eager and graph arms in every pair. Each native prompt +prefill uses full K=4. Measurements include logprob delivery and exclude model +loading and initial capture. Each speculative comparison uses the same speculative +algorithm in both arms. See [raw timing pairs](raw/shared-full-rollout-benchmark.json). + +CPU tests pass all five checks; CUDA/Triton tests pass all four applicable checks. +The independent FP32 oracle covers processed probabilities with temperature and +top-k/top-p, speculative target scores after publication, early exit, async +stop/abort, prefix invalidation, stable parameter addresses and failed publication +recovery. The PD control test confirms a newer complete epoch recovers from a +partial peer commit; delivery is in-process, so IPC and NIXL are not covered. + +Full-model VIME training and fresh-process checkpoint/HF export parity are +reported in [0z5a/vime#5](https://github.com/0z5a/vime/pull/5). +Real multi-GPU PD publication and rollout checks are recorded in the +[A100 report](../training-contract-a100-20261001/README.md); Thor has one GPU and no NIXL. +No packages, CUDA drivers or environments were updated. GPU runs use the existing +performance lock and finish naturally. Completed experiment weights/checkpoints +are removed after preserving logs and tensor hashes. diff --git a/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json b/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json new file mode 100644 index 00000000..22ae874c --- /dev/null +++ b/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json @@ -0,0 +1,98 @@ +{ + "native-2": { + "seconds": { + "eager": [ + 1.068922233011108, + 1.0962036999990232, + 1.0150359050021507, + 1.0776073549932335, + 1.0171921259898227 + ], + "graph": [ + 0.99387969498639, + 1.031959738000296, + 0.9904317960026674, + 1.0318653039867058, + 0.9868912409874611 + ] + }, + "medians": { + "eager": 1.068922233011108, + "graph": 0.99387969498639 + }, + "speedup": 1.0755046495096627, + "tokens_and_logprobs_exact": true + }, + "native-3": { + "seconds": { + "eager": [ + 1.5189770860015415, + 1.5267343849991448, + 1.4750143490091432, + 1.5399230089969933, + 1.4762638669926673 + ], + "graph": [ + 1.4206840230035596, + 1.4595294760074466, + 1.3996001509949565, + 1.4518495189840905, + 1.4113349709950853 + ] + }, + "medians": { + "eager": 1.5189770860015415, + "graph": 1.4206840230035596 + }, + "speedup": 1.0691871390164396, + "tokens_and_logprobs_exact": true + }, + "native-4": { + "seconds": { + "eager": [ + 1.9413879640051164, + 1.9600694489781745, + 1.8969329710234888, + 1.9612911650037859, + 1.8895282850135118 + ], + "graph": [ + 1.7855259570060298, + 1.8588317520043347, + 1.8060117769928183, + 1.8618553779961076, + 1.8234225899796002 + ] + }, + "medians": { + "eager": 1.9413879640051164, + "graph": 1.8234225899796002 + }, + "speedup": 1.0646944787641552, + "tokens_and_logprobs_exact": true + }, + "speculative-4": { + "seconds": { + "eager": [ + 1.4952338639996015, + 1.523701586003881, + 1.4740307310130447, + 1.5373682600038592, + 1.4766599040012807 + ], + "graph": [ + 1.3729902119957842, + 1.415411785012111, + 1.373799616994802, + 1.416570646018954, + 1.3740531750081573 + ] + }, + "medians": { + "eager": 1.4952338639996015, + "graph": 1.3740531750081573 + }, + "speedup": 1.0881921392821823, + "tokens_and_logprobs_exact": true + } +} \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..f4a55800 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Repository tests; keep imports separate from installed test packages.""" diff --git a/tests/test_pd.py b/tests/test_pd.py index 9b07ff14..71999078 100644 --- a/tests/test_pd.py +++ b/tests/test_pd.py @@ -239,6 +239,7 @@ def test_engine_yields_while_waiting_for_remote_kv(async_scheduling): from vllm_rlt.request import Request, Stage engine = object.__new__(LLMEngine) + engine._update_version = None engine.profiling = Profiler("cpu") engine.preemption = PreemptionManager(engine) engine.execution_config = ExecutionConfig(async_scheduling=async_scheduling) diff --git a/tests/test_pd_weight_control.py b/tests/test_pd_weight_control.py new file mode 100644 index 00000000..523efbbb --- /dev/null +++ b/tests/test_pd_weight_control.py @@ -0,0 +1,91 @@ +"""PD publication control with two real CPU engines; no NIXL transport claim.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from tests.helpers import tiny_ouro_config +from vllm_rlt import LLM, SamplingParams +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.models import OuroForCausalLM +from vllm_rlt.pd.config import PDConfig +from vllm_rlt.pd.engine import PDEngine +from vllm_rlt.pd.worker import PDWorker + + +def control_pair(): + coordinator = PDEngine.__new__(PDEngine) + coordinator.config = PDConfig() + coordinator.closed, coordinator.failure = False, None + coordinator.requests, coordinator.transfers = {}, {} + coordinator._weight_version, coordinator._update_version = 0, None + coordinator._update_failed, coordinator._paused = False, False + coordinator._weight_replies = {} + coordinator.peers = { + role: SimpleNamespace(name=role, role=role) for role in ("prefill", "decode") + } + workers = {} + for role, peer in coordinator.peers.items(): + worker = PDWorker.__new__(PDWorker) + worker.engine = LLMEngine(OuroForCausalLM(tiny_ouro_config())) + worker.work = {} + worker.send = lambda kind, peer=peer, **fields: coordinator._message( + peer, dict(kind=kind, **fields) + ) + workers[role] = worker + coordinator._send = lambda name, kind, **fields: workers[name].command( + dict(kind=kind, **fields) + ) + return coordinator, workers + + +class PDWeightControlTest(unittest.TestCase): + def test_all_workers_commit_and_partial_commit_requires_new_epoch(self): + coordinator, workers = control_pair() + facade = LLM(coordinator) + weights = list(workers["prefill"].engine.model.named_parameters()) + pointers = { + role: [p.data_ptr() for p in worker.engine.model.parameters()] + for role, worker in workers.items() + } + facade.start_weight_update() + facade.update_weights(weights) + facade.finish_weight_update() + self.assertEqual(facade.get_weight_version(), 1) + coordinator.start_weight_update() + coordinator.update_weights(weights) + with patch.object( + workers["decode"].engine, + "finish_weight_update", + side_effect=RuntimeError("copy failure"), + ): + with self.assertRaisesRegex(RuntimeError, "copy failure"): + coordinator.finish_weight_update() + self.assertEqual(workers["prefill"].engine.get_weight_version(), 2) + self.assertEqual(workers["decode"].engine.get_weight_version(), 1) + with self.assertRaises(RuntimeError): + coordinator.step() + with self.assertRaises(RuntimeError): + coordinator.add_request("blocked", [3]) + coordinator.start_weight_update() + coordinator.update_weights(weights) + coordinator.finish_weight_update() + self.assertEqual(coordinator.get_weight_version(), 3) + for role, worker in workers.items(): + self.assertEqual(worker.engine.get_weight_version(), 3) + self.assertEqual( + pointers[role], [p.data_ptr() for p in worker.engine.model.parameters()] + ) + output = LLM(worker.engine).generate( + [[3, 4]], SamplingParams(max_tokens=2, logprobs=0) + )[0] + self.assertEqual(output.weight_version, 3) + self.assertEqual(len(output.log_probs), len(output.token_ids)) + coordinator.reset_prefix_cache() + + +if __name__ == "__main__": + torch.set_num_threads(1) + unittest.main() diff --git a/tests/test_speculative.py b/tests/test_speculative.py index 412a8b13..162f54ec 100644 --- a/tests/test_speculative.py +++ b/tests/test_speculative.py @@ -395,7 +395,7 @@ def test_greedy_round_reads_back_once_with_ragged_requests(): # Draft IDs stay on the device; every speculative round, including ragged and # budget-shortened ones, synchronizes with the host exactly once. torch.manual_seed(123) - m = OuroForCausalLM(OuroConfig.tiny(hidden_size=256, head_dim=64)).to( + m = OuroForCausalLM(tiny_ouro_config(hidden_size=256, head_dim=64)).to( device="cuda", dtype=torch.bfloat16 ) e = engine(m, k=3, cache_config=CacheConfig(128, 16), attention_backend="triton") diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py new file mode 100644 index 00000000..803b8381 --- /dev/null +++ b/tests/test_training_contract.py @@ -0,0 +1,186 @@ +"""Training contract checks runnable with unittest on CPU or CUDA/Triton.""" + +import os +import unittest +from dataclasses import replace + +import torch + +from tests.reference import dense_reference +from vllm_rlt import ( + LLM, + CacheConfig, + ExecutionConfig, + ExitConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.models import OuroConfig, OuroForCausalLM +from vllm_rlt.request import Request +from vllm_rlt.worker.sampling import generator_for, probabilities + +DEVICE = os.environ.get("CONTRACT_DEVICE", "cpu") + + +def model(): + torch.manual_seed(17) + config = OuroConfig( + vocab_size=64, + hidden_size=256 if DEVICE == "cuda" else 32, + intermediate_size=512 if DEVICE == "cuda" else 64, + num_hidden_layers=2, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=64 if DEVICE == "cuda" else 8, + max_position_embeddings=128, + total_ut_steps=4, + eos_token_id=0, + ) + return OuroForCausalLM(config).to(DEVICE) + + +def llm(native, *, graphs=False, asynchronous=False, speculative=False, prefix=False): + return LLM( + native, + attention_backend="triton" if DEVICE == "cuda" else "torch", + cache_config=CacheConfig(num_blocks=128, block_size=2, enable_prefix_caching=prefix), + scheduler_config=SchedulerConfig(max_num_seqs=4, max_num_batched_tokens=32), + exit_config=ExitConfig("ouro_delayed" if asynchronous else "ouro"), + execution_config=ExecutionConfig(cuda_graphs=graphs, async_scheduling=asynchronous), + speculative_config=SpeculativeConfig(3) if speculative else None, + ) + + +class TrainingContractTest(unittest.TestCase): + def test_selected_probabilities_speculative_target_and_post_update(self): + native = model() + prompt = [3, 4, 5] + for graphs in [False, True] if DEVICE == "cuda" else [False]: + for speculative in (False, True): + rollout = llm(native, graphs=graphs, speculative=speculative) + pointers = [p.data_ptr() for p in native.parameters()] + for version in (0, 1): + if version: + weights = [(n, p.detach() * 0.97) for n, p in native.named_parameters()] + rollout.start_weight_update(version) + rollout.update_weights(weights[:5]) + rollout.update_weights(weights[5:]) + rollout.finish_weight_update() + for mode in ("raw", "processed"): + params = SamplingParams( + max_tokens=8, + temperature=0.8, + top_k=13, + top_p=0.9, + logprobs=0, + logprobs_mode=mode, + seed=29, + ignore_eos=True, + ) + output = rollout.generate([prompt], params)[0] + logits = dense_reference(native, prompt + output.token_ids[:-1], 4)[-1][2] + rows = logits[len(prompt) - 1 :] + expected = torch.stack( + [ + probabilities(row, params)[token].log() + if mode == "processed" + else row.float().log_softmax(-1)[token] + for row, token in zip(rows, output.token_ids, strict=True) + ] + ) + torch.testing.assert_close( + torch.tensor(output.log_probs, device=DEVICE), + expected, + atol=4e-5, + rtol=4e-5, + ) + self.assertEqual(output.weight_version, version) + self.assertEqual(output.sampling_params, replace(params, max_loops=4)) + self.assertEqual(pointers, [p.data_ptr() for p in native.parameters()]) + if graphs: + bank = ( + rollout.engine.speculative_runner.graphs + if speculative + else rollout.engine.model_runner.graphs + ) + self.assertGreater(bank.replays, 0) + + def test_early_exit_async_stop_and_abort(self): + for graphs in [False, True] if DEVICE == "cuda" else [False]: + for asynchronous in (False, True): + rollout = llm(model(), graphs=graphs, asynchronous=asynchronous) + rollout.start_weight_update(2) + rollout.update_weights(rollout.engine.model.named_parameters()) + rollout.finish_weight_update() + params = SamplingParams(max_tokens=5, exit_threshold=0, logprobs=0, ignore_eos=True) + output = rollout.generate([[3, 4]], params)[0] + self.assertEqual(output.weight_version, 2) + self.assertEqual(output.exit_depths, [4] + [3 if asynchronous else 2] * 4) + self.assertEqual(len(output.log_probs), 5) + stopped = rollout.generate( + [[3, 4]], replace(params, stop_token_ids=(output.token_ids[0],)) + )[0] + self.assertEqual(stopped.token_ids, output.token_ids[:1]) + self.assertEqual(len(stopped.log_probs), 1) + self.assertEqual(stopped.finish_reason, "stop") + rollout.engine.add_request("abort", [3, 4], params) + while not rollout.engine.step(): + pass + aborted = rollout.engine.abort_request("abort") + self.assertEqual(len(aborted.log_probs), len(aborted.token_ids)) + self.assertEqual(aborted.finish_reason, "abort") + + def test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy(self): + rollout = llm(model(), prefix=True) + native = rollout.engine.model + params = SamplingParams(max_tokens=3, logprobs=0, ignore_eos=True) + rollout.generate([[3, 4, 5, 6]], params) + self.assertTrue(rollout.engine.cache_manager.lookup_prefix([3, 4, 5, 6])) + rollout.pause_generation() + with self.assertRaises(RuntimeError): + rollout.generate([[3]], params) + rollout.resume_generation() + rollout.engine.add_request("active", [3], params) + with self.assertRaises(RuntimeError): + rollout.start_weight_update() + rollout.engine.abort_request("active") + rollout.start_weight_update(7) + self.assertEqual(rollout.engine.cache_manager.num_used_blocks, 0) + self.assertFalse(rollout.engine.cache_manager.lookup_prefix([3, 4, 5, 6])) + names = list(native.named_parameters()) + before = names[0][1].detach().clone() + with self.assertRaises(ValueError): + rollout.update_weights( + [(names[0][0], torch.zeros_like(before)), ("invalid", torch.zeros(1))] + ) + torch.testing.assert_close(before, names[0][1], rtol=0, atol=0) + with self.assertRaises(RuntimeError): + rollout.finish_weight_update() + with self.assertRaises(RuntimeError): + rollout.generate([[3]], params) + rollout.start_weight_update(7) + rollout.update_weights(names[:1]) + with self.assertRaises(ValueError): + rollout.finish_weight_update() + rollout.update_weights(names[1:]) + rollout.finish_weight_update() + self.assertEqual(rollout.generate([[3]], params)[0].weight_version, 7) + rollout.reset_prefix_cache() + self.assertEqual(rollout.engine.cache_manager.num_used_blocks, 0) + + def test_seed_none_and_validation(self): + params = SamplingParams(seed=None, temperature=1) + requests = [Request(str(i), [3], params) for i in range(2)] + seeds = [generator_for(r, DEVICE).initial_seed() for r in requests] + self.assertNotEqual(*seeds) + for values in ({"logprobs": 1}, {"logprobs_mode": "invalid"}, {"stop_token_ids": (-1,)}): + with self.assertRaises(ValueError): + SamplingParams(**values) + with self.assertRaises(ValueError): + llm(model()).generate([[3]], SamplingParams(stop_token_ids=(64,))) + + +if __name__ == "__main__": + torch.set_num_threads(1) + unittest.main() diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 968932ac..77fd211f 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -250,6 +250,12 @@ def lookup_prefix(self, tokens): found.append(blocks) return tuple(found) + def reset_prefix_cache(self): + self._pending_prefixes.clear() + for blocks in self._prefixes.values(): + self._drop_refs(blocks) + self._prefixes.clear() + def allocate(self, request_id: str, max_tokens: int, *, initial_tokens=None, prefix=()) -> bool: """Reserve a logical capacity; acquire physical pages for the current frontier.""" if not isinstance(request_id, str) or not request_id: diff --git a/vllm_rlt/engine/llm_engine.py b/vllm_rlt/engine/llm_engine.py index 8a7bb559..0f374d6d 100644 --- a/vllm_rlt/engine/llm_engine.py +++ b/vllm_rlt/engine/llm_engine.py @@ -1,5 +1,8 @@ +from collections.abc import Iterable from dataclasses import replace +import torch + from vllm_rlt.config import CacheConfig, ExecutionConfig, ExitConfig, SchedulerConfig from vllm_rlt.core.kv_cache_manager import KVCacheManager from vllm_rlt.core.memory import plan_cache @@ -28,6 +31,11 @@ def __init__( speculative_config=None, ): self.model = model + self._weight_version = 0 + self._update_version: int | None = None + self._update_failed = False + self._updated_names: set[str] = set() + self._paused = False self.profiling = Profiler(next(model.parameters()).device) cache_config = cache_config or CacheConfig() scheduler_config = scheduler_config or SchedulerConfig() @@ -132,6 +140,8 @@ def add_request( *, trace_id: str | None = None, ): + if self._update_version is not None or self._paused: + raise RuntimeError("Generation admission is paused") if not isinstance(request_id, str) or not request_id: raise ValueError("request_id must be a nonempty string") params = sampling_params or SamplingParams() @@ -140,9 +150,12 @@ def add_request( raise ValueError("prompt must contain at least one token") if any(type(t) is not int or not 0 <= t < config.vocab_size for t in prompt_token_ids): raise ValueError("prompt token IDs must be integers within the model vocabulary") + if any(t >= config.vocab_size for t in params.stop_token_ids): + raise ValueError("stop token IDs must be within the model vocabulary") max_loops = params.max_loops or config.total_ut_steps if max_loops > config.total_ut_steps or params.min_loops > max_loops: raise ValueError("requested loop bounds exceed the model's supported depth") + params = replace(params, max_loops=max_loops, top_k=min(params.top_k, config.vocab_size)) if self.speculative_config is not None and ( max_loops != self.speculative_config.target_loops or params.exit_threshold != 1.0 ): @@ -180,9 +193,76 @@ def add_request( "increase num_blocks or reduce prompt/max_tokens" ) self.scheduler.add_request( - Request(request_id, list(prompt_token_ids), params, exit_trace=trace) + Request( + request_id, + list(prompt_token_ids), + params, + exit_trace=trace, + weight_version=self._weight_version, + ) ) + def get_weight_version(self) -> int: + return self._weight_version + + def pause_generation(self): + # Existing requests can drain through step(); no new requests are admitted. + self._paused = True + + def resume_generation(self): + if self._update_version is not None: + raise RuntimeError("Complete the weight update before resuming generation") + self._paused = False + + def reset_prefix_cache(self): + if self.has_unfinished_requests(): + raise RuntimeError("Finish active requests before resetting cached KV") + self.model_runner.synchronize() + self.cache_manager.reset_prefix_cache() + + def start_weight_update(self, version: int | None = None): + if self.has_unfinished_requests(): + raise RuntimeError("Weight updates require an idle engine") + version = self._weight_version + 1 if version is None else version + if type(version) is not int or version <= self._weight_version: + raise ValueError("Weight version must increase") + self.reset_prefix_cache() + self._update_version, self._update_failed = version, False + self._updated_names.clear() + + @torch.no_grad() + def update_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + if self._update_version is None or self._update_failed: + raise RuntimeError("Start a full weight update first") + parameters = dict(self.model.named_parameters()) + weights = list(weights) + names = [name for name, _ in weights] + try: + if len(set(names)) != len(names) or self._updated_names.intersection(names): + raise ValueError("Duplicate weight in full update") + for name, value in weights: + if name not in parameters or value.shape != parameters[name].shape: + raise ValueError(f"Weight name or shape mismatch: {name}") + for name, value in weights: + parameters[name].copy_(value) + self._updated_names.update(names) + except Exception: + # A failed transfer may have copied some tensors. A fresh full update + # is the only recovery; generation never sees that mixed policy. + self._update_failed = True + raise + + def finish_weight_update(self): + if self._update_version is None or self._update_failed: + raise RuntimeError("No successful weight update to finish") + if self._updated_names != dict(self.model.named_parameters()).keys(): + raise ValueError("Full update must include every physical parameter") + device = next(self.model.parameters()).device + if device.type == "cuda": + torch.cuda.synchronize(device) + self._weight_version, self._update_version = self._update_version, None + self._updated_names.clear() + def has_unfinished_requests(self) -> bool: return self.scheduler.has_unfinished_requests @@ -208,6 +288,8 @@ def close(self): self.profiling.close() def step(self) -> list[RequestOutput]: + if self._update_version is not None: + raise RuntimeError("Generation is unavailable during a weight update") if not self.profiling.recording: return self._step() try: @@ -263,9 +345,11 @@ def _update_speculative(self, batch, results): reason = None for token in result.token_ids: request.generated_token_ids.append(token) + if params.logprobs is not None: + request.log_probs.append(result.log_probs[emitted]) request.exit_depths.append(self.speculative_config.target_loops) emitted += 1 - if token in eos_ids and not params.ignore_eos: + if token in params.stop_token_ids or (token in eos_ids and not params.ignore_eos): reason = FinishReason.STOP break if len(request.generated_token_ids) >= params.max_tokens: @@ -333,7 +417,9 @@ def _update(self, batch, result) -> list[RequestOutput]: request.exit_depths.append(request.loops_done) eos = self.model.config.eos_token_id eos_ids = eos if isinstance(eos, (tuple, list)) else [eos] - if token_id in eos_ids and not params.ignore_eos: + if token_id in params.stop_token_ids or ( + token_id in eos_ids and not params.ignore_eos + ): self._finish(request, FinishReason.STOP) elif len(request.generated_token_ids) >= params.max_tokens: self._finish(request, FinishReason.LENGTH) @@ -409,7 +495,7 @@ def _deliver_coda(self, ticket): params = request.sampling_params eos = self.model.config.eos_token_id eos_ids = eos if isinstance(eos, (tuple, list)) else [eos] - if token_id in eos_ids and not params.ignore_eos: + if token_id in params.stop_token_ids or (token_id in eos_ids and not params.ignore_eos): self._finish(request, FinishReason.STOP) elif len(request.generated_token_ids) >= params.max_tokens: self._finish(request, FinishReason.LENGTH) diff --git a/vllm_rlt/entrypoints/llm.py b/vllm_rlt/entrypoints/llm.py index a154477f..0895d0d1 100644 --- a/vllm_rlt/entrypoints/llm.py +++ b/vllm_rlt/entrypoints/llm.py @@ -1,3 +1,4 @@ +from collections.abc import Iterable from dataclasses import replace import torch @@ -9,6 +10,7 @@ resolve_model_config, resolve_model_source, ) +from vllm_rlt.pd.engine import PDEngine from vllm_rlt.sampling_params import SamplingParams @@ -31,6 +33,11 @@ def __init__( speculative_config=None, ): self._tokenizer_source = None + if isinstance(model, (LLMEngine, PDEngine)): + self.tokenizer = tokenizer + self.engine = model + self._next_request_id = 0 + return if isinstance(model, str): tokenizer_source = resolve_model_source(model) model_name, revision, _ = resolve_model_config(model, revision=revision) @@ -104,6 +111,27 @@ def generate(self, prompts, sampling_params=None): def start_profile(self, config, *, scheduled=False): return self.engine.start_profile(config, scheduled=scheduled) + def start_weight_update(self, version: int | None = None): + self.engine.start_weight_update(version) + + def update_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + self.engine.update_weights(weights) + + def finish_weight_update(self): + self.engine.finish_weight_update() + + def get_weight_version(self) -> int: + return self.engine.get_weight_version() + + def reset_prefix_cache(self): + self.engine.reset_prefix_cache() + + def pause_generation(self): + self.engine.pause_generation() + + def resume_generation(self): + self.engine.resume_generation() + def stop_profile(self): return self.engine.stop_profile() diff --git a/vllm_rlt/pd/engine.py b/vllm_rlt/pd/engine.py index 59b5da94..3fadd635 100644 --- a/vllm_rlt/pd/engine.py +++ b/vllm_rlt/pd/engine.py @@ -1,17 +1,20 @@ """CPU coordinator for independent prefill/decode GPU pools. D reservations precede P computation. Control channels carry only metadata and -outputs; NIXL moves KV directly between registered worker allocations. +outputs and CPU-staged full-weight chunks; NIXL moves KV between GPU pools. """ import multiprocessing as mp import time import uuid from collections import deque +from collections.abc import Iterable from dataclasses import asdict, dataclass, field, replace from pathlib import Path from types import SimpleNamespace +import torch + from vllm_rlt.config import CacheConfig, ExecutionConfig, ExitConfig, SchedulerConfig from vllm_rlt.models import OuroConfig, resolve_model_config from vllm_rlt.profiling import ProfileConfig @@ -55,6 +58,7 @@ class Pending: d_released: bool = False finished: bool = False timings: dict = field(default_factory=dict) + latest_output: RequestOutput | None = None class PDEngine: @@ -74,6 +78,7 @@ def __init__( revision=None, seed=0, speculative_config=None, + terminate_workers_on_failure: bool = True, ): if speculative_config is not None: raise ValueError("speculative decoding is not yet supported by PD") @@ -105,6 +110,12 @@ def __init__( self.outputs = deque() self.closed = False self.failure = None + self.terminate_workers_on_failure = terminate_workers_on_failure + self._weight_version = 0 + self._update_version: int | None = None + self._update_failed = False + self._paused = False + self._weight_replies = {} self._profile_session = None self._profile_replies = {} self._profile_statuses = {} @@ -190,6 +201,11 @@ def _poll(self): def _message(self, peer, m): kind = m["kind"] + if kind == "weight_reply": + replies = self._weight_replies.get(m["control_id"]) + if replies is not None: + replies[peer.name] = m + return if kind == "profile_reply": pending = self._profile_replies.get(m.get("control_id")) if pending is not None and m.get("session_id") == self._profile_session: @@ -249,6 +265,9 @@ def _message(self, peer, m): elif kind == "output": if not w.cancelled: output = replace(m["output"], request_id=w.request.request_id) + if output.weight_version != w.request.weight_version: + raise RuntimeError("PD workers returned a different policy version") + w.latest_output = output w.request.generated_token_ids = list(output.token_ids) w.request.exit_depths = list(output.exit_depths) w.timings.setdefault("first_token", now) @@ -296,6 +315,8 @@ def _blocks(self, peer, tokens): def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, trace_id=None): if self.closed or self.failure: raise RuntimeError("PD engine is unavailable") + if self._paused or self._update_version is not None or self._update_failed: + raise RuntimeError("Generation admission is paused") if not isinstance(request_id, str) or not request_id or request_id in self.requests: raise ValueError("request ID must be nonempty and unique among active requests") if len(self.transfers) >= self.config.max_pending_requests: @@ -306,9 +327,12 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra type(t) is not int or not 0 <= t < cfg.vocab_size for t in prompt_token_ids ): raise ValueError("prompt must contain valid model token IDs") + if any(t >= cfg.vocab_size for t in params.stop_token_ids): + raise ValueError("stop token IDs must be within the model vocabulary") maximum = params.max_loops or cfg.total_ut_steps if maximum > cfg.total_ut_steps or params.min_loops > maximum: raise ValueError("requested loop bounds exceed model depth") + params = replace(params, max_loops=maximum, top_k=min(params.top_k, cfg.vocab_size)) if len(prompt_token_ids) + params.max_tokens - 1 > cfg.max_position_embeddings: raise ValueError("prompt plus decode exceeds context capacity") if trace_id is not None and (not isinstance(trace_id, str) or not trace_id): @@ -335,7 +359,9 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra if p.role == role ): raise ValueError(f"request exceeds {role} KV pool capacity") - request = Request(request_id, list(prompt_token_ids), params) + request = Request( + request_id, list(prompt_token_ids), params, weight_version=self._weight_version + ) w = Pending(request, uuid.uuid4().hex, trace_id) self.requests[request_id] = request self.transfers[w.tid] = w @@ -412,14 +438,88 @@ def abort_request(self, request_id): self._send(w.p, "cancel", tid=w.tid) request.stage = Stage.FINISHED request.finish_reason = FinishReason.ABORT + if w.latest_output is not None: + return replace(w.latest_output, finished=True, finish_reason=FinishReason.ABORT.value) return RequestOutput.from_request(request) + def get_weight_version(self) -> int: + return self._weight_version + + def pause_generation(self): + self._paused = True + + def resume_generation(self): + if self._update_version is not None or self._update_failed: + raise RuntimeError("Complete the weight update before resuming generation") + self._paused = False + + def _weight_control(self, action, **arguments): + control_id = uuid.uuid4().hex + replies = self._weight_replies[control_id] = {} + try: + for name in self.peers: + self._send( + name, + "weight_control", + control_id=control_id, + action=action, + arguments=arguments, + ) + self._wait( + lambda: len(replies) == len(self.peers), + time.monotonic() + self.config.request_timeout, + ) + errors = [reply["error"] for reply in replies.values() if "error" in reply] + if errors: + raise RuntimeError("PD weight control failed: " + "; ".join(errors)) + return [reply["version"] for reply in replies.values()] + except Exception: + # Some peers may already have copied or committed the new policy. + # Keep admission blocked until every peer completes a newer full update. + self._update_failed = True + raise + finally: + del self._weight_replies[control_id] + + def reset_prefix_cache(self): + if self.has_unfinished_requests(): + raise RuntimeError("Finish active PD transfers before resetting cached KV") + self._weight_control("reset_prefix_cache") + + def start_weight_update(self, version: int | None = None): + if self.closed or self.failure or self.has_unfinished_requests(): + raise RuntimeError("Weight updates require an available, idle PD engine") + latest = max(self._weight_version, self._update_version or 0) + version = latest + 1 if version is None else version + if type(version) is not int or version <= latest: + raise ValueError("PD weight version must exceed all previous update attempts") + self._update_version, self._update_failed = version, False + self._weight_control("start_weight_update", version=version) + + def update_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + if self._update_version is None or self._update_failed: + raise RuntimeError("Start a full PD weight update first") + # CPU staging avoids CUDA tensor ownership across spawned worker processes. + weights = [(name, tensor.detach().cpu()) for name, tensor in weights] + self._weight_control("update_weights", weights=weights) + + def finish_weight_update(self): + if self._update_version is None or self._update_failed: + raise RuntimeError("No successful PD weight update to finish") + versions = self._weight_control("finish_weight_update") + if any(version != self._update_version for version in versions): + self._update_failed = True + raise RuntimeError("PD workers committed different policy versions") + self._weight_version, self._update_version = self._update_version, None + def has_unfinished_requests(self): return bool(self.transfers) def step(self): if self.closed: raise RuntimeError("PD engine is closed") + if self._update_version is not None or self._update_failed: + raise RuntimeError("Generation is unavailable during a PD weight update") try: self._poll() if any( @@ -573,6 +673,9 @@ def wait_for_profile_artifacts(self, timeout=None): return {"recording": False, "ranks": {}} def _terminate(self): + if not self.terminate_workers_on_failure: + self.closed = True + return if self._profile_session: self._profile_recording = False for name in self.peers: diff --git a/vllm_rlt/pd/worker.py b/vllm_rlt/pd/worker.py index db1df13d..7131c8d2 100644 --- a/vllm_rlt/pd/worker.py +++ b/vllm_rlt/pd/worker.py @@ -1,4 +1,4 @@ -"""One CUDA owner process per P/D worker. Control messages never carry tensors.""" +"""One CUDA owner per P/D worker; full updates use staged CPU tensor chunks.""" import hashlib import json @@ -170,6 +170,25 @@ def accept(self, command): def command(self, command): kind = command["kind"] + if kind == "weight_control": + try: + if self.work: + raise RuntimeError("Finish PD transfers before updating weights") + operations = { + "start_weight_update": self.engine.start_weight_update, + "update_weights": self.engine.update_weights, + "finish_weight_update": self.engine.finish_weight_update, + "reset_prefix_cache": self.engine.reset_prefix_cache, + } + operations[command["action"]](**command["arguments"]) + self.send( + "weight_reply", + control_id=command["control_id"], + version=self.engine.get_weight_version(), + ) + except Exception as error: + self.send("weight_reply", control_id=command["control_id"], error=str(error)) + return if kind.startswith("profile_"): try: action = kind.removeprefix("profile_") @@ -408,6 +427,16 @@ def run(self): if not self.work and not self.channel.poll(): time.sleep(0.001) self.engine.model_runner.synchronize() + graphs = self.engine.model_runner.graphs + graph_stats = ( + dict( + graph_captures=graphs.captures, + graph_replays=graphs.replays, + graph_fallbacks=graphs.fallbacks, + ) + if graphs is not None + else {} + ) self.engine.close() self.connector.close() self.send( @@ -417,6 +446,8 @@ def run(self): transfers=self.connector.transfers, transfer_seconds=self.connector.transfer_seconds, used_blocks=self.cache.num_used_blocks, + weight_version=self.engine.get_weight_version(), + **graph_stats, ) diff --git a/vllm_rlt/request.py b/vllm_rlt/request.py index c78eb51c..e8394589 100644 --- a/vllm_rlt/request.py +++ b/vllm_rlt/request.py @@ -44,6 +44,8 @@ class Request: generator: torch.Generator | None = field(default=None, repr=False) finish_reason: FinishReason | None = None exit_trace: tuple[int, ...] = field(default=(), repr=False) + weight_version: int = 0 + log_probs: list[torch.Tensor] = field(default_factory=list, repr=False) @property def num_scheduled_outputs(self) -> int: @@ -67,6 +69,9 @@ class RequestOutput: finished: bool finish_reason: str | None = None text: str = "" + log_probs: list[float] | None = None + weight_version: int = 0 + sampling_params: SamplingParams | None = None @classmethod def from_request(cls, request: Request): @@ -79,4 +84,13 @@ def from_request(cls, request: Request): finish_reason=request.finish_reason.value if request.finish_reason is not None else None, + log_probs=( + torch.stack(request.log_probs[: len(request.generated_token_ids)]).tolist() + if request.generated_token_ids + else [] + ) + if request.sampling_params.logprobs is not None + else None, + weight_version=request.weight_version, + sampling_params=request.sampling_params, ) diff --git a/vllm_rlt/sampling_params.py b/vllm_rlt/sampling_params.py index 30009bde..c5f75096 100644 --- a/vllm_rlt/sampling_params.py +++ b/vllm_rlt/sampling_params.py @@ -8,7 +8,10 @@ class SamplingParams: temperature: float = 0.0 top_p: float = 1.0 top_k: int = -1 - seed: int = 0 + seed: int | None = 0 + logprobs: int | None = None + logprobs_mode: str = "raw" + stop_token_ids: tuple[int, ...] = () min_loops: int = 2 max_loops: int | None = None exit_threshold: float = 1.0 @@ -18,8 +21,11 @@ class SamplingParams: def __post_init__(self): if type(self.priority) is not int: raise ValueError("priority must be an integer") - for name in ("max_tokens", "min_loops", "max_loops"): - value = getattr(self, name) + for name, value in ( + ("max_tokens", self.max_tokens), + ("min_loops", self.min_loops), + ("max_loops", self.max_loops), + ): if name == "max_loops" and value is None: continue if type(value) is not int or value < 1: @@ -34,5 +40,11 @@ def __post_init__(self): raise ValueError("top_k must be -1 or positive") if not 0 <= self.exit_threshold <= 1: raise ValueError("exit_threshold must be in [0, 1]") - if type(self.seed) is not int or not 0 <= self.seed < 2**63: + if self.seed is not None and (type(self.seed) is not int or not 0 <= self.seed < 2**63): raise ValueError("seed must be an integer in [0, 2**63)") + if self.logprobs is not None and (type(self.logprobs) is not int or self.logprobs != 0): + raise ValueError("logprobs=0 returns the selected-token probability") + if self.logprobs_mode not in ("raw", "processed"): + raise ValueError("logprobs_mode must be raw or processed") + if any(type(token) is not int or token < 0 for token in self.stop_token_ids): + raise ValueError("stop_token_ids must contain nonnegative integer IDs") diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index b0025137..640782e0 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -13,6 +13,7 @@ from vllm_rlt.worker.buffers import Workspace from vllm_rlt.worker.cuda_graph import RecurrentGraphs from vllm_rlt.worker.sampler import Sampler +from vllm_rlt.worker.sampling import selected_logprob @dataclass @@ -603,4 +604,6 @@ def _sample_tensor(self, logits: torch.Tensor, request: Request): # this method; the algorithm lives in Sampler. token, generator = self.sampler.sample(logits, request.sampling_params, request.generator) request.generator = generator + if request.sampling_params.logprobs is not None: + request.log_probs.append(selected_logprob(logits, token, request.sampling_params)) return token diff --git a/vllm_rlt/worker/sampler.py b/vllm_rlt/worker/sampler.py index 4bd8f31b..30506bf6 100644 --- a/vllm_rlt/worker/sampler.py +++ b/vllm_rlt/worker/sampler.py @@ -41,5 +41,9 @@ def sample( remove[0] = False logits = logits.scatter(0, indices, sorted_logits.masked_fill(remove, -torch.inf)) if generator is None: - generator = torch.Generator(device=self.device).manual_seed(params.seed) + generator = torch.Generator(device=self.device) + if params.seed is None: + generator.seed() + else: + generator.manual_seed(params.seed) return torch.multinomial(logits.softmax(-1), 1, generator=generator).squeeze(0), generator diff --git a/vllm_rlt/worker/sampling.py b/vllm_rlt/worker/sampling.py index 3aabb84d..54f59a89 100644 --- a/vllm_rlt/worker/sampling.py +++ b/vllm_rlt/worker/sampling.py @@ -21,10 +21,21 @@ def probabilities(logits, params): def generator_for(request, device): if request.generator is None: - request.generator = torch.Generator(device=device).manual_seed(request.sampling_params.seed) + request.generator = torch.Generator(device=device) + if request.sampling_params.seed is None: + request.generator.seed() + else: + request.generator.manual_seed(request.sampling_params.seed) return request.generator +def selected_logprob(logits, token, params): + if params.logprobs_mode == "processed" and params.temperature > 0: + return probabilities(logits, params)[token].log() + # Integer speculative tokens otherwise retain a view of the vocabulary row. + return logits.float().log_softmax(-1)[token].clone() + + def draw(probs, generator): return torch.multinomial(probs, 1, generator=generator).squeeze(0) diff --git a/vllm_rlt/worker/speculative.py b/vllm_rlt/worker/speculative.py index 9542ee70..0e5e940a 100644 --- a/vllm_rlt/worker/speculative.py +++ b/vllm_rlt/worker/speculative.py @@ -4,7 +4,7 @@ outputs; the engine applies returned tokens and owns KV commit/truncation. """ -from dataclasses import dataclass +from dataclasses import dataclass, field import torch @@ -14,6 +14,7 @@ generator_for, probabilities, rejection_sample, + selected_logprob, ) @@ -22,6 +23,7 @@ class SpeculativeResult: token_ids: list[int] accepted_count: int draft_count: int + log_probs: list[torch.Tensor] = field(default_factory=list) @dataclass @@ -163,6 +165,11 @@ def execute(self, batch): ) start += item.token_count results.append(SpeculativeResult(tokens, accepted, len(candidates[i]))) + if request.sampling_params.logprobs is not None: + results[-1].log_probs.extend( + selected_logprob(row, token, request.sampling_params) + for row, token in zip(rows, tokens) + ) continue start += item.token_count # Sampling keeps sequential rejection; RNG consumption order is unchanged. @@ -183,6 +190,11 @@ def execute(self, batch): ) tokens.append(int(token.item())) results.append(SpeculativeResult(tokens, accepted, len(candidates[i]))) + if request.sampling_params.logprobs is not None: + results[-1].log_probs.extend( + selected_logprob(row, token, request.sampling_params) + for row, token in zip(rows, tokens) + ) self.stats.rounds += len(items) self.stats.drafted_tokens += sum(len(c) for c in candidates) self.stats.verified_rows += len(ids) From e591bdab74df7e1482a36bd12fffce62bfad6bd0 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 1 Oct 2026 11:32:57 +0000 Subject: [PATCH 2/5] Report entropy seeds for reproducible training rollouts Signed-off-by: 0z5a --- docs/training_contract.md | 5 +- .../README.md | 51 ++++++++++++++++++ .../raw/minhao-cpu.log | 12 +++++ .../raw/minhao-seed-before.log | 14 +++++ tests/test_training_contract.py | 52 +++++++++++++++++++ vllm_rlt/request.py | 7 ++- 6 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 docs/validation/minhao-coordination-thor-20261001/README.md create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log diff --git a/docs/training_contract.md b/docs/training_contract.md index 9345cbf6..fa1d2b84 100644 --- a/docs/training_contract.md +++ b/docs/training_contract.md @@ -26,7 +26,10 @@ filtering. Greedy requests return the raw score in either mode. Speculative outputs use the verified **target** policy's score, including correction/bonus tokens; proposal or rejection-residual probabilities are never exposed as the policy score. Explicit `stop_token_ids` apply even with `ignore_eos=True`. -`seed=None` creates an independent entropy-seeded request generator. +`seed=None` resolves an independent seed in `[0, 2**63)` when the request is +created. The effective seed is returned in `sampling_params`, including for an +abort before sampling. Reusing those parameters replays the same request on the +same policy and execution configuration. The default remains `seed=0`. Publication requires an idle engine. Each chunk is checked for duplicate names, unknown names and mismatched shapes before that chunk is copied. `finish_weight_update` diff --git a/docs/validation/minhao-coordination-thor-20261001/README.md b/docs/validation/minhao-coordination-thor-20261001/README.md new file mode 100644 index 00000000..142e2799 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/README.md @@ -0,0 +1,51 @@ +# Phase 1 / VIME coordination checks — 2026-10-01 + +These checks exercise the semantics in +[MinhaoLi0318's PR 1 proposal](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5927860064) +on the existing Thor runtime. They test the contributor implementation, not an +unpublished Phase 1 branch. The +[coordination reply](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5930413915) +records the interface differences and division of upstream work. + +The replay requirement exposed a real omission: an entropy-seeded request +returned `sampling_params.seed=None`. The +[unfixed regression](raw/minhao-seed-before.log) fails on that missing effective +seed. `Request` now resolves an independent 63-bit seed when it is created and +retains it through output, preemption, abort and existing PD parameter transport. +Default `seed=0` is unchanged. An abort before any sampling also reports the seed; +the caller's immutable `SamplingParams(seed=None)` is not modified. + +| Thor CPU check | Result | +|---|---| +| Entropy seed reporting and exact token/score/depth replay | Ordinary, async and speculative paths pass | +| Abort before sampling | Effective seed retained; zero selected scores | +| Preemption | Token IDs, selected scores, depths and RNG continuity match uninterrupted execution | +| Existing probability/publication/cache/stop/abort checks | All pass | +| Full focused suite including in-process PD publication | **7 tests pass, 1.736 s** | + +See [CPU log](raw/minhao-cpu.log). Independent VIME FP32 mixed-depth replay, +recomputed gradients and generic physical-parameter conversion are checked +separately. No runtime packages are installed or updated. + +At the time of the initial coordination reply, new CUDA checks are queued behind +an existing Qwen-Image training reservation; they are not counted as passed here. +Full-model score-mismatch and selected-score-overhead measurements will be added +after that reservation permits execution. Earlier full-model/real-PD evidence +is retained in the [Thor](../training-contract-thor-20261001/README.md) and +[A100](../training-contract-a100-20261001/README.md) reports. + +## Interface coordination + +| Proposed Phase 1 interface | Current reference draft | VIME consumer plan | +|---|---|---| +| `RequestOutput.logprobs` | `RequestOutput.log_probs` | Use the upstream spelling once its PR 1 branch is available | +| Engine-level `raw_logprobs` / `processed_logprobs` | Request-level `raw` / `processed` | Configure processed scores for the trainer; avoid a second compatibility API | +| Effective seed reported in resolved parameters | Fixed by this follow-up | Keep seeded reproduction and opt-in entropy groups | +| Full-depth first token and executed decode depths | Already present | Full four-loop prefill and differentiable `last_exited` trace replay | +| PR 2 versioned full publication | Validated reference implementation | Reuse one agreed upstream contract | + +The old `RLEngine.set_loop_budget` mutated `total_ut_steps`; the current VIME +adapter supersedes it and does not need reduced-depth prefill. No new transport +layer is needed for the pilot: the consumer is in-process `LLM`, and existing PD +worker IPC remains inside the engine. MinhaoLi0318 and IDEA-V retain the proposed +Phase 1 upstream roles; this draft supplies reusable implementation and checks. diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log new file mode 100644 index 00000000..665bd66e --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log @@ -0,0 +1,12 @@ +test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok +test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok +test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok +test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok +test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok +test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok +test_all_workers_commit_and_partial_commit_requires_new_epoch (tests.test_pd_weight_control.PDWeightControlTest.test_all_workers_commit_and_partial_commit_requires_new_epoch) ... ok + +---------------------------------------------------------------------- +Ran 7 tests in 1.736s + +OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log new file mode 100644 index 00000000..1a90a902 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log @@ -0,0 +1,14 @@ +test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... FAIL + +====================================================================== +FAIL: test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/home/jwipc/experiments/ouro-vime-20260930/rlt-training-contract-20261001/tests/test_training_contract.py", line 190, in test_entropy_seed_is_reported_and_replays_the_same_policy + self.assertIsInstance(output.sampling_params.seed, int) +AssertionError: None is not an instance of + +---------------------------------------------------------------------- +Ran 1 test in 0.067s + +FAILED (failures=1) diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py index 803b8381..b536eaa5 100644 --- a/tests/test_training_contract.py +++ b/tests/test_training_contract.py @@ -16,6 +16,7 @@ SchedulerConfig, SpeculativeConfig, ) +from vllm_rlt.engine.llm_engine import LLMEngine from vllm_rlt.models import OuroConfig, OuroForCausalLM from vllm_rlt.request import Request from vllm_rlt.worker.sampling import generator_for, probabilities @@ -180,6 +181,57 @@ def test_seed_none_and_validation(self): with self.assertRaises(ValueError): llm(model()).generate([[3]], SamplingParams(stop_token_ids=(64,))) + def test_entropy_seed_is_reported_and_replays_the_same_policy(self): + params = SamplingParams( + max_tokens=8, temperature=0.9, seed=None, logprobs=0, ignore_eos=True + ) + for options in ({}, {"asynchronous": True}, {"speculative": True}): + rollout = llm(model(), graphs=DEVICE == "cuda", **options) + output = rollout.generate([[3, 4, 5]], params)[0] + self.assertIsInstance(output.sampling_params.seed, int) + self.assertLess(output.sampling_params.seed, 2**63) + replay = rollout.generate([[3, 4, 5]], output.sampling_params)[0] + self.assertEqual(output.token_ids, replay.token_ids) + self.assertEqual(output.log_probs, replay.log_probs) + self.assertEqual(output.exit_depths, replay.exit_depths) + rollout.engine.add_request("unstarted", [3], replace(params, temperature=0)) + aborted = rollout.engine.abort_request("unstarted") + self.assertIsInstance(aborted.sampling_params.seed, int) + self.assertEqual(aborted.log_probs, []) + rollout.close() + self.assertIsNone(params.seed) + + def test_preemption_preserves_selected_scores_and_rng(self): + native = model() + params = SamplingParams( + max_tokens=8, temperature=0.8, seed=45, logprobs=0, exit_threshold=0, ignore_eos=True + ) + results = [] + for suspend in (False, True): + engine = LLMEngine( + native, + cache_config=CacheConfig(64, 2), + scheduler_config=SchedulerConfig(enable_preemption=True), + attention_backend="triton" if DEVICE == "cuda" else "torch", + ) + engine.add_request("a", [1, 2, 3], params) + while len(engine.scheduler.requests["a"].generated_token_ids) < 3: + engine.step() + if suspend: + engine.add_request("b", [5], SamplingParams(max_tokens=1)) + engine.scheduler.selected_request_ids.clear() + self.assertTrue(engine.preemption.preempt(engine.scheduler.requests["b"])) + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.request_id == "a" and output.finished: + results.append(output) + if suspend: + self.assertEqual(engine.preemption.resumptions, 1) + engine.close() + self.assertEqual(results[0].token_ids, results[1].token_ids) + self.assertEqual(results[0].log_probs, results[1].log_probs) + self.assertEqual(results[0].exit_depths, results[1].exit_depths) + if __name__ == "__main__": torch.set_num_threads(1) diff --git a/vllm_rlt/request.py b/vllm_rlt/request.py index e8394589..e4d7ae21 100644 --- a/vllm_rlt/request.py +++ b/vllm_rlt/request.py @@ -1,5 +1,6 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from enum import Enum +from secrets import randbits import torch @@ -47,6 +48,10 @@ class Request: weight_version: int = 0 log_probs: list[torch.Tensor] = field(default_factory=list, repr=False) + def __post_init__(self): + if self.sampling_params.seed is None: + self.sampling_params = replace(self.sampling_params, seed=randbits(63)) + @property def num_scheduled_outputs(self) -> int: return len(self.generated_token_ids) + self.num_output_placeholders From bdce2876e3f78ac7efaa6cbf5783c185da2e9d19 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 1 Oct 2026 12:23:11 +0000 Subject: [PATCH 3/5] Measure selected-score cost and replay semantics on Thor Signed-off-by: 0z5a --- benchmarks/logprob_overhead.py | 125 + .../README.md | 93 +- .../raw/minhao-cleanup-smoke-removed.json | 4 + .../raw/minhao-cleanup-smoke.json | 2669 ++++++++++++++++ .../raw/minhao-cleanup-smoke.log | 40 + .../raw/minhao-cleanup.json | 7 + .../raw/minhao-cuda-parallel.log | 11 + .../raw/minhao-cuda.log | 11 + .../raw/minhao-driver-stack-wait.log | 19 + .../raw/minhao-driver-wait.log | 42 + .../raw/minhao-gpu-before.csv | 2 + .../raw/minhao-logprob-overhead.json | 2741 +++++++++++++++++ .../raw/minhao-logprob-overhead.log | 40 + .../raw/minhao-slow-cleanup.json | 4 + .../raw/minhao-tiny-overhead.json | 2669 ++++++++++++++++ 15 files changed, 8472 insertions(+), 5 deletions(-) create mode 100644 benchmarks/logprob_overhead.py create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json create mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json diff --git a/benchmarks/logprob_overhead.py b/benchmarks/logprob_overhead.py new file mode 100644 index 00000000..cdf5bdbd --- /dev/null +++ b/benchmarks/logprob_overhead.py @@ -0,0 +1,125 @@ +"""Alternate unprofiled score modes; report profiler operator snapshots separately.""" + +import argparse +import json +import statistics +import time +from dataclasses import replace +from pathlib import Path + +import torch +from transformers import AutoTokenizer + +from vllm_rlt import ( + LLM, + CacheConfig, + ExecutionConfig, + ExitConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) +from vllm_rlt.models import OuroForCausalLM + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("model", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--repeats", type=int, default=5) + args = parser.parse_args() + torch.set_num_threads(4) + native = OuroForCausalLM.from_pretrained(args.model, device="cuda", dtype=torch.bfloat16) + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False) + prompts = [tokenizer.encode(text) for text in ("Compute 17 + 25.\n", "Compute 9 * 8.\n")] + base = [ + SamplingParams(max_tokens=32, temperature=0.9, seed=31 + i, ignore_eos=True) + for i in range(2) + ] + modes = { + "none": base, + **{ + name: [replace(p, logprobs=0, logprobs_mode=name) for p in base] + for name in ("raw", "processed") + }, + } + report = {} + for path in ("sync", "async", "speculative"): + engine = LLM( + native, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=512), + scheduler_config=SchedulerConfig(max_num_seqs=2), + exit_config=ExitConfig("ouro_delayed" if path == "async" else "ouro"), + execution_config=ExecutionConfig(cuda_graphs=True, async_scheduling=path == "async"), + speculative_config=SpeculativeConfig(3) if path == "speculative" else None, + ) + for params in modes.values(): + engine.generate(prompts, params) + times = {name: [] for name in modes} + memory = {name: [] for name in modes} + for index in range(args.repeats): + names = list(modes) + names = names[index % 3 :] + names[: index % 3] + outputs = {} + for name in names: + torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + baseline = torch.cuda.memory_allocated() + started = time.monotonic() + outputs[name] = engine.generate(prompts, modes[name]) + torch.cuda.synchronize() + times[name].append(time.monotonic() - started) + memory[name].append(torch.cuda.max_memory_allocated() - baseline) + assert all(len(o.token_ids) == 32 for o in outputs["none"]) + assert all(o.log_probs is None for o in outputs["none"]) + for name in ("raw", "processed"): + assert [o.token_ids for o in outputs[name]] == [ + o.token_ids for o in outputs["none"] + ] + assert all(len(o.log_probs) == len(o.token_ids) for o in outputs[name]) + medians = {name: statistics.median(values) for name, values in times.items()} + profiles = {} + for name in ("none", "raw", "processed"): + for detail in ("ops", "full"): + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + record_shapes=detail == "full", + with_stack=detail == "full", + profile_memory=detail == "full", + ) as profile: + engine.generate(prompts, modes[name]) + torch.cuda.synchronize() + events = sorted( + profile.key_averages(), key=lambda e: e.self_cpu_time_total, reverse=True + ) + profiles[f"{name}-{detail}"] = [ + { + "operator": e.key, + "count": e.count, + "self_cpu_us": e.self_cpu_time_total, + "self_device_us": e.self_device_time_total, + "self_device_memory_bytes": e.self_device_memory_usage, + } + for e in events[:20] + ] + report[path] = { + "seconds": times, + "median_seconds": medians, + "tokens_per_second": {name: 64 / value for name, value in medians.items()}, + "peak_extra_allocated_bytes": memory, + "profiles": profiles, + "tokens_unchanged": True, + } + engine.close() + print(path, medians, flush=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + del engine + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/docs/validation/minhao-coordination-thor-20261001/README.md b/docs/validation/minhao-coordination-thor-20261001/README.md index 142e2799..59c4dce1 100644 --- a/docs/validation/minhao-coordination-thor-20261001/README.md +++ b/docs/validation/minhao-coordination-thor-20261001/README.md @@ -27,13 +27,96 @@ See [CPU log](raw/minhao-cpu.log). Independent VIME FP32 mixed-depth replay, recomputed gradients and generic physical-parameter conversion are checked separately. No runtime packages are installed or updated. -At the time of the initial coordination reply, new CUDA checks are queued behind -an existing Qwen-Image training reservation; they are not counted as passed here. -Full-model score-mismatch and selected-score-overhead measurements will be added -after that reservation permits execution. Earlier full-model/real-PD evidence -is retained in the [Thor](../training-contract-thor-20261001/README.md) and +The user authorized concurrent short CUDA validation while an existing +Qwen-Image training reservation remained active. **All six CUDA contract tests +pass in 135.474 s**, including entropy-seed replay with graphs, stop/abort, +preemption, speculative target scores and publication recovery; see +[CUDA log](raw/minhao-cuda-parallel.log). Startup is included in the suite time. The originally queued CUDA suite also +completed naturally after the reservation was released: **6/6 in 3.989 s**, +[log](raw/minhao-cuda.log). +Independent VIME FP32 mixed-depth replay/recomputed gradients and generic +physical-parameter conversion pass **two CPU tests in 0.302 s**. No existing +training process or reservation was interrupted. + +Earlier full-model training/save/resume and real-PD evidence is retained in the +[Thor](../training-contract-thor-20261001/README.md) and [A100](../training-contract-a100-20261001/README.md) reports. +## Selected-score cost + +`benchmarks/logprob_overhead.py` warms each mode and rotates the measurement +order over five repeats. Each unprofiled window generates two prompts with 32 +response tokens each, T=0.9, fixed per-request seeds, no filtering and CUDA +graphs enabled. Sync and speculative paths use Ouro exit; async uses delayed +Ouro exit. Token sequences must match across score modes; selected scores must +align with every committed token. Initialization and capture are excluded. + +Separately, all three modes receive an ops-only and a full-detail CPU/CUDA +profiler window (`record_shapes`, stacks and memory enabled for full detail). +The raw JSON retains the 20 operators with largest self CPU time per window, +with counts, self device time and memory. These are operator summaries, not +exported timeline traces. Profiled runs are excluded from throughput medians. +The [tiny smoke JSON](raw/minhao-tiny-overhead.json) checks all three paths and +18 profiler windows with one repeat; its timings are not full-model results. + +### Full Ouro-1.4B shared-GPU observations + +The full run overlaps another Qwen-Image training phase. GPU utilization +before this run was **98%**; [snapshot](raw/minhao-gpu-before.csv). +These measurements include resource contention and are not isolated throughput +results. Each mode has five unprofiled repeats. All three paths complete; +token IDs match across modes and selected scores align with committed tokens. + +| Path | None median (s) | Raw median (s) | Processed median (s) | Raw time overhead | Processed time overhead | +|---|---:|---:|---:|---:|---:| +| sync | 1.8017 | 1.8280 | 1.8281 | +1.46% | +1.47% | +| async | 1.6974 | 1.7849 | 1.7894 | +5.15% | +5.42% | +| speculative | 1.4366 | 1.4462 | 1.4486 | +0.67% | +0.83% | + +| Path / score mode | Tokens/s | Speed vs None | Median extra PyTorch allocation (KiB) | +|---|---:|---:|---:| +| sync / none | 35.52 | 1.000× | 933.0 | +| sync / raw | 35.01 | 0.986× | 965.0 | +| sync / processed | 35.01 | 0.986× | 933.0 | +| async / none | 37.70 | 1.000× | 965.0 | +| async / raw | 35.86 | 0.951× | 963.5 | +| async / processed | 35.77 | 0.949× | 965.0 | +| speculative / none | 44.55 | 1.000× | 2733.5 | +| speculative / raw | 44.25 | 0.993× | 2793.5 | +| speculative / processed | 44.18 | 0.992× | 2761.5 | + +[Raw timings / all 18 profiler operator summaries](raw/minhao-logprob-overhead.json), +[benchmark log](raw/minhao-logprob-overhead.log). Extra allocation is +per-window PyTorch peak above its starting allocation; it excludes resident +model/cache/graph storage and other processes. + +Between completed paths, the process temporarily waited in NVIDIA UVM external +allocation mapping (`os_acquire_rwlock_read/write`); an independent GPU status +query also waited. The [captured stack](raw/minhao-driver-stack-wait.log) and +[kernel log](raw/minhao-driver-wait.log) record concurrent `NV_ERR_NO_MEMORY` +allocation errors. Both waits resolved naturally; all paths completed without +process termination, driver reset or environment changes. Initialization waits +are outside the warmed measurement windows. The harness now drops the previous +engine and clears its allocator cache between paths to avoid retaining that +engine while constructing the next one. The recorded full run predates this +inter-path cleanup; its timed body is unchanged. +The updated cleanup harness then passes a separate tiny-model smoke across +all three paths and 18 profiler windows ([JSON](raw/minhao-cleanup-smoke.json), +[log](raw/minhao-cleanup-smoke.log)); its [57,039,421-byte model is removed](raw/minhao-cleanup-smoke-removed.json). + +All five full-model depth/effective-seed replay cases also complete, including +delayed async. The [VIME score-error table](https://github.com/0z5a/vime/blob/feat/ouro-rollout-cuda-graphs/docs/validation/minhao-coordination-thor-20261001/README.md) +records the remaining training/inference recomputation differences; full-depth +K=4 also has a nonzero residual. Exact seed replay and score-recomputation +agreement are separate checks. + +The [completed full-model copy and replacement shard](raw/minhao-cleanup.json) +are removed (3,232,344,846 bytes). The 358,667,054-byte local transfer copy +was also removed. The original slower download completed naturally; its +[2,873,677,792-byte model copy was removed](raw/minhao-slow-cleanup.json) by the +existing queue, without repeating GPU tests. All four task-owned model folders +and both replacement transfer copies are absent after validation. + ## Interface coordination | Proposed Phase 1 interface | Current reference draft | VIME consumer plan | diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json new file mode 100644 index 00000000..1f9931bd --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json @@ -0,0 +1,4 @@ +{ + "removed": "models/Ouro-minhao-cleanup-smoke", + "bytes": 57039421 +} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json new file mode 100644 index 00000000..ed52a20b --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json @@ -0,0 +1,2669 @@ +{ + "sync": { + "seconds": { + "none": [ + 0.08066848499993284 + ], + "raw": [ + 0.0914349139998194 + ], + "processed": [ + 0.08466345700026068 + ] + }, + "median_seconds": { + "none": 0.08066848499993284, + "raw": 0.0914349139998194, + "processed": 0.08466345700026068 + }, + "tokens_per_second": { + "none": 793.3705461315318, + "raw": 699.9514430573688, + "processed": 755.9341688563809 + }, + "peak_extra_allocated_bytes": { + "none": [ + 758784 + ], + "raw": [ + 823808 + ], + "processed": [ + 791040 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 24741.71000000011, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 14785.07000000005, + "self_device_us": 6.143999999963853, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2004, + "self_cpu_us": 10337.619999999994, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2020, + "self_cpu_us": 6443.9159999996045, + "self_device_us": 2353.745999999865, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1348, + "self_cpu_us": 4180.128999999671, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Activity Buffer Request", + "count": 3, + "self_cpu_us": 3175.8589999999667, + "self_device_us": 15.329000000011547, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 1981.3640000000255, + "self_device_us": 966.4389999999644, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 208, + "self_cpu_us": 1895.6409999999596, + "self_device_us": 293.95500000013635, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1556, + "self_cpu_us": 1581.7819999997378, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 716, + "self_cpu_us": 1457.2760000001927, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1335.993999999857, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1012, + "self_cpu_us": 1159.8320000000067, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 962.9140000004063, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 879.5000000000755, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 867.1529999999693, + "self_device_us": 227.0119999999879, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 830.8970000000882, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 824.2710000000934, + "self_device_us": 553.6069999999727, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2415, + "self_cpu_us": 754.1939999999772, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 744.7460000000592, + "self_device_us": 198.56199999995624, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 116, + "self_cpu_us": 645.1380000000013, + "self_device_us": 236.03399999999692, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 51961.9490000005, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 16149.892999999873, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2004, + "self_cpu_us": 10752.365000000542, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9578.167000000489, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 7612.398999999911, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2020, + "self_cpu_us": 7304.089999999687, + "self_device_us": 5244.179000000046, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1348, + "self_cpu_us": 4891.489000000262, + "self_device_us": 0, + "self_device_memory_bytes": 26818560 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 4097.280999999177, + "self_device_us": 0.0, + "self_device_memory_bytes": 162304 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 3075.5700000002166, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1556, + "self_cpu_us": 3032.1670000008203, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 716, + "self_cpu_us": 2489.1329999999634, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 2429.843999999812, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2326.7890000000816, + "self_device_us": 1660.2469999999175, + "self_device_memory_bytes": 6881280 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 2264.487000000183, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "aten::cat", + "count": 208, + "self_cpu_us": 2253.055999999902, + "self_device_us": 608.0070000000824, + "self_device_memory_bytes": 305152 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 2143.693999999712, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 2037.7939999994514, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1728.5120000003517, + "self_device_us": 0.0, + "self_device_memory_bytes": -12845056 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 1563.099999999995, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 1466.244999999848, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 119556.1379999991, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 23820.948999999957, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2194, + "self_cpu_us": 14371.875999999655, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2212, + "self_cpu_us": 11208.853000000732, + "self_device_us": 9572.278999999877, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 9940.423999999859, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1540, + "self_cpu_us": 7332.918000000681, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 3733.2030000000573, + "self_device_us": 1393.6519999997668, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 2742.5369999992313, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 2586.1810000003697, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2453.102000000058, + "self_device_us": 2902.2410000000755, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2360.366000000333, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2117.0190000001458, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1898.4860000001345, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 624, + "self_cpu_us": 1705.4810000000034, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3537, + "self_cpu_us": 1688.5239999999105, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1140, + "self_cpu_us": 1436.9140000001375, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1312.8229999998948, + "self_device_us": 794.6019999998643, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1280.3379999999306, + "self_device_us": 2046.0730000000694, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1207.4139999993881, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 1164.5660000002226, + "self_device_us": 719.3059999997731, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 115092.6299999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 29600.60599999951, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 25946.991000000064, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 17345.223999999165, + "self_device_us": 0.0, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cudaLaunchKernel", + "count": 2194, + "self_cpu_us": 15940.768999999744, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 13333.437000000138, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2212, + "self_cpu_us": 13191.647000002034, + "self_device_us": 8704.526000000209, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1540, + "self_cpu_us": 8662.159000000243, + "self_device_us": 0, + "self_device_memory_bytes": 39434240 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 7366.031999998762, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 4815.7890000007, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 4814.504000000853, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 4703.688000000431, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 4365.908999999834, + "self_device_us": 1169.2249999997575, + "self_device_memory_bytes": 337920 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 4116.904000000264, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 4072.4490000006735, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 3742.0259999994387, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 3527.933999998534, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 3314.13099999951, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 624, + "self_cpu_us": 3090.420999999369, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 3038.0740000001883, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 105856.23600000067, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 23095.000000000022, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2322, + "self_cpu_us": 15003.759999999633, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2148, + "self_cpu_us": 10396.17099999906, + "self_device_us": 9188.07700000014, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 8986.292999999905, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1476, + "self_cpu_us": 6711.9949999997025, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 3563.392000000484, + "self_device_us": 1348.502999999796, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 2623.3650000003126, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2498.1059999999925, + "self_device_us": 2908.510000000058, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 2357.0779999990714, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2048.535, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2041.7819999998937, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3537, + "self_cpu_us": 1853.5020000003638, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1701.4389999999712, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 192, + "self_cpu_us": 1573.1770000001234, + "self_device_us": 1020.2990000001446, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 1523.2590000001437, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1140, + "self_cpu_us": 1365.1570000002835, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1229.6910000001153, + "self_device_us": 799.3670000000275, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1220.0819999998166, + "self_device_us": 2031.6709999999912, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1216.561000000036, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 155410.87200000056, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 26514.28400000018, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 22682.278999999973, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2322, + "self_cpu_us": 17120.490999999194, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 16900.313999997426, + "self_device_us": 0.0, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 13033.013999998899, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2148, + "self_cpu_us": 12944.766999999709, + "self_device_us": 12005.580999999922, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1476, + "self_cpu_us": 8355.110999999586, + "self_device_us": 0, + "self_device_memory_bytes": 39401472 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 7274.417999999696, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 4893.769999999835, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 4797.883999998665, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 4710.996000000819, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 4480.034000000242, + "self_device_us": 1730.5530000000113, + "self_device_memory_bytes": 337920 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 4113.781999999035, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 3937.3690000012284, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 3691.073999999741, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 3555.606999998805, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 3436.052999998901, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 3407.9150000012155, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2940.6499999998923, + "self_device_us": 0.0, + "self_device_memory_bytes": -12845056 + } + ] + }, + "tokens_unchanged": true + }, + "async": { + "seconds": { + "none": [ + 0.1108772420002424 + ], + "raw": [ + 0.1287477310002032 + ], + "processed": [ + 0.12239944100019784 + ] + }, + "median_seconds": { + "none": 0.1108772420002424, + "raw": 0.1287477310002032, + "processed": 0.12239944100019784 + }, + "tokens_per_second": { + "none": 577.214934692009, + "raw": 497.0961391148632, + "processed": 522.8782049739635 + }, + "peak_extra_allocated_bytes": { + "none": [ + 790016 + ], + "raw": [ + 822272 + ], + "processed": [ + 822272 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 15284.012000000086, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 1850, + "self_cpu_us": 8500.083000000152, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 6639.298000000122, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1439, + "self_cpu_us": 5531.486999999959, + "self_device_us": 1906.9919999998415, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 4616.57799999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2769, + "self_cpu_us": 4578.959000000186, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 601, + "self_cpu_us": 2438.2479999999, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2059.4019999999846, + "self_device_us": 778.1539999999927, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3851, + "self_cpu_us": 1576.196000000137, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 629, + "self_cpu_us": 1306.6099999999087, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1285.6129999997502, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 1154.325999999974, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 996.3809999998794, + "self_device_us": 225.15500000009706, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 798.8650000000216, + "self_device_us": 569.2259999999969, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 751.5409999999792, + "self_device_us": 210.21199999991222, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::record_stream", + "count": 436, + "self_cpu_us": 730.9929999999513, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 116, + "self_cpu_us": 650.3899999999953, + "self_device_us": 229.37999999999693, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventQuery", + "count": 513, + "self_cpu_us": 642.4939999999581, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventRecordWithFlags", + "count": 698, + "self_cpu_us": 624.0679999998281, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 281, + "self_cpu_us": 616.0749999999405, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 16433.368999999864, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10018.040999999977, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9557.15300000049, + "self_device_us": 0.0, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cudaLaunchKernel", + "count": 1850, + "self_cpu_us": 9201.422999999402, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8294.379000000605, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 7561.7619999999515, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2769, + "self_cpu_us": 7053.931000000085, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6636.100999999369, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1439, + "self_cpu_us": 6178.882000000486, + "self_device_us": 6708.658999999896, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5287.671999999828, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 4708.536999999357, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4653.8080000003965, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4520.836999999436, + "self_device_us": 0.0, + "self_device_memory_bytes": -79872 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4280.746000000392, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 3811.028999999038, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3031.376999999363, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 601, + "self_cpu_us": 2754.339000000004, + "self_device_us": 0, + "self_device_memory_bytes": 26515968 + }, + { + "operator": "", + "count": 904, + "self_cpu_us": 2668.428999999229, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/__init__.py(1235): current_stream", + "count": 1126, + "self_cpu_us": 2653.2650000008134, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/_utils.py(869): _get_device_index", + "count": 1504, + "self_cpu_us": 2575.899000002003, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 42234.66600000008, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 17010.12299999994, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 11070.726999999686, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2040, + "self_cpu_us": 10325.780999999826, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1631, + "self_cpu_us": 6525.653000000375, + "self_device_us": 8328.703999999532, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 4947.713000000557, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 4864.119999999892, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 793, + "self_cpu_us": 3114.437000000318, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2199.381999999973, + "self_device_us": 3114.2829999999713, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2073.5459999992754, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1458.7159999996475, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 693, + "self_cpu_us": 1455.3750000004352, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 116, + "self_cpu_us": 1195.895999999781, + "self_device_us": 692.2980000000898, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1053.418000000198, + "self_device_us": 860.6630000000077, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 1023.9610000000209, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1004.951999999932, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 409, + "self_cpu_us": 958.3949999998126, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 96, + "self_cpu_us": 888.4549999998671, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 859.6940000000086, + "self_device_us": 2180.226000000088, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 837.9099999999289, + "self_device_us": 781.5500000001175, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 22104.463000000447, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 17926.967000000168, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 14532.418000000032, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2040, + "self_cpu_us": 10800.322000000502, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10283.571000000464, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9719.242999999962, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8455.242999999638, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 7985.128999999445, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1631, + "self_cpu_us": 7096.114999998659, + "self_device_us": 7530.1170000006005, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6655.402000001546, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5625.204000000713, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 4933.312999999274, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4771.646000000193, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4725.08799999866, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4402.655000000255, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 3951.7359999998625, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "aten::empty_strided", + "count": 793, + "self_cpu_us": 3523.918000000247, + "self_device_us": 0, + "self_device_memory_bytes": 39131648 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3151.4589999996992, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2998.801000000943, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 904, + "self_cpu_us": 2673.5620000006493, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 43409.94599999997, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 17950.804999999928, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 15988.948999999806, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2168, + "self_cpu_us": 11112.410000000078, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1567, + "self_cpu_us": 6483.044000000588, + "self_device_us": 8276.219999999763, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 4952.348999999742, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 4921.737000000263, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 729, + "self_cpu_us": 2952.603999999926, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2189.6210000000665, + "self_device_us": 3186.7999999999442, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2120.180999999159, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 693, + "self_cpu_us": 1494.2490000004798, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1457.0889999997416, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 116, + "self_cpu_us": 1239.0429999999242, + "self_device_us": 708.8769999999126, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 192, + "self_cpu_us": 1220.1639999996369, + "self_device_us": 1123.057999999999, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1135.0230000010429, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 1087.3439999998745, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1070.1700000001292, + "self_device_us": 874.7320000000436, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 409, + "self_cpu_us": 998.1089999999881, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 96, + "self_cpu_us": 941.3419999995094, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 902.430999999955, + "self_device_us": 2233.6359999999077, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 23466.177000000756, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 18052.96899999999, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 12057.866999999911, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2168, + "self_cpu_us": 11316.609999999708, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10142.197000000597, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9814.517999998712, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8444.050000000905, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 7757.864000000339, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1567, + "self_cpu_us": 6859.583000000366, + "self_device_us": 7261.939000000045, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6667.205999998861, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5536.804000000327, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 4946.131999999503, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4856.363000000383, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4753.402000000731, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4567.222000000331, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 3955.984000000677, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "aten::empty_strided", + "count": 729, + "self_cpu_us": 3271.6339999996685, + "self_device_us": 0, + "self_device_memory_bytes": 39098880 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3136.034000000277, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 3133.5739999998086, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 904, + "self_cpu_us": 2658.5939999993807, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + }, + "speculative": { + "seconds": { + "none": [ + 0.06240307000007306 + ], + "raw": [ + 0.0662938270002087 + ], + "processed": [ + 0.06682750300024054 + ] + }, + "median_seconds": { + "none": 0.06240307000007306, + "raw": 0.0662938270002087, + "processed": 0.06682750300024054 + }, + "tokens_per_second": { + "none": 1025.590567898744, + "raw": 965.3990860988991, + "processed": 957.6895309072021 + }, + "peak_extra_allocated_bytes": { + "none": [ + 2750976 + ], + "raw": [ + 2790912 + ], + "processed": [ + 2774528 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 14082.55900000005, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2245, + "self_cpu_us": 9969.27999999969, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1128, + "self_cpu_us": 8384.83099999996, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1277, + "self_cpu_us": 3738.911000000112, + "self_device_us": 1777.0809999999603, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 143, + "self_cpu_us": 2681.666000000013, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 592, + "self_cpu_us": 2131.4490000001315, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1464.813999999862, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1219, + "self_cpu_us": 1242.6320000000528, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1164.5480000000407, + "self_device_us": 348.58399999995163, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 977.9220000000182, + "self_device_us": 302.44099999999275, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 969.8469999999941, + "self_device_us": 148.32200000000057, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 893.5579999999991, + "self_device_us": 672.4589999999498, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 814.1800000000003, + "self_device_us": 175.17400000007547, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 797.685999999996, + "self_device_us": 602.407000000032, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2113, + "self_cpu_us": 754.8929999999518, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 77, + "self_cpu_us": 751.3600000000063, + "self_device_us": 112.54500000000348, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 743.7420000000911, + "self_device_us": 1537.3079999999263, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 742.4579999999497, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 348, + "self_cpu_us": 714.7729999999433, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_softmax", + "count": 119, + "self_cpu_us": 704.4650000000765, + "self_device_us": 1838.0370000000748, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaStreamSynchronize", + "count": 143, + "self_cpu_us": 19864.37699999997, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 14970.81799999996, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2245, + "self_cpu_us": 10701.993000000055, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1128, + "self_cpu_us": 10585.129000000112, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9116.944000000163, + "self_device_us": 0.0, + "self_device_memory_bytes": -1536 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6067.3340000002245, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 4490.072999999706, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1277, + "self_cpu_us": 4200.721000000049, + "self_device_us": 2472.8799999996913, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 592, + "self_cpu_us": 2475.1309999999894, + "self_device_us": 0, + "self_device_memory_bytes": 52925952 + }, + { + "operator": "aten::slice", + "count": 1219, + "self_cpu_us": 2209.2680000001774, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2199.3709999999483, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1887.626000000164, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1604.1319999999032, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1586.2180000000506, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1252.9259999999376, + "self_device_us": 522.2780000000967, + "self_device_memory_bytes": 25165824 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1165.056000000348, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + }, + { + "operator": "aten::_to_copy", + "count": 348, + "self_cpu_us": 1149.622000000063, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1149.0220000000054, + "self_device_us": 234.2420000000002, + "self_device_memory_bytes": 786432 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1108.9360000000352, + "self_device_us": 466.69400000002315, + "self_device_memory_bytes": 1404416 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 1083.3829999999834, + "self_device_us": 1011.7250000000568, + "self_device_memory_bytes": 79872 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1219, + "self_cpu_us": 32065.39100000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 15680.841000000019, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2394, + "self_cpu_us": 11380.852999999855, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 4496.739999999833, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1428, + "self_cpu_us": 4368.5809999999665, + "self_device_us": 4428.808999999856, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 743, + "self_cpu_us": 2724.119000000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1590.5000000001273, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 1249.5310000003237, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1218.5759999999027, + "self_device_us": 777.1980000000331, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 100, + "self_cpu_us": 1051.9999999999136, + "self_device_us": 428.0050000000215, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1017.5759999999709, + "self_device_us": 1260.8519999999708, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1012.5349999999967, + "self_device_us": 733.8019999999997, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 959.7169999995767, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 930.4999999999745, + "self_device_us": 1327.9919999999292, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 913.5249999998941, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 857.5730000000585, + "self_device_us": 328.99800000001414, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 831.2279999999759, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 829.5409999999483, + "self_device_us": 1179.376999999913, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 778.833999999988, + "self_device_us": 2838.9859999999917, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 736.6149999998129, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1219, + "self_cpu_us": 17222.49800000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 15855.267000000127, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 11819.789999999986, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2394, + "self_cpu_us": 11519.752999999928, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9310.95399999983, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6072.020999999813, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1428, + "self_cpu_us": 4727.487999999833, + "self_device_us": 4655.986000000061, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 4696.012000000468, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::empty_strided", + "count": 743, + "self_cpu_us": 3005.7680000001283, + "self_device_us": 0, + "self_device_memory_bytes": 65541632 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 2301.55100000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2113.589000000433, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1948.2030000000268, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 1625.992999999985, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1563.8129999999483, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1353.5050000000588, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1338.518000000211, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1312.731999999898, + "self_device_us": 807.4049999999152, + "self_device_memory_bytes": 25165824 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 1273.8249999999016, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1184.5459999999466, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1142.3969999999777, + "self_device_us": 341.6040000000071, + "self_device_memory_bytes": 786432 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1155, + "self_cpu_us": 30886.555999999768, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 15259.450999999935, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2522, + "self_cpu_us": 11694.530000000093, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 4923.203000000066, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1364, + "self_cpu_us": 4200.7139999999345, + "self_device_us": 4325.325999999722, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 679, + "self_cpu_us": 2475.386999999806, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1579.0149999997666, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 270, + "self_cpu_us": 1526.4960000000283, + "self_device_us": 958.5179999999909, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 1211.8220000002698, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1046.327999999994, + "self_device_us": 727.1810000000069, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 100, + "self_cpu_us": 1044.046000000038, + "self_device_us": 422.95099999992226, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_softmax", + "count": 183, + "self_cpu_us": 1043.5719999998073, + "self_device_us": 4847.890000000012, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1012.0560000000614, + "self_device_us": 1260.6269999999504, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 924.9700000000994, + "self_device_us": 1308.5010000000257, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 921.9240000003183, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 911.6380000003298, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 842.8640000000414, + "self_device_us": 329.63399999997273, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 816.9300000000476, + "self_device_us": 1178.6429999999946, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 807.9510000000835, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 778.4430000000539, + "self_device_us": 2829.236000000021, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1155, + "self_cpu_us": 42451.45200000021, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 16473.516999999978, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2522, + "self_cpu_us": 12428.987999999705, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9528.780999999806, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 7053.55099999987, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6385.314000000195, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 4901.061999999998, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1364, + "self_cpu_us": 4642.906999999959, + "self_device_us": 6678.909000000535, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 679, + "self_cpu_us": 2844.3409999998976, + "self_device_us": 0, + "self_device_memory_bytes": 65508864 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 2370.445000000463, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2189.775000000467, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 2057.2630000004756, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "aten::div", + "count": 270, + "self_cpu_us": 1714.8240000003643, + "self_device_us": 1486.07199999988, + "self_device_memory_bytes": 37748736 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 1671.7550000000952, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1589.7839999998428, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1412.5379999998395, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1349.7980000001553, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "vllm_rlt/worker/sampling.py(6): probabilities", + "count": 181, + "self_cpu_us": 1349.6579999995265, + "self_device_us": 0.0, + "self_device_memory_bytes": -35586048 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 1343.5690000002296, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1245.2050000000427, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + } + ] + }, + "tokens_unchanged": true + } +} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log new file mode 100644 index 00000000..17965cd3 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log @@ -0,0 +1,40 @@ +[transformers] You are using a model of type `ouro` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating. +USDT:2026-10-01 20:16:37 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:37 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:16:39 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:39 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:16:46 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:47 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:16:50 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:50 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:16:59 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:59 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:02 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:02 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +sync {'none': 0.08066848499993284, 'raw': 0.0914349139998194, 'processed': 0.08466345700026068} +USDT:2026-10-01 20:17:12 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:12 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:16 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:16 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:31 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:31 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:34 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:35 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:52 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:52 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:17:56 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:17:56 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +async {'none': 0.1108772420002424, 'raw': 0.1287477310002032, 'processed': 0.12239944100019784} +USDT:2026-10-01 20:18:14 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:14 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:16 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:16 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:22 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:22 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:25 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:25 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:32 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:32 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:18:35 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:18:35 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop +speculative {'none': 0.06240307000007306, 'raw': 0.0662938270002087, 'processed': 0.06682750300024054} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json new file mode 100644 index 00000000..8d18ff49 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json @@ -0,0 +1,7 @@ +{ + "removed": [ + "models/Ouro-minhao-full", + "minhao-fast-part-2" + ], + "bytes": 3232344846 +} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log new file mode 100644 index 00000000..8eef87ad --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log @@ -0,0 +1,11 @@ +test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok +test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok +test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok +test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok +test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok +test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 135.474s + +OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log new file mode 100644 index 00000000..ae3542fd --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log @@ -0,0 +1,11 @@ +test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok +test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok +test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok +test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok +test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok +test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 3.989s + +OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log new file mode 100644 index 00000000..855a3b10 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log @@ -0,0 +1,19 @@ +Kernel stack captured from PID 54942 during the observed driver wait on 2026-10-01: +[<0>] os_acquire_rwlock_write+0x38/0x64 [nvidia] +[<0>] portSyncRwLockAcquireWrite+0x10/0x30 [nvidia] +[<0>] rmapiLockAcquire+0x23c/0x340 [nvidia] +[<0>] _nvGpuOpsLocksAcquireWithClientLockFlags+0x98/0x1f0 [nvidia] +[<0>] dupMemory+0x88/0x690 [nvidia] +[<0>] nvUvmInterfaceDupMemory+0x34/0x40 [nvidia] +[<0>] uvm_map_external_allocation_on_gpu+0x1f0/0x3fc [nvidia_uvm] +[<0>] uvm_api_map_external_allocation+0x474/0x4e4 [nvidia_uvm] +[<0>] uvm_ioctl+0x117c/0x16a0 [nvidia_uvm] +[<0>] uvm_unlocked_ioctl_entry.part.0+0xdc/0x148 [nvidia_uvm] +[<0>] uvm_unlocked_ioctl_entry+0x78/0x9c [nvidia_uvm] +[<0>] __arm64_sys_ioctl+0xac/0xf0 +[<0>] invoke_syscall+0x48/0x114 +[<0>] el0_svc_common.constprop.0+0xc0/0xe0 +[<0>] do_el0_svc+0x1c/0x28 +[<0>] el0_svc+0x30/0xd8 +[<0>] el0t_64_sync_handler+0x120/0x12c +[<0>] el0t_64_sync+0x194/0x198 diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log new file mode 100644 index 00000000..b59b66e8 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log @@ -0,0 +1,42 @@ +Thu Oct 1 12:03:16 UTC 2026 +[<0>] futex_wait_queue+0x6c/0x98 +[<0>] __futex_wait+0xcc/0x160 +[<0>] futex_wait+0x88/0x110 +[<0>] do_futex+0xf8/0x1a0 +[<0>] __arm64_sys_futex+0xe0/0x13c +[<0>] invoke_syscall+0x48/0x114 +[<0>] el0_svc_common.constprop.0+0xc0/0xe0 +[<0>] do_el0_svc+0x1c/0x28 +[<0>] el0_svc+0x30/0xd8 +[<0>] el0t_64_sync_handler+0x120/0x12c +[<0>] el0t_64_sync+0x194/0x198 +[ 1630.645566] NVRM: GPU0 refcntRequestReference_IMPL: Failed to enter state 1 (current state: 0, status: 0x00000056) +[ 2474.164009] NVRM: GPU0 refcntRequestReference_IMPL: Failed to enter state 1 (current state: 0, status: 0x00000056) +[ 2478.973583] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2479.334052] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2479.459146] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2480.205368] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2480.300412] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2480.606304] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2480.823594] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2481.003826] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2481.261606] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2481.325949] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2482.079631] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2482.152349] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2482.562933] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2482.686092] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2483.701671] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2483.788894] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2484.193315] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2484.306663] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2484.419459] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.152876] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.231614] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.661010] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.775288] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2485.971025] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2486.389956] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2486.451899] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2486.841379] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 +[ 2486.904428] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv new file mode 100644 index 00000000..9c1a4867 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv @@ -0,0 +1,2 @@ +name, driver_version, memory.used [MiB], utilization.gpu [%] +NVIDIA Thor, 595.78, [N/A], 98 % diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json new file mode 100644 index 00000000..9ca1eca6 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json @@ -0,0 +1,2741 @@ +{ + "sync": { + "seconds": { + "none": [ + 1.797056486999736, + 1.8016931980000663, + 1.7812917080000261, + 1.8116626850001012, + 1.8313193210001373 + ], + "raw": [ + 1.8279988220001542, + 1.8234451930002251, + 1.810924204999992, + 1.8776579790001051, + 1.8599433469998985 + ], + "processed": [ + 1.8894610560000729, + 1.7770136750000347, + 1.8161140520001027, + 1.8662560849998044, + 1.8281205210000735 + ] + }, + "median_seconds": { + "none": 1.8016931980000663, + "raw": 1.8279988220001542, + "processed": 1.8281205210000735 + }, + "tokens_per_second": { + "none": 35.52214110096099, + "raw": 35.01096348080393, + "processed": 35.00863278149123 + }, + "peak_extra_allocated_bytes": { + "none": [ + 955392, + 955392, + 955392, + 955392, + 955392 + ], + "raw": [ + 988160, + 955392, + 988160, + 988160, + 955392 + ], + "processed": [ + 955392, + 955392, + 988160, + 955392, + 955392 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 3115292.1020000014, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 297542.58400000003, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 47216.64400000275, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6232, + "self_cpu_us": 46928.53299999721, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2724, + "self_cpu_us": 25592.631000006077, + "self_device_us": 7924.548000013914, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 10, + "self_cpu_us": 19917.72599999947, + "self_device_us": 2.1440000000875443, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16707.02599999922, + "self_device_us": 69237.73299999899, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2140, + "self_cpu_us": 16282.546999996724, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Activity Buffer Request", + "count": 11, + "self_cpu_us": 11960.367999999446, + "self_device_us": 3.072000000017397, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 9020.897999999828, + "self_device_us": 3486.1089999998353, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 384, + "self_cpu_us": 8604.2179999959, + "self_device_us": 1033.1409999988641, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1420, + "self_cpu_us": 7515.799999997356, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 800, + "self_cpu_us": 5476.198000001276, + "self_device_us": 164.86600000051476, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 5247.854999999913, + "self_device_us": 1053.979999999935, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2084, + "self_cpu_us": 5008.964999994238, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3880.791000000514, + "self_device_us": 2027.154000000004, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3850.2629999997826, + "self_device_us": 547.1479999999751, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 2068, + "self_cpu_us": 3678.7860000009327, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3572.775999995909, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4087, + "self_cpu_us": 3312.0390000030416, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 1714448.007000006, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 312503.2759999983, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 88477.1809999905, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaLaunchKernel", + "count": 6232, + "self_cpu_us": 45877.57399999528, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2724, + "self_cpu_us": 30653.4359999993, + "self_device_us": 8406.053999994654, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 17, + "self_cpu_us": 21553.933999999936, + "self_device_us": 113.12300000005052, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 21004.616999993712, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2140, + "self_cpu_us": 18041.15100000197, + "self_device_us": 0, + "self_device_memory_bytes": 120670208 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 15299.454999999656, + "self_device_us": 186853.4359999999, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 13051.557000005414, + "self_device_us": 0.0, + "self_device_memory_bytes": -1185792 + }, + { + "operator": "aten::_to_copy", + "count": 1420, + "self_cpu_us": 10733.110999994191, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 10510.103999998966, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 384, + "self_cpu_us": 9402.348000001257, + "self_device_us": 4816.147000000074, + "self_device_memory_bytes": 15486976 + }, + { + "operator": "aten::slice", + "count": 2084, + "self_cpu_us": 8692.440000006642, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 566, + "self_cpu_us": 8643.991000002294, + "self_device_us": 0.0, + "self_device_memory_bytes": -8029696 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 7680, + "self_cpu_us": 8545.6210000067, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 8124.672999999564, + "self_device_us": 15076.379999999674, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "Activity Buffer Request", + "count": 10, + "self_cpu_us": 7915.628000000057, + "self_device_us": 0.47999999999956344, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 7900.098000011858, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 7111.301000002073, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 1687563.1620000014, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 297471.37799999845, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6422, + "self_cpu_us": 38591.12499999725, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2916, + "self_cpu_us": 19720.34099999547, + "self_device_us": 4969.288999992985, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 11, + "self_cpu_us": 13895.465999999578, + "self_device_us": 2.6239999998360872, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2332, + "self_cpu_us": 13841.695999991633, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 12845.257000000787, + "self_device_us": 85966.08499999985, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 11226.992999998267, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 448, + "self_cpu_us": 7124.344999999742, + "self_device_us": 1127.6500000008473, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 6689.6719999999805, + "self_device_us": 3976.72400000012, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1548, + "self_cpu_us": 5447.389000004618, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 800, + "self_cpu_us": 4433.354999998611, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2086, + "self_cpu_us": 4181.640999996726, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 3946.4330000000346, + "self_device_us": 1249.5749999998784, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 5209, + "self_cpu_us": 3355.9950000011663, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 3335.78000000483, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3175.459000005998, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 2957.2929999998587, + "self_device_us": 2255.626000000015, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 2879.41300000012, + "self_device_us": 626.137999999959, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 2196, + "self_cpu_us": 2788.2449999992214, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 1712470.881999998, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 303259.42099999957, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 84949.75999997946, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 58914.68900000312, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 38, + "self_cpu_us": 55984.85400000001, + "self_device_us": 141.47600000002421, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6422, + "self_cpu_us": 39227.52699999942, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2916, + "self_cpu_us": 20689.506999994737, + "self_device_us": 8596.109999999157, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 17150.899999998324, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2332, + "self_cpu_us": 15443.806000003227, + "self_device_us": 0, + "self_device_memory_bytes": 133285888 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 14933.654000000508, + "self_device_us": 216397.0109999997, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 10250.292999994825, + "self_device_us": 0.0, + "self_device_memory_bytes": -1185792 + }, + { + "operator": "aten::_to_copy", + "count": 1548, + "self_cpu_us": 8689.791999999334, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 7680, + "self_cpu_us": 7914.010000004513, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2086, + "self_cpu_us": 7845.527000000181, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 7781.584000000366, + "self_device_us": 14088.113000000061, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "aten::cat", + "count": 448, + "self_cpu_us": 7187.911999997445, + "self_device_us": 1075.74999999817, + "self_device_memory_bytes": 15519744 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 6351.897999999905, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 566, + "self_cpu_us": 6237.029000000981, + "self_device_us": 0.0, + "self_device_memory_bytes": -8029696 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 6197.680000002206, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 7680, + "self_cpu_us": 5868.171000009923, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 1785390.3010000014, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 287422.5859999989, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 54, + "self_cpu_us": 132692.09599999993, + "self_device_us": 137.35000000000582, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 63230.2860000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6550, + "self_cpu_us": 41654.006999997655, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2852, + "self_cpu_us": 20811.043999990427, + "self_device_us": 5011.1320000056, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2268, + "self_cpu_us": 14158.613000006142, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 13688.341999998862, + "self_device_us": 237601.12799999924, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 448, + "self_cpu_us": 7512.926999999517, + "self_device_us": 1107.5330000010308, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 6855.801000000043, + "self_device_us": 3760.478000000774, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1548, + "self_cpu_us": 5959.337000000479, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 800, + "self_cpu_us": 4528.025000001053, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2086, + "self_cpu_us": 4388.27499998413, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4043.9099999990876, + "self_device_us": 6436.210999999603, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 5209, + "self_cpu_us": 3740.4920000093116, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3374.305000002234, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 3299.6420000069775, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3031.1940000002096, + "self_device_us": 5960.095999999843, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3006.850000000014, + "self_device_us": 629.8799999999992, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 2196, + "self_cpu_us": 3004.803000003427, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 1681418.688000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 313165.76199999935, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 89656.83900000324, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 55077.46799999963, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 46, + "self_cpu_us": 49129.840000000055, + "self_device_us": 177.34899999998743, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6550, + "self_cpu_us": 45109.648999988836, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2852, + "self_cpu_us": 23547.901000007907, + "self_device_us": 22245.57399999522, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 18967.310000011475, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 2268, + "self_cpu_us": 16894.335999999996, + "self_device_us": 0, + "self_device_memory_bytes": 133253120 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16466.671999998278, + "self_device_us": 191312.69199999896, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 11592.744999999559, + "self_device_us": 0.0, + "self_device_memory_bytes": -1185792 + }, + { + "operator": "aten::_to_copy", + "count": 1548, + "self_cpu_us": 9718.056999999331, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 7680, + "self_cpu_us": 8876.85000001891, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 448, + "self_cpu_us": 8632.131000004554, + "self_device_us": 1104.7309999994868, + "self_device_memory_bytes": 15519744 + }, + { + "operator": "aten::slice", + "count": 2086, + "self_cpu_us": 8531.177000011521, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 7987.785000000618, + "self_device_us": 16808.661000000444, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 566, + "self_cpu_us": 7880.714999992513, + "self_device_us": 0.0, + "self_device_memory_bytes": -8029696 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 7180.626999999522, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 6991.26000000355, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 7680, + "self_cpu_us": 6713.435000007084, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + }, + "async": { + "seconds": { + "none": [ + 1.695322036000107, + 1.6974444380002751, + 1.7046040849995734, + 1.6946781169999667, + 1.7272834199998215 + ], + "raw": [ + 1.7682591719999436, + 1.7628239639998355, + 1.7849022200002764, + 1.7954341879999447, + 1.8388650159999997 + ], + "processed": [ + 1.7741621370000757, + 1.770364053999856, + 1.7893874629999118, + 1.8360151079996285, + 1.8289056349999555 + ] + }, + "median_seconds": { + "none": 1.6974444380002751, + "raw": 1.7849022200002764, + "processed": 1.7893874629999118 + }, + "tokens_per_second": { + "none": 37.70373778796383, + "raw": 35.856305898924866, + "processed": 35.766429196225545 + }, + "peak_extra_allocated_bytes": { + "none": [ + 988160, + 988160, + 988160, + 988160, + 988160 + ], + "raw": [ + 986624, + 988160, + 986624, + 986624, + 988160 + ], + "processed": [ + 988160, + 988160, + 986624, + 988160, + 988160 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaEventSynchronize", + "count": 502, + "self_cpu_us": 1432410.3530000027, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 191290.37999999942, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 44, + "self_cpu_us": 34835.79299999999, + "self_device_us": 351.92999999994936, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6078, + "self_cpu_us": 33813.03999999954, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16364.334999999637, + "self_device_us": 183683.25199999977, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 11429.879000001887, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2143, + "self_cpu_us": 11020.502999999995, + "self_device_us": 4541.596999996408, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 10356.752000000413, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 8018.803000000195, + "self_device_us": 3866.8100000002287, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3297, + "self_cpu_us": 6736.454999988346, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1393, + "self_cpu_us": 6702.344000001056, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4626.323999999812, + "self_device_us": 9519.472000000073, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 5523, + "self_cpu_us": 3716.362000009337, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3477.173000000108, + "self_device_us": 619.724000000142, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3425.1280000001398, + "self_device_us": 12465.012999999697, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 1, + "self_cpu_us": 3281.159, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 985, + "self_cpu_us": 2562.969999997975, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 981, + "self_cpu_us": 2548.3720000009953, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2241.14700000608, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 228, + "self_cpu_us": 2224.8430000000544, + "self_device_us": 774.4480000006351, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaEventSynchronize", + "count": 502, + "self_cpu_us": 1334775.7140000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 213300.00299999965, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 37214.26000000528, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6078, + "self_cpu_us": 31441.57300000226, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16664.894000000244, + "self_device_us": 184634.85100000046, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "Command Buffer Full", + "count": 34, + "self_cpu_us": 16638.358999999953, + "self_device_us": 83.08099999994738, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 658, + "self_cpu_us": 14962.963000003772, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3297, + "self_cpu_us": 12126.546999992546, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 11609.655999997276, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 11161.335999998177, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2143, + "self_cpu_us": 10901.848999998805, + "self_device_us": 8184.701999998565, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 10622.565999999239, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 8326.869000005598, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 8074.22300000026, + "self_device_us": 0.0, + "self_device_memory_bytes": -79872 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 7691.808999999961, + "self_device_us": 7839.375999999642, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "", + "count": 658, + "self_cpu_us": 7110.883000002672, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1393, + "self_cpu_us": 7029.449000000028, + "self_device_us": 0, + "self_device_memory_bytes": 119249408 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 6454.799000000135, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 7680, + "self_cpu_us": 5952.708999990169, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 5775.1769999995595, + "self_device_us": 0.0, + "self_device_memory_bytes": -169984 + } + ], + "raw-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 865738.5560000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 440, + "self_cpu_us": 513537.4239999985, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 269982.26400000206, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6268, + "self_cpu_us": 37557.50699999495, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 19496.538000000193, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 15068.10300000109, + "self_device_us": 89085.99599999975, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2335, + "self_cpu_us": 15040.314000005079, + "self_device_us": 4926.857000000852, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 11644.150999997, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3299, + "self_cpu_us": 9157.908999995387, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1585, + "self_cpu_us": 9157.521999997385, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 7889.426999999985, + "self_device_us": 3726.571000000061, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 6645, + "self_cpu_us": 4706.0270000029495, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4578.1169999999665, + "self_device_us": 1194.1009999999906, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 1045, + "self_cpu_us": 3393.951000005082, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1113, + "self_cpu_us": 3387.474999996231, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3383.8789999999763, + "self_device_us": 2079.0089999998263, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 292, + "self_cpu_us": 3383.5519999986764, + "self_device_us": 885.724000001198, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3352.952999999814, + "self_device_us": 585.2520000001759, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3030.2580000000307, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::rsqrt", + "count": 388, + "self_cpu_us": 2112.1670000002123, + "self_device_us": 558.6339999998054, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 894514.4109999985, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 438, + "self_cpu_us": 356624.633000003, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 282386.8210000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 61673.15100000108, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 57640.45999999429, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6268, + "self_cpu_us": 37907.111000003744, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 658, + "self_cpu_us": 20129.23500000021, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 17286.161000000488, + "self_device_us": 209501.60500000042, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "Command Buffer Full", + "count": 24, + "self_cpu_us": 15990.03599999995, + "self_device_us": 94.47799999994459, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2335, + "self_cpu_us": 15931.423000005114, + "self_device_us": 5127.309000006906, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 15294.889999997744, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3299, + "self_cpu_us": 15165.398000015914, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 12728.342999995075, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 11771.589000000844, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1585, + "self_cpu_us": 10317.796999996372, + "self_device_us": 0, + "self_device_memory_bytes": 131865088 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 9568.225999996474, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "", + "count": 658, + "self_cpu_us": 8962.633000000906, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 8289.213999999814, + "self_device_us": 9734.966999999577, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 7695.529999998893, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 7675.931000003035, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 920932.4780000017, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 439, + "self_cpu_us": 482884.0049999996, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 305185.86299999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 73606.65199999898, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 58, + "self_cpu_us": 70443.25700000004, + "self_device_us": 161.2559999999503, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6396, + "self_cpu_us": 39018.222000004345, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2271, + "self_cpu_us": 15692.35799999762, + "self_device_us": 4989.233000002182, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 14275.767999999229, + "self_device_us": 203367.5870000003, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 12591.590999998225, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 3299, + "self_cpu_us": 10243.788999999848, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1521, + "self_cpu_us": 8787.994000000632, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 6978.011999999464, + "self_device_us": 7834.872999999812, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 6645, + "self_cpu_us": 5005.443999995054, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4159.8260000002265, + "self_device_us": 1241.8309999997346, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 1045, + "self_cpu_us": 3796.858999999785, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1113, + "self_cpu_us": 3539.445999995534, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 292, + "self_cpu_us": 3381.941999998534, + "self_device_us": 891.852000000601, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 3376.4410000027856, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3101.857999999801, + "self_device_us": 2076.3349999998336, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3023.192999999952, + "self_device_us": 613.0609999998796, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 860792.1709999986, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 439, + "self_cpu_us": 413039.0830000005, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 255455.0850000007, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 49905.24699999363, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6396, + "self_cpu_us": 37927.29899999841, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 658, + "self_cpu_us": 17880.09100000289, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 704, + "self_cpu_us": 16554.555999998465, + "self_device_us": 89435.19800000006, + "self_device_memory_bytes": 80609280 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 15447.242000002167, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2271, + "self_cpu_us": 13614.053999995802, + "self_device_us": 4926.6889999996565, + "self_device_memory_bytes": -1024 + }, + { + "operator": "aten::slice", + "count": 3299, + "self_cpu_us": 13604.380000005436, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 13420.60000000347, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 1362, + "self_cpu_us": 12200.938999997265, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 10094.503999997636, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1521, + "self_cpu_us": 8711.722999999149, + "self_device_us": 0, + "self_device_memory_bytes": 131832320 + }, + { + "operator": "aten::mul", + "count": 1260, + "self_cpu_us": 8518.236999999934, + "self_device_us": 3812.4549999999153, + "self_device_memory_bytes": 133613568 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 8272.147000002355, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "", + "count": 658, + "self_cpu_us": 8045.141999997961, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 6979.725999999505, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 6839.281000003541, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 6694.447999993816, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + }, + "speculative": { + "seconds": { + "none": [ + 1.4405976819998614, + 1.4354998989997512, + 1.4359760199999982, + 1.43785706799963, + 1.436613651999778 + ], + "raw": [ + 1.446159684000122, + 1.4462237470002037, + 1.4429731870000069, + 1.447812623000118, + 1.447334557999966 + ], + "processed": [ + 1.4485182270000223, + 1.4616416380004011, + 1.445917856000051, + 1.449624680999932, + 1.4485653670003558 + ] + }, + "median_seconds": { + "none": 1.436613651999778, + "raw": 1.4462237470002037, + "processed": 1.4485653670003558 + }, + "tokens_per_second": { + "none": 44.54920772255747, + "raw": 44.25318014086722, + "processed": 44.181644444895994 + }, + "peak_extra_allocated_bytes": { + "none": [ + 2799104, + 2799104, + 2799104, + 2799104, + 2799104 + ], + "raw": [ + 2860544, + 2827776, + 2860544, + 2860544, + 2827776 + ], + "processed": [ + 2827776, + 2827776, + 2860544, + 2827776, + 2827776 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 980, + "self_cpu_us": 1013133.5880000024, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 329889.5450000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6440, + "self_cpu_us": 44799.10400000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 14830.692999999728, + "self_device_us": 57957.05499999999, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1858, + "self_cpu_us": 11904.329999998092, + "self_device_us": 3719.296999995492, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 10062.069999999801, + "self_device_us": 3642.09899999938, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1349, + "self_cpu_us": 8686.187999998147, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 5437.313999999343, + "self_device_us": 1142.796999999984, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 5328.228999999939, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 4195.546000000275, + "self_device_us": 2040.650999999802, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 4130.804000000057, + "self_device_us": 541.9709999999882, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1036, + "self_cpu_us": 3498.193000000953, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 3478.147999996989, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1598, + "self_cpu_us": 3344.8489999989542, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 247, + "self_cpu_us": 3218.2849999998366, + "self_device_us": 773.1339999996012, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3629, + "self_cpu_us": 2757.4049999993445, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 204, + "self_cpu_us": 2495.4719999990484, + "self_device_us": 344.9210000009334, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::rsqrt", + "count": 388, + "self_cpu_us": 2447.126000000311, + "self_device_us": 568.7279999999582, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_index_put_impl_", + "count": 192, + "self_cpu_us": 2249.8519999999753, + "self_device_us": 1397.437999999971, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 134, + "self_cpu_us": 2245.1280000006864, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 980, + "self_cpu_us": 1004743.099000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 341942.8800000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 98, + "self_cpu_us": 65974.60500000956, + "self_device_us": 0.0, + "self_device_memory_bytes": -122880 + }, + { + "operator": "cudaLaunchKernel", + "count": 6440, + "self_cpu_us": 44354.85499999292, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 16, + "self_cpu_us": 43468.00100000008, + "self_device_us": 2872.282999999996, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 9, + "self_cpu_us": 17052.98900000006, + "self_device_us": 2.4319999997969717, + "self_device_memory_bytes": 122880 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 16737.56800000057, + "self_device_us": 133078.352, + "self_device_memory_bytes": 74514432 + }, + { + "operator": "aten::copy_", + "count": 1858, + "self_cpu_us": 12988.228999999212, + "self_device_us": 7612.869000000108, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 53, + "self_cpu_us": 12837.078999994766, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 10, + "self_cpu_us": 11424.281000001734, + "self_device_us": 0.0, + "self_device_memory_bytes": -41080832 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 10870.521999998786, + "self_device_us": 14274.18699999967, + "self_device_memory_bytes": 133641216 + }, + { + "operator": "aten::empty_strided", + "count": 1349, + "self_cpu_us": 9551.200000001178, + "self_device_us": 0, + "self_device_memory_bytes": 146153984 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 8481.077000000107, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 7766.478999998973, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "aten::slice", + "count": 1598, + "self_cpu_us": 7758.273000003454, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 9024, + "self_cpu_us": 6788.078000001582, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 6336, + "self_cpu_us": 6656.0759999922775, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", + "count": 96, + "self_cpu_us": 6457.112000000272, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 134, + "self_cpu_us": 6235.574999999869, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1036, + "self_cpu_us": 6220.435999998934, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1070, + "self_cpu_us": 1064068.1040000021, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 361931.5290000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 48, + "self_cpu_us": 104276.37, + "self_device_us": 164.09299999989162, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 158, + "self_cpu_us": 50917.7819999979, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6588, + "self_cpu_us": 44249.33099999894, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 9, + "self_cpu_us": 15985.770999999775, + "self_device_us": 3.1359999999403954, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2008, + "self_cpu_us": 13097.013999997875, + "self_device_us": 14456.707000001214, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 11822.664999999748, + "self_device_us": 173903.50500000032, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1499, + "self_cpu_us": 9125.855000001111, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 8022.662999998281, + "self_device_us": 14237.222000000518, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 4792.803000000362, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 4564.072000000597, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4263.941999999985, + "self_device_us": 1244.1820000001771, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1122, + "self_cpu_us": 3775.3999999982043, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 269, + "self_cpu_us": 3679.534000000115, + "self_device_us": 4876.952999999159, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1600, + "self_cpu_us": 3253.3939999918493, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3200.0180000000037, + "self_device_us": 5880.0210000000625, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3172.9669999997113, + "self_device_us": 606.8459999999995, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4139, + "self_cpu_us": 3093.79700000462, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 204, + "self_cpu_us": 2634.8199999991048, + "self_device_us": 346.6839999984659, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1070, + "self_cpu_us": 921827.2790000035, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 348470.8909999982, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 98, + "self_cpu_us": 82054.98900000838, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaLaunchKernel", + "count": 6588, + "self_cpu_us": 43856.03299999783, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 14546.498999999756, + "self_device_us": 58689.897, + "self_device_memory_bytes": 74514432 + }, + { + "operator": "aten::copy_", + "count": 2008, + "self_cpu_us": 13351.863999997171, + "self_device_us": 4054.520999999536, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 53, + "self_cpu_us": 13121.481000001226, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 10, + "self_cpu_us": 11731.000000006607, + "self_device_us": 0.0, + "self_device_memory_bytes": -41080832 + }, + { + "operator": "aten::empty_strided", + "count": 1499, + "self_cpu_us": 10030.803999998225, + "self_device_us": 0, + "self_device_memory_bytes": 158769664 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 9924.678000001335, + "self_device_us": 3753.6830000002865, + "self_device_memory_bytes": 133641216 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 7820.23400000037, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "aten::slice", + "count": 1600, + "self_cpu_us": 6671.56400000199, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 6336, + "self_cpu_us": 6460.941999995901, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1122, + "self_cpu_us": 6371.330999999439, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 9024, + "self_cpu_us": 6215.4499999884265, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", + "count": 96, + "self_cpu_us": 6019.520000000246, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/nn/modules/module.py(1782): _call_impl", + "count": 1397, + "self_cpu_us": 5992.312000000209, + "self_device_us": 0.0, + "self_device_memory_bytes": -99717120 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 5270.968999999644, + "self_device_us": 1170.8770000002514, + "self_device_memory_bytes": 28510208 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 5207.893999999975, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 4842.231000002561, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1006, + "self_cpu_us": 1086286.7079999999, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 366847.06800000044, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Command Buffer Full", + "count": 48, + "self_cpu_us": 81405.13199999991, + "self_device_us": 132.25700000009965, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 158, + "self_cpu_us": 49747.239000002395, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 6716, + "self_cpu_us": 44376.60800000187, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Buffer Flush", + "count": 9, + "self_cpu_us": 15859.352999999945, + "self_device_us": 2.464000000152737, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1944, + "self_cpu_us": 13164.344999999565, + "self_device_us": 7839.070999998783, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 12993.734999999626, + "self_device_us": 152600.5239999997, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1435, + "self_cpu_us": 8864.449999998058, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 8686.781999999766, + "self_device_us": 25866.910000000396, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 6254.039000003366, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 4607.503999999629, + "self_device_us": 1228.5170000003036, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 4396.7750000001415, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1122, + "self_cpu_us": 4098.707000003469, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 269, + "self_cpu_us": 3794.4410000006155, + "self_device_us": 858.2720000006775, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mean", + "count": 388, + "self_cpu_us": 3510.004999999982, + "self_device_us": 5943.891000000016, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1600, + "self_cpu_us": 3501.2590000032596, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::pow", + "count": 388, + "self_cpu_us": 3423.5350000000344, + "self_device_us": 600.6419999997925, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 268, + "self_cpu_us": 3386.080000000744, + "self_device_us": 443.71500000028755, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4139, + "self_cpu_us": 3143.4790000002977, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1006, + "self_cpu_us": 923970.0769999987, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 137, + "self_cpu_us": 349496.5669999999, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 98, + "self_cpu_us": 79725.7660000025, + "self_device_us": 0.0, + "self_device_memory_bytes": -8192 + }, + { + "operator": "cudaLaunchKernel", + "count": 6716, + "self_cpu_us": 45678.692999995874, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 673, + "self_cpu_us": 15432.809000000572, + "self_device_us": 59057.668000000325, + "self_device_memory_bytes": 74514432 + }, + { + "operator": "aten::copy_", + "count": 1944, + "self_cpu_us": 13113.157000004554, + "self_device_us": 3976.40699999536, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 53, + "self_cpu_us": 12922.643000007964, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 10, + "self_cpu_us": 11522.10300000626, + "self_device_us": 0.0, + "self_device_memory_bytes": -41080832 + }, + { + "operator": "aten::mul", + "count": 1314, + "self_cpu_us": 10494.68399999901, + "self_device_us": 3770.354000000312, + "self_device_memory_bytes": 133641216 + }, + { + "operator": "aten::empty_strided", + "count": 1435, + "self_cpu_us": 9768.620999997758, + "self_device_us": 0, + "self_device_memory_bytes": 158736896 + }, + { + "operator": "vllm_rlt/layers/normalization.py(20): forward", + "count": 388, + "self_cpu_us": 8493.670999999964, + "self_device_us": 0.0, + "self_device_memory_bytes": -143628288 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", + "count": 96, + "self_cpu_us": 6569.988999999679, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1600, + "self_cpu_us": 6520.582000002687, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 1122, + "self_cpu_us": 6445.759999999112, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 117, + "self_cpu_us": 6329.600000002482, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", + "count": 6336, + "self_cpu_us": 6263.172000000794, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/nn/modules/module.py(1782): _call_impl", + "count": 1397, + "self_cpu_us": 6239.434000000041, + "self_device_us": 0.0, + "self_device_memory_bytes": -99717120 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", + "count": 9024, + "self_cpu_us": 6183.9280000038, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::add", + "count": 772, + "self_cpu_us": 5625.210000000348, + "self_device_us": 1181.320999999678, + "self_device_memory_bytes": 28510208 + }, + { + "operator": "cuLaunchKernelEx", + "count": 769, + "self_cpu_us": 5410.53999999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + } +} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log new file mode 100644 index 00000000..b108ece5 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log @@ -0,0 +1,40 @@ +[transformers] You are using a model of type `ouro` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating. +USDT:2026-10-01 19:52:11 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:52:14 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:52:32 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:52:45 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:53:16 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:53:18 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:53:32 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:53:36 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:54:03 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:54:20 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 19:54:40 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 19:54:45 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +sync {'none': 1.8016931980000663, 'raw': 1.8279988220001542, 'processed': 1.8281205210000735} +USDT:2026-10-01 20:02:51 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:03:17 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:03:43 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:03:49 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:04:25 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:04:51 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:05:18 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:05:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:06:14 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:06:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:06:52 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:07:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +async {'none': 1.6974444380002751, 'raw': 1.7849022200002764, 'processed': 1.7893874629999118} +USDT:2026-10-01 20:11:40 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:12:07 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:12:35 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:12:37 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:13:07 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:13:09 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:13:50 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:14:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:14:49 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:14:51 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +USDT:2026-10-01 20:15:33 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start +USDT:2026-10-01 20:16:01 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop +speculative {'none': 1.436613651999778, 'raw': 1.4462237470002037, 'processed': 1.4485653670003558} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json new file mode 100644 index 00000000..b0c05fe2 --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json @@ -0,0 +1,4 @@ +{ + "removed": "models/Ouro-1.4B-parallel", + "bytes": 2873677792 +} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json new file mode 100644 index 00000000..96acb36b --- /dev/null +++ b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json @@ -0,0 +1,2669 @@ +{ + "sync": { + "seconds": { + "none": [ + 0.09519798499991339 + ], + "raw": [ + 0.11991658300007657 + ], + "processed": [ + 0.11633255899982942 + ] + }, + "median_seconds": { + "none": 0.09519798499991339, + "raw": 0.11991658300007657, + "processed": 0.11633255899982942 + }, + "tokens_per_second": { + "none": 672.2831370859186, + "raw": 533.7043334528564, + "processed": 550.1469283426821 + }, + "peak_extra_allocated_bytes": { + "none": [ + 758784 + ], + "raw": [ + 823808 + ], + "processed": [ + 791040 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 33297.56899999986, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 19800.59699999992, + "self_device_us": 11.039000000033411, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2004, + "self_cpu_us": 19130.309000000445, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2020, + "self_cpu_us": 9735.057999999846, + "self_device_us": 3466.580000000014, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1348, + "self_cpu_us": 6787.842999999959, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Activity Buffer Request", + "count": 3, + "self_cpu_us": 4397.798999999917, + "self_device_us": 17.24599999999691, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1556, + "self_cpu_us": 2589.577999999135, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2440.773999999882, + "self_device_us": 1127.5460000000476, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 208, + "self_cpu_us": 2438.501999999814, + "self_device_us": 416.11600000010685, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 2014.048999999999, + "self_device_us": 681.3790000000336, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 716, + "self_cpu_us": 2006.413000000081, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1787.3960000001898, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 1310.3229999998512, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1012, + "self_cpu_us": 1157.6350000001703, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 188, + "self_cpu_us": 1153.9850000001802, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1078.1530000000166, + "self_device_us": 288.4610000001521, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1031.721999999896, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2415, + "self_cpu_us": 989.9190000005906, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 935.0519999999015, + "self_device_us": 286.7849999999162, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 64, + "self_cpu_us": 901.8880000001336, + "self_device_us": 541.9650000000875, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1624, + "self_cpu_us": 96740.21499999995, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 29828.6660000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 19172.10899999903, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2004, + "self_cpu_us": 16537.040000000692, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 15321.057999998613, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2020, + "self_cpu_us": 13304.200000000319, + "self_device_us": 9814.611000000457, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 9246.910000000378, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1348, + "self_cpu_us": 8824.270000000583, + "self_device_us": 0, + "self_device_memory_bytes": 26818560 + }, + { + "operator": "(2): __init__", + "count": 780, + "self_cpu_us": 7972.586999999223, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 7952.3179999991335, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "torch/autograd/grad_mode.py(290): __enter__", + "count": 312, + "self_cpu_us": 5820.944000000607, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1556, + "self_cpu_us": 5617.165999999186, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 156, + "self_cpu_us": 5495.474999999722, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", + "count": 8, + "self_cpu_us": 4696.361000000001, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 716, + "self_cpu_us": 4452.799999999923, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 4376.564999998955, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 4297.467999999419, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(287): _make_scheduled_item", + "count": 376, + "self_cpu_us": 4069.740999999527, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "Activity Buffer Request", + "count": 2, + "self_cpu_us": 4064.9590000000353, + "self_device_us": 8.286999999976615, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 3937.9390000010226, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 108719.8060000003, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 24011.792999999805, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2194, + "self_cpu_us": 15252.708000000432, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 13507.98999999966, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2212, + "self_cpu_us": 11690.644999998814, + "self_device_us": 8416.399999999961, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1540, + "self_cpu_us": 7301.129000000332, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 4101.939999999672, + "self_device_us": 1255.3850000000248, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 2997.545000000003, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2510.9329999999395, + "self_device_us": 2639.002999999942, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 2464.008999998726, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2282.966000000388, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2134.255999999961, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1791.3630000006428, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3537, + "self_cpu_us": 1679.8659999990111, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 624, + "self_cpu_us": 1648.1559999993642, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1140, + "self_cpu_us": 1481.7340000004865, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1324.1299999998773, + "self_device_us": 683.1499999999578, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1257.5680000000211, + "self_device_us": 1779.3249999999916, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1212.7460000002857, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 1187.963000000027, + "self_device_us": 634.3850000001239, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1818, + "self_cpu_us": 101858.63600000048, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 19925.880999999914, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2194, + "self_cpu_us": 13766.511999999813, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 13020.369000001338, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "aten::copy_", + "count": 2212, + "self_cpu_us": 10548.945999998652, + "self_device_us": 9764.56400000026, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 10036.862999999756, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 8070.916999999871, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1540, + "self_cpu_us": 6907.167000000017, + "self_device_us": 0, + "self_device_memory_bytes": 39434240 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 5716.320000000458, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 3870.769999999612, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 3773.934000000846, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 3767.767999999496, + "self_device_us": 1359.2739999999485, + "self_device_memory_bytes": 337920 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 3737.9029999998384, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 3170.4539999998087, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 3001.5509999997157, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 2973.905000000177, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 2791.1890000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2571.306999999335, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2479.8980000000774, + "self_device_us": 2988.58499999993, + "self_device_memory_bytes": 6881280 + }, + { + "operator": "aten::clone", + "count": 624, + "self_cpu_us": 2234.690999999897, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 97750.7149999996, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 22683.500000000073, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2322, + "self_cpu_us": 20361.571000000113, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 13013.803999999898, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 2148, + "self_cpu_us": 11927.50900000012, + "self_device_us": 8301.904999999215, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1476, + "self_cpu_us": 6909.296999999731, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 4398.034999999721, + "self_device_us": 1229.2659999998846, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 3254.1110000007093, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2565.4240000000195, + "self_device_us": 2622.8850000000243, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 2342.3889999999733, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2110.396999999586, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 2086.5779999999377, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3537, + "self_cpu_us": 1777.3179999996833, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 192, + "self_cpu_us": 1635.8050000000076, + "self_device_us": 899.7680000000564, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1635.431000000337, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::to", + "count": 1140, + "self_cpu_us": 1513.9589999994594, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::clone", + "count": 560, + "self_cpu_us": 1491.8950000000768, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1314.6709999997838, + "self_device_us": 686.7049999998162, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 384, + "self_cpu_us": 1258.967999999892, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1243.792000000154, + "self_device_us": 1745.1790000000492, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1754, + "self_cpu_us": 103497.714000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 20846.46299999978, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2322, + "self_cpu_us": 15129.085999998966, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 13175.480000000473, + "self_device_us": 0.0, + "self_device_memory_bytes": -1024 + }, + { + "operator": "aten::copy_", + "count": 2148, + "self_cpu_us": 10448.986999999888, + "self_device_us": 10015.051999999127, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 128, + "self_cpu_us": 10255.892999999633, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 316, + "self_cpu_us": 8545.498999999993, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 1476, + "self_cpu_us": 6776.009000000365, + "self_device_us": 0, + "self_device_memory_bytes": 39401472 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 5816.4070000001375, + "self_device_us": 0.0, + "self_device_memory_bytes": -232448 + }, + { + "operator": "", + "count": 288, + "self_cpu_us": 3927.1879999986068, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 844, + "self_cpu_us": 3912.425999998626, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 272, + "self_cpu_us": 3884.787000000045, + "self_device_us": 1454.8620000000992, + "self_device_memory_bytes": 337920 + }, + { + "operator": "aten::slice", + "count": 1558, + "self_cpu_us": 3826.4770000002713, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(370): _update", + "count": 188, + "self_cpu_us": 3287.5009999989925, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 390, + "self_cpu_us": 3132.696000000161, + "self_device_us": 0.0, + "self_device_memory_bytes": -6689280 + }, + { + "operator": "", + "count": 744, + "self_cpu_us": 3023.1649999991896, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2872.8629999997174, + "self_device_us": 3133.3969999999745, + "self_device_memory_bytes": 6881280 + }, + { + "operator": "vllm_rlt/core/scheduler.py(323): _take", + "count": 188, + "self_cpu_us": 2871.759000000642, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 252, + "self_cpu_us": 2640.3929999992906, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 2423.540000000653, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ] + }, + "tokens_unchanged": true + }, + "async": { + "seconds": { + "none": [ + 0.11112063099994884 + ], + "raw": [ + 0.12377583400007097 + ], + "processed": [ + 0.12324414399995476 + ] + }, + "median_seconds": { + "none": 0.11112063099994884, + "raw": 0.12377583400007097, + "processed": 0.12324414399995476 + }, + "tokens_per_second": { + "none": 575.9506531242562, + "raw": 517.0637751466356, + "processed": 519.2944502095247 + }, + "peak_extra_allocated_bytes": { + "none": [ + 790016 + ], + "raw": [ + 822272 + ], + "processed": [ + 822272 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 16279.262000000004, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 1850, + "self_cpu_us": 8762.216999999695, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 6619.0009999997255, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1439, + "self_cpu_us": 5515.489000000218, + "self_device_us": 2130.588000000116, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2769, + "self_cpu_us": 4635.025999999778, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 4584.934999999944, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 601, + "self_cpu_us": 2461.7410000001278, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2147.010999999959, + "self_device_us": 809.0290000000505, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 3851, + "self_cpu_us": 1570.322000000206, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1314.6100000000988, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 629, + "self_cpu_us": 1276.088999999649, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 990.0759999999482, + "self_device_us": 233.5900000000729, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 940.6050000001692, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 864.0409999999702, + "self_device_us": 601.2849999999398, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 782.1960000001072, + "self_device_us": 233.108000000002, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::record_stream", + "count": 436, + "self_cpu_us": 745.8910000001142, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 116, + "self_cpu_us": 660.8229999999921, + "self_device_us": 266.3260000000107, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 281, + "self_cpu_us": 619.7590000002324, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventRecordWithFlags", + "count": 698, + "self_cpu_us": 613.4480000000897, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 64, + "self_cpu_us": 609.95099999999, + "self_device_us": 486.4459999999799, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 17218.51800000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 9886.790999998972, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9667.236999999815, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 1850, + "self_cpu_us": 9417.455000000573, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8362.155000000257, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1037, + "self_cpu_us": 7480.238000000069, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2769, + "self_cpu_us": 7326.50600000022, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6628.762999999737, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1439, + "self_cpu_us": 6172.138999999667, + "self_device_us": 7529.890999999854, + "self_device_memory_bytes": -1024 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5232.246999999894, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 4782.417000000076, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4636.270000000101, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4530.803999999982, + "self_device_us": 0.0, + "self_device_memory_bytes": -79872 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4478.493999999459, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 3908.4060000003046, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3038.3940000000453, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 601, + "self_cpu_us": 2837.559999999785, + "self_device_us": 0, + "self_device_memory_bytes": 26515968 + }, + { + "operator": "", + "count": 904, + "self_cpu_us": 2774.022999999871, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "torch/cuda/__init__.py(1235): current_stream", + "count": 1126, + "self_cpu_us": 2606.9919999999374, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2567.98200000016, + "self_device_us": 3231.842000000015, + "self_device_memory_bytes": 6881280 + } + ], + "raw-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 35043.07600000008, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 27261.345000000074, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 26212.683000000343, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2040, + "self_cpu_us": 17226.433999998942, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 9567.590000000226, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1631, + "self_cpu_us": 7822.165999999748, + "self_device_us": 7616.087999999582, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 5809.952999998892, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 793, + "self_cpu_us": 3797.753000000337, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2782.3519999998025, + "self_device_us": 2836.7000000000608, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2576.1720000005644, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1843.8050000004368, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 693, + "self_cpu_us": 1734.2529999999033, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 116, + "self_cpu_us": 1597.180999999936, + "self_device_us": 645.6439999999984, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 409, + "self_cpu_us": 1295.4429999995746, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1207.970999999644, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1207.7140000000436, + "self_device_us": 744.3319999997984, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 128, + "self_cpu_us": 1078.0610000000052, + "self_device_us": 684.6229999998395, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 96, + "self_cpu_us": 1074.36400000082, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 1062.3739999999489, + "self_device_us": 1878.7700000000477, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 1000.1569999997646, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1231, + "self_cpu_us": 20903.29400000006, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 18533.75499999998, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 11627.489999999798, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2040, + "self_cpu_us": 11486.406999999133, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10235.82400000188, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9755.430000000004, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 8791.236000000776, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 8558.017999999998, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8461.214000000378, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1631, + "self_cpu_us": 7077.14399999875, + "self_device_us": 7515.524999999724, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6816.739999999478, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5592.861000000013, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4809.259999999384, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4741.460000002217, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4577.247000000707, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 4044.835999999861, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "aten::empty_strided", + "count": 793, + "self_cpu_us": 3699.4469999996372, + "self_device_us": 0, + "self_device_memory_bytes": 39131648 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 3092.7339999997967, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3059.046000000157, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2879.6480000001075, + "self_device_us": 2809.507000000074, + "self_device_memory_bytes": 6881280 + } + ], + "processed-ops": [ + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 40731.565000000046, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 18052.8380000001, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2168, + "self_cpu_us": 11742.620000000232, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 10820.819999999676, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1567, + "self_cpu_us": 6499.077000000206, + "self_device_us": 8344.468000000634, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5100.179999999666, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 4953.838000000167, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 729, + "self_cpu_us": 2996.4449999999683, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2412.3269999999034, + "self_device_us": 3132.7200000000075, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 2234.7580000002854, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 64, + "self_cpu_us": 1542.5300000003408, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 693, + "self_cpu_us": 1528.0179999995985, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 192, + "self_cpu_us": 1305.0059999999976, + "self_device_us": 1111.3880000001209, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 116, + "self_cpu_us": 1274.8779999999783, + "self_device_us": 721.4030000000266, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::unsqueeze", + "count": 1132, + "self_cpu_us": 1105.1659999998597, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sigmoid", + "count": 124, + "self_cpu_us": 1061.8390000001273, + "self_device_us": 851.0689999999304, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 409, + "self_cpu_us": 1038.5240000001413, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 64, + "self_cpu_us": 956.9540000000325, + "self_device_us": 2173.455000000007, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::stack", + "count": 96, + "self_cpu_us": 929.1470000001209, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaEventSynchronize", + "count": 378, + "self_cpu_us": 908.5779999999498, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaGraphLaunch", + "count": 124, + "self_cpu_us": 19562.78000000015, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1167, + "self_cpu_us": 12463.96400000043, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2168, + "self_cpu_us": 11787.197999999762, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "triton/runtime/jit.py(708): run", + "count": 570, + "self_cpu_us": 10282.645999998535, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 124, + "self_cpu_us": 9901.121999998875, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 129, + "self_cpu_us": 9359.13800000002, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(450): submit", + "count": 188, + "self_cpu_us": 8651.15399999867, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 2771, + "self_cpu_us": 7960.596999999834, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1567, + "self_cpu_us": 6933.717000000461, + "self_device_us": 7324.400999999833, + "self_device_memory_bytes": -1024 + }, + { + "operator": "vllm_rlt/worker/async_state.py(169): prepare", + "count": 188, + "self_cpu_us": 6819.291999999174, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cuLaunchKernelEx", + "count": 658, + "self_cpu_us": 5430.379000000365, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 570, + "self_cpu_us": 5054.977999999375, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", + "count": 189, + "self_cpu_us": 4980.245000000294, + "self_device_us": 0.0, + "self_device_memory_bytes": -112640 + }, + { + "operator": "torch/cuda/_utils.py(557): _get_device_index", + "count": 1504, + "self_cpu_us": 4827.424999998944, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/async_state.py(46): transfer", + "count": 188, + "self_cpu_us": 4674.382999999931, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/model_runner.py(352): _execute", + "count": 188, + "self_cpu_us": 4062.3459999991246, + "self_device_us": 0.0, + "self_device_memory_bytes": -105472 + }, + { + "operator": "aten::empty_strided", + "count": 729, + "self_cpu_us": 3426.792000000013, + "self_device_us": 0, + "self_device_memory_bytes": 39098880 + }, + { + "operator": "", + "count": 688, + "self_cpu_us": 3211.9410000003777, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 4973, + "self_cpu_us": 3122.0590000002553, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 88, + "self_cpu_us": 2800.17300000012, + "self_device_us": 2851.515000000054, + "self_device_memory_bytes": 6881280 + } + ] + }, + "tokens_unchanged": true + }, + "speculative": { + "seconds": { + "none": [ + 0.0662261480001689 + ], + "raw": [ + 0.06795481500012102 + ], + "processed": [ + 0.06800570499990499 + ] + }, + "median_seconds": { + "none": 0.0662261480001689, + "raw": 0.06795481500012102, + "processed": 0.06800570499990499 + }, + "tokens_per_second": { + "none": 966.385663859489, + "raw": 941.8022843544792, + "processed": 941.0975152759523 + }, + "peak_extra_allocated_bytes": { + "none": [ + 2750976 + ], + "raw": [ + 2790912 + ], + "processed": [ + 2774528 + ] + }, + "profiles": { + "none-ops": [ + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 14250.309000000101, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2245, + "self_cpu_us": 10531.197000000051, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1128, + "self_cpu_us": 8562.952999999969, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1277, + "self_cpu_us": 3761.6589999999446, + "self_device_us": 1806.0290000000139, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 143, + "self_cpu_us": 2708.568000000006, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 592, + "self_cpu_us": 2201.785999999958, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1515.5650000001278, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1219, + "self_cpu_us": 1244.1550000002997, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1168.2059999999747, + "self_device_us": 362.9779999999464, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1026.8629999999969, + "self_device_us": 156.73599999999442, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 995.331999999973, + "self_device_us": 317.34799999996153, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 917.5149999999649, + "self_device_us": 683.0130000000008, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 843.7139999999272, + "self_device_us": 172.83400000001166, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 790.9519999999757, + "self_device_us": 621.252999999977, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 77, + "self_cpu_us": 775.0059999999964, + "self_device_us": 124.67199999998161, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2113, + "self_cpu_us": 760.4689999999889, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 759.0639999997917, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 756.3700000000226, + "self_device_us": 1531.8530000000064, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 348, + "self_cpu_us": 708.1709999999481, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_softmax", + "count": 119, + "self_cpu_us": 698.9639999999526, + "self_device_us": 1821.2669999999434, + "self_device_memory_bytes": 0 + } + ], + "none-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1128, + "self_cpu_us": 50386.84900000042, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 18524.925000000065, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2245, + "self_cpu_us": 12642.707000000097, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 10299.16400000074, + "self_device_us": 0.0, + "self_device_memory_bytes": -1536 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6868.614000000468, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 143, + "self_cpu_us": 6569.547999999943, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 5634.887000000228, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1277, + "self_cpu_us": 4802.597999998776, + "self_device_us": 6307.480999999992, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 592, + "self_cpu_us": 3066.6099999999415, + "self_device_us": 0, + "self_device_memory_bytes": 52925952 + }, + { + "operator": "aten::slice", + "count": 1219, + "self_cpu_us": 2758.779000000186, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2436.3480000002855, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 2412.122000000076, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 2321.6109999999353, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1901.2210000000298, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2113, + "self_cpu_us": 1567.9659999997698, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1554.9709999999468, + "self_device_us": 1259.2209999998413, + "self_device_memory_bytes": 25165824 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1469.638999999961, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + }, + { + "operator": "aten::_to_copy", + "count": 348, + "self_cpu_us": 1457.5210000002992, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1289.5729999998493, + "self_device_us": 1402.8940000000239, + "self_device_memory_bytes": 1404416 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 1265.5860000001667, + "self_device_us": 2400.320000000047, + "self_device_memory_bytes": 79872 + } + ], + "raw-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1219, + "self_cpu_us": 32791.54100000013, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 17470.00599999995, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2394, + "self_cpu_us": 12090.505000000023, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 4819.1389999999865, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1428, + "self_cpu_us": 4432.517000000222, + "self_device_us": 4649.904999999926, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 743, + "self_cpu_us": 2811.108999999853, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1714.3729999998868, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 1327.7300000001896, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1264.6509999999944, + "self_device_us": 774.6999999999425, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 100, + "self_cpu_us": 1105.010999999929, + "self_device_us": 438.56399999991436, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1051.4090000000483, + "self_device_us": 1278.6749999999424, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1050.9589999995637, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1045.2189999999996, + "self_device_us": 741.7150000000001, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 980.1500000000906, + "self_device_us": 1307.7370000000465, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 938.0559999998939, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 902.338000000078, + "self_device_us": 333.67199999995864, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 879.014999999983, + "self_device_us": 1192.532999999974, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 833.2999999999356, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 790.916999999994, + "self_device_us": 2837.7279999999555, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 769.0820000001117, + "self_device_us": 0, + "self_device_memory_bytes": 0 + } + ], + "raw-full": [ + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 17014.585000000065, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaMemcpyAsync", + "count": 1219, + "self_cpu_us": 15616.989000000358, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2394, + "self_cpu_us": 12305.728000000006, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9651.556000000543, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6259.108999999342, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 4917.951999999987, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1428, + "self_cpu_us": 4794.027999999778, + "self_device_us": 4887.427999999511, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 4222.749999999923, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 743, + "self_cpu_us": 3183.7439999999515, + "self_device_us": 0, + "self_device_memory_bytes": 65541632 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 2523.371999999762, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2202.935999999663, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 2086.883999999831, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 1783.7669999996692, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1756.5959999997158, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1639.7709999998492, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1436.0750000000262, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "aten::select", + "count": 579, + "self_cpu_us": 1346.087000000095, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 206, + "self_cpu_us": 1336.519999999975, + "self_device_us": 813.8999999999978, + "self_device_memory_bytes": 25165824 + }, + { + "operator": "torch/utils/_contextlib.py(120): decorate_context", + "count": 186, + "self_cpu_us": 1293.4269999999624, + "self_device_us": 0.0, + "self_device_memory_bytes": -20741632 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1222.6059999999607, + "self_device_us": 738.8599999998873, + "self_device_memory_bytes": 1404416 + } + ], + "processed-ops": [ + { + "operator": "cudaMemcpyAsync", + "count": 1155, + "self_cpu_us": 30564.47400000023, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 16297.44999999991, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2522, + "self_cpu_us": 12829.769999999564, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 6427.685000000046, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::copy_", + "count": 1364, + "self_cpu_us": 4445.575999999577, + "self_device_us": 4287.241000000021, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 679, + "self_cpu_us": 2640.5459999999166, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 1695.7159999999421, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 270, + "self_cpu_us": 1634.4419999999154, + "self_device_us": 895.1859999999397, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 1283.0529999999708, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::cat", + "count": 100, + "self_cpu_us": 1112.1259999999893, + "self_device_us": 597.6139999999978, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_softmax", + "count": 183, + "self_cpu_us": 1092.33499999993, + "self_device_us": 4761.238999999965, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mul", + "count": 175, + "self_cpu_us": 1076.8700000000263, + "self_device_us": 1272.6010000000606, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::mm", + "count": 57, + "self_cpu_us": 1041.7010000000118, + "self_device_us": 734.4430000000102, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1010.8409999999494, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::aminmax", + "count": 78, + "self_cpu_us": 1003.2569999999541, + "self_device_us": 1286.9970000000249, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 993.8319999999167, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::lt", + "count": 128, + "self_cpu_us": 919.8220000000947, + "self_device_us": 318.9200000000674, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::sum", + "count": 87, + "self_cpu_us": 879.4340000000484, + "self_device_us": 1155.8990000000413, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty", + "count": 315, + "self_cpu_us": 858.7320000000709, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::argmax", + "count": 110, + "self_cpu_us": 807.774000000074, + "self_device_us": 2808.9659999999603, + "self_device_memory_bytes": 0 + } + ], + "processed-full": [ + { + "operator": "cudaMemcpyAsync", + "count": 1155, + "self_cpu_us": 20902.823999999953, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaGraphLaunch", + "count": 156, + "self_cpu_us": 16891.75100000009, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaLaunchKernel", + "count": 2522, + "self_cpu_us": 15932.245000000374, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/cuda_graph.py(65): run", + "count": 112, + "self_cpu_us": 9363.272999999954, + "self_device_us": 0.0, + "self_device_memory_bytes": -512 + }, + { + "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", + "count": 60, + "self_cpu_us": 6491.891000000185, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "cudaStreamSynchronize", + "count": 168, + "self_cpu_us": 5420.91799999993, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/speculative.py(96): execute", + "count": 12, + "self_cpu_us": 5071.345999999881, + "self_device_us": 0.0, + "self_device_memory_bytes": -40443392 + }, + { + "operator": "aten::copy_", + "count": 1364, + "self_cpu_us": 4659.201999999755, + "self_device_us": 5583.503000000237, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::empty_strided", + "count": 679, + "self_cpu_us": 3039.1240000001817, + "self_device_us": 0, + "self_device_memory_bytes": 65508864 + }, + { + "operator": "aten::bitwise_and", + "count": 87, + "self_cpu_us": 2845.459999999992, + "self_device_us": 366.18600000001607, + "self_device_memory_bytes": 44544 + }, + { + "operator": "aten::slice", + "count": 1221, + "self_cpu_us": 2434.940000000655, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "", + "count": 716, + "self_cpu_us": 2247.8800000001356, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::eq", + "count": 87, + "self_cpu_us": 2212.9000000000815, + "self_device_us": 339.5000000001128, + "self_device_memory_bytes": 44544 + }, + { + "operator": "aten::multinomial", + "count": 78, + "self_cpu_us": 2182.421999999704, + "self_device_us": 0.0, + "self_device_memory_bytes": -15654912 + }, + { + "operator": "", + "count": 133, + "self_cpu_us": 1826.42999999996, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::div", + "count": 270, + "self_cpu_us": 1732.066999999999, + "self_device_us": 1221.016000000127, + "self_device_memory_bytes": 37748736 + }, + { + "operator": "aten::as_strided", + "count": 2669, + "self_cpu_us": 1720.8809999998857, + "self_device_us": 0, + "self_device_memory_bytes": 0 + }, + { + "operator": "aten::_to_copy", + "count": 435, + "self_cpu_us": 1615.1079999998499, + "self_device_us": 0.0, + "self_device_memory_bytes": 0 + }, + { + "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", + "count": 50, + "self_cpu_us": 1522.9489999999714, + "self_device_us": 0.0, + "self_device_memory_bytes": -3634176 + }, + { + "operator": "vllm_rlt/worker/sampling.py(6): probabilities", + "count": 181, + "self_cpu_us": 1390.739999999816, + "self_device_us": 0.0, + "self_device_memory_bytes": -35586048 + } + ] + }, + "tokens_unchanged": true + } +} From 08d1b8d53ed1c25fabb1ae0c1b973facab9ec999 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Thu, 1 Oct 2026 12:40:35 +0000 Subject: [PATCH 4/5] Keep training contract review focused on the public API Signed-off-by: 0z5a --- benchmarks/logprob_overhead.py | 125 - benchmarks/nixl_preflight.py | 73 - benchmarks/pd_training_contract.py | 172 -- benchmarks/training_contract.py | 94 - docs/training_contract.md | 28 - .../README.md | 134 - .../raw/minhao-cleanup-smoke-removed.json | 4 - .../raw/minhao-cleanup-smoke.json | 2669 ---------------- .../raw/minhao-cleanup-smoke.log | 40 - .../raw/minhao-cleanup.json | 7 - .../raw/minhao-cpu.log | 12 - .../raw/minhao-cuda-parallel.log | 11 - .../raw/minhao-cuda.log | 11 - .../raw/minhao-driver-stack-wait.log | 19 - .../raw/minhao-driver-wait.log | 42 - .../raw/minhao-gpu-before.csv | 2 - .../raw/minhao-logprob-overhead.json | 2741 ----------------- .../raw/minhao-logprob-overhead.log | 40 - .../raw/minhao-seed-before.log | 14 - .../raw/minhao-slow-cleanup.json | 4 - .../raw/minhao-tiny-overhead.json | 2669 ---------------- .../training-contract-a100-20261001/README.md | 77 - .../raw/contract-cpu-final.log | 10 - .../raw/cpu-regression-final.log | 9 - .../raw/lint-final.log | 4 - .../raw/nixl-preflight-ipc.log | 51 - .../raw/nixl-preflight-shm.log | 51 - .../raw/nixl-preflight-tcp.log | 6 - .../raw/nixl-preflight.log | 12 - .../raw/pd-full-graph-matched.json | 1189 ------- .../raw/pd-full-graph-matched.log | 11 - .../raw/pd-full-graph-tcp.log | 16 - .../raw/pd-tiny-eager-tcp.json | 1160 ------- .../raw/pd-tiny-eager-tcp.log | 11 - .../raw/pd-tiny-graph-tcp.json | 1163 ------- .../raw/pd-tiny-graph-tcp.log | 11 - .../raw/runtime.json | 13 - .../training-contract-thor-20261001/README.md | 35 - .../raw/shared-full-rollout-benchmark.json | 98 - tests/__init__.py | 1 - tests/test_pd.py | 1 - tests/test_pd_weight_control.py | 91 - tests/test_speculative.py | 2 +- tests/test_training_contract.py | 238 -- 44 files changed, 1 insertion(+), 13170 deletions(-) delete mode 100644 benchmarks/logprob_overhead.py delete mode 100644 benchmarks/nixl_preflight.py delete mode 100644 benchmarks/pd_training_contract.py delete mode 100644 benchmarks/training_contract.py delete mode 100644 docs/validation/minhao-coordination-thor-20261001/README.md delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json delete mode 100644 docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json delete mode 100644 docs/validation/training-contract-a100-20261001/README.md delete mode 100644 docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/lint-final.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json delete mode 100644 docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log delete mode 100644 docs/validation/training-contract-a100-20261001/raw/runtime.json delete mode 100644 docs/validation/training-contract-thor-20261001/README.md delete mode 100644 docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json delete mode 100644 tests/__init__.py delete mode 100644 tests/test_pd_weight_control.py delete mode 100644 tests/test_training_contract.py diff --git a/benchmarks/logprob_overhead.py b/benchmarks/logprob_overhead.py deleted file mode 100644 index cdf5bdbd..00000000 --- a/benchmarks/logprob_overhead.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Alternate unprofiled score modes; report profiler operator snapshots separately.""" - -import argparse -import json -import statistics -import time -from dataclasses import replace -from pathlib import Path - -import torch -from transformers import AutoTokenizer - -from vllm_rlt import ( - LLM, - CacheConfig, - ExecutionConfig, - ExitConfig, - SamplingParams, - SchedulerConfig, - SpeculativeConfig, -) -from vllm_rlt.models import OuroForCausalLM - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("model", type=Path) - parser.add_argument("output", type=Path) - parser.add_argument("--repeats", type=int, default=5) - args = parser.parse_args() - torch.set_num_threads(4) - native = OuroForCausalLM.from_pretrained(args.model, device="cuda", dtype=torch.bfloat16) - tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False) - prompts = [tokenizer.encode(text) for text in ("Compute 17 + 25.\n", "Compute 9 * 8.\n")] - base = [ - SamplingParams(max_tokens=32, temperature=0.9, seed=31 + i, ignore_eos=True) - for i in range(2) - ] - modes = { - "none": base, - **{ - name: [replace(p, logprobs=0, logprobs_mode=name) for p in base] - for name in ("raw", "processed") - }, - } - report = {} - for path in ("sync", "async", "speculative"): - engine = LLM( - native, - attention_backend="triton", - cache_config=CacheConfig(num_blocks=512), - scheduler_config=SchedulerConfig(max_num_seqs=2), - exit_config=ExitConfig("ouro_delayed" if path == "async" else "ouro"), - execution_config=ExecutionConfig(cuda_graphs=True, async_scheduling=path == "async"), - speculative_config=SpeculativeConfig(3) if path == "speculative" else None, - ) - for params in modes.values(): - engine.generate(prompts, params) - times = {name: [] for name in modes} - memory = {name: [] for name in modes} - for index in range(args.repeats): - names = list(modes) - names = names[index % 3 :] + names[: index % 3] - outputs = {} - for name in names: - torch.cuda.synchronize() - torch.cuda.reset_peak_memory_stats() - baseline = torch.cuda.memory_allocated() - started = time.monotonic() - outputs[name] = engine.generate(prompts, modes[name]) - torch.cuda.synchronize() - times[name].append(time.monotonic() - started) - memory[name].append(torch.cuda.max_memory_allocated() - baseline) - assert all(len(o.token_ids) == 32 for o in outputs["none"]) - assert all(o.log_probs is None for o in outputs["none"]) - for name in ("raw", "processed"): - assert [o.token_ids for o in outputs[name]] == [ - o.token_ids for o in outputs["none"] - ] - assert all(len(o.log_probs) == len(o.token_ids) for o in outputs[name]) - medians = {name: statistics.median(values) for name, values in times.items()} - profiles = {} - for name in ("none", "raw", "processed"): - for detail in ("ops", "full"): - with torch.profiler.profile( - activities=[ - torch.profiler.ProfilerActivity.CPU, - torch.profiler.ProfilerActivity.CUDA, - ], - record_shapes=detail == "full", - with_stack=detail == "full", - profile_memory=detail == "full", - ) as profile: - engine.generate(prompts, modes[name]) - torch.cuda.synchronize() - events = sorted( - profile.key_averages(), key=lambda e: e.self_cpu_time_total, reverse=True - ) - profiles[f"{name}-{detail}"] = [ - { - "operator": e.key, - "count": e.count, - "self_cpu_us": e.self_cpu_time_total, - "self_device_us": e.self_device_time_total, - "self_device_memory_bytes": e.self_device_memory_usage, - } - for e in events[:20] - ] - report[path] = { - "seconds": times, - "median_seconds": medians, - "tokens_per_second": {name: 64 / value for name, value in medians.items()}, - "peak_extra_allocated_bytes": memory, - "profiles": profiles, - "tokens_unchanged": True, - } - engine.close() - print(path, medians, flush=True) - args.output.write_text(json.dumps(report, indent=2) + "\n") - del engine - torch.cuda.empty_cache() - - -if __name__ == "__main__": - main() diff --git a/benchmarks/nixl_preflight.py b/benchmarks/nixl_preflight.py deleted file mode 100644 index a56156b9..00000000 --- a/benchmarks/nixl_preflight.py +++ /dev/null @@ -1,73 +0,0 @@ -import multiprocessing as mp -import time - -import torch -from nixl._api import nixl_agent - - -def worker(device, pipe): - torch.cuda.set_device(device) - tensor = ( - torch.arange(32, dtype=torch.float32, device=f"cuda:{device}") - if device == 0 - else torch.zeros(32, device=f"cuda:{device}") - ) - torch.cuda.synchronize() - name = f"ouro-nixl-{device}" - agent = nixl_agent(name) - registration = agent.register_memory(tensor) - pipe.send( - dict(name=name, metadata=agent.get_agent_metadata(), ptr=tensor.data_ptr(), device=device) - ) - peer = pipe.recv() - agent.add_remote_agent(peer["metadata"]) - pipe.send("ready") - pipe.recv() - if device == 0: - handle = agent.initialize_xfer( - "WRITE", - agent.get_xfer_descs(tensor), - agent.get_xfer_descs([(peer["ptr"], 128, peer["device"])], "VRAM"), - peer["name"], - notif_msg=b"done", - backends=["UCX"], - ) - agent.transfer(handle) - while agent.check_xfer_state(handle) != "DONE": - time.sleep(0.001) - agent.release_xfer_handle(handle) - else: - while not agent.get_new_notifs(): - time.sleep(0.001) - pipe.send(tensor.cpu().tolist()) - pipe.recv() - agent.remove_remote_agent(peer["name"]) - agent.deregister_memory(registration) - pipe.close() - - -if __name__ == "__main__": - context = mp.get_context("spawn") - peers = [] - for device in range(2): - parent, child = context.Pipe() - process = context.Process(target=worker, args=(device, child)) - process.start() - child.close() - peers.append((process, parent)) - infos = [p.recv() for _, p in peers] - for i, (_, p) in enumerate(peers): - p.send(infos[1 - i]) - assert all(p.recv() == "ready" for _, p in peers) - for _, p in peers: - p.send("go") - values = [p.recv() for _, p in peers] - print("VALUES", values, flush=True) - for _, p in peers: - p.send("stop") - for process, p in peers: - process.join() - p.close() - assert process.exitcode == 0 - assert values[0] == values[1] - print("MULTIPROCESS_NIXL_PASSED", flush=True) diff --git a/benchmarks/pd_training_contract.py b/benchmarks/pd_training_contract.py deleted file mode 100644 index 38830d64..00000000 --- a/benchmarks/pd_training_contract.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Two spawned CUDA owners: NIXL KV, policy publication, and local-engine parity.""" - -import argparse -import json -import time -from dataclasses import asdict, replace -from pathlib import Path - -import torch - -from vllm_rlt import LLM, CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig -from vllm_rlt.models import OuroConfig, OuroForCausalLM -from vllm_rlt.pd.config import PDConfig -from vllm_rlt.pd.engine import PDEngine - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("output", type=Path) - parser.add_argument("--model", type=Path) - parser.add_argument("--cuda-graphs", action="store_true") - parser.add_argument("--temperature", type=float, default=0.9) - parser.add_argument("--max-num-seqs", type=int, default=4) - args = parser.parse_args() - torch.set_num_threads(4) - torch.manual_seed(42) - dtype = torch.bfloat16 if args.model else torch.float32 - source = args.model or OuroConfig( - vocab_size=64, - hidden_size=32, - intermediate_size=64, - num_hidden_layers=2, - num_attention_heads=4, - num_key_value_heads=2, - head_dim=8, - max_position_embeddings=128, - total_ut_steps=4, - bos_token_id=0, - eos_token_id=0, - ) - native = ( - OuroForCausalLM.from_pretrained(source, device="cuda:0", dtype=dtype) - if args.model - else OuroForCausalLM(source).to(device="cuda:0", dtype=dtype) - ) - execution = ExecutionConfig(cuda_graphs=args.cuda_graphs, async_scheduling=True) - cache = CacheConfig(num_blocks=512, enable_prefix_caching=True) - scheduler = SchedulerConfig( - max_num_seqs=args.max_num_seqs, max_num_batched_tokens=64, prefill_chunk_size=16 - ) - common = dict( - execution_config=execution, - exit_config=ExitConfig("ouro_delayed"), - attention_backend="triton", - ) - local = LLM(native, cache_config=cache, scheduler_config=scheduler, **common) - pd = PDEngine( - source, - dtype="bfloat16" if args.model else "float32", - pd_config=PDConfig( - max_control_messages=1, startup_timeout=600, request_timeout=600, shutdown_timeout=120 - ), - prefill_cache_config=cache, - decode_cache_config=cache, - prefill_scheduler_config=scheduler, - decode_scheduler_config=scheduler, - terminate_workers_on_failure=False, - **common, - ) - disaggregated = LLM(pd) - prompts = [[3, 4, 5] * 9 + [6], [3, 4, 5] * 9 + [7]] - report = dict(model=asdict(native.config), cuda_graphs=args.cuda_graphs, iterations=[]) - report["max_num_seqs"] = args.max_num_seqs - try: - for version, loops in enumerate((2, 3, 4), 1): - with torch.no_grad(): - native.lm_head.weight.mul_(0.98) - for arm in (local, disaggregated): - arm.start_weight_update(version) - weights = list(native.named_parameters()) - for offset in range(0, len(weights), 8): - arm.update_weights(weights[offset : offset + 8]) - arm.finish_weight_update() - assert arm.get_weight_version() == version - for mode in ("raw", "processed"): - params = [ - SamplingParams( - max_tokens=6, - max_loops=loops, - seed=11 + i, - temperature=args.temperature, - top_k=8, - top_p=0.8, - ignore_eos=True, - logprobs=0, - logprobs_mode=mode, - ) - for i in range(len(prompts)) - ] - outputs, timings = {}, {} - for label, arm in (("local", local), ("pd", disaggregated)): - started = time.monotonic() - outputs[label] = arm.generate(prompts, params) - timings[label] = time.monotonic() - started - errors = [] - for expected, result in zip(outputs["local"], outputs["pd"], strict=True): - assert expected.token_ids == result.token_ids - assert expected.exit_depths == result.exit_depths - assert result.weight_version == version - assert result.sampling_params == expected.sampling_params - delta = ( - torch.tensor(expected.log_probs) - torch.tensor(result.log_probs) - ).abs() - errors.append(delta.max().item()) - torch.testing.assert_close( - torch.tensor(expected.log_probs), - torch.tensor(result.log_probs), - atol=2e-5, - rtol=2e-5, - ) - row = dict( - version=version, - loops=loops, - mode=mode, - seconds=timings, - max_logprob_error=max(errors), - outputs=[asdict(o) for o in outputs["pd"]], - ) - report["iterations"].append(row) - print("PARITY", version, loops, mode, timings, max(errors), flush=True) - # A failed chunk cannot expose a mixed policy; a newer full epoch recovers. - disaggregated.start_weight_update(4) - try: - disaggregated.update_weights([("missing_parameter", torch.zeros(1))]) - except RuntimeError as error: - report["rejected_chunk"] = str(error) - else: - raise AssertionError("Invalid publication unexpectedly succeeded") - disaggregated.start_weight_update(5) - disaggregated.update_weights(native.named_parameters()) - disaggregated.finish_weight_update() - recovered = disaggregated.generate(prompts[:1], params[:1])[0] - assert recovered.weight_version == 5 - stopped = disaggregated.generate( - prompts[:1], [replace(params[0], stop_token_ids=(recovered.token_ids[0],))] - )[0] - assert len(stopped.token_ids) == len(stopped.log_probs) == 1 - assert stopped.finish_reason == "stop" - pd.add_request("abort", prompts[0], params[0]) - while not pd.transfers[next(iter(pd.transfers))].latest_output: - pd.step() - aborted = pd.abort_request("abort") - assert aborted.finish_reason == "abort" - assert len(aborted.log_probs) == len(aborted.token_ids) - while pd.has_unfinished_requests(): - pd.step() - disaggregated.reset_prefix_cache() - report.update(recovered_version=5, stop=asdict(stopped), abort=asdict(aborted)) - finally: - disaggregated.close() - local.close() - report["workers"] = pd.worker_metrics - assert all( - m["used_blocks"] == 0 and m["weight_version"] == 5 for m in pd.worker_metrics.values() - ) - assert sum(m["bytes_sent"] for m in pd.worker_metrics.values()) > 0 - args.output.write_text(json.dumps(report, indent=2)) - print("PD_CONTRACT_PASSED", args.output, flush=True) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/training_contract.py b/benchmarks/training_contract.py deleted file mode 100644 index 0bd6fbe0..00000000 --- a/benchmarks/training_contract.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Alternating-order warmed rollout timings, including returned log probabilities.""" - -import argparse -import json -import statistics -import time -from pathlib import Path - -import torch -from transformers import AutoTokenizer - -from vllm_rlt import ( - LLM, - CacheConfig, - ExecutionConfig, - SamplingParams, - SchedulerConfig, - SpeculativeConfig, -) -from vllm_rlt.models import OuroForCausalLM - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("model", type=Path) - parser.add_argument("output", type=Path) - parser.add_argument("--repeats", type=int, default=5) - args = parser.parse_args() - torch.set_num_threads(4) - tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=False) - prompts = [tokenizer.encode(text) for text in ("Compute 17 + 25.\n", "Compute 9 * 8.\n")] - native = OuroForCausalLM.from_pretrained(args.model, device="cuda", dtype=torch.bfloat16) - report = {} - for speculative in (False, True): - arms = { - mode: LLM( - native, - attention_backend="triton", - cache_config=CacheConfig(num_blocks=512), - scheduler_config=SchedulerConfig(max_num_seqs=8, max_num_batched_tokens=128), - execution_config=ExecutionConfig(cuda_graphs=mode == "graph"), - speculative_config=SpeculativeConfig(3) if speculative else None, - ) - for mode in ("eager", "graph") - } - for loops in (4,) if speculative else (2, 3, 4): - params = [ - SamplingParams( - max_tokens=32, - max_loops=loops, - temperature=0.9, - seed=31 + i, - logprobs=0, - ignore_eos=True, - ) - for i in range(len(prompts)) - ] - for arm in arms.values(): - arm.generate(prompts, params) - arm.start_weight_update() - arm.update_weights(native.named_parameters()) - arm.finish_weight_update() - version = arms["eager"].get_weight_version() - times = {mode: [] for mode in arms} - for index in range(args.repeats): - outputs = {} - for mode in ("eager", "graph") if index % 2 == 0 else ("graph", "eager"): - torch.cuda.synchronize() - started = time.monotonic() - outputs[mode] = arms[mode].generate(prompts, params) - torch.cuda.synchronize() - times[mode].append(time.monotonic() - started) - for eager, graph in zip(outputs["eager"], outputs["graph"], strict=True): - assert eager.token_ids == graph.token_ids - assert eager.exit_depths == graph.exit_depths - assert eager.log_probs == graph.log_probs - assert eager.weight_version == graph.weight_version == version - medians = {mode: statistics.median(values) for mode, values in times.items()} - name = f"speculative-{loops}" if speculative else f"native-{loops}" - report[name] = { - "seconds": times, - "medians": medians, - "speedup": medians["eager"] / medians["graph"], - "tokens_and_logprobs_exact": True, - } - print(name, report[name], flush=True) - for arm in arms.values(): - arm.close() - del arms - args.output.write_text(json.dumps(report, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/docs/training_contract.md b/docs/training_contract.md index fa1d2b84..e5a73cfd 100644 --- a/docs/training_contract.md +++ b/docs/training_contract.md @@ -55,31 +55,3 @@ upstream self-speculation uses a full-depth target and synchronous execution; upstream PD does not implement self-speculation. Publication adds no such restriction to ordinary early-exit or async rollouts. Trainers must replay full-depth prompt prefill and the returned decode depths rather than assume the entire sequence used K. - -## Validation - -```bash -python -m unittest tests.test_training_contract tests.test_pd_weight_control -v -CONTRACT_DEVICE=cuda python -m unittest tests.test_training_contract -v -python benchmarks/training_contract.py /models/Ouro-1.4B timings.json -UCX_TLS=tcp,cuda_copy python benchmarks/nixl_preflight.py -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py pd.json --cuda-graphs -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py full-pd.json \ - --model /models/Ouro-1.4B --cuda-graphs --max-num-seqs 1 -``` - -CPU and Thor CUDA tests cover raw/processed probabilities against an independent -dense oracle, speculative correction/bonus alignment, publication, pointer stability, -prefix invalidation, early exit, async stop/abort and entropy seeds. The PD control -test drives two real CPU engines and checks recovery after one peer commits and -another fails. It uses in-process control delivery; it does **not** validate IPC or -NIXL transport. The separate two-A100 run uses real spawned workers and NIXL: -FP32 multi-request and full BF16 matched-batch checks cover three publications, -raw/processed scores, graph replay, prefix reuse, explicit stop/abort and recovery -to version five. See [PD evidence](validation/training-contract-a100-20261001/README.md). -No PD workers were launched on single-GPU Thor, where NIXL is absent. - -This is a draft implementation of the -[Phase 1 contract proposed in issue 70](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5888101273) -on upstream `ecb1f8b505b7e831815b40aec3b4598619cca23a`; formal upstream agreement -remains outside the scope of this contributor draft. diff --git a/docs/validation/minhao-coordination-thor-20261001/README.md b/docs/validation/minhao-coordination-thor-20261001/README.md deleted file mode 100644 index 59c4dce1..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/README.md +++ /dev/null @@ -1,134 +0,0 @@ -# Phase 1 / VIME coordination checks — 2026-10-01 - -These checks exercise the semantics in -[MinhaoLi0318's PR 1 proposal](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5927860064) -on the existing Thor runtime. They test the contributor implementation, not an -unpublished Phase 1 branch. The -[coordination reply](https://github.com/ThinkFlowLab/vllm-rlt/issues/70#issuecomment-5930413915) -records the interface differences and division of upstream work. - -The replay requirement exposed a real omission: an entropy-seeded request -returned `sampling_params.seed=None`. The -[unfixed regression](raw/minhao-seed-before.log) fails on that missing effective -seed. `Request` now resolves an independent 63-bit seed when it is created and -retains it through output, preemption, abort and existing PD parameter transport. -Default `seed=0` is unchanged. An abort before any sampling also reports the seed; -the caller's immutable `SamplingParams(seed=None)` is not modified. - -| Thor CPU check | Result | -|---|---| -| Entropy seed reporting and exact token/score/depth replay | Ordinary, async and speculative paths pass | -| Abort before sampling | Effective seed retained; zero selected scores | -| Preemption | Token IDs, selected scores, depths and RNG continuity match uninterrupted execution | -| Existing probability/publication/cache/stop/abort checks | All pass | -| Full focused suite including in-process PD publication | **7 tests pass, 1.736 s** | - -See [CPU log](raw/minhao-cpu.log). Independent VIME FP32 mixed-depth replay, -recomputed gradients and generic physical-parameter conversion are checked -separately. No runtime packages are installed or updated. - -The user authorized concurrent short CUDA validation while an existing -Qwen-Image training reservation remained active. **All six CUDA contract tests -pass in 135.474 s**, including entropy-seed replay with graphs, stop/abort, -preemption, speculative target scores and publication recovery; see -[CUDA log](raw/minhao-cuda-parallel.log). Startup is included in the suite time. The originally queued CUDA suite also -completed naturally after the reservation was released: **6/6 in 3.989 s**, -[log](raw/minhao-cuda.log). -Independent VIME FP32 mixed-depth replay/recomputed gradients and generic -physical-parameter conversion pass **two CPU tests in 0.302 s**. No existing -training process or reservation was interrupted. - -Earlier full-model training/save/resume and real-PD evidence is retained in the -[Thor](../training-contract-thor-20261001/README.md) and -[A100](../training-contract-a100-20261001/README.md) reports. - -## Selected-score cost - -`benchmarks/logprob_overhead.py` warms each mode and rotates the measurement -order over five repeats. Each unprofiled window generates two prompts with 32 -response tokens each, T=0.9, fixed per-request seeds, no filtering and CUDA -graphs enabled. Sync and speculative paths use Ouro exit; async uses delayed -Ouro exit. Token sequences must match across score modes; selected scores must -align with every committed token. Initialization and capture are excluded. - -Separately, all three modes receive an ops-only and a full-detail CPU/CUDA -profiler window (`record_shapes`, stacks and memory enabled for full detail). -The raw JSON retains the 20 operators with largest self CPU time per window, -with counts, self device time and memory. These are operator summaries, not -exported timeline traces. Profiled runs are excluded from throughput medians. -The [tiny smoke JSON](raw/minhao-tiny-overhead.json) checks all three paths and -18 profiler windows with one repeat; its timings are not full-model results. - -### Full Ouro-1.4B shared-GPU observations - -The full run overlaps another Qwen-Image training phase. GPU utilization -before this run was **98%**; [snapshot](raw/minhao-gpu-before.csv). -These measurements include resource contention and are not isolated throughput -results. Each mode has five unprofiled repeats. All three paths complete; -token IDs match across modes and selected scores align with committed tokens. - -| Path | None median (s) | Raw median (s) | Processed median (s) | Raw time overhead | Processed time overhead | -|---|---:|---:|---:|---:|---:| -| sync | 1.8017 | 1.8280 | 1.8281 | +1.46% | +1.47% | -| async | 1.6974 | 1.7849 | 1.7894 | +5.15% | +5.42% | -| speculative | 1.4366 | 1.4462 | 1.4486 | +0.67% | +0.83% | - -| Path / score mode | Tokens/s | Speed vs None | Median extra PyTorch allocation (KiB) | -|---|---:|---:|---:| -| sync / none | 35.52 | 1.000× | 933.0 | -| sync / raw | 35.01 | 0.986× | 965.0 | -| sync / processed | 35.01 | 0.986× | 933.0 | -| async / none | 37.70 | 1.000× | 965.0 | -| async / raw | 35.86 | 0.951× | 963.5 | -| async / processed | 35.77 | 0.949× | 965.0 | -| speculative / none | 44.55 | 1.000× | 2733.5 | -| speculative / raw | 44.25 | 0.993× | 2793.5 | -| speculative / processed | 44.18 | 0.992× | 2761.5 | - -[Raw timings / all 18 profiler operator summaries](raw/minhao-logprob-overhead.json), -[benchmark log](raw/minhao-logprob-overhead.log). Extra allocation is -per-window PyTorch peak above its starting allocation; it excludes resident -model/cache/graph storage and other processes. - -Between completed paths, the process temporarily waited in NVIDIA UVM external -allocation mapping (`os_acquire_rwlock_read/write`); an independent GPU status -query also waited. The [captured stack](raw/minhao-driver-stack-wait.log) and -[kernel log](raw/minhao-driver-wait.log) record concurrent `NV_ERR_NO_MEMORY` -allocation errors. Both waits resolved naturally; all paths completed without -process termination, driver reset or environment changes. Initialization waits -are outside the warmed measurement windows. The harness now drops the previous -engine and clears its allocator cache between paths to avoid retaining that -engine while constructing the next one. The recorded full run predates this -inter-path cleanup; its timed body is unchanged. -The updated cleanup harness then passes a separate tiny-model smoke across -all three paths and 18 profiler windows ([JSON](raw/minhao-cleanup-smoke.json), -[log](raw/minhao-cleanup-smoke.log)); its [57,039,421-byte model is removed](raw/minhao-cleanup-smoke-removed.json). - -All five full-model depth/effective-seed replay cases also complete, including -delayed async. The [VIME score-error table](https://github.com/0z5a/vime/blob/feat/ouro-rollout-cuda-graphs/docs/validation/minhao-coordination-thor-20261001/README.md) -records the remaining training/inference recomputation differences; full-depth -K=4 also has a nonzero residual. Exact seed replay and score-recomputation -agreement are separate checks. - -The [completed full-model copy and replacement shard](raw/minhao-cleanup.json) -are removed (3,232,344,846 bytes). The 358,667,054-byte local transfer copy -was also removed. The original slower download completed naturally; its -[2,873,677,792-byte model copy was removed](raw/minhao-slow-cleanup.json) by the -existing queue, without repeating GPU tests. All four task-owned model folders -and both replacement transfer copies are absent after validation. - -## Interface coordination - -| Proposed Phase 1 interface | Current reference draft | VIME consumer plan | -|---|---|---| -| `RequestOutput.logprobs` | `RequestOutput.log_probs` | Use the upstream spelling once its PR 1 branch is available | -| Engine-level `raw_logprobs` / `processed_logprobs` | Request-level `raw` / `processed` | Configure processed scores for the trainer; avoid a second compatibility API | -| Effective seed reported in resolved parameters | Fixed by this follow-up | Keep seeded reproduction and opt-in entropy groups | -| Full-depth first token and executed decode depths | Already present | Full four-loop prefill and differentiable `last_exited` trace replay | -| PR 2 versioned full publication | Validated reference implementation | Reuse one agreed upstream contract | - -The old `RLEngine.set_loop_budget` mutated `total_ut_steps`; the current VIME -adapter supersedes it and does not need reduced-depth prefill. No new transport -layer is needed for the pilot: the consumer is in-process `LLM`, and existing PD -worker IPC remains inside the engine. MinhaoLi0318 and IDEA-V retain the proposed -Phase 1 upstream roles; this draft supplies reusable implementation and checks. diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json deleted file mode 100644 index 1f9931bd..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke-removed.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "removed": "models/Ouro-minhao-cleanup-smoke", - "bytes": 57039421 -} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json deleted file mode 100644 index ed52a20b..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.json +++ /dev/null @@ -1,2669 +0,0 @@ -{ - "sync": { - "seconds": { - "none": [ - 0.08066848499993284 - ], - "raw": [ - 0.0914349139998194 - ], - "processed": [ - 0.08466345700026068 - ] - }, - "median_seconds": { - "none": 0.08066848499993284, - "raw": 0.0914349139998194, - "processed": 0.08466345700026068 - }, - "tokens_per_second": { - "none": 793.3705461315318, - "raw": 699.9514430573688, - "processed": 755.9341688563809 - }, - "peak_extra_allocated_bytes": { - "none": [ - 758784 - ], - "raw": [ - 823808 - ], - "processed": [ - 791040 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 24741.71000000011, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 14785.07000000005, - "self_device_us": 6.143999999963853, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2004, - "self_cpu_us": 10337.619999999994, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2020, - "self_cpu_us": 6443.9159999996045, - "self_device_us": 2353.745999999865, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1348, - "self_cpu_us": 4180.128999999671, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Activity Buffer Request", - "count": 3, - "self_cpu_us": 3175.8589999999667, - "self_device_us": 15.329000000011547, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 1981.3640000000255, - "self_device_us": 966.4389999999644, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 208, - "self_cpu_us": 1895.6409999999596, - "self_device_us": 293.95500000013635, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1556, - "self_cpu_us": 1581.7819999997378, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 716, - "self_cpu_us": 1457.2760000001927, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1335.993999999857, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1012, - "self_cpu_us": 1159.8320000000067, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 962.9140000004063, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 879.5000000000755, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 867.1529999999693, - "self_device_us": 227.0119999999879, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 830.8970000000882, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 824.2710000000934, - "self_device_us": 553.6069999999727, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2415, - "self_cpu_us": 754.1939999999772, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 744.7460000000592, - "self_device_us": 198.56199999995624, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 116, - "self_cpu_us": 645.1380000000013, - "self_device_us": 236.03399999999692, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 51961.9490000005, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 16149.892999999873, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2004, - "self_cpu_us": 10752.365000000542, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9578.167000000489, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 7612.398999999911, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2020, - "self_cpu_us": 7304.089999999687, - "self_device_us": 5244.179000000046, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1348, - "self_cpu_us": 4891.489000000262, - "self_device_us": 0, - "self_device_memory_bytes": 26818560 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 4097.280999999177, - "self_device_us": 0.0, - "self_device_memory_bytes": 162304 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 3075.5700000002166, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1556, - "self_cpu_us": 3032.1670000008203, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 716, - "self_cpu_us": 2489.1329999999634, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 2429.843999999812, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2326.7890000000816, - "self_device_us": 1660.2469999999175, - "self_device_memory_bytes": 6881280 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 2264.487000000183, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "aten::cat", - "count": 208, - "self_cpu_us": 2253.055999999902, - "self_device_us": 608.0070000000824, - "self_device_memory_bytes": 305152 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 2143.693999999712, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 2037.7939999994514, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1728.5120000003517, - "self_device_us": 0.0, - "self_device_memory_bytes": -12845056 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 1563.099999999995, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 1466.244999999848, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 119556.1379999991, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 23820.948999999957, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2194, - "self_cpu_us": 14371.875999999655, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2212, - "self_cpu_us": 11208.853000000732, - "self_device_us": 9572.278999999877, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 9940.423999999859, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1540, - "self_cpu_us": 7332.918000000681, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 3733.2030000000573, - "self_device_us": 1393.6519999997668, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 2742.5369999992313, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 2586.1810000003697, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2453.102000000058, - "self_device_us": 2902.2410000000755, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2360.366000000333, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2117.0190000001458, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1898.4860000001345, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 624, - "self_cpu_us": 1705.4810000000034, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3537, - "self_cpu_us": 1688.5239999999105, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1140, - "self_cpu_us": 1436.9140000001375, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1312.8229999998948, - "self_device_us": 794.6019999998643, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1280.3379999999306, - "self_device_us": 2046.0730000000694, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1207.4139999993881, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 1164.5660000002226, - "self_device_us": 719.3059999997731, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 115092.6299999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 29600.60599999951, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 25946.991000000064, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 17345.223999999165, - "self_device_us": 0.0, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cudaLaunchKernel", - "count": 2194, - "self_cpu_us": 15940.768999999744, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 13333.437000000138, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2212, - "self_cpu_us": 13191.647000002034, - "self_device_us": 8704.526000000209, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1540, - "self_cpu_us": 8662.159000000243, - "self_device_us": 0, - "self_device_memory_bytes": 39434240 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 7366.031999998762, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 4815.7890000007, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 4814.504000000853, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 4703.688000000431, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 4365.908999999834, - "self_device_us": 1169.2249999997575, - "self_device_memory_bytes": 337920 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 4116.904000000264, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 4072.4490000006735, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 3742.0259999994387, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 3527.933999998534, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 3314.13099999951, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 624, - "self_cpu_us": 3090.420999999369, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 3038.0740000001883, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 105856.23600000067, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 23095.000000000022, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2322, - "self_cpu_us": 15003.759999999633, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2148, - "self_cpu_us": 10396.17099999906, - "self_device_us": 9188.07700000014, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 8986.292999999905, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1476, - "self_cpu_us": 6711.9949999997025, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 3563.392000000484, - "self_device_us": 1348.502999999796, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 2623.3650000003126, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2498.1059999999925, - "self_device_us": 2908.510000000058, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 2357.0779999990714, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2048.535, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2041.7819999998937, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3537, - "self_cpu_us": 1853.5020000003638, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1701.4389999999712, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 192, - "self_cpu_us": 1573.1770000001234, - "self_device_us": 1020.2990000001446, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 1523.2590000001437, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1140, - "self_cpu_us": 1365.1570000002835, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1229.6910000001153, - "self_device_us": 799.3670000000275, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1220.0819999998166, - "self_device_us": 2031.6709999999912, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1216.561000000036, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 155410.87200000056, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 26514.28400000018, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 22682.278999999973, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2322, - "self_cpu_us": 17120.490999999194, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 16900.313999997426, - "self_device_us": 0.0, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 13033.013999998899, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2148, - "self_cpu_us": 12944.766999999709, - "self_device_us": 12005.580999999922, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1476, - "self_cpu_us": 8355.110999999586, - "self_device_us": 0, - "self_device_memory_bytes": 39401472 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 7274.417999999696, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 4893.769999999835, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 4797.883999998665, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 4710.996000000819, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 4480.034000000242, - "self_device_us": 1730.5530000000113, - "self_device_memory_bytes": 337920 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 4113.781999999035, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 3937.3690000012284, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 3691.073999999741, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 3555.606999998805, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 3436.052999998901, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 3407.9150000012155, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2940.6499999998923, - "self_device_us": 0.0, - "self_device_memory_bytes": -12845056 - } - ] - }, - "tokens_unchanged": true - }, - "async": { - "seconds": { - "none": [ - 0.1108772420002424 - ], - "raw": [ - 0.1287477310002032 - ], - "processed": [ - 0.12239944100019784 - ] - }, - "median_seconds": { - "none": 0.1108772420002424, - "raw": 0.1287477310002032, - "processed": 0.12239944100019784 - }, - "tokens_per_second": { - "none": 577.214934692009, - "raw": 497.0961391148632, - "processed": 522.8782049739635 - }, - "peak_extra_allocated_bytes": { - "none": [ - 790016 - ], - "raw": [ - 822272 - ], - "processed": [ - 822272 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 15284.012000000086, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 1850, - "self_cpu_us": 8500.083000000152, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 6639.298000000122, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1439, - "self_cpu_us": 5531.486999999959, - "self_device_us": 1906.9919999998415, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 4616.57799999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2769, - "self_cpu_us": 4578.959000000186, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 601, - "self_cpu_us": 2438.2479999999, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2059.4019999999846, - "self_device_us": 778.1539999999927, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3851, - "self_cpu_us": 1576.196000000137, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 629, - "self_cpu_us": 1306.6099999999087, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1285.6129999997502, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 1154.325999999974, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 996.3809999998794, - "self_device_us": 225.15500000009706, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 798.8650000000216, - "self_device_us": 569.2259999999969, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 751.5409999999792, - "self_device_us": 210.21199999991222, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::record_stream", - "count": 436, - "self_cpu_us": 730.9929999999513, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 116, - "self_cpu_us": 650.3899999999953, - "self_device_us": 229.37999999999693, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventQuery", - "count": 513, - "self_cpu_us": 642.4939999999581, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventRecordWithFlags", - "count": 698, - "self_cpu_us": 624.0679999998281, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 281, - "self_cpu_us": 616.0749999999405, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 16433.368999999864, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10018.040999999977, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9557.15300000049, - "self_device_us": 0.0, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cudaLaunchKernel", - "count": 1850, - "self_cpu_us": 9201.422999999402, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8294.379000000605, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 7561.7619999999515, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2769, - "self_cpu_us": 7053.931000000085, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6636.100999999369, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1439, - "self_cpu_us": 6178.882000000486, - "self_device_us": 6708.658999999896, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5287.671999999828, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 4708.536999999357, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4653.8080000003965, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4520.836999999436, - "self_device_us": 0.0, - "self_device_memory_bytes": -79872 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4280.746000000392, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 3811.028999999038, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3031.376999999363, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 601, - "self_cpu_us": 2754.339000000004, - "self_device_us": 0, - "self_device_memory_bytes": 26515968 - }, - { - "operator": "", - "count": 904, - "self_cpu_us": 2668.428999999229, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/__init__.py(1235): current_stream", - "count": 1126, - "self_cpu_us": 2653.2650000008134, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/_utils.py(869): _get_device_index", - "count": 1504, - "self_cpu_us": 2575.899000002003, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 42234.66600000008, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 17010.12299999994, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 11070.726999999686, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2040, - "self_cpu_us": 10325.780999999826, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1631, - "self_cpu_us": 6525.653000000375, - "self_device_us": 8328.703999999532, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 4947.713000000557, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 4864.119999999892, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 793, - "self_cpu_us": 3114.437000000318, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2199.381999999973, - "self_device_us": 3114.2829999999713, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2073.5459999992754, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1458.7159999996475, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 693, - "self_cpu_us": 1455.3750000004352, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 116, - "self_cpu_us": 1195.895999999781, - "self_device_us": 692.2980000000898, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1053.418000000198, - "self_device_us": 860.6630000000077, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 1023.9610000000209, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1004.951999999932, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 409, - "self_cpu_us": 958.3949999998126, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 96, - "self_cpu_us": 888.4549999998671, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 859.6940000000086, - "self_device_us": 2180.226000000088, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 837.9099999999289, - "self_device_us": 781.5500000001175, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 22104.463000000447, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 17926.967000000168, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 14532.418000000032, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2040, - "self_cpu_us": 10800.322000000502, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10283.571000000464, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9719.242999999962, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8455.242999999638, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 7985.128999999445, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1631, - "self_cpu_us": 7096.114999998659, - "self_device_us": 7530.1170000006005, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6655.402000001546, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5625.204000000713, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 4933.312999999274, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4771.646000000193, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4725.08799999866, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4402.655000000255, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 3951.7359999998625, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "aten::empty_strided", - "count": 793, - "self_cpu_us": 3523.918000000247, - "self_device_us": 0, - "self_device_memory_bytes": 39131648 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3151.4589999996992, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2998.801000000943, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 904, - "self_cpu_us": 2673.5620000006493, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 43409.94599999997, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 17950.804999999928, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 15988.948999999806, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2168, - "self_cpu_us": 11112.410000000078, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1567, - "self_cpu_us": 6483.044000000588, - "self_device_us": 8276.219999999763, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 4952.348999999742, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 4921.737000000263, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 729, - "self_cpu_us": 2952.603999999926, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2189.6210000000665, - "self_device_us": 3186.7999999999442, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2120.180999999159, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 693, - "self_cpu_us": 1494.2490000004798, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1457.0889999997416, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 116, - "self_cpu_us": 1239.0429999999242, - "self_device_us": 708.8769999999126, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 192, - "self_cpu_us": 1220.1639999996369, - "self_device_us": 1123.057999999999, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1135.0230000010429, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 1087.3439999998745, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1070.1700000001292, - "self_device_us": 874.7320000000436, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 409, - "self_cpu_us": 998.1089999999881, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 96, - "self_cpu_us": 941.3419999995094, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 902.430999999955, - "self_device_us": 2233.6359999999077, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 23466.177000000756, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 18052.96899999999, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 12057.866999999911, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2168, - "self_cpu_us": 11316.609999999708, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10142.197000000597, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9814.517999998712, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8444.050000000905, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 7757.864000000339, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1567, - "self_cpu_us": 6859.583000000366, - "self_device_us": 7261.939000000045, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6667.205999998861, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5536.804000000327, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 4946.131999999503, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4856.363000000383, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4753.402000000731, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4567.222000000331, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 3955.984000000677, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "aten::empty_strided", - "count": 729, - "self_cpu_us": 3271.6339999996685, - "self_device_us": 0, - "self_device_memory_bytes": 39098880 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3136.034000000277, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 3133.5739999998086, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 904, - "self_cpu_us": 2658.5939999993807, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - }, - "speculative": { - "seconds": { - "none": [ - 0.06240307000007306 - ], - "raw": [ - 0.0662938270002087 - ], - "processed": [ - 0.06682750300024054 - ] - }, - "median_seconds": { - "none": 0.06240307000007306, - "raw": 0.0662938270002087, - "processed": 0.06682750300024054 - }, - "tokens_per_second": { - "none": 1025.590567898744, - "raw": 965.3990860988991, - "processed": 957.6895309072021 - }, - "peak_extra_allocated_bytes": { - "none": [ - 2750976 - ], - "raw": [ - 2790912 - ], - "processed": [ - 2774528 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 14082.55900000005, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2245, - "self_cpu_us": 9969.27999999969, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1128, - "self_cpu_us": 8384.83099999996, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1277, - "self_cpu_us": 3738.911000000112, - "self_device_us": 1777.0809999999603, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 143, - "self_cpu_us": 2681.666000000013, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 592, - "self_cpu_us": 2131.4490000001315, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1464.813999999862, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1219, - "self_cpu_us": 1242.6320000000528, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1164.5480000000407, - "self_device_us": 348.58399999995163, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 977.9220000000182, - "self_device_us": 302.44099999999275, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 969.8469999999941, - "self_device_us": 148.32200000000057, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 893.5579999999991, - "self_device_us": 672.4589999999498, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 814.1800000000003, - "self_device_us": 175.17400000007547, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 797.685999999996, - "self_device_us": 602.407000000032, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2113, - "self_cpu_us": 754.8929999999518, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 77, - "self_cpu_us": 751.3600000000063, - "self_device_us": 112.54500000000348, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 743.7420000000911, - "self_device_us": 1537.3079999999263, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 742.4579999999497, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 348, - "self_cpu_us": 714.7729999999433, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_softmax", - "count": 119, - "self_cpu_us": 704.4650000000765, - "self_device_us": 1838.0370000000748, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaStreamSynchronize", - "count": 143, - "self_cpu_us": 19864.37699999997, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 14970.81799999996, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2245, - "self_cpu_us": 10701.993000000055, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1128, - "self_cpu_us": 10585.129000000112, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9116.944000000163, - "self_device_us": 0.0, - "self_device_memory_bytes": -1536 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6067.3340000002245, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 4490.072999999706, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1277, - "self_cpu_us": 4200.721000000049, - "self_device_us": 2472.8799999996913, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 592, - "self_cpu_us": 2475.1309999999894, - "self_device_us": 0, - "self_device_memory_bytes": 52925952 - }, - { - "operator": "aten::slice", - "count": 1219, - "self_cpu_us": 2209.2680000001774, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2199.3709999999483, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1887.626000000164, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1604.1319999999032, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1586.2180000000506, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1252.9259999999376, - "self_device_us": 522.2780000000967, - "self_device_memory_bytes": 25165824 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1165.056000000348, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - }, - { - "operator": "aten::_to_copy", - "count": 348, - "self_cpu_us": 1149.622000000063, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1149.0220000000054, - "self_device_us": 234.2420000000002, - "self_device_memory_bytes": 786432 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1108.9360000000352, - "self_device_us": 466.69400000002315, - "self_device_memory_bytes": 1404416 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 1083.3829999999834, - "self_device_us": 1011.7250000000568, - "self_device_memory_bytes": 79872 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1219, - "self_cpu_us": 32065.39100000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 15680.841000000019, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2394, - "self_cpu_us": 11380.852999999855, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 4496.739999999833, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1428, - "self_cpu_us": 4368.5809999999665, - "self_device_us": 4428.808999999856, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 743, - "self_cpu_us": 2724.119000000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1590.5000000001273, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 1249.5310000003237, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1218.5759999999027, - "self_device_us": 777.1980000000331, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 100, - "self_cpu_us": 1051.9999999999136, - "self_device_us": 428.0050000000215, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1017.5759999999709, - "self_device_us": 1260.8519999999708, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1012.5349999999967, - "self_device_us": 733.8019999999997, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 959.7169999995767, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 930.4999999999745, - "self_device_us": 1327.9919999999292, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 913.5249999998941, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 857.5730000000585, - "self_device_us": 328.99800000001414, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 831.2279999999759, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 829.5409999999483, - "self_device_us": 1179.376999999913, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 778.833999999988, - "self_device_us": 2838.9859999999917, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 736.6149999998129, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1219, - "self_cpu_us": 17222.49800000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 15855.267000000127, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 11819.789999999986, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2394, - "self_cpu_us": 11519.752999999928, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9310.95399999983, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6072.020999999813, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1428, - "self_cpu_us": 4727.487999999833, - "self_device_us": 4655.986000000061, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 4696.012000000468, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::empty_strided", - "count": 743, - "self_cpu_us": 3005.7680000001283, - "self_device_us": 0, - "self_device_memory_bytes": 65541632 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 2301.55100000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2113.589000000433, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1948.2030000000268, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 1625.992999999985, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1563.8129999999483, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1353.5050000000588, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1338.518000000211, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1312.731999999898, - "self_device_us": 807.4049999999152, - "self_device_memory_bytes": 25165824 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 1273.8249999999016, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1184.5459999999466, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1142.3969999999777, - "self_device_us": 341.6040000000071, - "self_device_memory_bytes": 786432 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1155, - "self_cpu_us": 30886.555999999768, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 15259.450999999935, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2522, - "self_cpu_us": 11694.530000000093, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 4923.203000000066, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1364, - "self_cpu_us": 4200.7139999999345, - "self_device_us": 4325.325999999722, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 679, - "self_cpu_us": 2475.386999999806, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1579.0149999997666, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 270, - "self_cpu_us": 1526.4960000000283, - "self_device_us": 958.5179999999909, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 1211.8220000002698, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1046.327999999994, - "self_device_us": 727.1810000000069, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 100, - "self_cpu_us": 1044.046000000038, - "self_device_us": 422.95099999992226, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_softmax", - "count": 183, - "self_cpu_us": 1043.5719999998073, - "self_device_us": 4847.890000000012, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1012.0560000000614, - "self_device_us": 1260.6269999999504, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 924.9700000000994, - "self_device_us": 1308.5010000000257, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 921.9240000003183, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 911.6380000003298, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 842.8640000000414, - "self_device_us": 329.63399999997273, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 816.9300000000476, - "self_device_us": 1178.6429999999946, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 807.9510000000835, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 778.4430000000539, - "self_device_us": 2829.236000000021, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1155, - "self_cpu_us": 42451.45200000021, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 16473.516999999978, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2522, - "self_cpu_us": 12428.987999999705, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9528.780999999806, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 7053.55099999987, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6385.314000000195, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 4901.061999999998, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1364, - "self_cpu_us": 4642.906999999959, - "self_device_us": 6678.909000000535, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 679, - "self_cpu_us": 2844.3409999998976, - "self_device_us": 0, - "self_device_memory_bytes": 65508864 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 2370.445000000463, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2189.775000000467, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 2057.2630000004756, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "aten::div", - "count": 270, - "self_cpu_us": 1714.8240000003643, - "self_device_us": 1486.07199999988, - "self_device_memory_bytes": 37748736 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 1671.7550000000952, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1589.7839999998428, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1412.5379999998395, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1349.7980000001553, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "vllm_rlt/worker/sampling.py(6): probabilities", - "count": 181, - "self_cpu_us": 1349.6579999995265, - "self_device_us": 0.0, - "self_device_memory_bytes": -35586048 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 1343.5690000002296, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1245.2050000000427, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - } - ] - }, - "tokens_unchanged": true - } -} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log deleted file mode 100644 index 17965cd3..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup-smoke.log +++ /dev/null @@ -1,40 +0,0 @@ -[transformers] You are using a model of type `ouro` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating. -USDT:2026-10-01 20:16:37 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:37 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:16:39 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:39 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:16:46 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:47 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:16:50 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:50 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:16:59 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:59 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:02 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:02 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -sync {'none': 0.08066848499993284, 'raw': 0.0914349139998194, 'processed': 0.08466345700026068} -USDT:2026-10-01 20:17:12 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:12 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:16 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:16 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:31 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:31 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:34 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:35 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:52 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:52 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:17:56 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:17:56 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -async {'none': 0.1108772420002424, 'raw': 0.1287477310002032, 'processed': 0.12239944100019784} -USDT:2026-10-01 20:18:14 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:14 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:16 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:16 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:22 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:22 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:25 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:25 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:32 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:32 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:18:35 72873:72873 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:18:35 72873:72873 SyncActivityProfilerHandler.cpp:59] profiler_stop -speculative {'none': 0.06240307000007306, 'raw': 0.0662938270002087, 'processed': 0.06682750300024054} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json deleted file mode 100644 index 8d18ff49..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cleanup.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "removed": [ - "models/Ouro-minhao-full", - "minhao-fast-part-2" - ], - "bytes": 3232344846 -} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log deleted file mode 100644 index 665bd66e..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cpu.log +++ /dev/null @@ -1,12 +0,0 @@ -test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok -test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok -test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok -test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok -test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok -test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok -test_all_workers_commit_and_partial_commit_requires_new_epoch (tests.test_pd_weight_control.PDWeightControlTest.test_all_workers_commit_and_partial_commit_requires_new_epoch) ... ok - ----------------------------------------------------------------------- -Ran 7 tests in 1.736s - -OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log deleted file mode 100644 index 8eef87ad..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda-parallel.log +++ /dev/null @@ -1,11 +0,0 @@ -test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok -test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok -test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok -test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok -test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok -test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok - ----------------------------------------------------------------------- -Ran 6 tests in 135.474s - -OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log deleted file mode 100644 index ae3542fd..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-cuda.log +++ /dev/null @@ -1,11 +0,0 @@ -test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok -test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... ok -test_preemption_preserves_selected_scores_and_rng (tests.test_training_contract.TrainingContractTest.test_preemption_preserves_selected_scores_and_rng) ... ok -test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok -test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok -test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok - ----------------------------------------------------------------------- -Ran 6 tests in 3.989s - -OK diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log deleted file mode 100644 index 855a3b10..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-stack-wait.log +++ /dev/null @@ -1,19 +0,0 @@ -Kernel stack captured from PID 54942 during the observed driver wait on 2026-10-01: -[<0>] os_acquire_rwlock_write+0x38/0x64 [nvidia] -[<0>] portSyncRwLockAcquireWrite+0x10/0x30 [nvidia] -[<0>] rmapiLockAcquire+0x23c/0x340 [nvidia] -[<0>] _nvGpuOpsLocksAcquireWithClientLockFlags+0x98/0x1f0 [nvidia] -[<0>] dupMemory+0x88/0x690 [nvidia] -[<0>] nvUvmInterfaceDupMemory+0x34/0x40 [nvidia] -[<0>] uvm_map_external_allocation_on_gpu+0x1f0/0x3fc [nvidia_uvm] -[<0>] uvm_api_map_external_allocation+0x474/0x4e4 [nvidia_uvm] -[<0>] uvm_ioctl+0x117c/0x16a0 [nvidia_uvm] -[<0>] uvm_unlocked_ioctl_entry.part.0+0xdc/0x148 [nvidia_uvm] -[<0>] uvm_unlocked_ioctl_entry+0x78/0x9c [nvidia_uvm] -[<0>] __arm64_sys_ioctl+0xac/0xf0 -[<0>] invoke_syscall+0x48/0x114 -[<0>] el0_svc_common.constprop.0+0xc0/0xe0 -[<0>] do_el0_svc+0x1c/0x28 -[<0>] el0_svc+0x30/0xd8 -[<0>] el0t_64_sync_handler+0x120/0x12c -[<0>] el0t_64_sync+0x194/0x198 diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log deleted file mode 100644 index b59b66e8..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-driver-wait.log +++ /dev/null @@ -1,42 +0,0 @@ -Thu Oct 1 12:03:16 UTC 2026 -[<0>] futex_wait_queue+0x6c/0x98 -[<0>] __futex_wait+0xcc/0x160 -[<0>] futex_wait+0x88/0x110 -[<0>] do_futex+0xf8/0x1a0 -[<0>] __arm64_sys_futex+0xe0/0x13c -[<0>] invoke_syscall+0x48/0x114 -[<0>] el0_svc_common.constprop.0+0xc0/0xe0 -[<0>] do_el0_svc+0x1c/0x28 -[<0>] el0_svc+0x30/0xd8 -[<0>] el0t_64_sync_handler+0x120/0x12c -[<0>] el0t_64_sync+0x194/0x198 -[ 1630.645566] NVRM: GPU0 refcntRequestReference_IMPL: Failed to enter state 1 (current state: 0, status: 0x00000056) -[ 2474.164009] NVRM: GPU0 refcntRequestReference_IMPL: Failed to enter state 1 (current state: 0, status: 0x00000056) -[ 2478.973583] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2479.334052] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2479.459146] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2480.205368] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2480.300412] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2480.606304] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2480.823594] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2481.003826] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2481.261606] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2481.325949] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2482.079631] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2482.152349] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2482.562933] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2482.686092] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2483.701671] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2483.788894] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2484.193315] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2484.306663] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2484.419459] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.152876] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.231614] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.661010] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.775288] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2485.971025] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2486.389956] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2486.451899] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2486.841379] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 -[ 2486.904428] NVRM: GPU0 nvCheckOkFailedNoLog: Check failed: Out of memory [NV_ERR_NO_MEMORY] (0x00000051) returned from _memdescAllocInternal(pMemDesc) @ mem_desc.c:1336 diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv deleted file mode 100644 index 9c1a4867..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-gpu-before.csv +++ /dev/null @@ -1,2 +0,0 @@ -name, driver_version, memory.used [MiB], utilization.gpu [%] -NVIDIA Thor, 595.78, [N/A], 98 % diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json deleted file mode 100644 index 9ca1eca6..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.json +++ /dev/null @@ -1,2741 +0,0 @@ -{ - "sync": { - "seconds": { - "none": [ - 1.797056486999736, - 1.8016931980000663, - 1.7812917080000261, - 1.8116626850001012, - 1.8313193210001373 - ], - "raw": [ - 1.8279988220001542, - 1.8234451930002251, - 1.810924204999992, - 1.8776579790001051, - 1.8599433469998985 - ], - "processed": [ - 1.8894610560000729, - 1.7770136750000347, - 1.8161140520001027, - 1.8662560849998044, - 1.8281205210000735 - ] - }, - "median_seconds": { - "none": 1.8016931980000663, - "raw": 1.8279988220001542, - "processed": 1.8281205210000735 - }, - "tokens_per_second": { - "none": 35.52214110096099, - "raw": 35.01096348080393, - "processed": 35.00863278149123 - }, - "peak_extra_allocated_bytes": { - "none": [ - 955392, - 955392, - 955392, - 955392, - 955392 - ], - "raw": [ - 988160, - 955392, - 988160, - 988160, - 955392 - ], - "processed": [ - 955392, - 955392, - 988160, - 955392, - 955392 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 3115292.1020000014, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 297542.58400000003, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 47216.64400000275, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6232, - "self_cpu_us": 46928.53299999721, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2724, - "self_cpu_us": 25592.631000006077, - "self_device_us": 7924.548000013914, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 10, - "self_cpu_us": 19917.72599999947, - "self_device_us": 2.1440000000875443, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16707.02599999922, - "self_device_us": 69237.73299999899, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2140, - "self_cpu_us": 16282.546999996724, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Activity Buffer Request", - "count": 11, - "self_cpu_us": 11960.367999999446, - "self_device_us": 3.072000000017397, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 9020.897999999828, - "self_device_us": 3486.1089999998353, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 384, - "self_cpu_us": 8604.2179999959, - "self_device_us": 1033.1409999988641, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1420, - "self_cpu_us": 7515.799999997356, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 800, - "self_cpu_us": 5476.198000001276, - "self_device_us": 164.86600000051476, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 5247.854999999913, - "self_device_us": 1053.979999999935, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2084, - "self_cpu_us": 5008.964999994238, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3880.791000000514, - "self_device_us": 2027.154000000004, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3850.2629999997826, - "self_device_us": 547.1479999999751, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 2068, - "self_cpu_us": 3678.7860000009327, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3572.775999995909, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4087, - "self_cpu_us": 3312.0390000030416, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 1714448.007000006, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 312503.2759999983, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 88477.1809999905, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaLaunchKernel", - "count": 6232, - "self_cpu_us": 45877.57399999528, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2724, - "self_cpu_us": 30653.4359999993, - "self_device_us": 8406.053999994654, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 17, - "self_cpu_us": 21553.933999999936, - "self_device_us": 113.12300000005052, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 21004.616999993712, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2140, - "self_cpu_us": 18041.15100000197, - "self_device_us": 0, - "self_device_memory_bytes": 120670208 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 15299.454999999656, - "self_device_us": 186853.4359999999, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 13051.557000005414, - "self_device_us": 0.0, - "self_device_memory_bytes": -1185792 - }, - { - "operator": "aten::_to_copy", - "count": 1420, - "self_cpu_us": 10733.110999994191, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 10510.103999998966, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 384, - "self_cpu_us": 9402.348000001257, - "self_device_us": 4816.147000000074, - "self_device_memory_bytes": 15486976 - }, - { - "operator": "aten::slice", - "count": 2084, - "self_cpu_us": 8692.440000006642, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 566, - "self_cpu_us": 8643.991000002294, - "self_device_us": 0.0, - "self_device_memory_bytes": -8029696 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 7680, - "self_cpu_us": 8545.6210000067, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 8124.672999999564, - "self_device_us": 15076.379999999674, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "Activity Buffer Request", - "count": 10, - "self_cpu_us": 7915.628000000057, - "self_device_us": 0.47999999999956344, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 7900.098000011858, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 7111.301000002073, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 1687563.1620000014, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 297471.37799999845, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6422, - "self_cpu_us": 38591.12499999725, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2916, - "self_cpu_us": 19720.34099999547, - "self_device_us": 4969.288999992985, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 11, - "self_cpu_us": 13895.465999999578, - "self_device_us": 2.6239999998360872, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2332, - "self_cpu_us": 13841.695999991633, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 12845.257000000787, - "self_device_us": 85966.08499999985, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 11226.992999998267, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 448, - "self_cpu_us": 7124.344999999742, - "self_device_us": 1127.6500000008473, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 6689.6719999999805, - "self_device_us": 3976.72400000012, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1548, - "self_cpu_us": 5447.389000004618, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 800, - "self_cpu_us": 4433.354999998611, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2086, - "self_cpu_us": 4181.640999996726, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 3946.4330000000346, - "self_device_us": 1249.5749999998784, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 5209, - "self_cpu_us": 3355.9950000011663, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 3335.78000000483, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3175.459000005998, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 2957.2929999998587, - "self_device_us": 2255.626000000015, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 2879.41300000012, - "self_device_us": 626.137999999959, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 2196, - "self_cpu_us": 2788.2449999992214, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 1712470.881999998, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 303259.42099999957, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 84949.75999997946, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 58914.68900000312, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 38, - "self_cpu_us": 55984.85400000001, - "self_device_us": 141.47600000002421, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6422, - "self_cpu_us": 39227.52699999942, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2916, - "self_cpu_us": 20689.506999994737, - "self_device_us": 8596.109999999157, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 17150.899999998324, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2332, - "self_cpu_us": 15443.806000003227, - "self_device_us": 0, - "self_device_memory_bytes": 133285888 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 14933.654000000508, - "self_device_us": 216397.0109999997, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 10250.292999994825, - "self_device_us": 0.0, - "self_device_memory_bytes": -1185792 - }, - { - "operator": "aten::_to_copy", - "count": 1548, - "self_cpu_us": 8689.791999999334, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 7680, - "self_cpu_us": 7914.010000004513, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2086, - "self_cpu_us": 7845.527000000181, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 7781.584000000366, - "self_device_us": 14088.113000000061, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "aten::cat", - "count": 448, - "self_cpu_us": 7187.911999997445, - "self_device_us": 1075.74999999817, - "self_device_memory_bytes": 15519744 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 6351.897999999905, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 566, - "self_cpu_us": 6237.029000000981, - "self_device_us": 0.0, - "self_device_memory_bytes": -8029696 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 6197.680000002206, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 7680, - "self_cpu_us": 5868.171000009923, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 1785390.3010000014, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 287422.5859999989, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 54, - "self_cpu_us": 132692.09599999993, - "self_device_us": 137.35000000000582, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 63230.2860000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6550, - "self_cpu_us": 41654.006999997655, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2852, - "self_cpu_us": 20811.043999990427, - "self_device_us": 5011.1320000056, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2268, - "self_cpu_us": 14158.613000006142, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 13688.341999998862, - "self_device_us": 237601.12799999924, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 448, - "self_cpu_us": 7512.926999999517, - "self_device_us": 1107.5330000010308, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 6855.801000000043, - "self_device_us": 3760.478000000774, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1548, - "self_cpu_us": 5959.337000000479, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 800, - "self_cpu_us": 4528.025000001053, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2086, - "self_cpu_us": 4388.27499998413, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4043.9099999990876, - "self_device_us": 6436.210999999603, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 5209, - "self_cpu_us": 3740.4920000093116, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3374.305000002234, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 3299.6420000069775, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3031.1940000002096, - "self_device_us": 5960.095999999843, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3006.850000000014, - "self_device_us": 629.8799999999992, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 2196, - "self_cpu_us": 3004.803000003427, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 1681418.688000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 313165.76199999935, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 89656.83900000324, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 55077.46799999963, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 46, - "self_cpu_us": 49129.840000000055, - "self_device_us": 177.34899999998743, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6550, - "self_cpu_us": 45109.648999988836, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2852, - "self_cpu_us": 23547.901000007907, - "self_device_us": 22245.57399999522, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 18967.310000011475, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 2268, - "self_cpu_us": 16894.335999999996, - "self_device_us": 0, - "self_device_memory_bytes": 133253120 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16466.671999998278, - "self_device_us": 191312.69199999896, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 11592.744999999559, - "self_device_us": 0.0, - "self_device_memory_bytes": -1185792 - }, - { - "operator": "aten::_to_copy", - "count": 1548, - "self_cpu_us": 9718.056999999331, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 7680, - "self_cpu_us": 8876.85000001891, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 448, - "self_cpu_us": 8632.131000004554, - "self_device_us": 1104.7309999994868, - "self_device_memory_bytes": 15519744 - }, - { - "operator": "aten::slice", - "count": 2086, - "self_cpu_us": 8531.177000011521, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 7987.785000000618, - "self_device_us": 16808.661000000444, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 566, - "self_cpu_us": 7880.714999992513, - "self_device_us": 0.0, - "self_device_memory_bytes": -8029696 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 7180.626999999522, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 6991.26000000355, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 7680, - "self_cpu_us": 6713.435000007084, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - }, - "async": { - "seconds": { - "none": [ - 1.695322036000107, - 1.6974444380002751, - 1.7046040849995734, - 1.6946781169999667, - 1.7272834199998215 - ], - "raw": [ - 1.7682591719999436, - 1.7628239639998355, - 1.7849022200002764, - 1.7954341879999447, - 1.8388650159999997 - ], - "processed": [ - 1.7741621370000757, - 1.770364053999856, - 1.7893874629999118, - 1.8360151079996285, - 1.8289056349999555 - ] - }, - "median_seconds": { - "none": 1.6974444380002751, - "raw": 1.7849022200002764, - "processed": 1.7893874629999118 - }, - "tokens_per_second": { - "none": 37.70373778796383, - "raw": 35.856305898924866, - "processed": 35.766429196225545 - }, - "peak_extra_allocated_bytes": { - "none": [ - 988160, - 988160, - 988160, - 988160, - 988160 - ], - "raw": [ - 986624, - 988160, - 986624, - 986624, - 988160 - ], - "processed": [ - 988160, - 988160, - 986624, - 988160, - 988160 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaEventSynchronize", - "count": 502, - "self_cpu_us": 1432410.3530000027, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 191290.37999999942, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 44, - "self_cpu_us": 34835.79299999999, - "self_device_us": 351.92999999994936, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6078, - "self_cpu_us": 33813.03999999954, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16364.334999999637, - "self_device_us": 183683.25199999977, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 11429.879000001887, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2143, - "self_cpu_us": 11020.502999999995, - "self_device_us": 4541.596999996408, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 10356.752000000413, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 8018.803000000195, - "self_device_us": 3866.8100000002287, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3297, - "self_cpu_us": 6736.454999988346, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1393, - "self_cpu_us": 6702.344000001056, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4626.323999999812, - "self_device_us": 9519.472000000073, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 5523, - "self_cpu_us": 3716.362000009337, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3477.173000000108, - "self_device_us": 619.724000000142, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3425.1280000001398, - "self_device_us": 12465.012999999697, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 1, - "self_cpu_us": 3281.159, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 985, - "self_cpu_us": 2562.969999997975, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 981, - "self_cpu_us": 2548.3720000009953, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2241.14700000608, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 228, - "self_cpu_us": 2224.8430000000544, - "self_device_us": 774.4480000006351, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaEventSynchronize", - "count": 502, - "self_cpu_us": 1334775.7140000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 213300.00299999965, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 37214.26000000528, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6078, - "self_cpu_us": 31441.57300000226, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16664.894000000244, - "self_device_us": 184634.85100000046, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "Command Buffer Full", - "count": 34, - "self_cpu_us": 16638.358999999953, - "self_device_us": 83.08099999994738, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 658, - "self_cpu_us": 14962.963000003772, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3297, - "self_cpu_us": 12126.546999992546, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 11609.655999997276, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 11161.335999998177, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2143, - "self_cpu_us": 10901.848999998805, - "self_device_us": 8184.701999998565, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 10622.565999999239, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 8326.869000005598, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 8074.22300000026, - "self_device_us": 0.0, - "self_device_memory_bytes": -79872 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 7691.808999999961, - "self_device_us": 7839.375999999642, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "", - "count": 658, - "self_cpu_us": 7110.883000002672, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1393, - "self_cpu_us": 7029.449000000028, - "self_device_us": 0, - "self_device_memory_bytes": 119249408 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 6454.799000000135, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 7680, - "self_cpu_us": 5952.708999990169, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 5775.1769999995595, - "self_device_us": 0.0, - "self_device_memory_bytes": -169984 - } - ], - "raw-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 865738.5560000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 440, - "self_cpu_us": 513537.4239999985, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 269982.26400000206, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6268, - "self_cpu_us": 37557.50699999495, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 19496.538000000193, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 15068.10300000109, - "self_device_us": 89085.99599999975, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2335, - "self_cpu_us": 15040.314000005079, - "self_device_us": 4926.857000000852, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 11644.150999997, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3299, - "self_cpu_us": 9157.908999995387, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1585, - "self_cpu_us": 9157.521999997385, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 7889.426999999985, - "self_device_us": 3726.571000000061, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 6645, - "self_cpu_us": 4706.0270000029495, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4578.1169999999665, - "self_device_us": 1194.1009999999906, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 1045, - "self_cpu_us": 3393.951000005082, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1113, - "self_cpu_us": 3387.474999996231, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3383.8789999999763, - "self_device_us": 2079.0089999998263, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 292, - "self_cpu_us": 3383.5519999986764, - "self_device_us": 885.724000001198, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3352.952999999814, - "self_device_us": 585.2520000001759, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3030.2580000000307, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::rsqrt", - "count": 388, - "self_cpu_us": 2112.1670000002123, - "self_device_us": 558.6339999998054, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 894514.4109999985, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 438, - "self_cpu_us": 356624.633000003, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 282386.8210000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 61673.15100000108, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 57640.45999999429, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6268, - "self_cpu_us": 37907.111000003744, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 658, - "self_cpu_us": 20129.23500000021, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 17286.161000000488, - "self_device_us": 209501.60500000042, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "Command Buffer Full", - "count": 24, - "self_cpu_us": 15990.03599999995, - "self_device_us": 94.47799999994459, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2335, - "self_cpu_us": 15931.423000005114, - "self_device_us": 5127.309000006906, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 15294.889999997744, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3299, - "self_cpu_us": 15165.398000015914, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 12728.342999995075, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 11771.589000000844, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1585, - "self_cpu_us": 10317.796999996372, - "self_device_us": 0, - "self_device_memory_bytes": 131865088 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 9568.225999996474, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "", - "count": 658, - "self_cpu_us": 8962.633000000906, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 8289.213999999814, - "self_device_us": 9734.966999999577, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 7695.529999998893, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 7675.931000003035, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 920932.4780000017, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 439, - "self_cpu_us": 482884.0049999996, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 305185.86299999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 73606.65199999898, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 58, - "self_cpu_us": 70443.25700000004, - "self_device_us": 161.2559999999503, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6396, - "self_cpu_us": 39018.222000004345, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2271, - "self_cpu_us": 15692.35799999762, - "self_device_us": 4989.233000002182, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 14275.767999999229, - "self_device_us": 203367.5870000003, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 12591.590999998225, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 3299, - "self_cpu_us": 10243.788999999848, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1521, - "self_cpu_us": 8787.994000000632, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 6978.011999999464, - "self_device_us": 7834.872999999812, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 6645, - "self_cpu_us": 5005.443999995054, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4159.8260000002265, - "self_device_us": 1241.8309999997346, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 1045, - "self_cpu_us": 3796.858999999785, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1113, - "self_cpu_us": 3539.445999995534, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 292, - "self_cpu_us": 3381.941999998534, - "self_device_us": 891.852000000601, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 3376.4410000027856, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3101.857999999801, - "self_device_us": 2076.3349999998336, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3023.192999999952, - "self_device_us": 613.0609999998796, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 860792.1709999986, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 439, - "self_cpu_us": 413039.0830000005, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 255455.0850000007, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 49905.24699999363, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6396, - "self_cpu_us": 37927.29899999841, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 658, - "self_cpu_us": 17880.09100000289, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 704, - "self_cpu_us": 16554.555999998465, - "self_device_us": 89435.19800000006, - "self_device_memory_bytes": 80609280 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 15447.242000002167, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2271, - "self_cpu_us": 13614.053999995802, - "self_device_us": 4926.6889999996565, - "self_device_memory_bytes": -1024 - }, - { - "operator": "aten::slice", - "count": 3299, - "self_cpu_us": 13604.380000005436, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 13420.60000000347, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 1362, - "self_cpu_us": 12200.938999997265, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 10094.503999997636, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1521, - "self_cpu_us": 8711.722999999149, - "self_device_us": 0, - "self_device_memory_bytes": 131832320 - }, - { - "operator": "aten::mul", - "count": 1260, - "self_cpu_us": 8518.236999999934, - "self_device_us": 3812.4549999999153, - "self_device_memory_bytes": 133613568 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 8272.147000002355, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "", - "count": 658, - "self_cpu_us": 8045.141999997961, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 6979.725999999505, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 6839.281000003541, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 6694.447999993816, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - }, - "speculative": { - "seconds": { - "none": [ - 1.4405976819998614, - 1.4354998989997512, - 1.4359760199999982, - 1.43785706799963, - 1.436613651999778 - ], - "raw": [ - 1.446159684000122, - 1.4462237470002037, - 1.4429731870000069, - 1.447812623000118, - 1.447334557999966 - ], - "processed": [ - 1.4485182270000223, - 1.4616416380004011, - 1.445917856000051, - 1.449624680999932, - 1.4485653670003558 - ] - }, - "median_seconds": { - "none": 1.436613651999778, - "raw": 1.4462237470002037, - "processed": 1.4485653670003558 - }, - "tokens_per_second": { - "none": 44.54920772255747, - "raw": 44.25318014086722, - "processed": 44.181644444895994 - }, - "peak_extra_allocated_bytes": { - "none": [ - 2799104, - 2799104, - 2799104, - 2799104, - 2799104 - ], - "raw": [ - 2860544, - 2827776, - 2860544, - 2860544, - 2827776 - ], - "processed": [ - 2827776, - 2827776, - 2860544, - 2827776, - 2827776 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 980, - "self_cpu_us": 1013133.5880000024, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 329889.5450000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6440, - "self_cpu_us": 44799.10400000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 14830.692999999728, - "self_device_us": 57957.05499999999, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1858, - "self_cpu_us": 11904.329999998092, - "self_device_us": 3719.296999995492, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 10062.069999999801, - "self_device_us": 3642.09899999938, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1349, - "self_cpu_us": 8686.187999998147, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 5437.313999999343, - "self_device_us": 1142.796999999984, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 5328.228999999939, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 4195.546000000275, - "self_device_us": 2040.650999999802, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 4130.804000000057, - "self_device_us": 541.9709999999882, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1036, - "self_cpu_us": 3498.193000000953, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 3478.147999996989, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1598, - "self_cpu_us": 3344.8489999989542, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 247, - "self_cpu_us": 3218.2849999998366, - "self_device_us": 773.1339999996012, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3629, - "self_cpu_us": 2757.4049999993445, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 204, - "self_cpu_us": 2495.4719999990484, - "self_device_us": 344.9210000009334, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::rsqrt", - "count": 388, - "self_cpu_us": 2447.126000000311, - "self_device_us": 568.7279999999582, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_index_put_impl_", - "count": 192, - "self_cpu_us": 2249.8519999999753, - "self_device_us": 1397.437999999971, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 134, - "self_cpu_us": 2245.1280000006864, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 980, - "self_cpu_us": 1004743.099000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 341942.8800000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 98, - "self_cpu_us": 65974.60500000956, - "self_device_us": 0.0, - "self_device_memory_bytes": -122880 - }, - { - "operator": "cudaLaunchKernel", - "count": 6440, - "self_cpu_us": 44354.85499999292, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 16, - "self_cpu_us": 43468.00100000008, - "self_device_us": 2872.282999999996, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 9, - "self_cpu_us": 17052.98900000006, - "self_device_us": 2.4319999997969717, - "self_device_memory_bytes": 122880 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 16737.56800000057, - "self_device_us": 133078.352, - "self_device_memory_bytes": 74514432 - }, - { - "operator": "aten::copy_", - "count": 1858, - "self_cpu_us": 12988.228999999212, - "self_device_us": 7612.869000000108, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 53, - "self_cpu_us": 12837.078999994766, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 10, - "self_cpu_us": 11424.281000001734, - "self_device_us": 0.0, - "self_device_memory_bytes": -41080832 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 10870.521999998786, - "self_device_us": 14274.18699999967, - "self_device_memory_bytes": 133641216 - }, - { - "operator": "aten::empty_strided", - "count": 1349, - "self_cpu_us": 9551.200000001178, - "self_device_us": 0, - "self_device_memory_bytes": 146153984 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 8481.077000000107, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 7766.478999998973, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "aten::slice", - "count": 1598, - "self_cpu_us": 7758.273000003454, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 9024, - "self_cpu_us": 6788.078000001582, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 6336, - "self_cpu_us": 6656.0759999922775, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", - "count": 96, - "self_cpu_us": 6457.112000000272, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 134, - "self_cpu_us": 6235.574999999869, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1036, - "self_cpu_us": 6220.435999998934, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1070, - "self_cpu_us": 1064068.1040000021, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 361931.5290000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 48, - "self_cpu_us": 104276.37, - "self_device_us": 164.09299999989162, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 158, - "self_cpu_us": 50917.7819999979, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6588, - "self_cpu_us": 44249.33099999894, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 9, - "self_cpu_us": 15985.770999999775, - "self_device_us": 3.1359999999403954, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2008, - "self_cpu_us": 13097.013999997875, - "self_device_us": 14456.707000001214, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 11822.664999999748, - "self_device_us": 173903.50500000032, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1499, - "self_cpu_us": 9125.855000001111, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 8022.662999998281, - "self_device_us": 14237.222000000518, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 4792.803000000362, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 4564.072000000597, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4263.941999999985, - "self_device_us": 1244.1820000001771, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1122, - "self_cpu_us": 3775.3999999982043, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 269, - "self_cpu_us": 3679.534000000115, - "self_device_us": 4876.952999999159, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1600, - "self_cpu_us": 3253.3939999918493, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3200.0180000000037, - "self_device_us": 5880.0210000000625, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3172.9669999997113, - "self_device_us": 606.8459999999995, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4139, - "self_cpu_us": 3093.79700000462, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 204, - "self_cpu_us": 2634.8199999991048, - "self_device_us": 346.6839999984659, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1070, - "self_cpu_us": 921827.2790000035, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 348470.8909999982, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 98, - "self_cpu_us": 82054.98900000838, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaLaunchKernel", - "count": 6588, - "self_cpu_us": 43856.03299999783, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 14546.498999999756, - "self_device_us": 58689.897, - "self_device_memory_bytes": 74514432 - }, - { - "operator": "aten::copy_", - "count": 2008, - "self_cpu_us": 13351.863999997171, - "self_device_us": 4054.520999999536, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 53, - "self_cpu_us": 13121.481000001226, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 10, - "self_cpu_us": 11731.000000006607, - "self_device_us": 0.0, - "self_device_memory_bytes": -41080832 - }, - { - "operator": "aten::empty_strided", - "count": 1499, - "self_cpu_us": 10030.803999998225, - "self_device_us": 0, - "self_device_memory_bytes": 158769664 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 9924.678000001335, - "self_device_us": 3753.6830000002865, - "self_device_memory_bytes": 133641216 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 7820.23400000037, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "aten::slice", - "count": 1600, - "self_cpu_us": 6671.56400000199, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 6336, - "self_cpu_us": 6460.941999995901, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1122, - "self_cpu_us": 6371.330999999439, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 9024, - "self_cpu_us": 6215.4499999884265, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", - "count": 96, - "self_cpu_us": 6019.520000000246, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/nn/modules/module.py(1782): _call_impl", - "count": 1397, - "self_cpu_us": 5992.312000000209, - "self_device_us": 0.0, - "self_device_memory_bytes": -99717120 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 5270.968999999644, - "self_device_us": 1170.8770000002514, - "self_device_memory_bytes": 28510208 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 5207.893999999975, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 4842.231000002561, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1006, - "self_cpu_us": 1086286.7079999999, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 366847.06800000044, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Command Buffer Full", - "count": 48, - "self_cpu_us": 81405.13199999991, - "self_device_us": 132.25700000009965, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 158, - "self_cpu_us": 49747.239000002395, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 6716, - "self_cpu_us": 44376.60800000187, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Buffer Flush", - "count": 9, - "self_cpu_us": 15859.352999999945, - "self_device_us": 2.464000000152737, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1944, - "self_cpu_us": 13164.344999999565, - "self_device_us": 7839.070999998783, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 12993.734999999626, - "self_device_us": 152600.5239999997, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1435, - "self_cpu_us": 8864.449999998058, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 8686.781999999766, - "self_device_us": 25866.910000000396, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 6254.039000003366, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 4607.503999999629, - "self_device_us": 1228.5170000003036, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 4396.7750000001415, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1122, - "self_cpu_us": 4098.707000003469, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 269, - "self_cpu_us": 3794.4410000006155, - "self_device_us": 858.2720000006775, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mean", - "count": 388, - "self_cpu_us": 3510.004999999982, - "self_device_us": 5943.891000000016, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1600, - "self_cpu_us": 3501.2590000032596, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::pow", - "count": 388, - "self_cpu_us": 3423.5350000000344, - "self_device_us": 600.6419999997925, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 268, - "self_cpu_us": 3386.080000000744, - "self_device_us": 443.71500000028755, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4139, - "self_cpu_us": 3143.4790000002977, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1006, - "self_cpu_us": 923970.0769999987, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 137, - "self_cpu_us": 349496.5669999999, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 98, - "self_cpu_us": 79725.7660000025, - "self_device_us": 0.0, - "self_device_memory_bytes": -8192 - }, - { - "operator": "cudaLaunchKernel", - "count": 6716, - "self_cpu_us": 45678.692999995874, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 673, - "self_cpu_us": 15432.809000000572, - "self_device_us": 59057.668000000325, - "self_device_memory_bytes": 74514432 - }, - { - "operator": "aten::copy_", - "count": 1944, - "self_cpu_us": 13113.157000004554, - "self_device_us": 3976.40699999536, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 53, - "self_cpu_us": 12922.643000007964, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 10, - "self_cpu_us": 11522.10300000626, - "self_device_us": 0.0, - "self_device_memory_bytes": -41080832 - }, - { - "operator": "aten::mul", - "count": 1314, - "self_cpu_us": 10494.68399999901, - "self_device_us": 3770.354000000312, - "self_device_memory_bytes": 133641216 - }, - { - "operator": "aten::empty_strided", - "count": 1435, - "self_cpu_us": 9768.620999997758, - "self_device_us": 0, - "self_device_memory_bytes": 158736896 - }, - { - "operator": "vllm_rlt/layers/normalization.py(20): forward", - "count": 388, - "self_cpu_us": 8493.670999999964, - "self_device_us": 0.0, - "self_device_memory_bytes": -143628288 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", - "count": 96, - "self_cpu_us": 6569.988999999679, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1600, - "self_cpu_us": 6520.582000002687, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 1122, - "self_cpu_us": 6445.759999999112, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 117, - "self_cpu_us": 6329.600000002482, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(623): _require_prefix", - "count": 6336, - "self_cpu_us": 6263.172000000794, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/nn/modules/module.py(1782): _call_impl", - "count": 1397, - "self_cpu_us": 6239.434000000041, - "self_device_us": 0.0, - "self_device_memory_bytes": -99717120 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(29): add", - "count": 9024, - "self_cpu_us": 6183.9280000038, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::add", - "count": 772, - "self_cpu_us": 5625.210000000348, - "self_device_us": 1181.320999999678, - "self_device_memory_bytes": 28510208 - }, - { - "operator": "cuLaunchKernelEx", - "count": 769, - "self_cpu_us": 5410.53999999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - } -} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log deleted file mode 100644 index b108ece5..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-logprob-overhead.log +++ /dev/null @@ -1,40 +0,0 @@ -[transformers] You are using a model of type `ouro` to instantiate a model of type ``. This may be expected if you are loading a checkpoint that shares a subset of the architecture (e.g., loading a `sam2_video` checkpoint into `Sam2Model`), but is otherwise not supported and can yield errors. Please verify that the checkpoint is compatible with the model you are instantiating. -USDT:2026-10-01 19:52:11 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:52:14 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:52:32 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:52:45 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:53:16 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:53:18 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:53:32 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:53:36 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:54:03 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:54:20 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 19:54:40 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 19:54:45 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -sync {'none': 1.8016931980000663, 'raw': 1.8279988220001542, 'processed': 1.8281205210000735} -USDT:2026-10-01 20:02:51 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:03:17 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:03:43 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:03:49 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:04:25 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:04:51 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:05:18 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:05:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:06:14 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:06:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:06:52 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:07:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -async {'none': 1.6974444380002751, 'raw': 1.7849022200002764, 'processed': 1.7893874629999118} -USDT:2026-10-01 20:11:40 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:12:07 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:12:35 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:12:37 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:13:07 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:13:09 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:13:50 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:14:21 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:14:49 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:14:51 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -USDT:2026-10-01 20:15:33 54942:54942 SyncActivityProfilerHandler.cpp:52] profiler_start -USDT:2026-10-01 20:16:01 54942:54942 SyncActivityProfilerHandler.cpp:59] profiler_stop -speculative {'none': 1.436613651999778, 'raw': 1.4462237470002037, 'processed': 1.4485653670003558} diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log deleted file mode 100644 index 1a90a902..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-seed-before.log +++ /dev/null @@ -1,14 +0,0 @@ -test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ... FAIL - -====================================================================== -FAIL: test_entropy_seed_is_reported_and_replays_the_same_policy (tests.test_training_contract.TrainingContractTest.test_entropy_seed_is_reported_and_replays_the_same_policy) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/home/jwipc/experiments/ouro-vime-20260930/rlt-training-contract-20261001/tests/test_training_contract.py", line 190, in test_entropy_seed_is_reported_and_replays_the_same_policy - self.assertIsInstance(output.sampling_params.seed, int) -AssertionError: None is not an instance of - ----------------------------------------------------------------------- -Ran 1 test in 0.067s - -FAILED (failures=1) diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json deleted file mode 100644 index b0c05fe2..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-slow-cleanup.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "removed": "models/Ouro-1.4B-parallel", - "bytes": 2873677792 -} \ No newline at end of file diff --git a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json b/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json deleted file mode 100644 index 96acb36b..00000000 --- a/docs/validation/minhao-coordination-thor-20261001/raw/minhao-tiny-overhead.json +++ /dev/null @@ -1,2669 +0,0 @@ -{ - "sync": { - "seconds": { - "none": [ - 0.09519798499991339 - ], - "raw": [ - 0.11991658300007657 - ], - "processed": [ - 0.11633255899982942 - ] - }, - "median_seconds": { - "none": 0.09519798499991339, - "raw": 0.11991658300007657, - "processed": 0.11633255899982942 - }, - "tokens_per_second": { - "none": 672.2831370859186, - "raw": 533.7043334528564, - "processed": 550.1469283426821 - }, - "peak_extra_allocated_bytes": { - "none": [ - 758784 - ], - "raw": [ - 823808 - ], - "processed": [ - 791040 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 33297.56899999986, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 19800.59699999992, - "self_device_us": 11.039000000033411, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2004, - "self_cpu_us": 19130.309000000445, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2020, - "self_cpu_us": 9735.057999999846, - "self_device_us": 3466.580000000014, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1348, - "self_cpu_us": 6787.842999999959, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Activity Buffer Request", - "count": 3, - "self_cpu_us": 4397.798999999917, - "self_device_us": 17.24599999999691, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1556, - "self_cpu_us": 2589.577999999135, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2440.773999999882, - "self_device_us": 1127.5460000000476, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 208, - "self_cpu_us": 2438.501999999814, - "self_device_us": 416.11600000010685, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 2014.048999999999, - "self_device_us": 681.3790000000336, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 716, - "self_cpu_us": 2006.413000000081, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1787.3960000001898, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 1310.3229999998512, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1012, - "self_cpu_us": 1157.6350000001703, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 188, - "self_cpu_us": 1153.9850000001802, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1078.1530000000166, - "self_device_us": 288.4610000001521, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1031.721999999896, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2415, - "self_cpu_us": 989.9190000005906, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 935.0519999999015, - "self_device_us": 286.7849999999162, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 64, - "self_cpu_us": 901.8880000001336, - "self_device_us": 541.9650000000875, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1624, - "self_cpu_us": 96740.21499999995, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 29828.6660000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 19172.10899999903, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2004, - "self_cpu_us": 16537.040000000692, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 15321.057999998613, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2020, - "self_cpu_us": 13304.200000000319, - "self_device_us": 9814.611000000457, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 9246.910000000378, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1348, - "self_cpu_us": 8824.270000000583, - "self_device_us": 0, - "self_device_memory_bytes": 26818560 - }, - { - "operator": "(2): __init__", - "count": 780, - "self_cpu_us": 7972.586999999223, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 7952.3179999991335, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "torch/autograd/grad_mode.py(290): __enter__", - "count": 312, - "self_cpu_us": 5820.944000000607, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1556, - "self_cpu_us": 5617.165999999186, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 156, - "self_cpu_us": 5495.474999999722, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(606): _write_prepared", - "count": 8, - "self_cpu_us": 4696.361000000001, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 716, - "self_cpu_us": 4452.799999999923, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 4376.564999998955, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 4297.467999999419, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(287): _make_scheduled_item", - "count": 376, - "self_cpu_us": 4069.740999999527, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "Activity Buffer Request", - "count": 2, - "self_cpu_us": 4064.9590000000353, - "self_device_us": 8.286999999976615, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 3937.9390000010226, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 108719.8060000003, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 24011.792999999805, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2194, - "self_cpu_us": 15252.708000000432, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 13507.98999999966, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2212, - "self_cpu_us": 11690.644999998814, - "self_device_us": 8416.399999999961, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1540, - "self_cpu_us": 7301.129000000332, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 4101.939999999672, - "self_device_us": 1255.3850000000248, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 2997.545000000003, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2510.9329999999395, - "self_device_us": 2639.002999999942, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 2464.008999998726, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2282.966000000388, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2134.255999999961, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1791.3630000006428, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3537, - "self_cpu_us": 1679.8659999990111, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 624, - "self_cpu_us": 1648.1559999993642, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1140, - "self_cpu_us": 1481.7340000004865, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1324.1299999998773, - "self_device_us": 683.1499999999578, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1257.5680000000211, - "self_device_us": 1779.3249999999916, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1212.7460000002857, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 1187.963000000027, - "self_device_us": 634.3850000001239, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1818, - "self_cpu_us": 101858.63600000048, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 19925.880999999914, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2194, - "self_cpu_us": 13766.511999999813, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 13020.369000001338, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "aten::copy_", - "count": 2212, - "self_cpu_us": 10548.945999998652, - "self_device_us": 9764.56400000026, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 10036.862999999756, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 8070.916999999871, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1540, - "self_cpu_us": 6907.167000000017, - "self_device_us": 0, - "self_device_memory_bytes": 39434240 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 5716.320000000458, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 3870.769999999612, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 3773.934000000846, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 3767.767999999496, - "self_device_us": 1359.2739999999485, - "self_device_memory_bytes": 337920 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 3737.9029999998384, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 3170.4539999998087, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 3001.5509999997157, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 2973.905000000177, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 2791.1890000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2571.306999999335, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2479.8980000000774, - "self_device_us": 2988.58499999993, - "self_device_memory_bytes": 6881280 - }, - { - "operator": "aten::clone", - "count": 624, - "self_cpu_us": 2234.690999999897, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 97750.7149999996, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 22683.500000000073, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2322, - "self_cpu_us": 20361.571000000113, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 13013.803999999898, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 2148, - "self_cpu_us": 11927.50900000012, - "self_device_us": 8301.904999999215, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1476, - "self_cpu_us": 6909.296999999731, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 4398.034999999721, - "self_device_us": 1229.2659999998846, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 3254.1110000007093, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2565.4240000000195, - "self_device_us": 2622.8850000000243, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 2342.3889999999733, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2110.396999999586, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 2086.5779999999377, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3537, - "self_cpu_us": 1777.3179999996833, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 192, - "self_cpu_us": 1635.8050000000076, - "self_device_us": 899.7680000000564, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1635.431000000337, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::to", - "count": 1140, - "self_cpu_us": 1513.9589999994594, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::clone", - "count": 560, - "self_cpu_us": 1491.8950000000768, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1314.6709999997838, - "self_device_us": 686.7049999998162, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 384, - "self_cpu_us": 1258.967999999892, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1243.792000000154, - "self_device_us": 1745.1790000000492, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1754, - "self_cpu_us": 103497.714000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 20846.46299999978, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2322, - "self_cpu_us": 15129.085999998966, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 13175.480000000473, - "self_device_us": 0.0, - "self_device_memory_bytes": -1024 - }, - { - "operator": "aten::copy_", - "count": 2148, - "self_cpu_us": 10448.986999999888, - "self_device_us": 10015.051999999127, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 128, - "self_cpu_us": 10255.892999999633, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 316, - "self_cpu_us": 8545.498999999993, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 1476, - "self_cpu_us": 6776.009000000365, - "self_device_us": 0, - "self_device_memory_bytes": 39401472 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 5816.4070000001375, - "self_device_us": 0.0, - "self_device_memory_bytes": -232448 - }, - { - "operator": "", - "count": 288, - "self_cpu_us": 3927.1879999986068, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 844, - "self_cpu_us": 3912.425999998626, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 272, - "self_cpu_us": 3884.787000000045, - "self_device_us": 1454.8620000000992, - "self_device_memory_bytes": 337920 - }, - { - "operator": "aten::slice", - "count": 1558, - "self_cpu_us": 3826.4770000002713, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(370): _update", - "count": 188, - "self_cpu_us": 3287.5009999989925, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 390, - "self_cpu_us": 3132.696000000161, - "self_device_us": 0.0, - "self_device_memory_bytes": -6689280 - }, - { - "operator": "", - "count": 744, - "self_cpu_us": 3023.1649999991896, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2872.8629999997174, - "self_device_us": 3133.3969999999745, - "self_device_memory_bytes": 6881280 - }, - { - "operator": "vllm_rlt/core/scheduler.py(323): _take", - "count": 188, - "self_cpu_us": 2871.759000000642, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 252, - "self_cpu_us": 2640.3929999992906, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 2423.540000000653, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ] - }, - "tokens_unchanged": true - }, - "async": { - "seconds": { - "none": [ - 0.11112063099994884 - ], - "raw": [ - 0.12377583400007097 - ], - "processed": [ - 0.12324414399995476 - ] - }, - "median_seconds": { - "none": 0.11112063099994884, - "raw": 0.12377583400007097, - "processed": 0.12324414399995476 - }, - "tokens_per_second": { - "none": 575.9506531242562, - "raw": 517.0637751466356, - "processed": 519.2944502095247 - }, - "peak_extra_allocated_bytes": { - "none": [ - 790016 - ], - "raw": [ - 822272 - ], - "processed": [ - 822272 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 16279.262000000004, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 1850, - "self_cpu_us": 8762.216999999695, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 6619.0009999997255, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1439, - "self_cpu_us": 5515.489000000218, - "self_device_us": 2130.588000000116, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2769, - "self_cpu_us": 4635.025999999778, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 4584.934999999944, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 601, - "self_cpu_us": 2461.7410000001278, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2147.010999999959, - "self_device_us": 809.0290000000505, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 3851, - "self_cpu_us": 1570.322000000206, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1314.6100000000988, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 629, - "self_cpu_us": 1276.088999999649, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 990.0759999999482, - "self_device_us": 233.5900000000729, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 940.6050000001692, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 864.0409999999702, - "self_device_us": 601.2849999999398, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 782.1960000001072, - "self_device_us": 233.108000000002, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::record_stream", - "count": 436, - "self_cpu_us": 745.8910000001142, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 116, - "self_cpu_us": 660.8229999999921, - "self_device_us": 266.3260000000107, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 281, - "self_cpu_us": 619.7590000002324, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventRecordWithFlags", - "count": 698, - "self_cpu_us": 613.4480000000897, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 64, - "self_cpu_us": 609.95099999999, - "self_device_us": 486.4459999999799, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 17218.51800000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 9886.790999998972, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9667.236999999815, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 1850, - "self_cpu_us": 9417.455000000573, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8362.155000000257, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1037, - "self_cpu_us": 7480.238000000069, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2769, - "self_cpu_us": 7326.50600000022, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6628.762999999737, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1439, - "self_cpu_us": 6172.138999999667, - "self_device_us": 7529.890999999854, - "self_device_memory_bytes": -1024 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5232.246999999894, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 4782.417000000076, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4636.270000000101, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4530.803999999982, - "self_device_us": 0.0, - "self_device_memory_bytes": -79872 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4478.493999999459, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 3908.4060000003046, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3038.3940000000453, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 601, - "self_cpu_us": 2837.559999999785, - "self_device_us": 0, - "self_device_memory_bytes": 26515968 - }, - { - "operator": "", - "count": 904, - "self_cpu_us": 2774.022999999871, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "torch/cuda/__init__.py(1235): current_stream", - "count": 1126, - "self_cpu_us": 2606.9919999999374, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2567.98200000016, - "self_device_us": 3231.842000000015, - "self_device_memory_bytes": 6881280 - } - ], - "raw-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 35043.07600000008, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 27261.345000000074, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 26212.683000000343, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2040, - "self_cpu_us": 17226.433999998942, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 9567.590000000226, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1631, - "self_cpu_us": 7822.165999999748, - "self_device_us": 7616.087999999582, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 5809.952999998892, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 793, - "self_cpu_us": 3797.753000000337, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2782.3519999998025, - "self_device_us": 2836.7000000000608, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2576.1720000005644, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1843.8050000004368, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 693, - "self_cpu_us": 1734.2529999999033, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 116, - "self_cpu_us": 1597.180999999936, - "self_device_us": 645.6439999999984, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 409, - "self_cpu_us": 1295.4429999995746, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1207.970999999644, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1207.7140000000436, - "self_device_us": 744.3319999997984, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 128, - "self_cpu_us": 1078.0610000000052, - "self_device_us": 684.6229999998395, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 96, - "self_cpu_us": 1074.36400000082, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 1062.3739999999489, - "self_device_us": 1878.7700000000477, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 1000.1569999997646, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1231, - "self_cpu_us": 20903.29400000006, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 18533.75499999998, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 11627.489999999798, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2040, - "self_cpu_us": 11486.406999999133, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10235.82400000188, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9755.430000000004, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 8791.236000000776, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 8558.017999999998, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8461.214000000378, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1631, - "self_cpu_us": 7077.14399999875, - "self_device_us": 7515.524999999724, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6816.739999999478, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5592.861000000013, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4809.259999999384, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4741.460000002217, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4577.247000000707, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 4044.835999999861, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "aten::empty_strided", - "count": 793, - "self_cpu_us": 3699.4469999996372, - "self_device_us": 0, - "self_device_memory_bytes": 39131648 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 3092.7339999997967, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3059.046000000157, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2879.6480000001075, - "self_device_us": 2809.507000000074, - "self_device_memory_bytes": 6881280 - } - ], - "processed-ops": [ - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 40731.565000000046, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 18052.8380000001, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2168, - "self_cpu_us": 11742.620000000232, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 10820.819999999676, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1567, - "self_cpu_us": 6499.077000000206, - "self_device_us": 8344.468000000634, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5100.179999999666, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 4953.838000000167, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 729, - "self_cpu_us": 2996.4449999999683, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2412.3269999999034, - "self_device_us": 3132.7200000000075, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 2234.7580000002854, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 64, - "self_cpu_us": 1542.5300000003408, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 693, - "self_cpu_us": 1528.0179999995985, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 192, - "self_cpu_us": 1305.0059999999976, - "self_device_us": 1111.3880000001209, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 116, - "self_cpu_us": 1274.8779999999783, - "self_device_us": 721.4030000000266, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::unsqueeze", - "count": 1132, - "self_cpu_us": 1105.1659999998597, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sigmoid", - "count": 124, - "self_cpu_us": 1061.8390000001273, - "self_device_us": 851.0689999999304, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 409, - "self_cpu_us": 1038.5240000001413, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 64, - "self_cpu_us": 956.9540000000325, - "self_device_us": 2173.455000000007, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::stack", - "count": 96, - "self_cpu_us": 929.1470000001209, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaEventSynchronize", - "count": 378, - "self_cpu_us": 908.5779999999498, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaGraphLaunch", - "count": 124, - "self_cpu_us": 19562.78000000015, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1167, - "self_cpu_us": 12463.96400000043, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2168, - "self_cpu_us": 11787.197999999762, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "triton/runtime/jit.py(708): run", - "count": 570, - "self_cpu_us": 10282.645999998535, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 124, - "self_cpu_us": 9901.121999998875, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 129, - "self_cpu_us": 9359.13800000002, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(450): submit", - "count": 188, - "self_cpu_us": 8651.15399999867, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 2771, - "self_cpu_us": 7960.596999999834, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1567, - "self_cpu_us": 6933.717000000461, - "self_device_us": 7324.400999999833, - "self_device_memory_bytes": -1024 - }, - { - "operator": "vllm_rlt/worker/async_state.py(169): prepare", - "count": 188, - "self_cpu_us": 6819.291999999174, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cuLaunchKernelEx", - "count": 658, - "self_cpu_us": 5430.379000000365, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 570, - "self_cpu_us": 5054.977999999375, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/engine/llm_engine.py(517): _step_async", - "count": 189, - "self_cpu_us": 4980.245000000294, - "self_device_us": 0.0, - "self_device_memory_bytes": -112640 - }, - { - "operator": "torch/cuda/_utils.py(557): _get_device_index", - "count": 1504, - "self_cpu_us": 4827.424999998944, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/async_state.py(46): transfer", - "count": 188, - "self_cpu_us": 4674.382999999931, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/model_runner.py(352): _execute", - "count": 188, - "self_cpu_us": 4062.3459999991246, - "self_device_us": 0.0, - "self_device_memory_bytes": -105472 - }, - { - "operator": "aten::empty_strided", - "count": 729, - "self_cpu_us": 3426.792000000013, - "self_device_us": 0, - "self_device_memory_bytes": 39098880 - }, - { - "operator": "", - "count": 688, - "self_cpu_us": 3211.9410000003777, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 4973, - "self_cpu_us": 3122.0590000002553, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 88, - "self_cpu_us": 2800.17300000012, - "self_device_us": 2851.515000000054, - "self_device_memory_bytes": 6881280 - } - ] - }, - "tokens_unchanged": true - }, - "speculative": { - "seconds": { - "none": [ - 0.0662261480001689 - ], - "raw": [ - 0.06795481500012102 - ], - "processed": [ - 0.06800570499990499 - ] - }, - "median_seconds": { - "none": 0.0662261480001689, - "raw": 0.06795481500012102, - "processed": 0.06800570499990499 - }, - "tokens_per_second": { - "none": 966.385663859489, - "raw": 941.8022843544792, - "processed": 941.0975152759523 - }, - "peak_extra_allocated_bytes": { - "none": [ - 2750976 - ], - "raw": [ - 2790912 - ], - "processed": [ - 2774528 - ] - }, - "profiles": { - "none-ops": [ - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 14250.309000000101, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2245, - "self_cpu_us": 10531.197000000051, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1128, - "self_cpu_us": 8562.952999999969, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1277, - "self_cpu_us": 3761.6589999999446, - "self_device_us": 1806.0290000000139, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 143, - "self_cpu_us": 2708.568000000006, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 592, - "self_cpu_us": 2201.785999999958, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1515.5650000001278, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1219, - "self_cpu_us": 1244.1550000002997, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1168.2059999999747, - "self_device_us": 362.9779999999464, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1026.8629999999969, - "self_device_us": 156.73599999999442, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 995.331999999973, - "self_device_us": 317.34799999996153, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 917.5149999999649, - "self_device_us": 683.0130000000008, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 843.7139999999272, - "self_device_us": 172.83400000001166, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 790.9519999999757, - "self_device_us": 621.252999999977, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 77, - "self_cpu_us": 775.0059999999964, - "self_device_us": 124.67199999998161, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2113, - "self_cpu_us": 760.4689999999889, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 759.0639999997917, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 756.3700000000226, - "self_device_us": 1531.8530000000064, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 348, - "self_cpu_us": 708.1709999999481, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_softmax", - "count": 119, - "self_cpu_us": 698.9639999999526, - "self_device_us": 1821.2669999999434, - "self_device_memory_bytes": 0 - } - ], - "none-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1128, - "self_cpu_us": 50386.84900000042, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 18524.925000000065, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2245, - "self_cpu_us": 12642.707000000097, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 10299.16400000074, - "self_device_us": 0.0, - "self_device_memory_bytes": -1536 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6868.614000000468, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 143, - "self_cpu_us": 6569.547999999943, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 5634.887000000228, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1277, - "self_cpu_us": 4802.597999998776, - "self_device_us": 6307.480999999992, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 592, - "self_cpu_us": 3066.6099999999415, - "self_device_us": 0, - "self_device_memory_bytes": 52925952 - }, - { - "operator": "aten::slice", - "count": 1219, - "self_cpu_us": 2758.779000000186, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2436.3480000002855, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 2412.122000000076, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 2321.6109999999353, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1901.2210000000298, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2113, - "self_cpu_us": 1567.9659999997698, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1554.9709999999468, - "self_device_us": 1259.2209999998413, - "self_device_memory_bytes": 25165824 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1469.638999999961, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - }, - { - "operator": "aten::_to_copy", - "count": 348, - "self_cpu_us": 1457.5210000002992, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1289.5729999998493, - "self_device_us": 1402.8940000000239, - "self_device_memory_bytes": 1404416 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 1265.5860000001667, - "self_device_us": 2400.320000000047, - "self_device_memory_bytes": 79872 - } - ], - "raw-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1219, - "self_cpu_us": 32791.54100000013, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 17470.00599999995, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2394, - "self_cpu_us": 12090.505000000023, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 4819.1389999999865, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1428, - "self_cpu_us": 4432.517000000222, - "self_device_us": 4649.904999999926, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 743, - "self_cpu_us": 2811.108999999853, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1714.3729999998868, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 1327.7300000001896, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1264.6509999999944, - "self_device_us": 774.6999999999425, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 100, - "self_cpu_us": 1105.010999999929, - "self_device_us": 438.56399999991436, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1051.4090000000483, - "self_device_us": 1278.6749999999424, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1050.9589999995637, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1045.2189999999996, - "self_device_us": 741.7150000000001, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 980.1500000000906, - "self_device_us": 1307.7370000000465, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 938.0559999998939, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 902.338000000078, - "self_device_us": 333.67199999995864, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 879.014999999983, - "self_device_us": 1192.532999999974, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 833.2999999999356, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 790.916999999994, - "self_device_us": 2837.7279999999555, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 769.0820000001117, - "self_device_us": 0, - "self_device_memory_bytes": 0 - } - ], - "raw-full": [ - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 17014.585000000065, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaMemcpyAsync", - "count": 1219, - "self_cpu_us": 15616.989000000358, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2394, - "self_cpu_us": 12305.728000000006, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9651.556000000543, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6259.108999999342, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 4917.951999999987, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1428, - "self_cpu_us": 4794.027999999778, - "self_device_us": 4887.427999999511, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 4222.749999999923, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 743, - "self_cpu_us": 3183.7439999999515, - "self_device_us": 0, - "self_device_memory_bytes": 65541632 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 2523.371999999762, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2202.935999999663, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 2086.883999999831, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 1783.7669999996692, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1756.5959999997158, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1639.7709999998492, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1436.0750000000262, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "aten::select", - "count": 579, - "self_cpu_us": 1346.087000000095, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 206, - "self_cpu_us": 1336.519999999975, - "self_device_us": 813.8999999999978, - "self_device_memory_bytes": 25165824 - }, - { - "operator": "torch/utils/_contextlib.py(120): decorate_context", - "count": 186, - "self_cpu_us": 1293.4269999999624, - "self_device_us": 0.0, - "self_device_memory_bytes": -20741632 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1222.6059999999607, - "self_device_us": 738.8599999998873, - "self_device_memory_bytes": 1404416 - } - ], - "processed-ops": [ - { - "operator": "cudaMemcpyAsync", - "count": 1155, - "self_cpu_us": 30564.47400000023, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 16297.44999999991, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2522, - "self_cpu_us": 12829.769999999564, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 6427.685000000046, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::copy_", - "count": 1364, - "self_cpu_us": 4445.575999999577, - "self_device_us": 4287.241000000021, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 679, - "self_cpu_us": 2640.5459999999166, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 1695.7159999999421, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 270, - "self_cpu_us": 1634.4419999999154, - "self_device_us": 895.1859999999397, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 1283.0529999999708, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::cat", - "count": 100, - "self_cpu_us": 1112.1259999999893, - "self_device_us": 597.6139999999978, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_softmax", - "count": 183, - "self_cpu_us": 1092.33499999993, - "self_device_us": 4761.238999999965, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mul", - "count": 175, - "self_cpu_us": 1076.8700000000263, - "self_device_us": 1272.6010000000606, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::mm", - "count": 57, - "self_cpu_us": 1041.7010000000118, - "self_device_us": 734.4430000000102, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1010.8409999999494, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::aminmax", - "count": 78, - "self_cpu_us": 1003.2569999999541, - "self_device_us": 1286.9970000000249, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 993.8319999999167, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::lt", - "count": 128, - "self_cpu_us": 919.8220000000947, - "self_device_us": 318.9200000000674, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::sum", - "count": 87, - "self_cpu_us": 879.4340000000484, - "self_device_us": 1155.8990000000413, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty", - "count": 315, - "self_cpu_us": 858.7320000000709, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::argmax", - "count": 110, - "self_cpu_us": 807.774000000074, - "self_device_us": 2808.9659999999603, - "self_device_memory_bytes": 0 - } - ], - "processed-full": [ - { - "operator": "cudaMemcpyAsync", - "count": 1155, - "self_cpu_us": 20902.823999999953, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaGraphLaunch", - "count": 156, - "self_cpu_us": 16891.75100000009, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaLaunchKernel", - "count": 2522, - "self_cpu_us": 15932.245000000374, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/cuda_graph.py(65): run", - "count": 112, - "self_cpu_us": 9363.272999999954, - "self_device_us": 0.0, - "self_device_memory_bytes": -512 - }, - { - "operator": "vllm_rlt/core/kv_cache_manager.py(442): _prepare_batches", - "count": 60, - "self_cpu_us": 6491.891000000185, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "cudaStreamSynchronize", - "count": 168, - "self_cpu_us": 5420.91799999993, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/speculative.py(96): execute", - "count": 12, - "self_cpu_us": 5071.345999999881, - "self_device_us": 0.0, - "self_device_memory_bytes": -40443392 - }, - { - "operator": "aten::copy_", - "count": 1364, - "self_cpu_us": 4659.201999999755, - "self_device_us": 5583.503000000237, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::empty_strided", - "count": 679, - "self_cpu_us": 3039.1240000001817, - "self_device_us": 0, - "self_device_memory_bytes": 65508864 - }, - { - "operator": "aten::bitwise_and", - "count": 87, - "self_cpu_us": 2845.459999999992, - "self_device_us": 366.18600000001607, - "self_device_memory_bytes": 44544 - }, - { - "operator": "aten::slice", - "count": 1221, - "self_cpu_us": 2434.940000000655, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "", - "count": 716, - "self_cpu_us": 2247.8800000001356, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::eq", - "count": 87, - "self_cpu_us": 2212.9000000000815, - "self_device_us": 339.5000000001128, - "self_device_memory_bytes": 44544 - }, - { - "operator": "aten::multinomial", - "count": 78, - "self_cpu_us": 2182.421999999704, - "self_device_us": 0.0, - "self_device_memory_bytes": -15654912 - }, - { - "operator": "", - "count": 133, - "self_cpu_us": 1826.42999999996, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::div", - "count": 270, - "self_cpu_us": 1732.066999999999, - "self_device_us": 1221.016000000127, - "self_device_memory_bytes": 37748736 - }, - { - "operator": "aten::as_strided", - "count": 2669, - "self_cpu_us": 1720.8809999998857, - "self_device_us": 0, - "self_device_memory_bytes": 0 - }, - { - "operator": "aten::_to_copy", - "count": 435, - "self_cpu_us": 1615.1079999998499, - "self_device_us": 0.0, - "self_device_memory_bytes": 0 - }, - { - "operator": "vllm_rlt/worker/sampling.py(51): rejection_sample", - "count": 50, - "self_cpu_us": 1522.9489999999714, - "self_device_us": 0.0, - "self_device_memory_bytes": -3634176 - }, - { - "operator": "vllm_rlt/worker/sampling.py(6): probabilities", - "count": 181, - "self_cpu_us": 1390.739999999816, - "self_device_us": 0.0, - "self_device_memory_bytes": -35586048 - } - ] - }, - "tokens_unchanged": true - } -} diff --git a/docs/validation/training-contract-a100-20261001/README.md b/docs/validation/training-contract-a100-20261001/README.md deleted file mode 100644 index 40473595..00000000 --- a/docs/validation/training-contract-a100-20261001/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Training contract with two A100 workers — 2026-10-01 - -Two A100-SXM4-40GB GPUs, driver 580.105.08, the existing `/venv/main` runtime, -PyTorch 2.13.0+cu130, Triton 3.7.1 and NIXL 1.4.1. Separate spawned P/D owners -register their GPU KV/hidden pools and exchange real NIXL WRITE transfers. -The public `LLM` interface stages physical policy chunks through CPU IPC and waits -for every worker to commit. No package, driver or service changes are made. - -| Case | Active slots | Maximum logprob error vs local | Published/recovered version | D graph captures / replays | P bytes transferred | -|---|---:|---:|---:|---:|---:| -| Random FP32, eager, K=2/3/4 | 4 | 4.77e-7 | 5 | Disabled | 300,928 | -| Random FP32, graphs, K=2/3/4 | 4 | 4.77e-7 | 5 | 2 / 116 | 300,928 | -| Full BF16 Ouro-1.4B, graphs, K=2/3/4 | 1 | 0 | 5 | 1 / 206 | 229,699,584 | - -Each case compares raw and processed scores at temperature 0.9, top-k 8 and -nucleus 0.8, with two prompts and six response tokens. Token IDs, physical exit -depths, effective parameters and policy versions agree. Prompt prefill is full -K=4; decode budgets change after each complete publication. Prefix reuse, -chunked updates, a rejected update, recovery into version five, explicit stop -and active abort are exercised. Both workers finish with zero used blocks and -version five. Graph captures remain valid through publication and recovery. - -The full-model source is `ByteDance/Ouro-1.4B` at -`574fa66cb8bf5abdc979642d01cf2b79b16bfab1`, safetensors SHA256 -`58872a72616c736595b8b7662079c5b12c5a162ec16eae94f21c348dfa9885af`. -See [full outputs/counters](raw/pd-full-graph-matched.json), -[tiny eager](raw/pd-tiny-eager-tcp.json), [tiny graphs](raw/pd-tiny-graph-tcp.json) -and [runtime](raw/runtime.json). - -## Transport and numerical limits - -Default UCX GPU IPC reports completion but leaves the destination zero in an -independent two-process, 128-byte probe. Explicit IPC and shared-memory-only -configurations also fail on this container. `UCX_TLS=tcp,cuda_copy` passes the -probe and every case above with the installed library. Transfers are host staged; -this report does not establish direct GPU IPC/RDMA performance. The workaround -is scoped to each command. [Probe source](../../../benchmarks/nixl_preflight.py), -[default failure](raw/nixl-preflight.log) and [passing probe](raw/nixl-preflight-tcp.log) -are retained. No processes are forcibly terminated; successful workers use the -normal stop/acknowledgment protocol. - -A full BF16 comparison with four slots has a maximum observed selected-score -difference of 0.11199 when local and PD prefill batches differ. The matched -one-slot run above is exact; the FP32 four-slot run is within 4.77e-7. -Different BF16 batching is therefore not claimed to be bitwise equivalent. See -[the rejected strict comparison](raw/pd-full-graph-tcp.log). -Other jobs were already running on both GPUs. Timings in these parity probes -include startup/capture/prefix-state differences and contention, and are not a -controlled throughput comparison. The [Thor table](../training-contract-thor-20261001/README.md) -contains alternating-order warmed eager/graph timings. - -## Reproduction - -From the repository root, in an existing CUDA/NIXL environment: - -```bash -UCX_TLS=tcp,cuda_copy python benchmarks/nixl_preflight.py -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py tiny.json -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py tiny-graph.json --cuda-graphs -UCX_TLS=tcp,cuda_copy python benchmarks/pd_training_contract.py full.json \ - --model /models/Ouro-1.4B --cuda-graphs --max-num-seqs 1 -``` - -The VIME training/save/resume and general HF export checks are recorded in -[0z5a/vime#5](https://github.com/0z5a/vime/pull/5). - -## Repository checks - -The complete default CPU regression suite passes: **421 passed, 141 skipped** -in 19.41 s; GPU tests, including the upstream forced-peer-failure test, are skipped -by the repository's ordinary collection rules. See -[regression log](raw/cpu-regression-final.log) and -[focused contract tests](raw/contract-cpu-final.log). -Ruff lint and format checks pass for all 88 Python files. -The existing runtimes lack `pre_commit`, so its three `language: system` hooks -are run directly without installing it: Python-file lint, format check and the -full default CPU suite. See [lint log](raw/lint-final.log). diff --git a/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log b/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log deleted file mode 100644 index 885048c7..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/contract-cpu-final.log +++ /dev/null @@ -1,10 +0,0 @@ -test_early_exit_async_stop_and_abort (tests.test_training_contract.TrainingContractTest.test_early_exit_async_stop_and_abort) ... ok -test_seed_none_and_validation (tests.test_training_contract.TrainingContractTest.test_seed_none_and_validation) ... ok -test_selected_probabilities_speculative_target_and_post_update (tests.test_training_contract.TrainingContractTest.test_selected_probabilities_speculative_target_and_post_update) ... ok -test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy (tests.test_training_contract.TrainingContractTest.test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy) ... ok -test_all_workers_commit_and_partial_commit_requires_new_epoch (tests.test_pd_weight_control.PDWeightControlTest.test_all_workers_commit_and_partial_commit_requires_new_epoch) ... ok - ----------------------------------------------------------------------- -Ran 5 tests in 2.095s - -OK diff --git a/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log b/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log deleted file mode 100644 index d10641bc..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/cpu-regression-final.log +++ /dev/null @@ -1,9 +0,0 @@ -.......s.s.s.sssssssssssssssssss....sssssssssssssss.s................... [ 12%] -............................................ssssss......s..............s [ 25%] -ssssssss.sssssssssssssssssssssssss...................................... [ 38%] -....ssssssss.ssss....sssssss...............................s............ [ 51%] -..sssssssssssssssss.................sss................................. [ 64%] -..ssssss..sss.....ss...ss............................................... [ 76%] -........................................................................ [ 89%] -.............ss.ss...............................sssss.... [100%] -421 passed, 141 skipped in 19.41s diff --git a/docs/validation/training-contract-a100-20261001/raw/lint-final.log b/docs/validation/training-contract-a100-20261001/raw/lint-final.log deleted file mode 100644 index 3d3e3c6d..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/lint-final.log +++ /dev/null @@ -1,4 +0,0 @@ -Required pre-commit hooks use language: system. pre_commit is absent; no installation performed. -ruff 0.16.8 -All checks passed! -88 files already formatted diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log deleted file mode 100644 index 11287164..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-ipc.log +++ /dev/null @@ -1,51 +0,0 @@ -[1790822017.515227] [25f216012e8e:21943:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy, cuda_ipc/cuda - no am bcopy -W1001 02:33:37.515292 21943 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable -E1001 02:33:37.515377 21943 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep -E1001 02:33:37.515387 21943 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND -Process SpawnProcess-1: -Traceback (most recent call last): - File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap - self.run() - File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run - self._target(*self._args, **self._kwargs) - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker - agent = nixl_agent(name) - ^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ - self.create_backend(bknd, init) - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend - self.backends[backend] = self.agent.createBackend(backend, initParams) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND -[1790822017.583102] [25f216012e8e:21944:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy, cuda_ipc/cuda - no am bcopy -W1001 02:33:37.583151 21944 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable -E1001 02:33:37.583303 21944 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep -E1001 02:33:37.583311 21944 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND -Process SpawnProcess-2: -Traceback (most recent call last): - File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap - self.run() - File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run - self._target(*self._args, **self._kwargs) - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker - agent = nixl_agent(name) - ^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ - self.create_backend(bknd, init) - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend - self.backends[backend] = self.agent.createBackend(backend, initParams) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND -Traceback (most recent call last): - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 42, in - infos=[p.recv() for _,p in peers] - ^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 250, in recv - buf = self._recv_bytes() - ^^^^^^^^^^^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes - buf = self._recv(4) - ^^^^^^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 399, in _recv - raise EOFError -EOFError diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log deleted file mode 100644 index d2072821..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-shm.log +++ /dev/null @@ -1,51 +0,0 @@ -[1790822145.345936] [25f216012e8e:23279:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy -W1001 02:35:45.345994 23279 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable -E1001 02:35:45.346127 23279 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep -E1001 02:35:45.346146 23279 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND -Process SpawnProcess-1: -Traceback (most recent call last): - File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap - self.run() - File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run - self._target(*self._args, **self._kwargs) - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker - agent = nixl_agent(name) - ^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ - self.create_backend(bknd, init) - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend - self.backends[backend] = self.agent.createBackend(backend, initParams) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND -[1790822145.362990] [25f216012e8e:23280:0] select.c:721 UCX ERROR no active messages transport to : sysv/memory - no peer failure handler, posix/memory - no peer failure handler, cma/memory - no am bcopy, cuda_copy/cuda - no am bcopy -W1001 02:35:45.363052 23280 ucx_enums.cpp:58] Unexpected UCX error: Destination is unreachable -E1001 02:35:45.363210 23280 ucx_utils.cpp:543] UCX endpoint create failed: failed to create ep -E1001 02:35:45.363225 23280 nixl_agent.cpp:433] createBackend: backend 'UCX' encountered error during intra-agent transfer setup with status NIXL_ERR_BACKEND -Process SpawnProcess-2: -Traceback (most recent call last): - File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap - self.run() - File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run - self._target(*self._args, **self._kwargs) - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 13, in worker - agent = nixl_agent(name) - ^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 269, in __init__ - self.create_backend(bknd, init) - File "/usr/local/lib/python3.12/dist-packages/nixl_cu13/_api.py", line 400, in create_backend - self.backends[backend] = self.agent.createBackend(backend, initParams) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -nixl_cu13._bindings.nixlBackendError: NIXL_ERR_BACKEND -Traceback (most recent call last): - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 42, in - infos=[p.recv() for _,p in peers] - ^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 250, in recv - buf = self._recv_bytes() - ^^^^^^^^^^^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 430, in _recv_bytes - buf = self._recv(4) - ^^^^^^^^^^^^^ - File "/usr/lib/python3.12/multiprocessing/connection.py", line 399, in _recv - raise EOFError -EOFError diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log deleted file mode 100644 index c5bfffe1..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight-tcp.log +++ /dev/null @@ -1,6 +0,0 @@ -2026-10-01 02:12:24 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:12:24 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-1 -2026-10-01 02:12:24 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:12:24 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-0 -VALUES [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0]] -MULTIPROCESS_NIXL_PASSED diff --git a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log b/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log deleted file mode 100644 index ce2d6594..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/nixl-preflight.log +++ /dev/null @@ -1,12 +0,0 @@ -2026-10-01 02:11:20 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:11:20 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-0 -2026-10-01 02:11:20 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:11:20 NIXL INFO _api.py:285 Initialized NIXL agent: ouro-nixl-1 -VALUES [[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0, 31.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]] -[1790820681.044462] [25f216012e8e:15986:0] cuda_ctx.c:23 UCX ERROR cuDevicePrimaryCtxGetState(cuda_device, &flags, &active) failed: unrecognized error code 4 -[1790820681.060647] [25f216012e8e:15985:0] cuda_ctx.c:23 UCX ERROR cuDevicePrimaryCtxGetState(cuda_device, &flags, &active) failed: unrecognized error code 4 -Traceback (most recent call last): - File "/workspace/0z5a/work/ouro-vime-contract-20261001/nixl_preflight.py", line 50, in - assert values[0]==values[1] - ^^^^^^^^^^^^^^^^^^^^ -AssertionError diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json deleted file mode 100644 index 03da93de..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.json +++ /dev/null @@ -1,1189 +0,0 @@ -{ - "model": { - "vocab_size": 49152, - "hidden_size": 2048, - "intermediate_size": 5632, - "num_hidden_layers": 24, - "num_attention_heads": 16, - "num_key_value_heads": 16, - "head_dim": 128, - "hidden_act": "silu", - "max_position_embeddings": 65536, - "initializer_range": 0.02, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "total_ut_steps": 4, - "early_exit_threshold": 1.0, - "bos_token_id": 0, - "eos_token_id": 0, - "pad_token_id": null, - "tie_word_embeddings": false, - "attention_dropout": 0.0, - "rope_scaling": null, - "use_sliding_window": false, - "sliding_window": null, - "layer_types": [ - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention", - "full_attention" - ] - }, - "cuda_graphs": true, - "iterations": [ - { - "version": 1, - "loops": 2, - "mode": "raw", - "seconds": { - "local": 1.501969379838556, - "pd": 15.95300066191703 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "0", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.4585607051849365, - -1.3445959091186523, - -0.6607269048690796, - -0.4440094828605652, - -0.288143515586853, - -0.2284386157989502 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "1", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 4, - 4, - 4, - 198 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.4905070662498474, - -1.852401614189148, - -1.8474926948547363, - -4.226284027099609, - -2.1031107902526855, - -3.4267778396606445 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 1, - "loops": 2, - "mode": "processed", - "seconds": { - "local": 0.544591479934752, - "pd": 6.968707168009132 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "2", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.5379835367202759, - -0.22262510657310486, - 0.0, - 0.0, - 0.0 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "3", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 4, - 4, - 4, - 198 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.5063599348068237, - -1.3956828117370605, - -1.77802312374115, - -1.5563204288482666, - -0.19239495694637299 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "raw", - "seconds": { - "local": 0.7185850627720356, - "pd": 14.518414614256471 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "4", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.5055561065673828, - -0.5541938543319702, - -0.48413512110710144, - -0.3641066551208496, - -0.22259598970413208, - -0.19062571227550507 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "5", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 4, - 3, - 4, - 3 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.54900062084198, - -0.6617550849914551, - -1.9724218845367432, - -0.34641382098197937, - -1.458582878112793, - -0.42156851291656494 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "processed", - "seconds": { - "local": 0.66817781701684, - "pd": 7.080546109005809 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "6", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.1622927039861679, - -0.15219146013259888, - 0.0, - 0.0, - 0.0 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "7", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 4, - 3, - 4, - 3 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.2678171992301941, - -1.781590223312378, - 0.0, - -1.1942179203033447, - 0.0 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "raw", - "seconds": { - "local": 0.849096744786948, - "pd": 14.640745197888464 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "8", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.5625201463699341, - -0.37419480085372925, - -0.2975218594074249, - -0.2469334453344345, - -0.20516522228717804, - -0.1743372529745102 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "9", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -0.616585373878479, - -0.5554823875427246, - -0.43136879801750183, - -0.33480215072631836, - -0.2979239821434021, - -0.46127814054489136 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "processed", - "seconds": { - "local": 0.7937315646559, - "pd": 7.206241376232356 - }, - "max_logprob_error": 0.0, - "outputs": [ - { - "request_id": "10", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "11", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 3, - 3, - 3, - 3, - 3, - 3 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - 0.0, - -0.1622924953699112, - 0.0, - 0.0, - 0.0, - 0.0 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - } - ], - "max_num_seqs": 1, - "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", - "recovered_version": 5, - "stop": { - "request_id": "13", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "stop", - "text": "", - "log_probs": [ - 0.0 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [ - 3 - ], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "abort": { - "request_id": "abort", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 3 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "abort", - "text": "", - "log_probs": [ - 0.0 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "workers": { - "cae75e7035744e8795024750bd65b22e-decode-1": { - "kind": "stopped", - "prefill_tokens": 0, - "bytes_sent": 0, - "transfers": 0, - "transfer_seconds": 0.0, - "used_blocks": 0, - "weight_version": 5, - "graph_captures": 1, - "graph_replays": 206, - "graph_fallbacks": 0 - }, - "cae75e7035744e8795024750bd65b22e-prefill-0": { - "kind": "stopped", - "prefill_tokens": 244, - "bytes_sent": 229699584, - "transfers": 22, - "transfer_seconds": 207.36763043235987, - "used_blocks": 0, - "weight_version": 5 - } - } -} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log deleted file mode 100644 index d37a3c92..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-matched.log +++ /dev/null @@ -1,11 +0,0 @@ -2026-10-01 02:30:31 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:30:31 NIXL INFO _api.py:285 Initialized NIXL agent: cae75e7035744e8795024750bd65b22e-decode-1 -2026-10-01 02:30:31 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:30:31 NIXL INFO _api.py:285 Initialized NIXL agent: cae75e7035744e8795024750bd65b22e-prefill-0 -PARITY 1 2 raw {'local': 1.501969379838556, 'pd': 15.95300066191703} 0.0 -PARITY 1 2 processed {'local': 0.544591479934752, 'pd': 6.968707168009132} 0.0 -PARITY 2 3 raw {'local': 0.7185850627720356, 'pd': 14.518414614256471} 0.0 -PARITY 2 3 processed {'local': 0.66817781701684, 'pd': 7.080546109005809} 0.0 -PARITY 3 4 raw {'local': 0.849096744786948, 'pd': 14.640745197888464} 0.0 -PARITY 3 4 processed {'local': 0.7937315646559, 'pd': 7.206241376232356} 0.0 -PD_CONTRACT_PASSED pd-full-graph-matched.json diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log deleted file mode 100644 index 03b98612..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-full-graph-tcp.log +++ /dev/null @@ -1,16 +0,0 @@ -2026-10-01 02:19:01 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:19:01 NIXL INFO _api.py:285 Initialized NIXL agent: 35f3775d875246919e95de82111ac017-prefill-0 -2026-10-01 02:19:01 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:19:01 NIXL INFO _api.py:285 Initialized NIXL agent: 35f3775d875246919e95de82111ac017-decode-1 -Traceback (most recent call last): - File "/workspace/0z5a/work/ouro-vime-contract-20261001/rlt/benchmarks/pd_training_contract.py", line 168, in - main() - File "/workspace/0z5a/work/ouro-vime-contract-20261001/rlt/benchmarks/pd_training_contract.py", line 111, in main - torch.testing.assert_close( - File "/usr/local/lib/python3.12/dist-packages/torch/testing/_comparison.py", line 1631, in assert_close - raise error_metas[0].to_error(msg) -AssertionError: Tensor-likes are not close! - -Mismatched elements: 6 / 6 (100.0%) -Greatest absolute difference: 0.11199069023132324 at index (1,) (up to 2e-05 allowed) -Greatest relative difference: 0.07553135603666306 at index (1,) (up to 2e-05 allowed) diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json deleted file mode 100644 index c5c64fd2..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.json +++ /dev/null @@ -1,1160 +0,0 @@ -{ - "model": { - "vocab_size": 64, - "hidden_size": 32, - "intermediate_size": 64, - "num_hidden_layers": 2, - "num_attention_heads": 4, - "num_key_value_heads": 2, - "head_dim": 8, - "hidden_act": "silu", - "max_position_embeddings": 128, - "initializer_range": 0.02, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "total_ut_steps": 4, - "early_exit_threshold": 1.0, - "bos_token_id": 0, - "eos_token_id": 0, - "pad_token_id": null, - "tie_word_embeddings": false, - "attention_dropout": 0.0, - "rope_scaling": null, - "use_sliding_window": false, - "sliding_window": null, - "layer_types": null - }, - "cuda_graphs": false, - "iterations": [ - { - "version": 1, - "loops": 2, - "mode": "raw", - "seconds": { - "local": 1.1838269280269742, - "pd": 2.0369991376064718 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "0", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 42, - 42, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.992061138153076, - -3.8517885208129883, - -3.85848069190979, - -3.9884941577911377, - -3.8614566326141357, - -3.8722712993621826 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "1", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 51, - 15, - 55 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.861236810684204, - -3.9610671997070312, - -3.9557578563690186, - -4.008734226226807, - -3.9762279987335205, - -4.012908458709717 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 1, - "loops": 2, - "mode": "processed", - "seconds": { - "local": 0.23476281203329563, - "pd": 0.41898630978539586 - }, - "max_logprob_error": 1.1920928955078125e-07, - "outputs": [ - { - "request_id": "2", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 42, - 42, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.969618320465088, - -1.8275089263916016, - -1.835660457611084, - -1.980717658996582, - -1.8372858762741089, - -1.8491119146347046 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "3", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 51, - 15, - 55 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8221769332885742, - -1.9443289041519165, - -1.942601203918457, - -2.004945993423462, - -1.9664429426193237, - -2.008206367492676 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "raw", - "seconds": { - "local": 0.21088168676942587, - "pd": 0.45077179512009025 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "4", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 12, - 61, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.9952800273895264, - -4.035950660705566, - -3.7910518646240234, - -3.8775575160980225, - -3.849231719970703, - -3.8591208457946777 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "5", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 56, - 15, - 56 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.8670716285705566, - -3.976059913635254, - -3.975245237350464, - -4.008485317230225, - -3.941457986831665, - -4.011442184448242 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "processed", - "seconds": { - "local": 0.24939796887338161, - "pd": 0.38997333077713847 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "6", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 12, - 61, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.9691096544265747, - -2.072470188140869, - -1.8017675876617432, - -1.8991870880126953, - -1.8665740489959717, - -1.8762420415878296 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "7", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 56, - 15, - 56 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8246119022369385, - -2.0073468685150146, - -2.00469970703125, - -2.0388402938842773, - -1.96330726146698, - -2.036628007888794 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "raw", - "seconds": { - "local": 0.22700949804857373, - "pd": 0.45369746908545494 - }, - "max_logprob_error": 4.76837158203125e-07, - "outputs": [ - { - "request_id": "8", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 60, - 9, - 25, - 60, - 42 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.9984395503997803, - -4.025251865386963, - -4.024652004241943, - -3.9303245544433594, - -4.02800178527832, - -4.010705947875977 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "9", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 60, - 60, - 9, - 15, - 56 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.8727941513061523, - -4.021127223968506, - -4.020061016082764, - -4.025237083435059, - -3.993036985397339, - -4.017432689666748 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "processed", - "seconds": { - "local": 0.25585177494212985, - "pd": 0.42146871984004974 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "10", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 60, - 9, - 25, - 60, - 42 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.9686124324798584, - -1.9942327737808228, - -1.9923925399780273, - -1.8824831247329712, - -1.9928045272827148, - -1.9732145071029663 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "11", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 60, - 60, - 9, - 15, - 56 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8269999027252197, - -1.9946293830871582, - -1.9918233156204224, - -1.995558738708496, - -1.9558247327804565, - -1.983024001121521 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - } - ], - "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", - "recovered_version": 5, - "stop": { - "request_id": "13", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "stop", - "text": "", - "log_probs": [ - -1.9686124324798584 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [ - 42 - ], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "abort": { - "request_id": "abort", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "abort", - "text": "", - "log_probs": [ - -1.9686124324798584 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "workers": { - "e7b52c471c5a4a7383476001ca2853cc-decode-1": { - "kind": "stopped", - "prefill_tokens": 0, - "bytes_sent": 0, - "transfers": 0, - "transfer_seconds": 0.0, - "used_blocks": 0, - "weight_version": 5 - }, - "e7b52c471c5a4a7383476001ca2853cc-prefill-0": { - "kind": "stopped", - "prefill_tokens": 292, - "bytes_sent": 300928, - "transfers": 22, - "transfer_seconds": 2.774511670228094, - "used_blocks": 0, - "weight_version": 5 - } - } -} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log deleted file mode 100644 index 9ae443a5..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-eager-tcp.log +++ /dev/null @@ -1,11 +0,0 @@ -2026-10-01 02:14:34 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:14:34 NIXL INFO _api.py:285 Initialized NIXL agent: e7b52c471c5a4a7383476001ca2853cc-decode-1 -2026-10-01 02:14:34 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:14:34 NIXL INFO _api.py:285 Initialized NIXL agent: e7b52c471c5a4a7383476001ca2853cc-prefill-0 -PARITY 1 2 raw {'local': 1.1838269280269742, 'pd': 2.0369991376064718} 2.384185791015625e-07 -PARITY 1 2 processed {'local': 0.23476281203329563, 'pd': 0.41898630978539586} 1.1920928955078125e-07 -PARITY 2 3 raw {'local': 0.21088168676942587, 'pd': 0.45077179512009025} 2.384185791015625e-07 -PARITY 2 3 processed {'local': 0.24939796887338161, 'pd': 0.38997333077713847} 2.384185791015625e-07 -PARITY 3 4 raw {'local': 0.22700949804857373, 'pd': 0.45369746908545494} 4.76837158203125e-07 -PARITY 3 4 processed {'local': 0.25585177494212985, 'pd': 0.42146871984004974} 2.384185791015625e-07 -PD_CONTRACT_PASSED pd-tiny-eager-tcp.json diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json deleted file mode 100644 index e95c674c..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.json +++ /dev/null @@ -1,1163 +0,0 @@ -{ - "model": { - "vocab_size": 64, - "hidden_size": 32, - "intermediate_size": 64, - "num_hidden_layers": 2, - "num_attention_heads": 4, - "num_key_value_heads": 2, - "head_dim": 8, - "hidden_act": "silu", - "max_position_embeddings": 128, - "initializer_range": 0.02, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "total_ut_steps": 4, - "early_exit_threshold": 1.0, - "bos_token_id": 0, - "eos_token_id": 0, - "pad_token_id": null, - "tie_word_embeddings": false, - "attention_dropout": 0.0, - "rope_scaling": null, - "use_sliding_window": false, - "sliding_window": null, - "layer_types": null - }, - "cuda_graphs": true, - "iterations": [ - { - "version": 1, - "loops": 2, - "mode": "raw", - "seconds": { - "local": 1.2744349539279938, - "pd": 2.329219114035368 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "0", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 42, - 42, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.992061138153076, - -3.8517885208129883, - -3.85848069190979, - -3.9884941577911377, - -3.8614566326141357, - -3.8722712993621826 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "1", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 51, - 15, - 55 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.861236810684204, - -3.9610671997070312, - -3.9557578563690186, - -4.008734226226807, - -3.9762279987335205, - -4.012908458709717 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 1, - "loops": 2, - "mode": "processed", - "seconds": { - "local": 0.23766471119597554, - "pd": 0.436434387229383 - }, - "max_logprob_error": 1.1920928955078125e-07, - "outputs": [ - { - "request_id": "2", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 42, - 42, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.969618320465088, - -1.8275089263916016, - -1.835660457611084, - -1.980717658996582, - -1.8372858762741089, - -1.8491119146347046 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "3", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 51, - 15, - 55 - ], - "exit_depths": [ - 4, - 2, - 2, - 2, - 2, - 2 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8221769332885742, - -1.9443289041519165, - -1.942601203918457, - -2.004945993423462, - -1.9664429426193237, - -2.008206367492676 - ], - "weight_version": 1, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 2, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "raw", - "seconds": { - "local": 0.22012542001903057, - "pd": 0.4495909558609128 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "4", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 12, - 61, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.9952800273895264, - -4.035950660705566, - -3.7910518646240234, - -3.8775575160980225, - -3.849231719970703, - -3.8591208457946777 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "5", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 56, - 15, - 56 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.8670716285705566, - -3.976059913635254, - -3.975245237350464, - -4.008485317230225, - -3.941457986831665, - -4.011442184448242 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 2, - "loops": 3, - "mode": "processed", - "seconds": { - "local": 0.24550614319741726, - "pd": 0.40186773892492056 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "6", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 12, - 61, - 25, - 42, - 42 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.9691096544265747, - -2.072470188140869, - -1.8017675876617432, - -1.8991870880126953, - -1.8665740489959717, - -1.8762420415878296 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "7", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 47, - 47, - 56, - 15, - 56 - ], - "exit_depths": [ - 4, - 3, - 3, - 3, - 3, - 3 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8246119022369385, - -2.0073468685150146, - -2.00469970703125, - -2.0388402938842773, - -1.96330726146698, - -2.036628007888794 - ], - "weight_version": 2, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 3, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "raw", - "seconds": { - "local": 0.23163969395682216, - "pd": 0.4639628282748163 - }, - "max_logprob_error": 4.76837158203125e-07, - "outputs": [ - { - "request_id": "8", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 60, - 9, - 25, - 60, - 42 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.9984395503997803, - -4.025251865386963, - -4.024652004241943, - -3.9303245544433594, - -4.02800178527832, - -4.010705947875977 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "9", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 60, - 60, - 9, - 15, - 56 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -3.8727941513061523, - -4.021127223968506, - -4.020061016082764, - -4.025237083435059, - -3.993036985397339, - -4.017432689666748 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "raw", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - }, - { - "version": 3, - "loops": 4, - "mode": "processed", - "seconds": { - "local": 0.2783969510346651, - "pd": 0.46485115215182304 - }, - "max_logprob_error": 2.384185791015625e-07, - "outputs": [ - { - "request_id": "10", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42, - 60, - 9, - 25, - 60, - 42 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.9686124324798584, - -1.9942327737808228, - -1.9923925399780273, - -1.8824831247329712, - -1.9928045272827148, - -1.9732145071029663 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - { - "request_id": "11", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 7 - ], - "token_ids": [ - 61, - 60, - 60, - 9, - 15, - 56 - ], - "exit_depths": [ - 4, - 4, - 4, - 4, - 4, - 4 - ], - "finished": true, - "finish_reason": "length", - "text": "", - "log_probs": [ - -1.8269999027252197, - -1.9946293830871582, - -1.9918233156204224, - -1.995558738708496, - -1.9558247327804565, - -1.983024001121521 - ], - "weight_version": 3, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 12, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - } - ] - } - ], - "rejected_chunk": "PD weight control failed: Weight name or shape mismatch: missing_parameter; Weight name or shape mismatch: missing_parameter", - "recovered_version": 5, - "stop": { - "request_id": "13", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "stop", - "text": "", - "log_probs": [ - -1.9686124324798584 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [ - 42 - ], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "abort": { - "request_id": "abort", - "prompt_token_ids": [ - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 3, - 4, - 5, - 6 - ], - "token_ids": [ - 42 - ], - "exit_depths": [ - 4 - ], - "finished": true, - "finish_reason": "abort", - "text": "", - "log_probs": [ - -1.9686124324798584 - ], - "weight_version": 5, - "sampling_params": { - "max_tokens": 6, - "temperature": 0.9, - "top_p": 0.8, - "top_k": 8, - "seed": 11, - "logprobs": 0, - "logprobs_mode": "processed", - "stop_token_ids": [], - "min_loops": 2, - "max_loops": 4, - "exit_threshold": 1.0, - "ignore_eos": true, - "priority": 0 - } - }, - "workers": { - "0759283ec0764cee8d1bd7369fa54644-prefill-0": { - "kind": "stopped", - "prefill_tokens": 292, - "bytes_sent": 300928, - "transfers": 22, - "transfer_seconds": 2.949465547222644, - "used_blocks": 0, - "weight_version": 5 - }, - "0759283ec0764cee8d1bd7369fa54644-decode-1": { - "kind": "stopped", - "prefill_tokens": 0, - "bytes_sent": 0, - "transfers": 0, - "transfer_seconds": 0.0, - "used_blocks": 0, - "weight_version": 5, - "graph_captures": 2, - "graph_replays": 116, - "graph_fallbacks": 0 - } - } -} \ No newline at end of file diff --git a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log b/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log deleted file mode 100644 index e391a5ae..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/pd-tiny-graph-tcp.log +++ /dev/null @@ -1,11 +0,0 @@ -2026-10-01 02:16:37 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:16:37 NIXL INFO _api.py:285 Initialized NIXL agent: 0759283ec0764cee8d1bd7369fa54644-prefill-0 -2026-10-01 02:16:37 NIXL INFO _api.py:407 Backend UCX was instantiated -2026-10-01 02:16:37 NIXL INFO _api.py:285 Initialized NIXL agent: 0759283ec0764cee8d1bd7369fa54644-decode-1 -PARITY 1 2 raw {'local': 1.2744349539279938, 'pd': 2.329219114035368} 2.384185791015625e-07 -PARITY 1 2 processed {'local': 0.23766471119597554, 'pd': 0.436434387229383} 1.1920928955078125e-07 -PARITY 2 3 raw {'local': 0.22012542001903057, 'pd': 0.4495909558609128} 2.384185791015625e-07 -PARITY 2 3 processed {'local': 0.24550614319741726, 'pd': 0.40186773892492056} 2.384185791015625e-07 -PARITY 3 4 raw {'local': 0.23163969395682216, 'pd': 0.4639628282748163} 4.76837158203125e-07 -PARITY 3 4 processed {'local': 0.2783969510346651, 'pd': 0.46485115215182304} 2.384185791015625e-07 -PD_CONTRACT_PASSED pd-tiny-graph-tcp.json diff --git a/docs/validation/training-contract-a100-20261001/raw/runtime.json b/docs/validation/training-contract-a100-20261001/raw/runtime.json deleted file mode 100644 index f4813a5e..00000000 --- a/docs/validation/training-contract-a100-20261001/raw/runtime.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "torch": "2.13.0+cu130", - "triton": "3.7.1", - "nixl": "1.4.1", - "transformers": "5.17.0", - "numpy": "2.3.5", - "safetensors": "0.8.0", - "gpus": [ - "NVIDIA A100-SXM4-40GB, 580.105.08, 40960 MiB", - "NVIDIA A100-SXM4-40GB, 580.105.08, 40960 MiB" - ], - "UCX_TLS": "tcp,cuda_copy" -} \ No newline at end of file diff --git a/docs/validation/training-contract-thor-20261001/README.md b/docs/validation/training-contract-thor-20261001/README.md deleted file mode 100644 index 52920253..00000000 --- a/docs/validation/training-contract-thor-20261001/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Training contract on Thor — 2026-10-01 - -These timings use full BF16 `ByteDance/Ouro-1.4B` at revision -`574fa66cb8bf5abdc979642d01cf2b79b16bfab1`, with the public `LLM` rollout/update -API on upstream `ecb1f8b505b7e831815b40aec3b4598619cca23a`. - -| Rollout | Eager (s) | CUDA graphs (s) | Speedup | Time saved | -|---|---:|---:|---:|---:| -| Native K=2 decode | 1.0689 | 0.9939 | 1.076× | 7.02% | -| Native K=3 decode | 1.5190 | 1.4207 | 1.069× | 6.47% | -| Native K=4 decode | 1.9414 | 1.8234 | 1.065× | 6.08% | -| Self-speculative target K=4, draft K=2, 3 candidates | 1.4952 | 1.3741 | 1.088× | 8.11% | - -Five alternating-order pairs after per-arm warmup and full-weight publication; -two prompts, 32 response tokens each, temperature=0.9 and fixed per-request seeds. -Returned selected-token raw logprobs, tokens, exit depths and weight versions -match exactly between eager and graph arms in every pair. Each native prompt -prefill uses full K=4. Measurements include logprob delivery and exclude model -loading and initial capture. Each speculative comparison uses the same speculative -algorithm in both arms. See [raw timing pairs](raw/shared-full-rollout-benchmark.json). - -CPU tests pass all five checks; CUDA/Triton tests pass all four applicable checks. -The independent FP32 oracle covers processed probabilities with temperature and -top-k/top-p, speculative target scores after publication, early exit, async -stop/abort, prefix invalidation, stable parameter addresses and failed publication -recovery. The PD control test confirms a newer complete epoch recovers from a -partial peer commit; delivery is in-process, so IPC and NIXL are not covered. - -Full-model VIME training and fresh-process checkpoint/HF export parity are -reported in [0z5a/vime#5](https://github.com/0z5a/vime/pull/5). -Real multi-GPU PD publication and rollout checks are recorded in the -[A100 report](../training-contract-a100-20261001/README.md); Thor has one GPU and no NIXL. -No packages, CUDA drivers or environments were updated. GPU runs use the existing -performance lock and finish naturally. Completed experiment weights/checkpoints -are removed after preserving logs and tensor hashes. diff --git a/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json b/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json deleted file mode 100644 index 22ae874c..00000000 --- a/docs/validation/training-contract-thor-20261001/raw/shared-full-rollout-benchmark.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "native-2": { - "seconds": { - "eager": [ - 1.068922233011108, - 1.0962036999990232, - 1.0150359050021507, - 1.0776073549932335, - 1.0171921259898227 - ], - "graph": [ - 0.99387969498639, - 1.031959738000296, - 0.9904317960026674, - 1.0318653039867058, - 0.9868912409874611 - ] - }, - "medians": { - "eager": 1.068922233011108, - "graph": 0.99387969498639 - }, - "speedup": 1.0755046495096627, - "tokens_and_logprobs_exact": true - }, - "native-3": { - "seconds": { - "eager": [ - 1.5189770860015415, - 1.5267343849991448, - 1.4750143490091432, - 1.5399230089969933, - 1.4762638669926673 - ], - "graph": [ - 1.4206840230035596, - 1.4595294760074466, - 1.3996001509949565, - 1.4518495189840905, - 1.4113349709950853 - ] - }, - "medians": { - "eager": 1.5189770860015415, - "graph": 1.4206840230035596 - }, - "speedup": 1.0691871390164396, - "tokens_and_logprobs_exact": true - }, - "native-4": { - "seconds": { - "eager": [ - 1.9413879640051164, - 1.9600694489781745, - 1.8969329710234888, - 1.9612911650037859, - 1.8895282850135118 - ], - "graph": [ - 1.7855259570060298, - 1.8588317520043347, - 1.8060117769928183, - 1.8618553779961076, - 1.8234225899796002 - ] - }, - "medians": { - "eager": 1.9413879640051164, - "graph": 1.8234225899796002 - }, - "speedup": 1.0646944787641552, - "tokens_and_logprobs_exact": true - }, - "speculative-4": { - "seconds": { - "eager": [ - 1.4952338639996015, - 1.523701586003881, - 1.4740307310130447, - 1.5373682600038592, - 1.4766599040012807 - ], - "graph": [ - 1.3729902119957842, - 1.415411785012111, - 1.373799616994802, - 1.416570646018954, - 1.3740531750081573 - ] - }, - "medians": { - "eager": 1.4952338639996015, - "graph": 1.3740531750081573 - }, - "speedup": 1.0881921392821823, - "tokens_and_logprobs_exact": true - } -} \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index f4a55800..00000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Repository tests; keep imports separate from installed test packages.""" diff --git a/tests/test_pd.py b/tests/test_pd.py index 71999078..9b07ff14 100644 --- a/tests/test_pd.py +++ b/tests/test_pd.py @@ -239,7 +239,6 @@ def test_engine_yields_while_waiting_for_remote_kv(async_scheduling): from vllm_rlt.request import Request, Stage engine = object.__new__(LLMEngine) - engine._update_version = None engine.profiling = Profiler("cpu") engine.preemption = PreemptionManager(engine) engine.execution_config = ExecutionConfig(async_scheduling=async_scheduling) diff --git a/tests/test_pd_weight_control.py b/tests/test_pd_weight_control.py deleted file mode 100644 index 523efbbb..00000000 --- a/tests/test_pd_weight_control.py +++ /dev/null @@ -1,91 +0,0 @@ -"""PD publication control with two real CPU engines; no NIXL transport claim.""" - -import unittest -from types import SimpleNamespace -from unittest.mock import patch - -import torch - -from tests.helpers import tiny_ouro_config -from vllm_rlt import LLM, SamplingParams -from vllm_rlt.engine.llm_engine import LLMEngine -from vllm_rlt.models import OuroForCausalLM -from vllm_rlt.pd.config import PDConfig -from vllm_rlt.pd.engine import PDEngine -from vllm_rlt.pd.worker import PDWorker - - -def control_pair(): - coordinator = PDEngine.__new__(PDEngine) - coordinator.config = PDConfig() - coordinator.closed, coordinator.failure = False, None - coordinator.requests, coordinator.transfers = {}, {} - coordinator._weight_version, coordinator._update_version = 0, None - coordinator._update_failed, coordinator._paused = False, False - coordinator._weight_replies = {} - coordinator.peers = { - role: SimpleNamespace(name=role, role=role) for role in ("prefill", "decode") - } - workers = {} - for role, peer in coordinator.peers.items(): - worker = PDWorker.__new__(PDWorker) - worker.engine = LLMEngine(OuroForCausalLM(tiny_ouro_config())) - worker.work = {} - worker.send = lambda kind, peer=peer, **fields: coordinator._message( - peer, dict(kind=kind, **fields) - ) - workers[role] = worker - coordinator._send = lambda name, kind, **fields: workers[name].command( - dict(kind=kind, **fields) - ) - return coordinator, workers - - -class PDWeightControlTest(unittest.TestCase): - def test_all_workers_commit_and_partial_commit_requires_new_epoch(self): - coordinator, workers = control_pair() - facade = LLM(coordinator) - weights = list(workers["prefill"].engine.model.named_parameters()) - pointers = { - role: [p.data_ptr() for p in worker.engine.model.parameters()] - for role, worker in workers.items() - } - facade.start_weight_update() - facade.update_weights(weights) - facade.finish_weight_update() - self.assertEqual(facade.get_weight_version(), 1) - coordinator.start_weight_update() - coordinator.update_weights(weights) - with patch.object( - workers["decode"].engine, - "finish_weight_update", - side_effect=RuntimeError("copy failure"), - ): - with self.assertRaisesRegex(RuntimeError, "copy failure"): - coordinator.finish_weight_update() - self.assertEqual(workers["prefill"].engine.get_weight_version(), 2) - self.assertEqual(workers["decode"].engine.get_weight_version(), 1) - with self.assertRaises(RuntimeError): - coordinator.step() - with self.assertRaises(RuntimeError): - coordinator.add_request("blocked", [3]) - coordinator.start_weight_update() - coordinator.update_weights(weights) - coordinator.finish_weight_update() - self.assertEqual(coordinator.get_weight_version(), 3) - for role, worker in workers.items(): - self.assertEqual(worker.engine.get_weight_version(), 3) - self.assertEqual( - pointers[role], [p.data_ptr() for p in worker.engine.model.parameters()] - ) - output = LLM(worker.engine).generate( - [[3, 4]], SamplingParams(max_tokens=2, logprobs=0) - )[0] - self.assertEqual(output.weight_version, 3) - self.assertEqual(len(output.log_probs), len(output.token_ids)) - coordinator.reset_prefix_cache() - - -if __name__ == "__main__": - torch.set_num_threads(1) - unittest.main() diff --git a/tests/test_speculative.py b/tests/test_speculative.py index 162f54ec..412a8b13 100644 --- a/tests/test_speculative.py +++ b/tests/test_speculative.py @@ -395,7 +395,7 @@ def test_greedy_round_reads_back_once_with_ragged_requests(): # Draft IDs stay on the device; every speculative round, including ragged and # budget-shortened ones, synchronizes with the host exactly once. torch.manual_seed(123) - m = OuroForCausalLM(tiny_ouro_config(hidden_size=256, head_dim=64)).to( + m = OuroForCausalLM(OuroConfig.tiny(hidden_size=256, head_dim=64)).to( device="cuda", dtype=torch.bfloat16 ) e = engine(m, k=3, cache_config=CacheConfig(128, 16), attention_backend="triton") diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py deleted file mode 100644 index b536eaa5..00000000 --- a/tests/test_training_contract.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Training contract checks runnable with unittest on CPU or CUDA/Triton.""" - -import os -import unittest -from dataclasses import replace - -import torch - -from tests.reference import dense_reference -from vllm_rlt import ( - LLM, - CacheConfig, - ExecutionConfig, - ExitConfig, - SamplingParams, - SchedulerConfig, - SpeculativeConfig, -) -from vllm_rlt.engine.llm_engine import LLMEngine -from vllm_rlt.models import OuroConfig, OuroForCausalLM -from vllm_rlt.request import Request -from vllm_rlt.worker.sampling import generator_for, probabilities - -DEVICE = os.environ.get("CONTRACT_DEVICE", "cpu") - - -def model(): - torch.manual_seed(17) - config = OuroConfig( - vocab_size=64, - hidden_size=256 if DEVICE == "cuda" else 32, - intermediate_size=512 if DEVICE == "cuda" else 64, - num_hidden_layers=2, - num_attention_heads=4, - num_key_value_heads=2, - head_dim=64 if DEVICE == "cuda" else 8, - max_position_embeddings=128, - total_ut_steps=4, - eos_token_id=0, - ) - return OuroForCausalLM(config).to(DEVICE) - - -def llm(native, *, graphs=False, asynchronous=False, speculative=False, prefix=False): - return LLM( - native, - attention_backend="triton" if DEVICE == "cuda" else "torch", - cache_config=CacheConfig(num_blocks=128, block_size=2, enable_prefix_caching=prefix), - scheduler_config=SchedulerConfig(max_num_seqs=4, max_num_batched_tokens=32), - exit_config=ExitConfig("ouro_delayed" if asynchronous else "ouro"), - execution_config=ExecutionConfig(cuda_graphs=graphs, async_scheduling=asynchronous), - speculative_config=SpeculativeConfig(3) if speculative else None, - ) - - -class TrainingContractTest(unittest.TestCase): - def test_selected_probabilities_speculative_target_and_post_update(self): - native = model() - prompt = [3, 4, 5] - for graphs in [False, True] if DEVICE == "cuda" else [False]: - for speculative in (False, True): - rollout = llm(native, graphs=graphs, speculative=speculative) - pointers = [p.data_ptr() for p in native.parameters()] - for version in (0, 1): - if version: - weights = [(n, p.detach() * 0.97) for n, p in native.named_parameters()] - rollout.start_weight_update(version) - rollout.update_weights(weights[:5]) - rollout.update_weights(weights[5:]) - rollout.finish_weight_update() - for mode in ("raw", "processed"): - params = SamplingParams( - max_tokens=8, - temperature=0.8, - top_k=13, - top_p=0.9, - logprobs=0, - logprobs_mode=mode, - seed=29, - ignore_eos=True, - ) - output = rollout.generate([prompt], params)[0] - logits = dense_reference(native, prompt + output.token_ids[:-1], 4)[-1][2] - rows = logits[len(prompt) - 1 :] - expected = torch.stack( - [ - probabilities(row, params)[token].log() - if mode == "processed" - else row.float().log_softmax(-1)[token] - for row, token in zip(rows, output.token_ids, strict=True) - ] - ) - torch.testing.assert_close( - torch.tensor(output.log_probs, device=DEVICE), - expected, - atol=4e-5, - rtol=4e-5, - ) - self.assertEqual(output.weight_version, version) - self.assertEqual(output.sampling_params, replace(params, max_loops=4)) - self.assertEqual(pointers, [p.data_ptr() for p in native.parameters()]) - if graphs: - bank = ( - rollout.engine.speculative_runner.graphs - if speculative - else rollout.engine.model_runner.graphs - ) - self.assertGreater(bank.replays, 0) - - def test_early_exit_async_stop_and_abort(self): - for graphs in [False, True] if DEVICE == "cuda" else [False]: - for asynchronous in (False, True): - rollout = llm(model(), graphs=graphs, asynchronous=asynchronous) - rollout.start_weight_update(2) - rollout.update_weights(rollout.engine.model.named_parameters()) - rollout.finish_weight_update() - params = SamplingParams(max_tokens=5, exit_threshold=0, logprobs=0, ignore_eos=True) - output = rollout.generate([[3, 4]], params)[0] - self.assertEqual(output.weight_version, 2) - self.assertEqual(output.exit_depths, [4] + [3 if asynchronous else 2] * 4) - self.assertEqual(len(output.log_probs), 5) - stopped = rollout.generate( - [[3, 4]], replace(params, stop_token_ids=(output.token_ids[0],)) - )[0] - self.assertEqual(stopped.token_ids, output.token_ids[:1]) - self.assertEqual(len(stopped.log_probs), 1) - self.assertEqual(stopped.finish_reason, "stop") - rollout.engine.add_request("abort", [3, 4], params) - while not rollout.engine.step(): - pass - aborted = rollout.engine.abort_request("abort") - self.assertEqual(len(aborted.log_probs), len(aborted.token_ids)) - self.assertEqual(aborted.finish_reason, "abort") - - def test_transaction_invalidates_prefixes_and_recovers_without_mixed_policy(self): - rollout = llm(model(), prefix=True) - native = rollout.engine.model - params = SamplingParams(max_tokens=3, logprobs=0, ignore_eos=True) - rollout.generate([[3, 4, 5, 6]], params) - self.assertTrue(rollout.engine.cache_manager.lookup_prefix([3, 4, 5, 6])) - rollout.pause_generation() - with self.assertRaises(RuntimeError): - rollout.generate([[3]], params) - rollout.resume_generation() - rollout.engine.add_request("active", [3], params) - with self.assertRaises(RuntimeError): - rollout.start_weight_update() - rollout.engine.abort_request("active") - rollout.start_weight_update(7) - self.assertEqual(rollout.engine.cache_manager.num_used_blocks, 0) - self.assertFalse(rollout.engine.cache_manager.lookup_prefix([3, 4, 5, 6])) - names = list(native.named_parameters()) - before = names[0][1].detach().clone() - with self.assertRaises(ValueError): - rollout.update_weights( - [(names[0][0], torch.zeros_like(before)), ("invalid", torch.zeros(1))] - ) - torch.testing.assert_close(before, names[0][1], rtol=0, atol=0) - with self.assertRaises(RuntimeError): - rollout.finish_weight_update() - with self.assertRaises(RuntimeError): - rollout.generate([[3]], params) - rollout.start_weight_update(7) - rollout.update_weights(names[:1]) - with self.assertRaises(ValueError): - rollout.finish_weight_update() - rollout.update_weights(names[1:]) - rollout.finish_weight_update() - self.assertEqual(rollout.generate([[3]], params)[0].weight_version, 7) - rollout.reset_prefix_cache() - self.assertEqual(rollout.engine.cache_manager.num_used_blocks, 0) - - def test_seed_none_and_validation(self): - params = SamplingParams(seed=None, temperature=1) - requests = [Request(str(i), [3], params) for i in range(2)] - seeds = [generator_for(r, DEVICE).initial_seed() for r in requests] - self.assertNotEqual(*seeds) - for values in ({"logprobs": 1}, {"logprobs_mode": "invalid"}, {"stop_token_ids": (-1,)}): - with self.assertRaises(ValueError): - SamplingParams(**values) - with self.assertRaises(ValueError): - llm(model()).generate([[3]], SamplingParams(stop_token_ids=(64,))) - - def test_entropy_seed_is_reported_and_replays_the_same_policy(self): - params = SamplingParams( - max_tokens=8, temperature=0.9, seed=None, logprobs=0, ignore_eos=True - ) - for options in ({}, {"asynchronous": True}, {"speculative": True}): - rollout = llm(model(), graphs=DEVICE == "cuda", **options) - output = rollout.generate([[3, 4, 5]], params)[0] - self.assertIsInstance(output.sampling_params.seed, int) - self.assertLess(output.sampling_params.seed, 2**63) - replay = rollout.generate([[3, 4, 5]], output.sampling_params)[0] - self.assertEqual(output.token_ids, replay.token_ids) - self.assertEqual(output.log_probs, replay.log_probs) - self.assertEqual(output.exit_depths, replay.exit_depths) - rollout.engine.add_request("unstarted", [3], replace(params, temperature=0)) - aborted = rollout.engine.abort_request("unstarted") - self.assertIsInstance(aborted.sampling_params.seed, int) - self.assertEqual(aborted.log_probs, []) - rollout.close() - self.assertIsNone(params.seed) - - def test_preemption_preserves_selected_scores_and_rng(self): - native = model() - params = SamplingParams( - max_tokens=8, temperature=0.8, seed=45, logprobs=0, exit_threshold=0, ignore_eos=True - ) - results = [] - for suspend in (False, True): - engine = LLMEngine( - native, - cache_config=CacheConfig(64, 2), - scheduler_config=SchedulerConfig(enable_preemption=True), - attention_backend="triton" if DEVICE == "cuda" else "torch", - ) - engine.add_request("a", [1, 2, 3], params) - while len(engine.scheduler.requests["a"].generated_token_ids) < 3: - engine.step() - if suspend: - engine.add_request("b", [5], SamplingParams(max_tokens=1)) - engine.scheduler.selected_request_ids.clear() - self.assertTrue(engine.preemption.preempt(engine.scheduler.requests["b"])) - while engine.has_unfinished_requests(): - for output in engine.step(): - if output.request_id == "a" and output.finished: - results.append(output) - if suspend: - self.assertEqual(engine.preemption.resumptions, 1) - engine.close() - self.assertEqual(results[0].token_ids, results[1].token_ids) - self.assertEqual(results[0].log_probs, results[1].log_probs) - self.assertEqual(results[0].exit_depths, results[1].exit_depths) - - -if __name__ == "__main__": - torch.set_num_threads(1) - unittest.main() From a2423c2b908c88a131984b722227ef132998fb5b Mon Sep 17 00:00:00 2001 From: 0z5a Date: Sun, 1 Nov 2026 22:57:26 +0100 Subject: [PATCH 5/5] Keep PD cleanup reachable when forced termination is disabled With terminate_workers_on_failure=False, a failure in step() marked the coordinator closed although no worker was stopped or joined, so a later close() returned immediately. A failure now only records the failed state: step() reports the engine unavailable, and close() still drains transfers, requests a stop and joins the workers, so remote writers quiesce before receive resources are released. Signed-off-by: 0z5a --- tests/test_training_contract.py | 30 ++++++++++++++++++++++++++++++ vllm_rlt/pd/engine.py | 4 +++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/tests/test_training_contract.py b/tests/test_training_contract.py index e6b03c1d..6710ae57 100644 --- a/tests/test_training_contract.py +++ b/tests/test_training_contract.py @@ -211,6 +211,36 @@ def test_cpu_stage_clients_publish_to_all_pd_peers_and_recover_control_errors(): client.close() +def test_failure_without_forced_termination_keeps_cooperative_close(): + client = StageClient( + tiny_ouro_model(), device="cpu", dtype="float32", cache_config=CacheConfig(64, 2) + ) + pd = object.__new__(PDEngine) + pd.config = PDConfig() + pd.peers = {"0": SimpleNamespace(name="0", role="decode", client=client)} + pd.requests, pd.transfers, pd.outputs, pd.worker_metrics = {}, {}, [], {} + pd.closed, pd.failure, pd.terminate_workers_on_failure = False, None, False + pd._update_version, pd._update_failed, pd._profile_session = None, False, None + + def timeout(): + raise TimeoutError("PD request/transfer deadline exceeded") + + pd._admit = timeout + try: + with pytest.raises(TimeoutError): + pd.step() + # The failed coordinator neither signalled the worker nor counts as shut down. + assert not pd.closed and client.process.is_alive() and not client.stopped + with pytest.raises(RuntimeError, match="unavailable"): + pd.step() + pd.close() + assert pd.closed and client.stopped + assert not client.process.is_alive() + pd.close() + finally: + client.close() + + @pytest.mark.parametrize("asynchronous", [False, True]) def test_mixed_score_admission_and_unspecified_seed_replay(asynchronous): model = tiny_ouro_model() diff --git a/vllm_rlt/pd/engine.py b/vllm_rlt/pd/engine.py index 1823d516..dba649e9 100644 --- a/vllm_rlt/pd/engine.py +++ b/vllm_rlt/pd/engine.py @@ -508,6 +508,8 @@ def has_unfinished_requests(self): def step(self): if self.closed: raise RuntimeError("PD engine is closed") + if self.failure: + raise RuntimeError("PD engine is unavailable") if self._update_version is not None or self._update_failed: raise RuntimeError("Generation is unavailable during a PD weight update") try: @@ -672,7 +674,7 @@ def wait_for_profile_artifacts(self, timeout=None): def _terminate(self): if not self.terminate_workers_on_failure: - self.closed = True + # Failed is not closed: close() keeps its cooperative drain, stop and join. return if self._profile_session: self._profile_recording = False