From 5f94a3e40dfc62270b6a31a21966470c057adb84 Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:04:45 -0700 Subject: [PATCH] [https://nvbugs/7004364][fix] Extend GPU lifecycle startup budget Allow 600 seconds for the four native MPI ranks to become ready and increase the enclosing pytest timeout to 1080 seconds. Preserve the phase gates, transfer deadlines, and lifecycle assertions. Unwaive baseline, cancel_before_publication, cancel_late_aux, and cancel_late_kv, covering NVBUGs 7004364 and 7011346. Keep the separate 6902353 waivers and synchronize the documented startup deadline. This adjusts the test startup budget; it does not establish or fix the underlying cause of slow initialization. Verification is recorded in the pull request for the tested revision. Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 4 ---- tests/unittest/disaggregated/lifecycle_acceptance.md | 2 +- tests/unittest/disaggregated/test_transfer_lifecycle_gpu.py | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 750101c4db30..b02dd9e96ce7 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -180,10 +180,6 @@ unittest/_torch/test_connector_adp.py SKIP (https://nvbugs/7003528) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_wait_in_progress_on_zero_timeout SKIP (https://nvbugs/6517836) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_gpu_tensor SKIP (https://nvbugs/6517836) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_multiple_chunks SKIP (https://nvbugs/6517836) -unittest/disaggregated/test_transfer_lifecycle_gpu.py::test_transfer_lifecycle_gpu[baseline] SKIP (https://nvbugs/7004364) -unittest/disaggregated/test_transfer_lifecycle_gpu.py::test_transfer_lifecycle_gpu[cancel_before_publication] SKIP (https://nvbugs/7004364) -unittest/disaggregated/test_transfer_lifecycle_gpu.py::test_transfer_lifecycle_gpu[cancel_late_aux] SKIP (https://nvbugs/7011346) -unittest/disaggregated/test_transfer_lifecycle_gpu.py::test_transfer_lifecycle_gpu[cancel_late_kv] SKIP (https://nvbugs/7011346) unittest/disaggregated/test_transfer_lifecycle_gpu.py::test_transfer_lifecycle_gpu[destination_fatal] SKIP (https://nvbugs/6902353) unittest/disaggregated/test_transfer_lifecycle_gpu.py::test_transfer_lifecycle_gpu[source_fatal] SKIP (https://nvbugs/6902353) unittest/disaggregated/test_transfer_lifecycle_gpu.py::test_transfer_lifecycle_gpu[timeout_late_aux] SKIP (https://nvbugs/6902353) diff --git a/tests/unittest/disaggregated/lifecycle_acceptance.md b/tests/unittest/disaggregated/lifecycle_acceptance.md index 534758c5e569..9ba868fa9dc1 100644 --- a/tests/unittest/disaggregated/lifecycle_acceptance.md +++ b/tests/unittest/disaggregated/lifecycle_acceptance.md @@ -211,7 +211,7 @@ or reused identities mean the original process no longer executes; a live or unobservable rank fails. This process observation is not GPU/RDMA fencing proof. The [exit-observer tests](test_transfer_lifecycle_process_exit.py) exercise all nine scenarios on each rank. The [startup tests](test_transfer_lifecycle_startup.py) -check that both workers and the supervisor share one 120-second startup deadline, +check that both workers and the supervisor share one 600-second startup deadline, reject readiness observed at or after expiry, and leave relative transfer waits unchanged; startup failures print bounded worker logs and missing readiness markers before cleanup. diff --git a/tests/unittest/disaggregated/test_transfer_lifecycle_gpu.py b/tests/unittest/disaggregated/test_transfer_lifecycle_gpu.py index 46a9334c10c0..af7579770cc9 100644 --- a/tests/unittest/disaggregated/test_transfer_lifecycle_gpu.py +++ b/tests/unittest/disaggregated/test_transfer_lifecycle_gpu.py @@ -65,7 +65,7 @@ ) _PHASE_TIMEOUT_S = 30.0 _TIMEOUT_S = 4 * _PHASE_TIMEOUT_S -_WORKER_TIMEOUT_S = 120.0 +_WORKER_TIMEOUT_S = 600.0 def _wait( @@ -961,7 +961,7 @@ def retired() -> bool: _record(directory, f"{role}.reused_after_done", stable_outcome=session.status.name, sentinel=77) -@pytest.mark.timeout(600, method="signal") +@pytest.mark.timeout(_WORKER_TIMEOUT_S + 4 * _TIMEOUT_S, method="signal") @pytest.mark.parametrize("case", _CASES) def test_transfer_lifecycle_gpu(case: str, tmp_path: Path) -> None: """Qualify real GPU/NIXL software lifecycle with four GPUs and separate MPI jobs.