Repository navigation
fix(gms): honor configured timeout during initial weights admission #14877
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
galletas1712
merged 11 commits into
ai-dynamo:main
from
glamr-agent:dyn-4439-gms-interpod-shadow-readiness-hang-9a98fd0d5a08
Sep 16, 2026
Merged
Changes from 10 commits
Commits
Show all changes
11 commits
Select commit
Hold shift + click to select a range
9664f9f
fix(vllm): bound a GMS shadow engine's wait to enter standby
glamr-agent ac7585e
test(vllm): cover the init_device timeout forward and fix the bound l…
glamr-agent 4ca9a79
fix(vllm): address review on the GMS shadow standby bound
glamr-agent b2a82a8
chore: remove factory working notes
glamr-agent e01eb6c
fix(gms): reject non-finite shadow pause bounds and cap the connect wait
glamr-agent aeb1b71
fix(gms): reserve the unbounded standby wait for exactly "0"
glamr-agent 511b04d
test(vllm): keep one case per rejected-bound path
glamr-agent 6ebf73c
docs(gms): clarify shadow timeout controls and recovery scope
galletas1712 29c2439
fix(gms): isolate timeout forwarding test from NVML
glamr-agent bf7f35e
refactor(gms): limit timeout fix to configured admission
galletas1712 8b36da0
refactor(gms): minimize initial admission timeout fix
galletas1712 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| """Unit tests for the vLLM GMS worker's weights-admission deadline.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| from types import SimpleNamespace | ||
|
|
||
| import pytest | ||
| from _deps import HAS_GMS, HAS_TORCH | ||
|
|
||
| if not HAS_GMS: | ||
| pytest.skip( | ||
| "gpu_memory_service package is not available in this test image", | ||
| allow_module_level=True, | ||
| ) | ||
|
|
||
| if not HAS_TORCH: | ||
| pytest.skip("torch is required", allow_module_level=True) | ||
|
|
||
| pytest.importorskip("vllm", reason="vLLM is required") | ||
|
|
||
| from gpu_memory_service.common.locks import RequestedLockType # noqa: E402 | ||
| from gpu_memory_service.common.vmm import VMMDeviceType # noqa: E402 | ||
| from gpu_memory_service.integrations.vllm import worker as gms_worker # noqa: E402 | ||
|
|
||
| pytestmark = [ | ||
| pytest.mark.pre_merge, | ||
| pytest.mark.unit, | ||
| pytest.mark.vllm, | ||
| pytest.mark.core, | ||
| pytest.mark.gpu_0, | ||
| ] | ||
|
|
||
| _RO_CONNECT_TIMEOUT_MS = 4200 | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def init_device_calls(monkeypatch, tmp_path): | ||
| """Run ``GMSWorker.init_device`` with every device collaborator replaced. | ||
|
|
||
| Returns the keyword arguments the worker handed to the client-manager | ||
| factory. | ||
| """ | ||
| calls: list[dict[str, object]] = [] | ||
|
|
||
| def fake_factory(socket_path, device, mode=None, *, tag=None, timeout_ms=None): | ||
| calls.append( | ||
| { | ||
| "socket_path": socket_path, | ||
| "device": device, | ||
| "mode": mode, | ||
| "tag": tag, | ||
| "timeout_ms": timeout_ms, | ||
| } | ||
| ) | ||
| return SimpleNamespace() | ||
|
|
||
| monkeypatch.setattr( | ||
| gms_worker, "get_or_create_gms_client_memory_manager", fake_factory | ||
| ) | ||
| monkeypatch.setattr(gms_worker, "get_vmm_device_type", lambda: VMMDeviceType.CUDA) | ||
| # Socket discovery queries NVML for a GPU UUID, which gpu_0 runners lack. | ||
| monkeypatch.setattr( | ||
| gms_worker, | ||
| "get_socket_path", | ||
| lambda device, tag: str(tmp_path / f"gms_{device}_{tag}.sock"), | ||
| ) | ||
| monkeypatch.setattr( | ||
| "vllm.platforms.current_platform", | ||
| SimpleNamespace(set_device=lambda _device: None), | ||
| ) | ||
| # The parent's device init allocates on a real GPU; this test is only about | ||
| # the arguments GMS resolves before it runs. | ||
| monkeypatch.setattr(gms_worker._BaseWorker, "init_device", lambda self: None) | ||
|
|
||
| def run(extra_config: dict) -> list[dict[str, object]]: | ||
| worker = gms_worker.GMSWorker.__new__(gms_worker.GMSWorker) | ||
| worker.local_rank = 0 | ||
| worker.parallel_config = SimpleNamespace( | ||
| distributed_executor_backend="mp", | ||
| data_parallel_backend="mp", | ||
| nnodes_within_dp=1, | ||
| data_parallel_rank_local=0, | ||
| data_parallel_index=0, | ||
| pipeline_parallel_size=1, | ||
| tensor_parallel_size=1, | ||
| ) | ||
| worker.vllm_config = SimpleNamespace( | ||
| load_config=SimpleNamespace(model_loader_extra_config=extra_config) | ||
| ) | ||
| worker.init_device() | ||
| return calls | ||
|
|
||
| return run | ||
|
|
||
|
|
||
| def test_init_device_forwards_the_resolved_ro_connect_timeout(init_device_calls): | ||
| """A shadow's weights admission must carry the deadline, not drop it. | ||
|
|
||
| ``init_device`` resolves ``gms_ro_connect_timeout_ms`` for its own use on | ||
| the wake path. If it does not also hand the value to the factory, the very | ||
| first weights handshake is unbounded, and a shadow refused admission parks | ||
| there with no log and no traceback. | ||
| """ | ||
| calls = init_device_calls( | ||
| { | ||
| "gms_read_only": True, | ||
| "gms_ro_connect_timeout_ms": _RO_CONNECT_TIMEOUT_MS, | ||
| } | ||
| ) | ||
|
|
||
| assert len(calls) == 1 | ||
| assert calls[0]["tag"] == "weights" | ||
| assert calls[0]["mode"] == RequestedLockType.RO | ||
| assert calls[0]["timeout_ms"] == _RO_CONNECT_TIMEOUT_MS |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.