Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/filters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,7 @@ snapshot_vllm:
- 'deploy/operator/internal/dynamo/backend_vllm.go'

snapshot_sglang:
- 'components/src/dynamo/sglang/args.py'
- 'components/src/dynamo/sglang/snapshot.py'
- 'deploy/operator/internal/dynamo/backend_sglang.go'
Comment thread
galletas1712 marked this conversation as resolved.
Resolved

Expand Down
3 changes: 3 additions & 0 deletions components/src/dynamo/sglang/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,9 @@ async def parse_args(args: list[str]) -> Config:

if is_snapshot_enabled():
configure_snapshot_capture_env()
# SGLang's scheduler launcher reads the raw field before late
# resolution, so snapshot mode must enable it before ServerArgs creation.
parsed_args.enable_memory_saver = True

# TODO: sglang downloads the model in `from_cli_args`, which means we had to
# fetch_model (download the model) here, in `parse_args`. `parse_args` should not
Expand Down
41 changes: 41 additions & 0 deletions components/src/dynamo/sglang/tests/test_sglang_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -517,6 +517,47 @@ def resolve(parsed_args):
await parse_args(sys.argv[1:])


@pytest.mark.asyncio
@pytest.mark.parametrize(
Comment thread
galletas1712 marked this conversation as resolved.
Resolved
("snapshot_enabled", "expected"),
[
(False, False),
(True, True),
],
)
async def test_parse_args_sets_raw_memory_saver_before_resolution(
monkeypatch, mock_sglang_cli, tmp_path, snapshot_enabled, expected
):
monkeypatch.setattr(
"dynamo.sglang.args.configure_snapshot_capture_env", lambda: None
)
monkeypatch.delenv("DYN_GMS_USE_V1", raising=False)
if snapshot_enabled:
monkeypatch.setenv(SNAPSHOT_CONTROL_DIR_ENV, str(tmp_path))
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "0")
else:
monkeypatch.delenv(SNAPSHOT_CONTROL_DIR_ENV, raising=False)
server_args = SimpleNamespace(
disaggregation_mode="null",
dllm_algorithm=None,
kv_events_config=None,
get_model_config=lambda: SimpleNamespace(is_multimodal=False),
)

def resolve(parsed_args):
# SGLang 0.5.19 copies this raw field unchanged; late resolution does
# not update it before the parent process launches the scheduler.
server_args.enable_memory_saver = parsed_args.enable_memory_saver
return server_args

monkeypatch.setattr("dynamo.sglang.args.ServerArgs.from_cli_args", resolve)
mock_sglang_cli(model=str(tmp_path))

config = await parse_args(sys.argv[1:])

assert config.server_args.enable_memory_saver is expected


def test_compat_filters_async_generate_kwargs_for_older_engines():
class OldEngine:
async def async_generate(self, input_ids=None, sampling_params=None):
Expand Down
Loading