Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion vllm/v1/worker/gpu_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
import gc
import os
from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from contextlib import AbstractContextManager, contextmanager, nullcontext
from datetime import timedelta
from types import NoneType
from typing import TYPE_CHECKING, Any

import numpy as np
import regex as re
import torch
import torch.nn as nn

Expand Down Expand Up @@ -208,6 +209,30 @@ def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
)
return allocator.use_memory_pool(tag=tag)

@contextmanager
def _scoped_allocator_max_split(self, max_split_size_mb: int):
"""Temporarily set max_split_size_mb to reduce allocator fragmentation at the
cost of more cudaMalloc calls (negligible in practice). Restores the original
value on exit."""
if not current_platform.is_cuda():
yield
return

conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
match = re.search(r"max_split_size_mb:(\d+)", conf)
original_value = match.group(1) if match else None
Comment thread
MatthewBonanni marked this conversation as resolved.
Resolved

torch._C._accelerator_setAllocatorSettings(
f"max_split_size_mb:{max_split_size_mb}"
)
try:
yield
finally:
# PyTorch defaults to SIZE_MAX (no limit).
_SIZE_MAX_MB = (2**64 - 1) // (1024 * 1024)
restore = original_value if original_value else str(_SIZE_MAX_MB)
torch._C._accelerator_setAllocatorSettings(f"max_split_size_mb:{restore}")

@instrument(span_name="Init device")
def init_device(self):
if self.device_config.device_type == "cuda":
Expand Down Expand Up @@ -312,6 +337,8 @@ def load_model(self, *, load_dummy_weights: bool = False) -> None:
with (
self._maybe_get_memory_pool_context(tag="weights"),
set_current_vllm_config(self.vllm_config),
# 20 MiB is the minimum PyTorch allows for max_split_size_mb.
self._scoped_allocator_max_split(max_split_size_mb=20),
):
self.model_runner.load_model(load_dummy_weights=load_dummy_weights)

Expand Down
Loading