Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -283,3 +283,6 @@ test/registered/xpu/test_nvidia_nemotron_3_nano.py
artifacts/
.claude/scheduled_tasks.lock
.humanize/

# Internal, non-published docs
internal-docs/
8 changes: 3 additions & 5 deletions python/sglang/srt/elastic_ep/expert_backup_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,12 +159,10 @@ def update_weights(self, weight_name_filter=None):
param = param.narrow(
0, param.shape[0] // 2, param.shape[0] // 2
)
server_ptr_list.append(weight_info["weight_ptr"])
server_ptr_list.append(weight_info.weight_ptr)
local_ptr_list.append(param.data_ptr())
assert (
param.numel() * param.element_size() == weight_info["byte_size"]
)
weight_size_list.append(weight_info["byte_size"])
assert param.numel() * param.element_size() == weight_info.byte_size
weight_size_list.append(weight_info.byte_size)
before_transfer = time.time()
ret = self.transfer_engine.engine.batch_transfer_sync_read(
self.session_id_list[i],
Expand Down
20 changes: 10 additions & 10 deletions python/sglang/srt/elastic_ep/expert_backup_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@
from sglang.srt.configs.load_config import LoadConfig
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.environ import envs
from sglang.srt.managers.io_struct import BackupDramReq, sock_recv, sock_send
from sglang.srt.managers.io_struct import (
BackupDramReq,
ExpertWeightPointer,
sock_recv,
sock_send,
)
from sglang.srt.model_loader.loader import DefaultModelLoader, get_model_loader
from sglang.srt.model_loader.utils import set_default_torch_dtype
from sglang.srt.server_args import (
Expand Down Expand Up @@ -128,15 +133,10 @@ def backup_weights_from_disk(self):
end_byte = current_byte_offset + byte_size
weight_ptr = buffer_base_ptr + current_byte_offset
self.continuous_buffer[start_byte:end_byte].copy_(weight_bytes)
self.weight_pointer_map[name] = {
"name": name,
"weight_ptr": weight_ptr,
"shape": weight_info["shape"],
"numel": weight_info["numel"],
"dtype": weight_info["dtype"],
"element_size": weight_info["element_size"],
"byte_size": byte_size,
}
self.weight_pointer_map[name] = ExpertWeightPointer(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why should we drop the shape, numel, dtype element_size etc info here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't find any consumer that reads them in the codebase. For example, ExpertBackupClient.update_weights uses only weight_ptr and byte_size. Every other field is derivable: name comes from the map key, and shape/numel/dtype/element_size are re-derived from the local parameter tensor (we even assert numel * element_size == byte_size).

So those five were adding only weight on the wire. With one entry per expert weight, ExpertWeightPointer is array_like too, so each entry is just [weight_ptr, byte_size].

weight_ptr=weight_ptr,
byte_size=byte_size,
)

current_byte_offset = end_byte

Expand Down
5 changes: 5 additions & 0 deletions python/sglang/srt/entrypoints/http_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,11 @@ async def set_internal_state(
async def _dumper_control_handler(method: str, request: Request):
body_bytes = await request.body()
body = await request.json() if body_bytes else {}
if not isinstance(body, dict):
return ORJSONResponse(
status_code=400,
content={"error": "Request body must be a JSON object."},
)
obj = DumperControlReqInput(method=method, body=body)
results = await _global_state.tokenizer_manager.dumper_control(obj)
if any(not r.success for r in results):
Expand Down
1 change: 1 addition & 0 deletions python/sglang/srt/environ.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@ class Envs:

# IPC
SGLANG_USE_PICKLE_IPC = EnvBool(True)
# Log top-level PickleWrapper frames unwrapped on msgpack IPC decode.
SGLANG_LOG_PICKLE_IPC_OBJECTS = EnvBool(False)

# SGLang CI
Expand Down
94 changes: 52 additions & 42 deletions python/sglang/srt/managers/io_struct.py
Original file line number Diff line number Diff line change
Expand Up @@ -1551,7 +1551,7 @@ class UpdateWeightFromDiskReqInput(BaseReq, kw_only=True):
token_step: int = 0
# Whether to flush the cache after updating weights
flush_cache: bool = True
# Tensor metadata
# Tensor metadata from the JSON request body, so it is already msgpack-native.
manifest: Optional[Dict[str, Any]] = None


Expand Down Expand Up @@ -1669,9 +1669,17 @@ class UpdateExpertBackupReq(BaseReq, kw_only=True):
pass


class ExpertWeightPointer(msgspec.Struct, kw_only=True, array_like=True):
# One expert weight's pointer + byte length in the DRAM backup buffer.
# array_like: the map has tens of thousands of entries, so positional
# encoding drops the repeated field names from the wire.
weight_ptr: int
byte_size: int


class BackupDramReq(BaseReq, kw_only=True):
rank: int
weight_pointer_map: Dict[str, Any]
weight_pointer_map: Dict[str, ExpertWeightPointer]
session_id: str
buffer_size: int

Expand Down Expand Up @@ -1718,7 +1726,9 @@ class GetWeightsByNameReqInput(BaseReq, kw_only=True):


class GetWeightsByNameReqOutput(BaseReq, kw_only=True):
parameter: Optional[List[Any]]
# A flat List[float] or a per-row List[List[float]]. The union is on the
# element: Union[List[float], List[List[float]]] is invalid msgspec.
parameter: Optional[List[Union[float, List[float]]]]


class ReleaseMemoryOccupationReqInput(BaseReq, kw_only=True):
Expand Down Expand Up @@ -1746,10 +1756,32 @@ class CheckWeightsReqInput(BaseReq, kw_only=True):
allow_quant_error: bool = False


# Wire versions of the pydantic ParallelismInfo/ChecksumInfo in
# sglang.srt.utils.weight_checker. Not array_like: the payload is read by field
# name and re-serialized to JSON, so it must stay a {field: value} map.
class ParallelismInfo(msgspec.Struct, kw_only=True):
tp_rank: int
tp_size: int
dp_rank: int
dp_size: int
pp_rank: int
pp_size: int
rank: int
size: int


class ChecksumInfo(msgspec.Struct, kw_only=True):
checksums: Dict[str, str]
per_gpu_checksum: str
parallelism_info: ParallelismInfo


class CheckWeightsReqOutput(BaseReq, kw_only=True):
success: bool
message: str
payload: Optional[Dict[str, Any]] = None
# One ChecksumInfo per TP rank. The producer wraps the tp==1 result in a
# one-element list so the shape is always a list.
payload: Optional[List[ChecksumInfo]] = None


class SlowDownReqInput(BaseReq, kw_only=True):
Expand Down Expand Up @@ -1782,16 +1814,19 @@ class GetInternalStateReq(BaseReq, kw_only=True):


class GetInternalStateReqOutput(BaseReq, kw_only=True):
# A vars() dump of ServerArgs, left untyped because a struct would drift. The
# producer sanitizes it with msgspec_to_builtins so every value is
# msgpack-native.
internal_state: Dict[str, Any]


class SetInternalStateReq(BaseReq, kw_only=True):
server_args: Dict[str, Any]
# Only numeric scheduler knobs are accepted (see Scheduler.set_internal_state).
server_args: Dict[str, Union[int, float]]


class SetInternalStateReqOutput(BaseReq, kw_only=True):
updated: bool
server_args: Dict[str, Any]


class ProfileReqType(Enum):
Expand Down Expand Up @@ -1922,13 +1957,16 @@ class SeparateReasoningReqInput(BaseReq, kw_only=True):


class VertexGenerateReqInput(BaseReq, kw_only=True):
# Both fields come from the JSON request body, so they are already
# msgpack-native.
instances: List[Dict[str, Any]]
parameters: Optional[Dict[str, Any]] = None


class RpcReqInput(BaseReq, kw_only=True):
method: str
parameters: Optional[Dict[str, Any]] = None
# collective_rpc kwargs are flat scalars across all in-tree callers.
parameters: Optional[Dict[str, Union[bool, int, float, str, None]]] = None


class RpcReqOutput(BaseReq, kw_only=True):
Expand Down Expand Up @@ -1970,10 +2008,12 @@ def to_ref(self) -> LoRARef:

class LoadLoRAAdapterFromTensorsReqInput(BaseReq, kw_only=True):
lora_name: str
# The PEFT adapter_config.json, already JSON — a tighter type would only add
# decode strictness with no benefit.
config_dict: Dict[str, Any]
serialized_tensors: str
pinned: bool = False
added_tokens_config: Optional[Dict[str, Any]] = None
added_tokens_config: Optional[Dict[str, int]] = None
lora_id: Optional[str] = None
load_format: Optional[str] = None

Expand Down Expand Up @@ -2006,10 +2046,6 @@ class BlockReqInput(BaseReq, kw_only=True):
req_type: BlockReqType


class SetInjectDumpMetadataReqInput(BaseReq, kw_only=True):
dump_metadata: Dict[str, Any]


class SetInjectDumpMetadataReqOutput(BaseReq, kw_only=True):
success: bool

Expand All @@ -2024,11 +2060,13 @@ class LazyDumpTensorsReqOutput(BaseReq, kw_only=True):

class DumperControlReqInput(BaseReq, kw_only=True):
method: str
# JSON request body (guarded to be a dict at the /dumper endpoint).
body: Dict[str, Any]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the real data type for the dictionary, will those type not able to be serialized to bytes?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

response is what handle_request returns (typed -> list[dict]), e.g. get_state()'s config dict, or {} for configure/reset. It's built and re-serialized with json.dumps, so every value is JSON-native and msgpack-safe.

Like body, it stays Dict[str, Any] because the dumper response has no fixed schema.



class DumperControlReqOutput(BaseReq, kw_only=True):
success: bool
# JSON-native per-worker response dicts.
response: List[Dict[str, Any]]

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the real data type for the dictionary, will those type not able to be serialized to bytes?

@jorgeantonio21 jorgeantonio21 Jul 8, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In this case, the body is the raw request.json() (afaik, the dumper method endpoint rejects it unless it is a dict). This value is then passed to dumper.configure(**body). Since it comes straight from json.loads, every value is already JSON-native and encodes fine over msgpack.

I think it should stay Dict[str, Any] because configure() takes open-ended kwargs with no fixed schema.

error: str = ""

Expand Down Expand Up @@ -2068,28 +2106,6 @@ def _check_all_req_types():

_check_all_req_types()

# IPC struct types whose fields still use opaque annotations (Any, Dict[str, Any],
# List[Any], etc.) instead of precise types. Keep these on explicit pickle
# transport until their field schemas are tightened, and keep the registry
# explicit so opaque usage can be audited and gradually narrowed.
# NOTE: GenerateReqInput and EmbeddingReqInput are standalone (not BaseReq/
# BaseBatchReq subclasses) and are tracked separately.
_REQ_TYPES_WITH_OPAQUE_FIELDS: tuple[Type[msgspec.Struct], ...] = (
UpdateWeightFromDiskReqInput, # manifest: Optional[Dict[str, Any]]
BackupDramReq, # weight_pointer_map: Dict[str, Any]
GetWeightsByNameReqOutput, # parameter: Optional[List[Any]]
CheckWeightsReqOutput, # payload: Optional[Dict[str, Any]]
GetInternalStateReqOutput, # internal_state: Dict[str, Any]
SetInternalStateReq, # server_args: Dict[str, Any]
SetInternalStateReqOutput, # server_args: Dict[str, Any]
VertexGenerateReqInput, # instances, parameters: Dict[str, Any]
RpcReqInput, # parameters: Optional[Dict[str, Any]]
LoadLoRAAdapterFromTensorsReqInput, # config_dict, added_tokens_config: Dict[str, Any]
SetInjectDumpMetadataReqInput, # dump_metadata: Dict[str, Any]
DumperControlReqInput, # body: Dict[str, Any]
DumperControlReqOutput, # response: List[Dict[str, Any]]
)


def wrap_as_pickle(obj: object) -> object:
if obj is None:
Expand Down Expand Up @@ -2180,19 +2196,13 @@ def hook_custom_types(*new_types: Type):


def _maybe_wrap_pickle(obj: Any) -> Any:
if isinstance(obj, _REQ_TYPES_WITH_OPAQUE_FIELDS):
if envs.SGLANG_LOG_PICKLE_IPC_OBJECTS.get():
logger.info(f"Object of type {type(obj)} is wrapped via PickleWrapper.")
return PickleWrapper(pickle.dumps(obj))

if isinstance(obj, (msgspec.Struct, *_primitive_types)):
return obj

raise TypeError(
f"Cannot serialize object of type {type(obj)} over msgpack IPC. "
"Add a precise msgspec-compatible type, use an explicit PickleWrapper "
"field for the opaque payload, or add the struct to "
"_REQ_TYPES_WITH_OPAQUE_FIELDS with an audit comment."
"Add a precise msgspec-compatible type, or use an explicit PickleWrapper "
"field via wrap_as_pickle(...) for the opaque payload."
)


Expand Down
12 changes: 4 additions & 8 deletions python/sglang/srt/managers/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -3825,8 +3825,10 @@ def get_internal_state(self, recv_req: GetInternalStateReq):
if info_record is not None:
ret["dspark_info_record"] = info_record

# This field is not serializable.
# These fields are not msgpack-serializable (a config object and a bound
# signal handler); no reader consumes them.
ret.pop("model_config", None)
ret.pop("custom_sigquit_handler", None)

return GetInternalStateReqOutput(internal_state=msgspec_to_builtins(ret))

Expand Down Expand Up @@ -3908,13 +3910,7 @@ def set_internal_state(self, recv_req: SetInternalStateReq):
get_server_args().override(source="update_server_args", **remaining)
logger.info(f"Global server args updated! {get_server_args()=}")

server_args = dict(vars(get_server_args()))
# This field is not serializable.
server_args.pop("model_config", None)
return SetInternalStateReqOutput(
updated=if_success,
server_args=msgspec_to_builtins(server_args),
)
return SetInternalStateReqOutput(updated=if_success)

def save_remote_model(self, **kwargs):
self.weight_updater.save_remote_model(kwargs)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, Iterator, Optional, Tuple

import msgspec
import torch

from sglang.srt.constants import (
Expand All @@ -18,6 +19,7 @@
)
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.io_struct import (
ChecksumInfo,
CheckWeightsReqInput,
CheckWeightsReqOutput,
DestroyWeightsUpdateGroupReqInput,
Expand Down Expand Up @@ -289,6 +291,11 @@ def check_weights(self, recv_req: CheckWeightsReqInput):
all_payloads, payload, group=self.tp_cpu_group
)
payload = all_payloads
if payload is not None:
# Normalize to one ChecksumInfo per rank so the wire shape is a
# uniform List[ChecksumInfo] (tp==1 becomes a single-element list).
per_rank = payload if isinstance(payload, list) else [payload]
payload = [msgspec.convert(p, ChecksumInfo) for p in per_rank]
Comment thread
jorgeantonio21 marked this conversation as resolved.
Resolved
return CheckWeightsReqOutput(
success=True, message="Success.", payload=payload
)
Expand Down
15 changes: 8 additions & 7 deletions python/sglang/srt/managers/tokenizer_control_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
AddExternalCorpusReqOutput,
AttachHiCacheStorageReqInput,
AttachHiCacheStorageReqOutput,
ChecksumInfo,
CheckWeightsReqInput,
CheckWeightsReqOutput,
ClearHiCacheReqInput,
Expand Down Expand Up @@ -77,6 +78,7 @@
get_bool_env_var,
normalize_serialized_named_tensor_payloads,
)
from sglang.srt.utils.msgspec_utils import msgspec_to_builtins
from sglang.utils import TypeBasedDispatcher

if TYPE_CHECKING:
Expand Down Expand Up @@ -760,16 +762,15 @@ async def check_weights(
ranks: Optional[List[Dict]] = None
per_engine_checksum: Optional[str] = None
if any(r.payload is not None for r in results):
ranks = []
rank_infos: List[ChecksumInfo] = []
for r in results:
if isinstance(r.payload, list):
ranks.extend(r.payload)
else:
ranks.append(r.payload)
if r.payload is not None:
rank_infos.extend(r.payload)
h = hashlib.sha256()
for rank in ranks:
h.update(rank["per_gpu_checksum"].encode())
for info in rank_infos:
h.update(info.per_gpu_checksum.encode())
per_engine_checksum = h.hexdigest()
ranks = [msgspec_to_builtins(info) for info in rank_infos]
return success, message, ranks, per_engine_checksum

async def slow_down(
Expand Down
Loading
Loading