Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions lib/gpu_memory_service/client/torch/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,3 +221,64 @@ def materialize_module_from_gms(
len(meta_tensors),
meta_tensors[:10],
)


def rebind_nonparameter_tensors(
gms_client_memory_manager: "GMSClientMemoryManager",
model: torch.nn.Module,
) -> int:
"""Re-bind GMS-resident non-parameter tensors to private clones.

The publisher builds the whole model inside the GMS memory pool, so
buffers and tensor attributes (fp8 KV scales, quantization ranges, ...)
land in the same committed allocations as the weights, which are
remapped read-only after publish. Unlike parameters, these tensors can
be written after load (for example ``init_fp8_kv_scales`` on wake),
which faults on the read-only mapping. Cloning them into ordinary CUDA
memory gives the publisher the same binding semantics importers get
from ``materialize_module_from_gms``: parameters stay on the shared
read-only mapping, everything else is private and writable. The GMS
copies stay registered so importers can still materialize from them.

Must run before CUDA graph capture: the clones live at new addresses.

Returns the number of bytes rebound, i.e. how much memory is duplicated
between the read-only GMS copies and the private clones.
"""
mappings = gms_client_memory_manager.mappings
rebound_bytes = 0
for name, tensor, tensor_type in list(_iter_module_tensors(model)):
if tensor_type == "parameter":
continue
ptr = int(tensor.data_ptr())
if not any(
va <= ptr < va + mapping.aligned_size
for va, mapping in mappings.items()
):
# Allocated outside the GMS pool; already private.
continue

mod, attr = _resolve_module_attr(model, name)
clone = tensor.detach().clone()
if (
tensor_type == "buffer"
and hasattr(mod, "_buffers")
and attr in mod._buffers
):
mod._buffers[attr] = clone
elif attr.isdigit() and not isinstance(mod, torch.nn.Module):
# Element of a tensor list/tuple attribute.
if not isinstance(mod, list):
logger.debug("[GMS] Cannot rebind immutable container %r", name)
continue
Comment thread
galletas1712 marked this conversation as resolved.
Resolved
mod[int(attr)] = clone
else:
if isinstance(getattr(type(mod), attr, None), property):
# Read-only derived attribute; the underlying tensor is
# iterated (and rebound) separately.
logger.debug("[GMS] Skipping property attribute %r", name)
continue
setattr(mod, attr, clone)
rebound_bytes += clone.numel() * clone.element_size()
Comment thread
galletas1712 marked this conversation as resolved.
Resolved

return rebound_bytes
117 changes: 117 additions & 0 deletions lib/gpu_memory_service/docs/mutable-state-and-memory-classes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
# Mutable model state and GMS memory classes

Status: near-term fix implemented (publisher rebind in `finalize_gms_write`);
medium-term plan (misc pool + KV semantics convergence) not yet scheduled.

## Problem

The publisher builds the entire model inside the GMS `weights` memory pool,
so the committed allocations contain more than parameters: torch buffers
(`expert_map`, Mamba `conv_weights`, ...) and tensor attributes (fp8 KV
scales `_k_scale`/`_v_scale`/`_q_scale`/`_prob_scale`, quantization ranges)
are swept into the same caching-allocator segments. That sweep is
intentional — a fresh importer never runs `load_weights`, so everything the
forward pass references must be materializable from GMS.

The bug is that import semantics are uniform: after publish (and after any
RO reconnect — wake, restore), the whole tag is remapped
`CU_MEM_ACCESS_FLAGS_PROT_READ`. Some captured tensors are written *after*
load:

- `post_kv_cache_wake_up` → `init_fp8_kv_scales` rewrites the fp8 scale
attrs on every wake (any `--kv-cache-dtype fp8` model);
- fp8 range attrs are updated on the forward path.

A write through a `PROT_READ` mapping is a fatal MMU fault
(NVRM Xid 31 `FAULT_RO_VIOLATION`). Cold engines only avoid it by timing:
every write lands inside the RW load window, and the late writers first run
after a sleep/wake — or after a snapshot restore, which is how this was
found (Nemotron NVFP4-MoE, fp8 KV, DP8).

## Current lifecycle classes

| tag | lifecycle | sleep | wake/restore |
|---|---|---|---|
| `weights` | committed-immutable | unmap (bytes live server-side) | remap same VAs, `PROT_READ` |
| `kv_cache` | ephemeral RW | unmap + abort | fresh physical at preserved VAs |

Mutable-durable state (the scale/range attrs and, conservatively, all
non-parameter tensors) fits neither class.

## Near-term fix: publisher rebind (implemented)

`materialize_module_from_gms` already gives importers the right binding:
parameters bind to the shared read-only mapping; buffers/tensor attrs are
`detach().clone()`d into ordinary CUDA memory. The fix makes the publisher
adopt the same rule: at the end of `finalize_gms_write` (after
`register_module_tensors` records the GMS copies for importers, and after
the RO remap), `rebind_nonparameter_tensors` re-binds every GMS-resident
non-parameter tensor to a private clone.

Consequences:

- late writers hit ordinary CUDA memory — preserved byte-exact and
VA-stable by cuda-checkpoint/CRIU in the snapshot flow;
- the weights tag can be honestly `PROT_READ` for every consumer,
including snapshot-restored publishers;
- the server's committed copy can never be mutated;
- the GMS copies stay registered, so importer materialization is unchanged.

Constraints:

- the rebind must run before CUDA graph capture (clones live at new
addresses; captured graphs bake raw pointers);
- classification is deliberately blanket — *every* non-parameter is treated
as mutable, because proving immutability per-buffer is impractical. The
cost is the clone bytes, logged at finalize as
`rebound X MiB of non-parameter tensors to private memory` to keep the
duplication visible per model.

## Medium-term plan: `misc` pool and KV semantics convergence

Rather than keeping mutable state as process-private clones forever, the
plan is a third pool (`misc`) for non-parameter tensors, paired with a
change to KV semantics so it is *not* a third lifecycle:

- **Converged mutable class**: server-side allocations whose backing is
*reused* across sleep/wake instead of aborted and reallocated. Contents
are best-effort on failure (KV contents are disposable after a failure
anyway; misc contents are re-derivable: scales/ranges are recomputed at
wake, load-time constants are re-materializable from the artifact).
`kv_cache` moves to these semantics too — same VAs, same physical
backing, no realloc churn on wake.
- **Weights pool becomes parameters-only** at allocation time (misc pool
active for non-parameter creation) or via a post-load dense repack
(packed-slabs pattern), which also drops dead load-transient blocks from
the artifact.
- **Importer/shadow dedup**: shadows import misc read-only sources and keep
private working copies only where they write.

Open design questions to resolve before building:

1. **Snapshot capture point**: the saver copies committed state at
load-commit time. Mutable misc contents at dump time differ from
load-time values. Either the contract is "restore = load-time values +
recompute at wake" (holds for the known set), or the saver needs a
second, checkpoint-barrier-time pass over misc.
2. **Multi-consumer isolation**: misc is written per-engine; concurrent
engines (shadow failover) must not share writable physical pages —
copy-on-import or per-engine instances are required.
3. **Commit/staleness semantics**: misc is uncommitted-mutable; the layout
hash and `StaleMemoryLayoutError` rules need a class-aware definition.
4. **Placement**: non-parameter tensors are created inside model
`__init__`/load, interleaved with parameters under one mempool context;
routing them to misc at creation needs either a tag-switching hook or a
post-load migration pass (which must also complete before graph
capture).

## References

- Fault forensics: NVRM Xid 31 `FAULT_RO_VIOLATION ACCESS_TYPE_VIRT_WRITE`
on all ranks at first post-restore wake/forward; fault addresses at the
final 4 KiB page of 2 MiB small-pool segments (fp8 scale blocks at the
segment tail).
- Code: `client/torch/module.py` (`rebind_nonparameter_tensors`,
`materialize_module_from_gms`), `integrations/common/utils.py`
(`finalize_gms_write`), `client/memory_manager.py` (VA-stable
unmap/remap, `cumem_set_access`).
18 changes: 16 additions & 2 deletions lib/gpu_memory_service/integrations/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@

import torch
from gpu_memory_service.client.torch.allocator import prune_allocations
from gpu_memory_service.client.torch.module import register_module_tensors
from gpu_memory_service.client.torch.module import (
rebind_nonparameter_tensors,
register_module_tensors,
)
from gpu_memory_service.common.locks import RequestedLockType

if TYPE_CHECKING:
Expand Down Expand Up @@ -82,6 +85,13 @@ def finalize_gms_write(
"""Finalize GMS write mode: register tensors, commit, reconnect in read mode.

Flow: register tensors -> sync -> unmap + commit -> connect(RO) -> remap
-> rebind non-parameter tensors to private clones

The rebind mirrors the importer binding semantics from
``materialize_module_from_gms``: after publish, the read-only GMS
mapping backs parameters only, while buffers and tensor attributes that
may be written later (fp8 KV scale re-initialization on wake,
quantization range updates) live in ordinary CUDA memory.

Args:
allocator: The GMS client memory manager in write mode.
Expand Down Expand Up @@ -110,13 +120,17 @@ def finalize_gms_write(
allocator.connect(RequestedLockType.RO)
allocator.remap_all_vas()

rebound_bytes = rebind_nonparameter_tensors(allocator, model)

logger.info(
"[GMS] Committed %.2f GiB, switched to read mode with %d mappings "
"(pruned %d allocations / %.2f GiB before commit)",
"(pruned %d allocations / %.2f GiB before commit; rebound %.2f MiB "
"of non-parameter tensors to private memory)",
total_bytes / (1 << 30),
len(allocator.mappings),
pruned_count,
pruned_bytes / (1 << 30),
rebound_bytes / (1 << 20),
)

return GMSCommittedMemoryStats(
Expand Down
99 changes: 99 additions & 0 deletions lib/gpu_memory_service/tests/test_module_rebind.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Publisher-side rebind of non-parameter tensors to private clones."""

from types import SimpleNamespace

import pytest

torch = pytest.importorskip("torch")

from gpu_memory_service.client.torch.module import ( # noqa: E402
rebind_nonparameter_tensors,
)
Comment thread
galletas1712 marked this conversation as resolved.
Resolved

requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA"
)
Comment thread
galletas1712 marked this conversation as resolved.
Resolved


class _Layer(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.randn(4, 4, device="cuda"))
self.register_buffer(
"expert_map", torch.arange(4, device="cuda", dtype=torch.int32)
)
self._k_scale = torch.ones(1, device="cuda")
self.scales = [
torch.full((1,), 2.0, device="cuda"),
torch.full((1,), 3.0, device="cuda"),
]
self.private_scale = torch.ones(1, device="cuda")


def _fake_manager(*tensors: torch.Tensor) -> SimpleNamespace:
"""Manager double whose mappings cover exactly the given tensors."""
mappings = {
tensor.data_ptr(): SimpleNamespace(
aligned_size=tensor.numel() * tensor.element_size()
)
for tensor in tensors
}
return SimpleNamespace(mappings=mappings)


@requires_cuda
def test_rebind_moves_gms_resident_nonparameters_only():
Comment thread
galletas1712 marked this conversation as resolved.
Resolved
model = _Layer()
manager = _fake_manager(
model.weight, model.expert_map, model._k_scale, model.scales[0]
)

weight_ptr = model.weight.data_ptr()
buffer_ptr = model.expert_map.data_ptr()
scale_ptr = model._k_scale.data_ptr()
list_ptr = model.scales[0].data_ptr()
untracked_list_ptr = model.scales[1].data_ptr()
private_ptr = model.private_scale.data_ptr()

rebound_bytes = rebind_nonparameter_tensors(manager, model)

expected_bytes = sum(
t.numel() * t.element_size()
for t in (model.expert_map, model._k_scale, model.scales[0])
)
assert rebound_bytes == expected_bytes

# Parameters keep their (shared, read-only) binding.
assert model.weight.data_ptr() == weight_ptr

# GMS-resident buffer, tensor attr, and list element are rebound...
assert model.expert_map.data_ptr() != buffer_ptr
assert model._k_scale.data_ptr() != scale_ptr
assert model.scales[0].data_ptr() != list_ptr

# ...preserving values, and the clones are writable.
assert torch.equal(
model.expert_map.cpu(), torch.arange(4, dtype=torch.int32)
)
assert model.scales[0].item() == 2.0
model._k_scale.fill_(4.0)
assert model._k_scale.item() == 4.0

# Tensors outside the GMS mappings are already private and untouched.
assert model.scales[1].data_ptr() == untracked_list_ptr
assert model.private_scale.data_ptr() == private_ptr


@requires_cuda
def test_rebind_is_idempotent_after_clones_leave_gms():
Comment thread
galletas1712 marked this conversation as resolved.
Resolved
model = _Layer()
manager = _fake_manager(model.expert_map, model._k_scale)

first = rebind_nonparameter_tensors(manager, model)
assert first > 0

# The clones no longer point into the tracked mappings.
assert rebind_nonparameter_tensors(manager, model) == 0
Loading