diff --git a/src/megatron/bridge/peft/multi_lora_layers.py b/src/megatron/bridge/peft/multi_lora_layers.py
index c0addc1841..95d923025f 100644
--- a/src/megatron/bridge/peft/multi_lora_layers.py
+++ b/src/megatron/bridge/peft/multi_lora_layers.py
@@ -19,8 +19,9 @@
 via per-layer ``tokens_per_adapter`` set by :func:`set_tokens_per_adapter_slot`.
 
 Forward stacks the raw weights of all adapters and uses ``torch._grouped_mm``
-for a single fused kernel; TP/SP collectives are issued once around the two
-GEMMs to match the layout of the wrapped base linear.
+when its 16-byte alignment contract is satisfied, otherwise falling back to
+per-slot linear projections. TP/SP collectives are still issued once around
+the two projections to match the layout of the wrapped base linear.
 
 :class:`MultiLoRAGroupedExpertLinear` is the MoE counterpart, wrapping a grouped
 expert linear (``mlp.experts.linear_fc{1,2}`` of a ``TEGroupedMLP``) with one
@@ -55,12 +56,93 @@
 )
 
 
+# ``torch._grouped_mm`` operand contract. Both values mirror the CUDA layout
+# checks in PyTorch ATen's GroupedMMUtils.h — see
+# https://github.com/pytorch/pytorch/blob/ab5fb26f8ffc6e4dc97b51b5611bce957645b1db/aten/src/ATen/native/GroupedMMUtils.h#L24-L48
+# — and are not user-configurable tuning parameters. The failure modes are
+# asymmetric: gating too strictly only routes an eligible tensor to the safe
+# per-slot fallback, while gating too loosely sends it to the fast path where
+# PyTorch's host-side TORCH_CHECK fails loudly.
+_PYTORCH_GROUPED_MM_ALIGNMENT_BYTES = 16
+_GROUPED_MM_SUPPORTED_DTYPES = frozenset((torch.float16, torch.bfloat16))
+
+
+def _has_aligned_grouped_mm_layout(tensor: torch.Tensor) -> bool:
+    """Return whether a tensor's address and non-unit strides are 16-byte aligned."""
+
+    element_size = tensor.element_size()
+    return tensor.data_ptr() % _PYTORCH_GROUPED_MM_ALIGNMENT_BYTES == 0 and all(
+        stride == 1 or stride * element_size % _PYTORCH_GROUPED_MM_ALIGNMENT_BYTES == 0 for stride in tensor.stride()
+    )
+
+
+def _can_use_grouped_mm(input_: torch.Tensor, weights: torch.Tensor) -> bool:
+    """Return whether grouped MM is safe for one dense multi-LoRA projection.
+
+    ``weights`` has the ``[groups, output_features, input_features]`` layout
+    consumed by :func:`torch.nn.functional.linear`. Grouped MM receives its
+    transpose, and its backward may receive a contiguous output gradient after
+    a TP collective. Conservatively require that layout to remain 16-byte
+    aligned as well as the current input and weight strides.
+    """
+
+    # PyTorch 2.11 accepts FP32 grouped MM in forward but its backward fails
+    # for this layout, so keep FP32 on the autograd-safe fallback.
+    if (
+        not input_.is_cuda
+        or not weights.is_cuda
+        or input_.device != weights.device
+        or input_.shape[0] == 0
+        or not hasattr(torch, "_grouped_mm")
+        or input_.dtype != weights.dtype
+        or input_.dtype not in _GROUPED_MM_SUPPORTED_DTYPES
+    ):
+        return False
+    grouped_weights = weights.transpose(-2, -1)
+    if not _has_aligned_grouped_mm_layout(input_) or not _has_aligned_grouped_mm_layout(grouped_weights):
+        return False
+    contiguous_output_stride_bytes = weights.shape[-2] * input_.element_size()
+    return contiguous_output_stride_bytes % _PYTORCH_GROUPED_MM_ALIGNMENT_BYTES == 0
+
+
+def _apply_per_slot_linear(
+    input_: torch.Tensor,
+    weights: torch.Tensor,
+    token_counts: Sequence[int],
+) -> torch.Tensor:
+    """Apply one weight per contiguous adapter slot without grouped MM."""
+
+    if len(token_counts) != weights.shape[0] or sum(token_counts) != input_.shape[0]:
+        raise RuntimeError(
+            f"Per-slot projection received {input_.shape[0]} rows, {weights.shape[0]} weight groups, "
+            f"and token counts {tuple(token_counts)}."
+        )
+    chunks = input_.split(tuple(token_counts), dim=0)
+    return torch.cat(
+        [nn.functional.linear(chunk, weight) for chunk, weight in zip(chunks, weights)],
+        dim=0,
+    )
+
+
+def _apply_multi_lora_projection(
+    input_: torch.Tensor,
+    weights: torch.Tensor,
+    offsets: torch.Tensor,
+    token_counts: Sequence[int],
+) -> torch.Tensor:
+    """Apply a dense multi-LoRA projection through grouped MM or its safe fallback."""
+
+    if _can_use_grouped_mm(input_, weights):
+        return torch._grouped_mm(input_, weights.transpose(-2, -1), offsets)
+    return _apply_per_slot_linear(input_, weights, token_counts)
+
+
 class MultiLoRALinear(AdapterWrapper):
     """Megatron parallel linear wrapped with *N* concurrent LoRA adapters.
 
     Each adapter slot is a :class:`ParallelLinearAdapter` stored in an
-    ``nn.ModuleList``. Forward uses grouped GEMM with a single set of
-    TP/SP comms for efficiency.
+    ``nn.ModuleList``. Forward uses grouped GEMM where its operand strides are
+    supported, with a safe per-slot fallback and a single set of TP/SP comms.
 
     For bridge export compatibility, use :func:`expose_adapter_slot` to
     temporarily expose one slot as ``.adapter``.
@@ -145,6 +227,10 @@ def __init__(
         )
 
         self.tokens_per_adapter: Optional[torch.Tensor] = None
+        # Host copy written by set_tokens_per_adapter_slot. The safe fallback
+        # needs Python split sizes, and retaining the setter's one synchronization
+        # avoids a device-to-host sync in every adapted layer.
+        self.tokens_per_adapter_host: Optional[Tuple[int, ...]] = None
         # Host-side sum of tokens_per_adapter (set alongside it); lets forward
         # detect an SP-sharded input without a per-layer device sync.
         self.tokens_per_adapter_total: Optional[int] = None
@@ -163,6 +249,11 @@ def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> Tuple[torch.Ten
             return linear_output, bias
 
         tokens_per_adapter = self.tokens_per_adapter
+        token_counts = self.tokens_per_adapter_host
+        if tokens_per_adapter is None or token_counts is None:
+            raise RuntimeError(
+                f"{self.base_linear_name}: set_tokens_per_adapter_slot() must run before every forward."
+            )
         x = layernorm_output.contiguous()
 
         # SP gather (once) — for column-parallel base layers without an LN-fused
@@ -190,13 +281,14 @@ def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> Tuple[torch.Ten
                 )
             start = parallel_state.get_tensor_model_parallel_rank() * x_flat.shape[0]
             tokens_per_adapter = _narrow_token_counts_to_window(tokens_per_adapter, start, x_flat.shape[0])
+            token_counts = _narrow_token_counts_to_window_host(token_counts, start, x_flat.shape[0])
 
         offsets = tokens_per_adapter.cumsum(dim=0, dtype=torch.int32)
 
         stacked_A = torch.stack([a.linear_in.weight for a in self.adapters])
         stacked_B = torch.stack([a.linear_out.weight for a in self.adapters])
 
-        mid = torch._grouped_mm(x_flat, stacked_A.transpose(-2, -1), offsets)
+        mid = _apply_multi_lora_projection(x_flat, stacked_A, offsets, token_counts)
 
         # TP collective between A and B: row-parallel base needs an all-reduce
         # of the partial sums; every other base (column-parallel and replicated
@@ -208,7 +300,7 @@ def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> Tuple[torch.Ten
         else:
             mid = gather_from_tensor_model_parallel_region(mid)
 
-        out = torch._grouped_mm(mid, stacked_B.transpose(-2, -1), offsets)
+        out = _apply_multi_lora_projection(mid, stacked_B, offsets, token_counts)
 
         # Per-token scaling is applied *before* the output-side TP/SP comms.
         # ``per_token_scaling`` is indexed by the full token count
@@ -483,6 +575,7 @@ def __init__(
         )
 
         self.tokens_per_adapter: Optional[torch.Tensor] = None
+        self.tokens_per_adapter_host: Optional[Tuple[int, ...]] = None
         # Written by set_tokens_per_adapter_slot alongside tokens_per_adapter;
         # unused here (the slot-routing hook owns the expert-side SP narrow).
         self.tokens_per_adapter_total: Optional[int] = None
@@ -608,6 +701,23 @@ def _narrow_token_counts_to_window(counts: torch.Tensor, start: int, num_rows: i
     return (cum.clamp(max=start + num_rows) - (cum - counts).clamp(min=start)).clamp(min=0).to(counts.dtype)
 
 
+def _narrow_token_counts_to_window_host(
+    counts: Sequence[int],
+    start: int,
+    num_rows: int,
+) -> Tuple[int, ...]:
+    """Host-side equivalent of :func:`_narrow_token_counts_to_window`."""
+
+    end = start + num_rows
+    slot_start = 0
+    narrowed = []
+    for count in counts:
+        slot_end = slot_start + count
+        narrowed.append(max(0, min(slot_end, end) - max(slot_start, start)))
+        slot_start = slot_end
+    return tuple(narrowed)
+
+
 def _iter_multi_lora_modules(model):
     models = model if isinstance(model, list) else [model]
     for model_chunk in models:
@@ -626,13 +736,10 @@ def set_tokens_per_adapter_slot(model, tokens_per_adapter: torch.Tensor) -> None
     """
     if tokens_per_adapter.dim() != 1:
         raise ValueError(
-            f"tokens_per_adapter must be a 1-D tensor of per-slot counts; "
-            f"got shape {tuple(tokens_per_adapter.shape)}"
+            f"tokens_per_adapter must be a 1-D tensor of per-slot counts; got shape {tuple(tokens_per_adapter.shape)}"
         )
     if tokens_per_adapter.is_floating_point() or tokens_per_adapter.is_complex():
-        raise ValueError(
-            f"tokens_per_adapter must be an integer tensor; got dtype {tokens_per_adapter.dtype}"
-        )
+        raise ValueError(f"tokens_per_adapter must be an integer tensor; got dtype {tokens_per_adapter.dtype}")
     # One host sync per micro-batch (the tolist doubles as the sync the SP-shard
     # narrowing needs): layers whose base linear consumes the SP-sharded sequence
     # compare their row count against this total to narrow the spans to their
@@ -644,13 +751,13 @@ def set_tokens_per_adapter_slot(model, tokens_per_adapter: torch.Tensor) -> None
             f"non-monotonic grouped-GEMM offsets); got {counts}"
         )
     total = int(sum(counts))
+    host_counts = tuple(counts)
     modules = list(_iter_multi_lora_modules(model))
     if modules:
         n_adapters = modules[0].n_adapters
         if len(counts) != n_adapters:
             raise ValueError(
-                f"tokens_per_adapter has {len(counts)} entries but the model was "
-                f"built with n_adapters={n_adapters}"
+                f"tokens_per_adapter has {len(counts)} entries but the model was built with n_adapters={n_adapters}"
             )
         # The dense grouped GEMM consumes the counts on the model's device; the
         # MoE routing already moves them defensively — do it once here for both.
@@ -659,6 +766,7 @@ def set_tokens_per_adapter_slot(model, tokens_per_adapter: torch.Tensor) -> None
             tokens_per_adapter = tokens_per_adapter.to(first_param.device)
     for module in modules:
         module.tokens_per_adapter = tokens_per_adapter
+        module.tokens_per_adapter_host = host_counts
         module.tokens_per_adapter_total = total
 
 
diff --git a/tests/unit_tests/peft/test_multi_lora_layers.py b/tests/unit_tests/peft/test_multi_lora_layers.py
index b1e9979b48..b6dc38a1cd 100644
--- a/tests/unit_tests/peft/test_multi_lora_layers.py
+++ b/tests/unit_tests/peft/test_multi_lora_layers.py
@@ -51,6 +51,7 @@
 from megatron.bridge.peft import multi_lora_layers as multi_lora_layers_module
 from megatron.bridge.peft.multi_lora import MultiLoRA
 from megatron.bridge.peft.multi_lora_layers import (
+    _PYTORCH_GROUPED_MM_ALIGNMENT_BYTES,
     MultiLoRALinear,
     _iter_multi_lora_modules,
     clear_adapter_slot,
@@ -332,6 +333,7 @@ def test_set_tokens_per_adapter_slot(self) -> None:
 
         for module in container.mods:
             assert module.tokens_per_adapter is tokens
+            assert module.tokens_per_adapter_host == (3, 5)
 
     def test_set_tokens_per_adapter_slot_validates_input(self) -> None:
         # A wrong length silently mis-groups the grouped GEMM; negative counts
@@ -681,9 +683,12 @@ def test_load_adapter_unused_keys_raises():
 # an out-of-bounds grouped GEMM, not a shape error.
 # --------------------------------------------------------------------------- #
 def _narrow(counts, start, num_rows):
-    return multi_lora_layers_module._narrow_token_counts_to_window(
+    device_counts = multi_lora_layers_module._narrow_token_counts_to_window(
         torch.tensor(counts, dtype=torch.int32), start, num_rows
     ).tolist()
+    host_counts = list(multi_lora_layers_module._narrow_token_counts_to_window_host(counts, start, num_rows))
+    assert host_counts == device_counts
+    return device_counts
 
 
 def test_narrow_window_spanning_a_slot_boundary():
@@ -710,6 +715,25 @@ def test_narrow_window_covering_many_small_slots():
     assert sum(_narrow([4, 3, 5, 0], start=3, num_rows=6)) == 6
 
 
+def test_narrow_twins_agree_on_random_windows():
+    # Property check that the device- and host-side narrows are true twins:
+    # random per-slot counts (including zeros) against random windows that come
+    # out empty, interior, or overshooting the total token count.
+    generator = torch.Generator().manual_seed(20260819)
+    for _ in range(50):
+        n_slots = int(torch.randint(1, 6, (1,), generator=generator))
+        counts = torch.randint(0, 9, (n_slots,), dtype=torch.int32, generator=generator)
+        total = int(counts.sum())
+        start = int(torch.randint(0, total + 4, (1,), generator=generator))
+        num_rows = int(torch.randint(0, total + 4, (1,), generator=generator))
+
+        device_counts = multi_lora_layers_module._narrow_token_counts_to_window(counts, start, num_rows)
+        host_counts = multi_lora_layers_module._narrow_token_counts_to_window_host(
+            tuple(counts.tolist()), start, num_rows
+        )
+        assert tuple(device_counts.tolist()) == host_counts
+
+
 # --------------------------------------------------------------------------- #
 # Forward smoke / B4 reset: single-GPU integration through a real
 # ColumnParallelLinear.
@@ -830,8 +854,13 @@ def test_backward_matches_per_slot_reference(self):
         set_tokens_per_adapter_slot([mlora], torch.tensor(counts, dtype=torch.int32, device="cuda"))
         x = torch.randn(sum(counts), 16, dtype=torch.bfloat16, device="cuda")
 
-        out, _ = mlora(x)
-        out.float().sum().backward()
+        with patch.object(
+            multi_lora_layers_module,
+            "_apply_per_slot_linear",
+            side_effect=AssertionError("aligned rank unexpectedly used the fallback"),
+        ):
+            out, _ = mlora(x)
+            out.float().sum().backward()
 
         # Per-slot reference on cloned leaf weights: same math, no grouping.
         a_refs = [a.linear_in.weight.detach().clone().requires_grad_(True) for a in mlora.adapters]
@@ -857,6 +886,219 @@ def test_backward_matches_per_slot_reference(self):
         assert mlora.adapters[2].linear_out.weight.grad is not None
         assert torch.count_nonzero(mlora.adapters[2].linear_out.weight.grad) == 0
 
+    def test_unaligned_local_rank_backward_matches_per_slot_reference(self):
+        """A physical BF16 rank of two must bypass grouped MM for both projections."""
+
+        from megatron.bridge.peft.multi_lora_layers import (
+            init_adapter_slot,
+            set_tokens_per_adapter_slot,
+        )
+
+        mlora = self._build(dim=2, n_adapters=3, alpha=4)
+        init_adapter_slot([mlora], 0, rank=1, alpha=2)
+        init_adapter_slot([mlora], 1, rank=2, alpha=4)
+        with torch.no_grad():
+            for slot in range(3):
+                mlora.adapters[slot].linear_out.weight.normal_(std=0.02)
+            mlora._apply_rank_mask(0)
+
+        counts = [3, 5, 0]
+        set_tokens_per_adapter_slot([mlora], torch.tensor(counts, dtype=torch.int32, device="cuda"))
+        x = torch.randn(sum(counts), 16, dtype=torch.bfloat16, device="cuda", requires_grad=True)
+        grad_output = torch.randn(sum(counts), 16, dtype=torch.bfloat16, device="cuda")
+
+        with patch.object(
+            torch,
+            "_grouped_mm",
+            side_effect=AssertionError("unaligned rank unexpectedly used grouped MM"),
+        ):
+            out, _ = mlora(x)
+            out.backward(grad_output)
+
+        x_ref = x.detach().clone().requires_grad_(True)
+        a_refs = [adapter.linear_in.weight.detach().clone().requires_grad_(True) for adapter in mlora.adapters]
+        b_refs = [adapter.linear_out.weight.detach().clone().requires_grad_(True) for adapter in mlora.adapters]
+        scaling = (mlora.alpha_values / mlora.rank_values).tolist()
+        ref_rows = []
+        start = 0
+        for slot, count in enumerate(counts):
+            slot_input = x_ref.narrow(0, start, count)
+            hidden = nn.functional.linear(slot_input, a_refs[slot])
+            ref_rows.append(scaling[slot] * nn.functional.linear(hidden, b_refs[slot]))
+            start += count
+        ref_out = mlora.to_wrap(x_ref)[0] + torch.cat(ref_rows, dim=0)
+        ref_out.backward(grad_output)
+
+        torch.testing.assert_close(out, ref_out)
+        torch.testing.assert_close(x.grad, x_ref.grad)
+        for slot in range(2):
+            torch.testing.assert_close(mlora.adapters[slot].linear_in.weight.grad, a_refs[slot].grad)
+            torch.testing.assert_close(mlora.adapters[slot].linear_out.weight.grad, b_refs[slot].grad)
+        assert mlora.adapters[2].linear_in.weight.grad is not None
+        assert torch.count_nonzero(mlora.adapters[2].linear_in.weight.grad) == 0
+        assert mlora.adapters[2].linear_out.weight.grad is not None
+        assert torch.count_nonzero(mlora.adapters[2].linear_out.weight.grad) == 0
+
+    def test_misaligned_storage_offset_uses_fallback(self):
+        """Aligned strides are insufficient when an operand's data pointer is offset."""
+
+        counts = [3, 5]
+        # Slicing one BF16 element keeps a contiguous (8, 1) stride but moves
+        # the data pointer by two bytes, which torch._grouped_mm rejects.
+        storage = torch.randn(sum(counts) * 8 + 1, dtype=torch.bfloat16, device="cuda")
+        input_ = storage[1:].view(sum(counts), 8).requires_grad_(True)
+        weights = torch.randn(2, 8, 8, dtype=torch.bfloat16, device="cuda", requires_grad=True)
+        offsets = torch.tensor(counts, dtype=torch.int32, device="cuda").cumsum(dim=0)
+
+        assert input_.stride() == (8, 1)
+        assert input_.data_ptr() % _PYTORCH_GROUPED_MM_ALIGNMENT_BYTES != 0
+        assert not multi_lora_layers_module._can_use_grouped_mm(input_, weights)
+
+        with patch.object(
+            torch,
+            "_grouped_mm",
+            side_effect=AssertionError("misaligned pointer unexpectedly used grouped MM"),
+        ):
+            out = multi_lora_layers_module._apply_multi_lora_projection(input_, weights, offsets, counts)
+            out.float().sum().backward()
+
+        reference = torch.cat(
+            [
+                nn.functional.linear(chunk, weight)
+                for chunk, weight in zip(input_.detach().split(counts), weights.detach())
+            ]
+        )
+        torch.testing.assert_close(out, reference)
+        assert input_.grad is not None
+        assert weights.grad is not None
+
+    @pytest.mark.skipif(not hasattr(torch, "_grouped_mm"), reason="needs torch._grouped_mm")
+    def test_exactly_16_byte_aligned_input_stays_on_grouped_mm(self):
+        """Boundary canary: 16-but-not-32-byte alignment must stay on the fast path.
+
+        Allocator-natural CUDA tensors are 256-byte aligned, so ordinary inputs
+        could never reveal PyTorch tightening its alignment contract beyond
+        ``_PYTORCH_GROUPED_MM_ALIGNMENT_BYTES``; this probes the exact boundary.
+        """
+
+        counts = [3, 5]
+        features = 8
+        # A storage offset of 8 BF16 elements = 16 bytes from the allocator's
+        # 256-byte-aligned base leaves the data pointer exactly 16-byte aligned
+        # but not 32-byte aligned.
+        storage = torch.randn(sum(counts) * features + 8, dtype=torch.bfloat16, device="cuda")
+        assert storage.data_ptr() % (2 * _PYTORCH_GROUPED_MM_ALIGNMENT_BYTES) == 0
+        input_ = storage[8:].view(sum(counts), features)
+        weights = torch.randn(2, features, features, dtype=torch.bfloat16, device="cuda")
+        offsets = torch.tensor(counts, dtype=torch.int32, device="cuda").cumsum(dim=0, dtype=torch.int32)
+
+        assert input_.data_ptr() % _PYTORCH_GROUPED_MM_ALIGNMENT_BYTES == 0
+        assert input_.data_ptr() % (2 * _PYTORCH_GROUPED_MM_ALIGNMENT_BYTES) != 0
+        assert multi_lora_layers_module._can_use_grouped_mm(input_, weights)
+
+        out = torch._grouped_mm(input_, weights.transpose(-2, -1), offsets)
+
+        reference = torch.cat(
+            [nn.functional.linear(chunk, weight) for chunk, weight in zip(input_.split(counts), weights)]
+        )
+        torch.testing.assert_close(out, reference)
+
+    def test_fp32_uses_fallback(self):
+        """FP32 grouped-MM backward is unsupported on the validated PyTorch stack."""
+
+        counts = [3, 5]
+        input_ = torch.randn(sum(counts), 8, dtype=torch.float32, device="cuda", requires_grad=True)
+        weights = torch.randn(2, 8, 8, dtype=torch.float32, device="cuda", requires_grad=True)
+        offsets = torch.tensor(counts, dtype=torch.int32, device="cuda").cumsum(dim=0)
+
+        assert not multi_lora_layers_module._can_use_grouped_mm(input_, weights)
+        with patch.object(
+            torch,
+            "_grouped_mm",
+            side_effect=AssertionError("FP32 unexpectedly used grouped MM"),
+        ):
+            out = multi_lora_layers_module._apply_multi_lora_projection(input_, weights, offsets, counts)
+            out.sum().backward()
+
+        assert torch.isfinite(out).all()
+        assert input_.grad is not None
+        assert weights.grad is not None
+
+    def test_all_zero_counts_use_fallback(self):
+        """An empty grouped-MM output cannot safely participate in backward."""
+
+        counts = [0, 0, 0]
+        input_ = torch.empty(0, 8, dtype=torch.bfloat16, device="cuda", requires_grad=True)
+        weights = torch.randn(3, 8, 8, dtype=torch.bfloat16, device="cuda", requires_grad=True)
+        offsets = torch.zeros(3, dtype=torch.int32, device="cuda")
+
+        assert not multi_lora_layers_module._can_use_grouped_mm(input_, weights)
+        with patch.object(
+            torch,
+            "_grouped_mm",
+            side_effect=AssertionError("empty batch unexpectedly used grouped MM"),
+        ):
+            out = multi_lora_layers_module._apply_multi_lora_projection(input_, weights, offsets, counts)
+            out.float().sum().backward()
+
+        assert out.shape == (0, 8)
+        assert input_.grad is not None
+        assert weights.grad is not None
+        assert torch.count_nonzero(weights.grad) == 0
+
+    def test_unaligned_fallback_uses_sp_narrowed_host_counts(self):
+        """The fallback follows a sequence-parallel window that crosses slots."""
+
+        from megatron.bridge.peft.multi_lora_layers import set_tokens_per_adapter_slot
+
+        mlora = self._build(dim=2, n_adapters=3, alpha=4)
+        with torch.no_grad():
+            for adapter in mlora.adapters:
+                adapter.linear_out.weight.normal_(std=0.02)
+
+        # Full spans [3, 5, 0] narrow to [3, 1, 0] for rank 0's four-row
+        # sequence-parallel window, crossing the slot-0/slot-1 boundary.
+        full_counts = [3, 5, 0]
+        local_counts = [3, 1, 0]
+        set_tokens_per_adapter_slot([mlora], torch.tensor(full_counts, dtype=torch.int32, device="cuda"))
+        x = torch.randn(sum(local_counts), 16, dtype=torch.bfloat16, device="cuda", requires_grad=True)
+
+        with (
+            patch.object(
+                multi_lora_layers_module.parallel_state, "get_tensor_model_parallel_world_size", return_value=2
+            ),
+            patch.object(multi_lora_layers_module.parallel_state, "get_tensor_model_parallel_rank", return_value=0),
+            patch.object(
+                multi_lora_layers_module, "gather_from_tensor_model_parallel_region", side_effect=lambda value: value
+            ),
+            patch.object(
+                torch, "_grouped_mm", side_effect=AssertionError("unaligned rank unexpectedly used grouped MM")
+            ),
+        ):
+            out, _ = mlora(x)
+            out.float().sum().backward()
+
+        a_refs = [adapter.linear_in.weight.detach().clone().requires_grad_(True) for adapter in mlora.adapters]
+        b_refs = [adapter.linear_out.weight.detach().clone().requires_grad_(True) for adapter in mlora.adapters]
+        x_ref = x.detach().clone().requires_grad_(True)
+        scaling = (mlora.alpha_values / mlora.rank_values).tolist()
+        rows = []
+        start = 0
+        for slot, count in enumerate(local_counts):
+            slot_input = x_ref.narrow(0, start, count)
+            rows.append(
+                scaling[slot] * nn.functional.linear(nn.functional.linear(slot_input, a_refs[slot]), b_refs[slot])
+            )
+            start += count
+        reference = mlora.to_wrap(x_ref)[0] + torch.cat(rows)
+        reference.float().sum().backward()
+
+        torch.testing.assert_close(out, reference)
+        torch.testing.assert_close(x.grad, x_ref.grad)
+        for slot in range(3):
+            torch.testing.assert_close(mlora.adapters[slot].linear_in.weight.grad, a_refs[slot].grad)
+            torch.testing.assert_close(mlora.adapters[slot].linear_out.weight.grad, b_refs[slot].grad)
+
     def test_consecutive_forwards_with_different_counts(self):
         # Counts are per-micro-batch state stashed on the layer; a second
         # forward with a different split (and total) must not see the first
@@ -885,9 +1127,7 @@ def reference(x, counts):
             return mlora.to_wrap(x)[0] + torch.cat(rows, dim=0)
 
         for counts in ([3, 5], [6, 2], [0, 4]):
-            set_tokens_per_adapter_slot(
-                [mlora], torch.tensor(counts, dtype=torch.int32, device="cuda")
-            )
+            set_tokens_per_adapter_slot([mlora], torch.tensor(counts, dtype=torch.int32, device="cuda"))
             x = torch.randn(sum(counts), 16, dtype=torch.bfloat16, device="cuda")
             out, _ = mlora(x)
             torch.testing.assert_close(out, reference(x, counts))