Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions megatron/core/resharding/copy_services/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,29 @@
from __future__ import annotations

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Optional

import torch
import torch.distributed as dist


@dataclass
class SendOp:
"""Single send operation pending in a CopyService queue."""

task_id: int | None
tensor: torch.Tensor
dest_rank: int


@dataclass
class RecvOp:
"""Single receive operation pending in a CopyService queue."""

task_id: int | None
tensor: torch.Tensor
src_rank: int


class CopyService(ABC):
Expand All @@ -17,6 +37,13 @@ class CopyService(ABC):
remote transfers simply ignore it.
"""

def __init__(self, group=None):
self.group = group
# group.rank()/size() supports cross-cluster ProcessGroups where members
# have independent default PGs.
self.rank = group.rank() if group is not None else dist.get_rank()
self.world_size = group.size() if group is not None else dist.get_world_size()

@abstractmethod
def submit_send(self, src_tensor: torch.Tensor, dest_rank: int, task_id: Optional[int] = None):
"""Register a tensor send from the current rank to ``dest_rank``."""
Expand All @@ -31,3 +58,38 @@ def submit_recv(self, dest_tensor: torch.Tensor, src_rank: int, task_id: Optiona
def run(self):
"""Execute all previously submitted send/recv operations as a single batch."""
...

def close(self) -> None:
"""Release backend-owned resources. Default no-op; NVSHMEM overrides."""


def match_local_ops_by_task_id(
local_sends: list, local_recvs: list, backend_name: str, rank: int
) -> list[tuple]:
"""Pair same-rank send/recv ops by task_id, raising on any mismatch.

Returns a list of ``(send_op, recv_op)`` tuples for the caller to apply
backend-specific local-copy logic. Either op type may be a backend-local
wrapper as long as it exposes ``.task_id``.
"""
sends_by_id = {op.task_id: op for op in local_sends}
recvs_by_id = {op.task_id: op for op in local_recvs}
if None in sends_by_id or None in recvs_by_id:
raise RuntimeError(
f"{backend_name}: local (same-rank) transfer requires a task_id "
"to match sends with recvs"
)
if len(sends_by_id) != len(local_sends) or len(recvs_by_id) != len(local_recvs):
raise RuntimeError(
f"{backend_name}: unmatched local ops on rank {rank}: "
f"{len(local_sends)} local sends vs {len(local_recvs)} local recvs"
)
pairs = []
for task_id, recv_op in recvs_by_id.items():
send_op = sends_by_id.get(task_id)
if send_op is None:
raise RuntimeError(
f"{backend_name}: missing local send for task_id={task_id} on rank {rank}"
)
pairs.append((send_op, recv_op))
return pairs
115 changes: 40 additions & 75 deletions megatron/core/resharding/copy_services/gloo_copy_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,52 +2,35 @@
from __future__ import annotations

import logging
from dataclasses import dataclass
from typing import List, Optional, Tuple

import torch
import torch.distributed as dist

from .base import CopyService
from .base import CopyService, RecvOp, SendOp, match_local_ops_by_task_id

logger = logging.getLogger(__name__)


@dataclass
class SendOp:
"""Simple container describing a single send operation."""

task_id: int | None
tensor: torch.Tensor
dest_rank: int


@dataclass
class RecvOp:
"""Simple container describing a single receive operation."""

task_id: int | None
tensor: torch.Tensor
src_rank: int


class GlooCopyService(CopyService):
"""
CopyService implementation that routes refit traffic over a CPU/Gloo
process group instead of NCCL.
"""

def __init__(self, group=None):
super().__init__(group=group)
if group is not None:
self.gloo_pg = group
self.rank = group.rank()
self.world_size = group.size()
else:
self.rank = dist.get_rank()
self.world_size = dist.get_world_size()
self.gloo_pg = dist.new_group(backend="gloo")
self.send_ops: List[SendOp] = []
# Each recv op is paired with its GPU destination tensor; the SendOp/RecvOp
# itself carries a pinned-CPU staging buffer for Gloo's CPU PG.
self.recv_ops: List[Tuple[RecvOp, torch.Tensor]] = []
# Dedicated stream for GPU-side work (local same-rank copies and the
# final CPU->GPU writebacks) so they overlap with the GPU->CPU staging
# copies issued on the default stream during ``run()``.
self._copy_stream = torch.cuda.Stream()
if self.rank == 0:
logger.info(
Expand All @@ -58,7 +41,7 @@ def submit_send(self, src_tensor: torch.Tensor, dest_rank: int, task_id: Optiona
self.send_ops.append(SendOp(task_id=task_id, tensor=src_tensor, dest_rank=dest_rank))

def submit_recv(self, dest_tensor: torch.Tensor, src_rank: int, task_id: Optional[int] = None):
# Allocate a pinned CPU buffer for faster CPU↔GPU transfer.
# Pinned CPU staging buffer for the Gloo recv; copied back to dest_tensor in run().
cpu_buffer = torch.empty(
dest_tensor.shape, dtype=dest_tensor.dtype, device="cpu", pin_memory=True
)
Expand All @@ -74,67 +57,50 @@ def run(self):
f"{len(self.send_ops)} sends + {len(self.recv_ops)} recvs = {total_ops} ops"
)

p2p_ops: List[dist.P2POp] = []

# Short-circuit self transfers into local device copies.
local_sends = [op for op in self.send_ops if op.dest_rank == self.rank]
remote_sends = [op for op in self.send_ops if op.dest_rank != self.rank]
local_recvs = [(recv, dst) for (recv, dst) in self.recv_ops if recv.src_rank == self.rank]
remote_recvs = [(recv, dst) for (recv, dst) in self.recv_ops if recv.src_rank != self.rank]

# Local copies run on a dedicated stream so they overlap with the
# GPU->CPU staging copies issued on the default stream below.
if local_sends or local_recvs:
local_sends_by_id = {op.task_id: op for op in local_sends}
if None in local_sends_by_id:
raise RuntimeError(
"GlooCopyService: local (same-rank) transfer requires a task_id "
"to match sends with recvs"
)
local_recvs_by_id = {recv.task_id: (recv, dst) for (recv, dst) in local_recvs}
if None in local_recvs_by_id:
raise RuntimeError(
"GlooCopyService: local (same-rank) transfer requires a task_id "
"to match sends with recvs"
)
if len(local_sends_by_id) != len(local_sends) or len(local_recvs_by_id) != len(
local_recvs
):
raise RuntimeError(
f"GlooCopyService: unmatched local ops on rank {self.rank}: "
f"{len(local_sends)} local sends vs {len(local_recvs)} local recvs"
)
for task_id, (recv_op, dst_tensor) in local_recvs_by_id.items():
send_op = local_sends_by_id.get(task_id)
if send_op is None:
raise RuntimeError(
f"GlooCopyService: missing local send for task_id={task_id} "
f"on rank {self.rank}"
)
with torch.no_grad():
local_recv_objs = [recv for recv, _ in local_recvs]
dst_by_task_id = {recv.task_id: dst for recv, dst in local_recvs}
pairs = match_local_ops_by_task_id(
local_sends, local_recv_objs, "GlooCopyService", self.rank
)
with torch.no_grad(), torch.cuda.stream(self._copy_stream):
for send_op, recv_op in pairs:
src_tensor = send_op.tensor
dst_tensor = dst_by_task_id[recv_op.task_id]
if dst_tensor.device != src_tensor.device:
dst_tensor.copy_(src_tensor.to(dst_tensor.device))
else:
dst_tensor.copy_(src_tensor)

# Build Gloo P2P ops over CPU tensors. For sends we stage all
# GPU→CPU copies with non_blocking, sync once, then build P2P ops.
# Use group_peer (not peer) to pass ranks directly in group space,
# avoiding the global-to-group rank conversion in P2POp which doesn't
# work for cross-world ProcessGroups.
# Stage all remote sends GPU->CPU on the default stream with non_blocking
# copies, then wait on a single event so other streams (e.g. _copy_stream)
# aren't blocked. Use group_peer (not peer) to pass ranks directly in
# group space, avoiding the global-to-group rank conversion in P2POp
# which doesn't work for cross-world ProcessGroups.
cpu_send_bufs: List[torch.Tensor] = []
for op in remote_sends:
cpu_tensor = torch.empty(
op.tensor.shape, dtype=op.tensor.dtype, device="cpu", pin_memory=True
)
cpu_tensor.copy_(op.tensor.detach(), non_blocking=True)
cpu_send_bufs.append(cpu_tensor)
# Single sync after all GPU→CPU copies are issued.
if cpu_send_bufs:
torch.cuda.synchronize()
stage_event = torch.cuda.Event()
stage_event.record() # default stream
stage_event.synchronize()

for op in remote_sends:
# Drop the GPU reference now that staging is complete.
op.tensor = None

p2p_ops: List[dist.P2POp] = []
for cpu_tensor, op in zip(cpu_send_bufs, remote_sends):
p2p_ops.append(
dist.P2POp(dist.isend, cpu_tensor, group=self.gloo_pg, group_peer=op.dest_rank)
Expand All @@ -149,19 +115,18 @@ def run(self):
for req in reqs:
req.wait()

# Copy received CPU buffers back into the original destination tensors.
# Use non_blocking with pinned memory for overlap.
for recv, dst_tensor in remote_recvs:
if dst_tensor.is_cuda:
dst_tensor.copy_(recv.tensor, non_blocking=True)
else:
dst_tensor.copy_(recv.tensor)

if self._copy_stream is not None:
torch.cuda.current_stream().wait_stream(self._copy_stream)

# Ensure all async CPU→GPU copies are complete.
torch.cuda.synchronize()
# CPU->GPU writebacks run on _copy_stream so they pipeline with each
# other and don't block the default stream.
with torch.cuda.stream(self._copy_stream):
for recv, dst_tensor in remote_recvs:
if dst_tensor.is_cuda:
dst_tensor.copy_(recv.tensor, non_blocking=True)
else:
dst_tensor.copy_(recv.tensor)

# Join _copy_stream into the default stream so subsequent default-stream
# work sees the local copies and the writebacks.
torch.cuda.current_stream().wait_stream(self._copy_stream)

if self.rank == 0:
logger.info("GlooCopyService: batched communication completed")
Expand Down
62 changes: 8 additions & 54 deletions megatron/core/resharding/copy_services/nccl_copy_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,46 +2,24 @@
from __future__ import annotations

import logging
from dataclasses import dataclass
from typing import List, Optional

import torch
import torch.distributed as dist

from .base import CopyService
from .base import CopyService, RecvOp, SendOp, match_local_ops_by_task_id

logger = logging.getLogger(__name__)


@dataclass
class SendOp:
"""Simple container describing a single send operation."""

task_id: int | None
tensor: torch.Tensor
dest_rank: int


@dataclass
class RecvOp:
"""Simple container describing a single receive operation."""

task_id: int | None
tensor: torch.Tensor
src_rank: int


class NCCLCopyService(CopyService):
"""
Thin wrapper around torch.distributed batch_isend_irecv to submit and execute
a batch of point-to-point sends and recvs.
"""

def __init__(self, group=None):
self.group = group
# Use group.rank()/size() to support cross-cluster ProcessGroups
self.rank = group.rank() if group is not None else dist.get_rank()
self.world_size = group.size() if group is not None else dist.get_world_size()
super().__init__(group=group)
self.send_ops: List[SendOp] = []
self.recv_ops: List[RecvOp] = []
# Dedicated stream for local (same-rank) copies to avoid unnecessary
Expand Down Expand Up @@ -72,35 +50,12 @@ def run(self):
remote_recvs = [op for op in self.recv_ops if op.src_rank != self.rank]

if local_sends or local_recvs:
local_sends_by_id = {op.task_id: op for op in local_sends}
if None in local_sends_by_id:
raise RuntimeError(
"NCCLCopyService: local (same-rank) transfer requires a task_id "
"to match sends with recvs"
)
local_recvs_by_id = {op.task_id: op for op in local_recvs}
if None in local_recvs_by_id:
raise RuntimeError(
"NCCLCopyService: local (same-rank) transfer requires a task_id "
"to match sends with recvs"
)
if len(local_sends_by_id) != len(local_sends) or len(local_recvs_by_id) != len(
local_recvs
):
raise RuntimeError(
f"NCCLCopyService: unmatched local ops on rank {self.rank}: "
f"{len(local_sends)} local sends vs {len(local_recvs)} local recvs"
)
for task_id, recv_op in local_recvs_by_id.items():
send_op = local_sends_by_id.get(task_id)
if send_op is None:
raise RuntimeError(
f"NCCLCopyService: missing local send for task_id={task_id} "
f"on rank {self.rank}"
)
with torch.no_grad():
with torch.cuda.stream(self._copy_stream):
recv_op.tensor.copy_(send_op.tensor)
pairs = match_local_ops_by_task_id(
local_sends, local_recvs, "NCCLCopyService", self.rank
)
with torch.no_grad(), torch.cuda.stream(self._copy_stream):
for send_op, recv_op in pairs:
recv_op.tensor.copy_(send_op.tensor)

p2p_ops = []
for op in remote_sends:
Expand All @@ -113,7 +68,6 @@ def run(self):
for req in reqs:
req.wait()

# Make sure the copy stream is finished
torch.cuda.current_stream().wait_stream(self._copy_stream)

if self.rank == 0:
Expand Down
Loading
Loading