Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,6 @@ class _OwnedRange:
buffer_relative_offset: int


def _validate_mesh_axis(mesh: DeviceMesh, axis: int) -> None:
if not isinstance(axis, int) or isinstance(axis, bool):
raise TypeError(f"Mesh axis must be an int, got {type(axis).__name__}.")
if axis < 0 or axis >= mesh.ndim:
raise ValueError(f"Mesh axis {axis} is out of bounds for mesh ndim {mesh.ndim}.")


def _validate_placements(placements: Iterable[Placement]) -> None:
seen_flat = False
for placement in placements:
Expand Down Expand Up @@ -309,7 +302,6 @@ def redistribute(

def allgather(self, mesh_axis: int, *, out: "DBuffer | None" = None) -> "DBuffer":
"""All-gather a sharded axis into Replicate placement."""
_validate_mesh_axis(self.mesh, mesh_axis)
if not isinstance(self.placements[mesh_axis], Flat):
raise ValueError(
f"allgather() currently requires Flat placement on axis {mesh_axis!r}."
Expand All @@ -328,7 +320,6 @@ def allgather(self, mesh_axis: int, *, out: "DBuffer | None" = None) -> "DBuffer

def allreduce(self, mesh_axis: int, *, out: "DBuffer | None" = None) -> "DBuffer":
"""All-reduce a Partial axis into Replicate placement."""
_validate_mesh_axis(self.mesh, mesh_axis)
axis = mesh_axis
partial_placement = self.placements[axis]
if not isinstance(partial_placement, Partial):
Expand All @@ -347,7 +338,6 @@ def reduce_scatter(
self, mesh_axis: int, new_placement: Placement, *, out: "DBuffer | None" = None
) -> "DBuffer":
"""Reduce-scatter a Partial axis into ``new_placement``."""
_validate_mesh_axis(self.mesh, mesh_axis)
axis = mesh_axis
if not isinstance(new_placement, Flat):
raise NotImplementedError("DBuffer currently supports reduce_scatter() to Flat only.")
Expand All @@ -371,7 +361,6 @@ def scatter(
self, mesh_axis: int, new_placement: Placement, *, out: "DBuffer | None" = None
) -> "DBuffer":
"""Locally chunk a Replicate axis into ``new_placement``."""
_validate_mesh_axis(self.mesh, mesh_axis)
axis = mesh_axis
if not isinstance(new_placement, Flat):
raise NotImplementedError("DBuffer currently supports scatter() to Flat only.")
Expand Down
21 changes: 0 additions & 21 deletions tests/unit_tests/distributed/megatron_fsdp/test_dbuffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
"""Unit tests for Megatron-FSDP DBuffer."""

from collections.abc import Iterable
from typing import cast

import pytest
import torch
Expand Down Expand Up @@ -244,26 +243,6 @@ def test_sharded_allgather_into_existing_buffer(distributed_setup):
_assert_dbuffer_local_tensors_close(destination, tensors)


@pytest.mark.distributed
def test_mesh_axis_must_be_non_negative_int(distributed_setup):
"""DBuffer communication methods require explicit non-negative integer mesh axes."""
mesh = init_device_mesh(distributed_setup.device.type, (distributed_setup.world_size,))
buffer = DBuffer(
mesh=mesh,
placements=[Replicate()],
tensor_shapes=[torch.Size((4,))],
dtype=torch.float32,
device=distributed_setup.device,
)

with pytest.raises(TypeError, match="Mesh axis must be an int"):
buffer.allgather(cast(int, "dp"))
with pytest.raises(TypeError, match="Mesh axis must be an int"):
buffer.allgather(True)
with pytest.raises(ValueError, match="Mesh axis -1 is out of bounds"):
buffer.allgather(-1)


@pytest.mark.distributed
def test_replicate_scatter_round_trip(distributed_setup):
"""Replicated buffers locally chunk into sharded buffers and all-gather back."""
Expand Down
Loading