Skip to content
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
027e064
drop most of base.py strategy
dimapihtar Mar 23, 2026
17168af
code clenaup
dimapihtar Mar 24, 2026
8621dde
fix docstring
dimapihtar Mar 24, 2026
8687661
remove base import
dimapihtar Mar 24, 2026
bb1ed10
fix imports
dimapihtar Mar 24, 2026
358d076
fix imports
dimapihtar Mar 24, 2026
cf3f930
fix async
dimapihtar Mar 24, 2026
557a1d9
fix async
dimapihtar Mar 24, 2026
39fd4a2
Merge branch 'main' into simplify_base_strategy
dimapihtar Mar 30, 2026
36c1711
fix unit test
dimapihtar Mar 30, 2026
2436abc
remove super init call
dimapihtar Mar 30, 2026
25b15a5
fix unit tests
dimapihtar Mar 30, 2026
b0990ed
fix serialization unit test
dimapihtar Mar 30, 2026
a3f8d6d
remove super init call
dimapihtar Mar 30, 2026
f1d3f87
fix import
dimapihtar Mar 30, 2026
67e70bf
Merge branch 'main' into simplify_base_strategy
dimapihtar Mar 30, 2026
80283cc
fix code style
dimapihtar Mar 30, 2026
2132326
remove func
dimapihtar Mar 30, 2026
d0899ae
Merge branch 'main' into simplify_base_strategy
dimapihtar Mar 30, 2026
f051146
Merge branch 'main' into simplify_base_strategy
dimapihtar Mar 31, 2026
2192dae
minor fixes
dimapihtar Mar 31, 2026
91b0706
Merge branch 'main' into simplify_base_strategy
dimapihtar Mar 31, 2026
ca20c94
disable async save for some tests
dimapihtar Mar 31, 2026
8894445
fix code style
dimapihtar Mar 31, 2026
9aa2ffc
fix import
dimapihtar Mar 31, 2026
f2a3842
back removed modules and add deprecation warnings
dimapihtar Apr 2, 2026
39b7aca
Merge branch 'main' into simplify_base_strategy
dimapihtar Apr 2, 2026
6579e03
fix code style
dimapihtar Apr 2, 2026
b1c4f49
fix code style
dimapihtar Apr 2, 2026
3ee2f72
fix imports
dimapihtar Apr 2, 2026
1d394b0
fix unit test
dimapihtar Apr 2, 2026
c03e751
fix depreaction warnings
dimapihtar Apr 3, 2026
d8f2bcb
fix deprecation warnings
dimapihtar Apr 3, 2026
778a158
Merge branch 'main' into simplify_base_strategy
dimapihtar Apr 3, 2026
1e4e81e
fix code style
dimapihtar Apr 3, 2026
4aea010
fix unit tests
dimapihtar Apr 3, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion megatron/core/dist_checkpointing/exchange_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import numpy as np
import torch

from ..utils import get_pg_rank, get_pg_size, log_single_rank
from .core import CheckpointingException
from .dict_utils import nested_values
from .mapping import ShardedStateDict, ShardedTensor, is_main_replica
Expand Down Expand Up @@ -197,6 +196,8 @@ def determine_main_replica_uniform_distribution(
parallelization. Returns None if the process_group is trivial (1 rank)
"""
from ..utils import get_pg_size

if parallelization_group is None:
parallelization_group = torch.distributed.group.WORLD
group_size = get_pg_size(group=parallelization_group)
Expand Down Expand Up @@ -285,6 +286,8 @@ def exchange_loaded_tensors_gather_rounds(
needed by this rank to load a given state dict. Includes
previously loaded tensors (from `loaded_tensors` input)
"""
from ..utils import get_pg_rank, get_pg_size

if parallelization_group is None:
parallelization_group = torch.distributed.group.WORLD
main_rank_for_shard, _, shard_to_metadata, all_ranks_for_shard = shard_distribution
Expand Down Expand Up @@ -398,6 +401,8 @@ def exchange_loaded_tensors_gather_object(
previously loaded tensors (from `loaded_tensors` input)
"""
from ..utils import log_single_rank

all_loaded_tensors_list = [None] * torch.distributed.get_world_size(group=parallelization_group)
torch.distributed.all_gather_object(
all_loaded_tensors_list, loaded_tensors, group=parallelization_group
Expand Down Expand Up @@ -431,6 +436,8 @@ def exchange_loaded_objects_gather_object(
Dict[_ShardId, Any]: dictionary mapping shard ids to objects needed by this rank to
load a given state dict.
"""
from ..utils import log_single_rank

all_loaded_objects_list = [None] * torch.distributed.get_world_size()
torch.distributed.all_gather_object(all_loaded_objects_list, loaded_objects, group=None)
all_loaded_objects_list = cast(List[Dict[_ShardId, Any]], all_loaded_objects_list)
Expand Down
6 changes: 0 additions & 6 deletions megatron/core/dist_checkpointing/serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
from .core import CheckpointingConfig, save_config
from .dict_utils import merge
from .mapping import (
CheckpointingException,
CommonStateDict,
ShardedObject,
ShardedStateDict,
Expand All @@ -30,7 +29,6 @@
)
from .state_dict_utils import load_preprocess, save_preprocess
from .strategies.async_utils import AsyncRequest
from .strategies.base import AsyncSaveShardedStrategy
from .strategies.common import load_common, save_common
from .strategies.torch import TorchDistLoadShardedStrategy, TorchDistSaveShardedStrategy
from .utils import extract_sharded_base, force_all_tensors_to_non_fp8
Expand Down Expand Up @@ -393,10 +391,6 @@ def metadata_finalize_fn():
metadata_finalize_fn()
return None

if not isinstance(sharded_strategy, AsyncSaveShardedStrategy):
raise CheckpointingException(
f'Cannot apply async_save to non-async strategy {sharded_strategy}'
)
async_request = sharded_strategy.async_save(sharded_state_dict, checkpoint_dir, async_strategy)
async_request.finalize_fns.append(metadata_finalize_fn)
return async_request
Expand Down
192 changes: 32 additions & 160 deletions megatron/core/dist_checkpointing/strategies/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,178 +2,50 @@

""" Strategies base interfaces. """

from abc import ABC, abstractmethod
from collections import defaultdict
import logging
from enum import Enum
from pathlib import Path
from typing import Any, DefaultDict, Union

from ..mapping import CheckpointingException, ShardedStateDict
from .async_utils import AsyncCallsQueue, AsyncRequest
from .torch import TorchDistLoadShardedStrategy, TorchDistSaveShardedStrategy

logger = logging.getLogger(__name__)


class LoadShardedStrategy:
"""Base class for load strategies to be removed in future releases."""

pass


class SaveShardedStrategy:
"""Base class for save strategies to be removed in future releases."""

def __init__(self, backend: str, version: int):
pass
Comment thread
dimapihtar marked this conversation as resolved.
Resolved


class StrategyAction(Enum):
"""Specifies save vs load action."""
"""Specifies save vs load and sharded vs common action.
To be removed in future releases."""

LOAD_COMMON = 'load_common'
LOAD_SHARDED = 'load_sharded'
SAVE_COMMON = 'save_common'
SAVE_SHARDED = 'save_sharded'


default_strategies: DefaultDict[str, dict[tuple, Any]] = defaultdict(dict)

async_calls = AsyncCallsQueue()


def get_default_strategy(action: StrategyAction, backend: str, version: int):
"""Retrieves a default strategy for a given action, backend and version."""
error_hint: str = ""
try:
error_hint = ' Please use PyTorch version >=2.1'
from .torch import register_default_torch_strategies

register_default_torch_strategies()
except ImportError as e:
raise CheckpointingException(
f'Cannot import a default strategy for: {(action.value, backend, version)}. '
f'Error: {e}. Hint: {error_hint}'
) from e
try:
return default_strategies[action.value][(backend, version)]
except KeyError as e:
raise CheckpointingException(
f'Cannot find a default strategy for: {(action.value, backend, version)}'
) from e


def register_default_strategy(
action: StrategyAction,
backend: str,
version: int,
strategy: Union['SaveStrategyBase', 'LoadStrategyBase'],
):
"""Adds a given strategy to the registry of default strategies.

Args:
action (StrategyAction): specifies save/load and sharded
backend (str): backend that the strategy becomes a default for
version (int): version that the strategy becomes a default for
strategy (SaveStrategyBase, LoadStrategyBase): strategy to register
"""
default_strategies[action.value][(backend, version)] = strategy


class LoadStrategyBase(ABC):
"""Base class for a load strategy. Requires implementing checks for compatibility with a
given checkpoint version."""

@abstractmethod
def check_backend_compatibility(self, loaded_backend):
"""Verifies if this strategy is compatible with `loaded_backend`."""
raise NotImplementedError

@abstractmethod
def check_version_compatibility(self, loaded_version):
"""Verifies if this strategy is compatible with `loaded_version`."""
raise NotImplementedError

@property
def can_handle_sharded_objects(self):
"""Returns whether or not this strategy can handle loading ShardedObjects."""
return False


class SaveStrategyBase(ABC):
"""Base class for a save strategy. Requires defining a backend type and
version of the saved format."""

def __init__(self, backend: str, version: int):
self.backend = backend
self.version = version

@property
def can_handle_sharded_objects(self):
"""Returns whether or not this strategy can handle saving ShardedObjects."""
return False

def __str__(self):
return f'{self.__class__.__name__}({self.backend}, {self.version})'


class LoadShardedStrategy(LoadStrategyBase):
"""Load strategy for sharded tensors"""

@abstractmethod
def load(self, sharded_state_dict: ShardedStateDict, checkpoint_dir: Union[str, Path]):
"""Load the sharded part of the checkpoint."""
raise NotImplementedError

@abstractmethod
def load_tensors_metadata(self, checkpoint_dir: Union[str, Path]):
"""Load tensors metadata from the checkpoint for ShardedTensors.

Returns a dictionary similar to a sharded state dict, but note that
the dictionary keys are simply ShardedTensor keys (contrary to the
actual sharded state dicts where keys correspond to state dict keys).

Dict values are ShardedTensors without any data and sharding (so, the
only useful information is tensors global shape and dtype).
"""
raise NotImplementedError(
f'Loading only tensors metadata not implemented for {self.__class__.__name__}'
)

def load_sharded_metadata(self, checkpoint_dir: Union[str, Path]):
"""Load sharded metadata from the checkpoint for ShardedTensors and ShardedObjects.

Returns a dictionary similar to a sharded state dict, but note that
the dictionary keys are simply sharded keys (contrary to the
actual sharded state dicts where keys correspond to state dict keys).

Dict values are ShardedTensors or ShardedObjects without any data and sharding.
"""
if not self.can_handle_sharded_objects:
return self.load_tensors_metadata(checkpoint_dir)
raise NotImplementedError(
f'Loading only sharded metadata not implemented for {self.__class__.__name__}'
)

def remove_sharded_tensors(self, checkpoint_dir: Union[str, Path], key_prefix: str):
"""Remove all tensors whose key starts with key_prefix"""
raise NotImplementedError


class SaveShardedStrategy(SaveStrategyBase):
"""Save strategy for sharded tensors"""

@abstractmethod
def save(self, sharded_state_dict: ShardedStateDict, checkpoint_dir: Union[str, Path]):
"""Save the sharded part of the state dict."""
raise NotImplementedError


class AsyncSaveShardedStrategy(SaveShardedStrategy):
"""Save strategy suitable for async save."""

@abstractmethod
def async_save(
self, sharded_state_dict: ShardedStateDict, checkpoint_dir: Union[str, Path]
) -> AsyncRequest:
"""Perform preparation and return an AsyncRequest to the external caller.

Args:
sharded_state_dict (ShardedStateDict): sharded state dict to save
checkpoint_dir (Path): checkpoint target directory

Returns:
AsyncRequest: represents the async save function and finalization function.
It is the caller responsibility to actually schedule the async save.
"""
raise NotImplementedError

def save(self, sharded_state_dict: ShardedStateDict, checkpoint_dir: Union[str, Path]):
"""Each async strategy can be trivially used as a sync strategy."""
async_request = self.async_save(sharded_state_dict, checkpoint_dir)
async_request.execute_sync()
del async_request
logger.warning(
'megatron.core.dist_checkpointing.strategies.base.get_default_strategy'
' is deprecated and will be removed in the future releases. Please use'
' TorchDistLoadShardedStrategy() and TorchDistSaveShardedStrategy()'
' to get the default load and save sharded strategies.'
)
if backend != 'torch_dist':
logger.warning(f'{backend} is not supported. `torch_dist` backend will be used.')
if action == StrategyAction.LOAD_SHARDED:
return TorchDistLoadShardedStrategy()
else:
assert action == StrategyAction.SAVE_SHARDED, f'{action} is not supported'
return TorchDistSaveShardedStrategy()
Loading
Loading