Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion megatron/training/arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -904,7 +904,7 @@ def validate_args(args, defaults={}):

# Map string data-type to torch.dtype.
dtype_map = {
'fp32': torch.float32, 'bf16': torch.bfloat16, 'fp16': torch.float16, 'fp8': torch.uint8, 'auto': None,
'fp32': torch.float32, 'bf16': torch.bfloat16, 'fp16': torch.float16, 'fp8': torch.uint8, 'auto': None, None: None,
}
map_dtype = lambda d: d if isinstance(d, torch.dtype) else dtype_map[d]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ MODEL_ARGS:
--untie-embeddings-and-output-weights: true
--deterministic-mode: true
--no-gradient-accumulation-fusion: true
--no-async-tensor-model-parallel-allreduce: true
--attention-softmax-in-fp32: true
--use-checkpoint-opt_param-scheduler: true
--use-mcore-models: true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ MODEL_ARGS:
--use-torch-fsdp2: true
--deterministic-mode: true
--no-gradient-accumulation-fusion: true
--no-async-tensor-model-parallel-allreduce: true
--use-checkpoint-opt_param-scheduler: true
--use-mcore-models: true
--ckpt-format: torch_dist
Expand Down
1 change: 0 additions & 1 deletion tools/checkpoint/loader_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,6 @@ def build_sys_argv(self):
'--no-masked-softmax-fusion',
'--no-bias-gelu-fusion',
'--no-bias-dropout-fusion',
'--no-async-tensor-model-parallel-allreduce',
'--use-cpu-initialization',
'--micro-batch-size', '1',
'--no-load-optim',
Expand Down
1 change: 0 additions & 1 deletion tools/checkpoint/loader_legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ def _load_checkpoint(queue, args):
'--no-masked-softmax-fusion',
'--no-bias-gelu-fusion',
'--no-bias-dropout-fusion',
'--no-async-tensor-model-parallel-allreduce',
'--use-cpu-initialization',
'--micro-batch-size', '1',
'--no-load-optim',
Expand Down
1 change: 0 additions & 1 deletion tools/checkpoint/loader_llama_mistral.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,6 @@ def _load_checkpoint(queue, args):
'--no-masked-softmax-fusion',
'--no-bias-gelu-fusion',
'--no-bias-dropout-fusion',
'--no-async-tensor-model-parallel-allreduce',
'--use-cpu-initialization',
'--micro-batch-size', '1',
'--no-load-optim',
Expand Down
1 change: 0 additions & 1 deletion tools/checkpoint/loader_mixtral_hf.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,6 @@ def _load_checkpoint(queue, args):
'--no-masked-softmax-fusion',
'--no-bias-gelu-fusion',
'--no-bias-dropout-fusion',
'--no-async-tensor-model-parallel-allreduce',
'--use-cpu-initialization',
'--micro-batch-size', '1',
'--no-load-optim',
Expand Down
10 changes: 10 additions & 0 deletions tools/checkpoint/saver_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,14 @@ def initialize_megatron_env(self):

self.import_model_provider()

# Initialize torch.distributed with a minimal single-process backend so that
# process-group size queries (get_pg_size / get_tensor_model_parallel_group_if_none)
# return the fake groups below rather than falling back to world_size=1.
if not torch.distributed.is_initialized():
os.environ.setdefault('MASTER_ADDR', 'localhost')
os.environ.setdefault('MASTER_PORT', '12356')
torch.distributed.init_process_group(backend='gloo', rank=0, world_size=1)

# fake initializing distributed
mpu.set_tensor_model_parallel_world_size(self.args.target_tensor_parallel_size)
mpu.set_pipeline_model_parallel_world_size(self.args.target_pipeline_parallel_size)
Expand All @@ -172,12 +180,14 @@ def initialize_megatron_env(self):
fake_pp_group = _ConverterFakeProcessGroup(size=self.args.target_pipeline_parallel_size)
fake_ep_group = _ConverterFakeProcessGroup(size=self.args.target_expert_parallel_size)
fake_dp_group = _ConverterFakeProcessGroup(size=1)
fake_dp_ep_group = _ConverterFakeProcessGroup(size=1)
mpu._TENSOR_MODEL_PARALLEL_GROUP = fake_tp_group
mpu._PIPELINE_MODEL_PARALLEL_GROUP = fake_pp_group
mpu._EXPERT_MODEL_PARALLEL_GROUP = fake_ep_group
mpu._DATA_PARALLEL_GROUP = fake_dp_group
mpu._DATA_PARALLEL_GROUP_WITH_CP = fake_dp_group
mpu._INTRA_PARTIAL_DATA_PARALLEL_GROUP_WITH_CP = fake_dp_group
mpu._EXPERT_DATA_PARALLEL_GROUP = fake_dp_ep_group

try:
import torch_llm_debug_tools
Expand Down
Loading