Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion 3rdparty/Megatron-LM
Submodule Megatron-LM updated 67 files
+5 −3 .gitlab/stages/00.pre.yml
+4 −2 .gitlab/stages/02.test.yml
+3 −1 gpt_builders.py
+3 −7 megatron/core/datasets/bert_dataset.py
+11 −0 megatron/core/datasets/blended_megatron_dataset_config.py
+0 −8 megatron/core/datasets/gpt_dataset.py
+47 −0 megatron/core/datasets/megatron_dataset.py
+2 −2 megatron/core/datasets/t5_dataset.py
+14 −0 megatron/core/fp4_utils.py
+290 −175 megatron/core/models/gpt/gpt_layer_specs.py
+39 −0 megatron/core/models/gpt/linear_attention_module_specs.py
+4 −2 megatron/core/models/gpt/moe_module_specs.py
+6 −5 megatron/core/optimizer/__init__.py
+26 −18 megatron/core/optimizer/layer_wise_optimizer.py
+13 −9 megatron/core/parallel_state.py
+84 −19 megatron/core/process_groups_config.py
+549 −0 megatron/core/ssm/gated_delta_net.py
+68 −28 megatron/core/transformer/attention.py
+2 −2 megatron/core/transformer/moe/README.md
+29 −18 megatron/core/transformer/moe/experts.py
+4 −1 megatron/core/transformer/moe/moe_layer.py
+27 −6 megatron/core/transformer/moe/token_dispatcher.py
+9 −1 megatron/core/transformer/spec_utils.py
+105 −6 megatron/core/transformer/transformer_config.py
+84 −6 megatron/training/arguments.py
+17 −7 megatron/training/checkpointing.py
+37 −15 megatron/training/tokenizer/tokenizer.py
+115 −19 megatron/training/training.py
+4 −0 megatron/training/utils.py
+1 −0 pretrain_bert.py
+1 −0 pretrain_gpt.py
+1 −0 pretrain_mamba.py
+1 −0 pretrain_retro.py
+1 −0 pretrain_t5.py
+1 −0 pretrain_vlm.py
+1 −0 pyproject.toml
+2 −2 ...es/moe/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/model_config.yaml
+2 −2 ...est_cases/moe/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/model_config.yaml
+55 −55 ...functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgx_h100.json
+250 −250 ...l_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgxh100_coreweave.json
+250 −250 ...ctional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgxh100_eos.json
+2 −3 tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/model_config.yaml
+2 −2 ...moe/gpt3_mr_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_dgx_a100_1N8G/model_config.yaml
+2 −2 ...tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4_dgx_a100_1N8G/model_config.yaml
+344 −0 ...es/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgxh100_coreweave.json
+344 −0 ...st_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgxh100_eos.json
+2 −3 ...nctional_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/model_config.yaml
+249 −249 ...gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgxh100_coreweave.json
+249 −249 ...s/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgxh100_eos.json
+4 −5 ...l_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/model_config.yaml
+344 −0 ...gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_selective_recompute_experimental/golden_values_dev_dgxh100_coreweave.json
+344 −0 ...s/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_selective_recompute_experimental/golden_values_dev_dgxh100_eos.json
+2 −3 ...l_tests/test_cases/moe/gpt3_mr_mcore_te_tp2_pp2_ep4_etp1_selective_recompute_experimental/model_config.yaml
+0 −101 tests/test_utils/recipes/bert.yaml
+10 −64 tests/test_utils/recipes/moe.yaml
+0 −116 tests/test_utils/recipes/t5.yaml
+319 −0 tests/unit_tests/ssm/test_gated_delta_net.py
+394 −0 tests/unit_tests/test_layer_wise_optimizer.py
+539 −114 tests/unit_tests/test_muon_optimizer.py
+15 −2 tests/unit_tests/test_optimizer.py
+33 −0 tests/unit_tests/test_process_groups_config.py
+7 −2 tests/unit_tests/transformer/moe/test_shared_experts.py
+6 −6 tests/unit_tests/transformer/moe/test_token_dispatcher.py
+15 −5 tests/unit_tests/transformer/test_attention.py
+10 −1 tests/unit_tests/transformer/test_transformer_block_custom_pgs.py
+2 −0 tools/autoformat.sh
+207 −14 uv.lock
Loading
Loading