Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 61 additions & 73 deletions gpt_builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,6 @@
from megatron.training.arguments import core_transformer_config_from_args
from megatron.training.yaml_arguments import core_transformer_config_from_yaml

import megatron.legacy.model # isort: skip

# NOTE: Loading `megatron.legacy.model` earlier fails due to circular import


def gpt_builder(args, pre_process, post_process, vp_stage=None, config=None, pg_collection=None):
print_rank_0('building GPT model ...')
Expand All @@ -32,84 +28,76 @@ def gpt_builder(args, pre_process, post_process, vp_stage=None, config=None, pg_
config = core_transformer_config_from_yaml(args, "language_model")
else:
config = core_transformer_config_from_args(args)
if args.use_legacy_models:
model = megatron.legacy.model.GPTModel(
config,
num_tokentypes=0,
parallel_output=True,
pre_process=pre_process,
post_process=post_process,
)
else: # using core models
if args.spec is not None:
transformer_layer_spec = import_module(args.spec)
else:
use_te = args.transformer_impl == "transformer_engine"

if args.experimental_attention_variant is not None:
transformer_layer_spec = (
get_transformer_block_with_experimental_attention_variant_spec(
config=config, vp_stage=vp_stage
)
)
elif args.num_experts:
# Define the decoder block spec
transformer_layer_spec = get_gpt_decoder_block_spec(
config,
use_transformer_engine=use_te,
normalization=args.normalization,
qk_l2_norm=args.qk_l2_norm,
vp_stage=vp_stage,
)
elif args.heterogeneous_layers_config_path is not None:
assert not (config.transformer_impl == "inference_optimized")
transformer_layer_spec = get_gpt_heterogeneous_layer_spec(config, use_te)
else:
# Define the decoder layer spec
transformer_layer_spec = _get_transformer_layer_spec(use_te, config)
mtp_block_spec = None
if args.mtp_num_layers is not None:
assert not (config.transformer_impl == "inference_optimized")
if (
hasattr(transformer_layer_spec, 'layer_specs')
and len(transformer_layer_spec.layer_specs) == 0
):
# Get the decoder layer spec explicitly if no decoder layer in the last stage,
# Only happens with block spec (TransformerBlockSubmodules) when using MoE.
transformer_layer_spec_for_mtp = _get_transformer_layer_spec(use_te, config)
else:
# Define the decoder block spec
decoder_layer_specs = get_gpt_decoder_layer_specs(
config, use_transformer_engine=use_te, normalization=args.normalization, qk_l2_norm=args.qk_l2_norm, vp_stage=vp_stage
if args.spec is not None:
transformer_layer_spec = import_module(args.spec)
else:
use_te = args.transformer_impl == "transformer_engine"

if args.experimental_attention_variant is not None:
transformer_layer_spec = (
get_transformer_block_with_experimental_attention_variant_spec(
config=config, vp_stage=vp_stage
)
transformer_layer_spec_for_mtp = decoder_layer_specs[-1]
# Use spec of the last layer in decoder block as spec of the transformer layer in MTP
mtp_block_spec = get_gpt_mtp_block_spec(
)
elif args.num_experts:
# Define the decoder block spec
transformer_layer_spec = get_gpt_decoder_block_spec(
config,
transformer_layer_spec_for_mtp,
use_transformer_engine=use_te,
normalization=args.normalization,
qk_l2_norm=args.qk_l2_norm,
vp_stage=vp_stage,
)

model = GPTModel(
config=config,
transformer_layer_spec=transformer_layer_spec,
vocab_size=args.padded_vocab_size,
max_sequence_length=args.max_position_embeddings,
pre_process=pre_process,
post_process=post_process,
fp16_lm_cross_entropy=args.fp16_lm_cross_entropy,
parallel_output=True,
share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights,
position_embedding_type=args.position_embedding_type,
rotary_percent=args.rotary_percent,
rotary_base=args.rotary_base,
rope_scaling=args.use_rope_scaling,
mtp_block_spec=mtp_block_spec,
elif args.heterogeneous_layers_config_path is not None:
assert not (config.transformer_impl == "inference_optimized")
transformer_layer_spec = get_gpt_heterogeneous_layer_spec(config, use_te)
else:
# Define the decoder layer spec
transformer_layer_spec = _get_transformer_layer_spec(use_te, config)
mtp_block_spec = None
if args.mtp_num_layers is not None:
assert not (config.transformer_impl == "inference_optimized")
if (
hasattr(transformer_layer_spec, 'layer_specs')
and len(transformer_layer_spec.layer_specs) == 0
):
# Get the decoder layer spec explicitly if no decoder layer in the last stage,
# Only happens with block spec (TransformerBlockSubmodules) when using MoE.
transformer_layer_spec_for_mtp = _get_transformer_layer_spec(use_te, config)
else:
# Define the decoder block spec
decoder_layer_specs = get_gpt_decoder_layer_specs(
config, use_transformer_engine=use_te, normalization=args.normalization, qk_l2_norm=args.qk_l2_norm, vp_stage=vp_stage
)
transformer_layer_spec_for_mtp = decoder_layer_specs[-1]
# Use spec of the last layer in decoder block as spec of the transformer layer in MTP
mtp_block_spec = get_gpt_mtp_block_spec(
config,
transformer_layer_spec_for_mtp,
use_transformer_engine=use_te,
vp_stage=vp_stage,
pg_collection=pg_collection,
)

model = GPTModel(
config=config,
transformer_layer_spec=transformer_layer_spec,
vocab_size=args.padded_vocab_size,
max_sequence_length=args.max_position_embeddings,
pre_process=pre_process,
post_process=post_process,
fp16_lm_cross_entropy=args.fp16_lm_cross_entropy,
parallel_output=True,
share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights,
position_embedding_type=args.position_embedding_type,
rotary_percent=args.rotary_percent,
rotary_base=args.rotary_base,
rope_scaling=args.use_rope_scaling,
mtp_block_spec=mtp_block_spec,
vp_stage=vp_stage,
pg_collection=pg_collection,
)

return model


Expand Down
3 changes: 0 additions & 3 deletions megatron/legacy/model/__init__.py

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removing LayerNorm and RMSNorm also breaks existing model files. We need to either remove the models, or merge in main if they've already been removed.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a few things that still depend on this module:

  • megatron/training/checkpointing.py
  • tools/checkpoint/loader_legacy.py
  • examples/academic_paper_scripts/detxoify_lm/generate_samples_gpt.py
  • examples/academic_paper_scripts/detxoify_lm/finetune_gpt.py
  • examples/gpt3/gpt_config.yaml

Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.

from .fused_layer_norm import MixedFusedLayerNorm as LayerNorm
from .rms_norm import RMSNorm

from .gpt_model import GPTModel
from .language_model import get_language_model
44 changes: 0 additions & 44 deletions megatron/legacy/model/fused_bias_gelu.py

This file was deleted.

99 changes: 0 additions & 99 deletions megatron/legacy/model/fused_layer_norm.py

This file was deleted.

Loading
Loading