Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions scripts/performance/argument_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,13 @@ def parse_cli_args():
help="List available config variants for the specified model/task/gpu/dtype and interactively select one (with 15s timeout).",
)

# Testing parameters
_testing_args(parser)

return parser


def _testing_args(parser):
"""Add testing-related arguments to the parser."""
testing_args = parser.add_argument_group("Testing arguments")
testing_args.add_argument(
"--is_long_convergence_run",
Expand Down Expand Up @@ -892,5 +898,3 @@ def parse_cli_args():
default=None,
help="End step (0-indexed, exclusive) for timing average window. If None, averages to end.",
)

return parser
100 changes: 100 additions & 0 deletions scripts/performance/utils/evaluate.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import json
import logging
import math
Expand All @@ -22,6 +23,11 @@
from typing import Any, Dict, List, Optional


try:
from scripts.performance.argument_parser import _testing_args
except (ImportError, ModuleNotFoundError):
from .scripts.performance.argument_parser import _testing_args
Comment thread
gautham-kollu marked this conversation as resolved.
Resolved

try:
import numpy as np

Expand Down Expand Up @@ -840,3 +846,97 @@ def calc_convergence_and_performance(

_logger.info(f"Convergence check completed successfully for {model_family_name}_{model_recipe_name}")
return has_validation_failures is False, error_msg


def main():
"""
Evaluate the performance of a model against golden values.
"""
logging.basicConfig(level=logging.INFO)

parser = argparse.ArgumentParser(
description="Evaluate against golden values",
argument_default=None,
)

# add any new testing args here
_testing_args(parser)

parser.add_argument(
"--log_paths",
type=str,
nargs="+",
required=True,
help="Paths to training log files to evaluate",
)
parser.add_argument(
"--assets_dir",
type=str,
required=True,
help="Directory containing job results / where new golden values are written",
)

parser.add_argument(
"-m",
"--model_family_name",
type=str,
help="Model family name to use for experiment. E.g. `--model_family_name llama` (not llama3)",
required=True,
)
parser.add_argument(
"-mr",
"--model_recipe_name",
type=str,
help="Model recipe name to use for experiment. E.g. `--model_recipe_name llama31_405b`",
required=True,
)

args = parser.parse_args()

convergence_config = {
"correlation_threshold": args.correlation_threshold,
"high_loss_tolerance": args.high_loss_tolerance,
"medium_loss_tolerance": args.medium_loss_tolerance,
"low_loss_tolerance": args.low_loss_tolerance,
"final_loss_tolerance": args.final_loss_tolerance,
"max_outlier_ratio": args.max_outlier_ratio,
"outlier_threshold": args.outlier_threshold,
"skip_first_percent_loss": args.skip_first_percent_loss,
}

performance_config = {
"timing_threshold": args.timing_threshold,
"skip_first_percent_time": args.skip_first_percent_time,
"eval_time_start_step": args.eval_time_start_step,
"eval_time_end_step": args.eval_time_end_step,
}

memory_config = {
"memory_threshold": args.memory_threshold,
}

passed, error_msg = calc_convergence_and_performance(
model_family_name=args.model_family_name,
model_recipe_name=args.model_recipe_name,
assets_dir=args.assets_dir,
log_paths=args.log_paths,
loss_metric="lm loss",
timing_metric="elapsed time per iteration (ms)",
alloc_metric="alloc",
max_alloc_metric="max_alloc",
golden_values_path=args.golden_values_path,
convergence_config=convergence_config,
performance_config=performance_config,
memory_config=memory_config,
wandb_run=None,
)

if not passed:
logger.error(f"Evaluation FAILED:\n{error_msg}")
sys.exit(1)

logger.info("Evaluation PASSED")


if __name__ == "__main__":
main()
13 changes: 13 additions & 0 deletions src/megatron/bridge/training/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1217,6 +1217,19 @@ def validate(self) -> None:
if self.dist.use_megatron_fsdp or self.ddp.use_megatron_fsdp:
self._validate_and_apply_megatron_fsdp_configs()

# Validate reuse_grad_buf_for_mxfp8_param_ag when FSDP is not enabled
is_fsdp = self.dist.use_megatron_fsdp or self.ddp.use_megatron_fsdp
if (
not is_fsdp
and self.mixed_precision is not None
and self.mixed_precision.fp8_param_gather
and self.mixed_precision.fp8_recipe == "mxfp8"
):
assert self.mixed_precision.reuse_grad_buf_for_mxfp8_param_ag, (
"When fp8_param_gather=True and fp8_recipe='mxfp8', "
"reuse_grad_buf_for_mxfp8_param_ag must be set to True"
)

# Deterministic mode validations and settings
self._validate_and_apply_deterministic_mode()

Expand Down
6 changes: 0 additions & 6 deletions src/megatron/bridge/training/mixed_precision.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,6 @@ def finalize(self):
if self.fp4_param is None:
self.fp4_param = self.fp4_param_gather

# Validate that mxfp8 recipe requires reuse_grad_buf_for_mxfp8_param_ag=True when fp8_param_gather=True
if self.fp8_param_gather and self.fp8_recipe == "mxfp8":
assert self.reuse_grad_buf_for_mxfp8_param_ag, (
"When fp8_param_gather=True and fp8_recipe='mxfp8', "
"reuse_grad_buf_for_mxfp8_param_ag must be set to True"
)
# FP4 and FP8 are mutually exclusive
if self.fp4 and self.fp8:
raise ValueError("fp4 and fp8 cannot be used simultaneously. Please choose one.")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# GPU_COUNT=x4
#!/bin/bash
Comment thread
gautham-kollu marked this conversation as resolved.
Resolved
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -xeuo pipefail # Exit immediately if a command exits with a non-zero status

export CUDA_VISIBLE_DEVICES="0,1,2,3"
Comment thread
gautham-kollu marked this conversation as resolved.
Resolved
MEGATRON_BRIDGE_ROOT="/opt/Megatron-Bridge/"

# Run DeepSeek V3 perf recipe functional tests on 4 GPUs (expert_model_parallel_size=4)
# This script tests the GB300 proxy configuration (FSDP + EP=4 + MoE optimizations)
# to ensure perf-related features can run basic training without crashes
output_log_file="/tmp/test_deepseek_recipes_pretrain_perf_gb200.log"
uv run python -m torch.distributed.run --nproc_per_node=4 --nnodes=1 -m coverage run --data-file=/opt/Megatron-Bridge/.coverage --source=/opt/Megatron-Bridge/ --parallel-mode -m pytest -s -o log_cli=true -o log_cli_level=INFO -v -s -x -m "not pleasefixme" --tb=short -rA tests/functional_tests/test_groups/recipes/test_deepseek_recipes_pretrain_fsdp.py 2>&1 | tee -a $output_log_file
coverage combine -q

golden_values_path="$MEGATRON_BRIDGE_ROOT/tests/functional_tests/test_groups/recipes/golden_values/test_deepseek_recipes_pretrain_fsdp_gb200.json"
# Setting the threshold to 0.25 as the GPU utilization is flaky.
uv run python -m scripts.performance.utils.evaluate --log_paths $output_log_file --golden_values_path $golden_values_path --assets_dir /tmp \
--model_family_name deepseek --model_recipe_name deepseek_fsdp_1node_gb200 --timing_threshold 0.25
Loading
Loading