diff --git a/tests/performance_tests/shell_test_utils/compare_to_baseline.py b/tests/performance_tests/shell_test_utils/compare_to_baseline.py index be0c99c847f..70ccc0885d4 100644 --- a/tests/performance_tests/shell_test_utils/compare_to_baseline.py +++ b/tests/performance_tests/shell_test_utils/compare_to_baseline.py @@ -78,12 +78,28 @@ def main() -> int: ap.add_argument( "--config", required=True, help="Path to model_config.yaml (for tolerance + metrics list)." ) + ap.add_argument( + "--platform", + required=True, + help="Hardware platform key (e.g. h100, gb200). baseline_values.json is a " + "{platform: {batch_key: {metrics}}} mapping; this picks the subtree to compare against.", + ) args = ap.parse_args() results = json.loads(Path(args.results).read_text()) - baseline = json.loads(Path(args.baseline).read_text()) + full_baseline = json.loads(Path(args.baseline).read_text()) config = yaml.safe_load(Path(args.config).read_text()) + if args.platform not in full_baseline: + available = ", ".join(sorted(full_baseline.keys())) or "" + print( + f"ERROR: no baseline for platform '{args.platform}' in {args.baseline}. " + f"Recorded platforms: {available}.\n" + f" Run once with RECORD_BASELINE=1 on a '{args.platform}' node to bootstrap." + ) + return 1 + baseline = full_baseline[args.platform] + tol = float(config.get("TOLERANCE_PCT", 10)) / 100.0 upper_tol = float(config.get("UPPER_TOLERANCE_PCT", 20)) / 100.0 metrics: list[str] = list(config.get("METRICS") or sorted(THROUGHPUT_METRICS | LATENCY_METRICS)) diff --git a/tests/performance_tests/shell_test_utils/run_perf_test.sh b/tests/performance_tests/shell_test_utils/run_perf_test.sh index 8fd68be8099..60d95314b0f 100755 --- a/tests/performance_tests/shell_test_utils/run_perf_test.sh +++ b/tests/performance_tests/shell_test_utils/run_perf_test.sh @@ -10,8 +10,12 @@ # RESULTS_ROOT=/path/where/results.json/and/server-logs/go # # Optional: -# RECORD_BASELINE=1 (skip the comparison; copy results.json over baseline_values.json) +# RECORD_BASELINE=1 (skip the comparison; merge results.json into baseline_values.json +# under the detected PLATFORM key, preserving other platforms.) # SKIP_COMPARE=1 (skip the comparison step entirely) +# PLATFORM= (override the platform key used to look up / write the baseline. +# Defaults to auto-detection via `nvidia-smi -L` — +# recognizes "h100", "gb200", "b200", "a100".) # # Expects /usr/local/bin/yq (present in mcore_ci_dev image, NOT in bare NGC PyTorch). @@ -33,6 +37,27 @@ done : "${CHECKPOINT_LOAD_PATH:?CHECKPOINT_LOAD_PATH is required}" : "${RESULTS_ROOT:?RESULTS_ROOT is required}" +# Resolve PLATFORM. Caller can override via env; otherwise inspect the first GPU. +# baseline_values.json is a {platform: {batch_key: {metrics}}} mapping, so this +# value picks which subtree to read / write. +if [[ -z "${PLATFORM:-}" ]]; then + GPU_NAME=$(nvidia-smi -L 2>/dev/null | head -1 || true) + case "$GPU_NAME" in + *GB200*|*"Grace Blackwell"*) PLATFORM=gb200 ;; + *B200*) PLATFORM=b200 ;; + *H100*) PLATFORM=h100 ;; + *A100*) PLATFORM=a100 ;; + *) + echo "[run_perf_test] error: could not auto-detect PLATFORM from nvidia-smi (\"$GPU_NAME\")." >&2 + echo " Pass PLATFORM= explicitly." >&2 + exit 2 + ;; + esac + echo "[run_perf_test] auto-detected PLATFORM=$PLATFORM from \"$GPU_NAME\"" +else + echo "[run_perf_test] using caller-provided PLATFORM=$PLATFORM" +fi + ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" PERF_DIR="$ROOT_DIR/tests/performance_tests" YQ=/usr/local/bin/yq @@ -93,8 +118,15 @@ while IFS= read -r LINE; do done done < "$ARGS_FILE" -# Override TP/PP from config (the args file ships defaults; config wins). -MODEL_ARGS+=(--tensor-model-parallel-size "$TP" --pipeline-model-parallel-size "$PP") +# Override TP/PP/EP from config (the args file ships defaults; config wins). +# EP must be overridden too — args files for MoE checkpoints (e.g. hybrid_nanov3_3b.args +# hardcodes --expert-model-parallel-size 8) would otherwise force a world size that +# doesn't fit smaller-node platforms like GB200 (4 GPUs/node). +MODEL_ARGS+=( + --tensor-model-parallel-size "$TP" + --pipeline-model-parallel-size "$PP" + --expert-model-parallel-size "$EP" +) # ── Make image-bundled extras (mamba-ssm) visible to the cog venv ───────────── # cog's auto-managed venv uses `uv sync --extra dev --extra mlm` and inherits @@ -118,10 +150,21 @@ if [[ ( "$MODEL" == hybrid_* || "$MODEL" == mamba_* ) && -n "${VIRTUAL_ENV:-}" ] [[ -d "$REAL_VENV/lib/python3.12/site-packages" ]] || REAL_VENV="$VIRTUAL_ENV" PTH_FILE="$REAL_VENV/lib/python3.12/site-packages/_cog_perf_mamba_shim.pth" if [[ -d "$OPT_VENV_SITE" ]] && [[ -d "$REAL_VENV/lib/python3.12/site-packages" ]]; then + # H100 / mcore_ci_dev path: append the image's prebuilt venv to sys.path + # via a .pth file so mamba-ssm + causal-conv1d are visible to the cog venv. echo "[run_perf_test] installing mamba-ssm shim .pth: $PTH_FILE -> $OPT_VENV_SITE" echo "import sys; sys.path.append('$OPT_VENV_SITE')" > "$PTH_FILE" + elif python -c "import mamba_ssm" 2>/dev/null; then + echo "[run_perf_test] mamba_ssm already importable; skipping install" else - echo "[run_perf_test] warning: cannot install mamba shim (REAL_VENV=$REAL_VENV, OPT_VENV_SITE=$OPT_VENV_SITE)" >&2 + # GB200 / bare-NGC path: /opt/venv doesn't exist on this image, so install + # mamba-ssm + causal-conv1d into the cog venv at runtime. Relies on + # prebuilt wheels (PyPI ships linux_aarch64 wheels for both starting + # mamba-ssm 2.2.2 / causal-conv1d 1.4.0). + echo "[run_perf_test] /opt/venv missing; installing mamba-ssm + causal-conv1d via uv pip" + uv pip install --no-build-isolation mamba-ssm causal-conv1d || { + echo "[run_perf_test] warning: mamba-ssm install failed — hybrid tests will fail with ImportError" >&2 + } fi fi @@ -227,8 +270,23 @@ CASE_DIR="$(dirname "$CONFIG_PATH")" BASELINE_PATH="$CASE_DIR/baseline_values.json" if [[ "${RECORD_BASELINE:-0}" == "1" ]]; then - echo "[run_perf_test] RECORD_BASELINE=1 → copying results.json over $BASELINE_PATH" - cp "$RESULTS_JSON" "$BASELINE_PATH" + echo "[run_perf_test] RECORD_BASELINE=1 → merging results.json into $BASELINE_PATH under key '$PLATFORM'" + uv run --no-sync python - "$RESULTS_JSON" "$BASELINE_PATH" "$PLATFORM" <<'PY' +import json +import sys +from pathlib import Path + +results_path, baseline_path, platform = sys.argv[1], sys.argv[2], sys.argv[3] +results = json.loads(Path(results_path).read_text()) +baseline = {} +if Path(baseline_path).exists(): + baseline = json.loads(Path(baseline_path).read_text()) +# Merge: overwrite only the current platform's subtree, leave others intact. +baseline[platform] = results +Path(baseline_path).write_text(json.dumps(baseline, indent=2) + "\n") +print(f"[run_perf_test] wrote {len(results)} batch entries under '{platform}' " + f"({sorted(baseline.keys())} platforms recorded total)") +PY exit 0 fi @@ -246,4 +304,5 @@ fi uv run --no-sync python "$PERF_DIR/shell_test_utils/compare_to_baseline.py" \ --results "$RESULTS_JSON" \ --baseline "$BASELINE_PATH" \ - --config "$CONFIG_PATH" + --config "$CONFIG_PATH" \ + --platform "$PLATFORM" diff --git a/tests/performance_tests/test_cases/gpt/gpt_16b_perf/baseline_values.json b/tests/performance_tests/test_cases/gpt/gpt_16b_perf/baseline_values.json index 5fb7e092109..4f8afb34494 100644 --- a/tests/performance_tests/test_cases/gpt/gpt_16b_perf/baseline_values.json +++ b/tests/performance_tests/test_cases/gpt/gpt_16b_perf/baseline_values.json @@ -1,38 +1,78 @@ { - "batch_1": { - "batch_size": 1, - "dataset": "gsm8k", - "num_input_tokens_avg": 60.2, - "num_output_tokens": 256, - "num_iters": 5, - "throughput_tok_per_sec": 10.769140788963465, - "avg_latency_ms": 23771.573346015066, - "p50_latency_ms": 23764.9713489227, - "p99_latency_ms": 24204.065156169236, - "tpot_ms_per_tok": 92.85791871388938 + "h100": { + "batch_1": { + "batch_size": 1, + "dataset": "gsm8k", + "num_input_tokens_avg": 60.2, + "num_output_tokens": 256, + "num_iters": 5, + "throughput_tok_per_sec": 10.769140788963465, + "avg_latency_ms": 23771.573346015066, + "p50_latency_ms": 23764.9713489227, + "p99_latency_ms": 24204.065156169236, + "tpot_ms_per_tok": 92.85791871388938 + }, + "batch_8": { + "batch_size": 8, + "dataset": "gsm8k", + "num_input_tokens_avg": 59.625, + "num_output_tokens": 256, + "num_iters": 5, + "throughput_tok_per_sec": 84.46829823481039, + "avg_latency_ms": 24243.28422200051, + "p50_latency_ms": 24333.226206013933, + "p99_latency_ms": 24442.530745174736, + "tpot_ms_per_tok": 94.71008848504425 + }, + "batch_32": { + "batch_size": 32, + "dataset": "gsm8k", + "num_input_tokens_avg": 62.475, + "num_output_tokens": 256, + "num_iters": 5, + "throughput_tok_per_sec": 324.7526388613147, + "avg_latency_ms": 25220.91832018632, + "p50_latency_ms": 25256.027992116287, + "p99_latency_ms": 25858.840166125447, + "tpot_ms_per_tok": 98.53653572208714 + } }, - "batch_8": { - "batch_size": 8, - "dataset": "gsm8k", - "num_input_tokens_avg": 59.625, - "num_output_tokens": 256, - "num_iters": 5, - "throughput_tok_per_sec": 84.46829823481039, - "avg_latency_ms": 24243.28422200051, - "p50_latency_ms": 24333.226206013933, - "p99_latency_ms": 24442.530745174736, - "tpot_ms_per_tok": 94.71008848504425 - }, - "batch_32": { - "batch_size": 32, - "dataset": "gsm8k", - "num_input_tokens_avg": 62.475, - "num_output_tokens": 256, - "num_iters": 5, - "throughput_tok_per_sec": 324.7526388613147, - "avg_latency_ms": 25220.91832018632, - "p50_latency_ms": 25256.027992116287, - "p99_latency_ms": 25858.840166125447, - "tpot_ms_per_tok": 98.53653572208714 + "gb200": { + "batch_1": { + "batch_size": 1, + "dataset": "gsm8k", + "num_input_tokens_avg": 60.2, + "num_output_tokens": 256, + "num_iters": 5, + "throughput_tok_per_sec": 9.304504276642584, + "avg_latency_ms": 27513.491276372224, + "p50_latency_ms": 27654.788297135383, + "p99_latency_ms": 27841.648617759347, + "tpot_ms_per_tok": 107.47482834849507 + }, + "batch_8": { + "batch_size": 8, + "dataset": "gsm8k", + "num_input_tokens_avg": 59.625, + "num_output_tokens": 256, + "num_iters": 5, + "throughput_tok_per_sec": 74.52925322314233, + "avg_latency_ms": 27477.007888664957, + "p50_latency_ms": 27517.11248792708, + "p99_latency_ms": 27547.496049664915, + "tpot_ms_per_tok": 107.34040197676222 + }, + "batch_32": { + "batch_size": 32, + "dataset": "gsm8k", + "num_input_tokens_avg": 62.475, + "num_output_tokens": 256, + "num_iters": 5, + "throughput_tok_per_sec": 288.46324079164685, + "avg_latency_ms": 28394.489749116474, + "p50_latency_ms": 28259.915568865836, + "p99_latency_ms": 28714.433840010315, + "tpot_ms_per_tok": 110.93267867399845 + } } -} \ No newline at end of file +} diff --git a/tests/performance_tests/test_cases/gpt/gpt_583m_perf/baseline_values.json b/tests/performance_tests/test_cases/gpt/gpt_583m_perf/baseline_values.json index ed702bda257..5c83450644f 100644 --- a/tests/performance_tests/test_cases/gpt/gpt_583m_perf/baseline_values.json +++ b/tests/performance_tests/test_cases/gpt/gpt_583m_perf/baseline_values.json @@ -1,46 +1,48 @@ { - "batch_1": { - "batch_size": 1, - "num_input_tokens": 512, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 22.08203581766323, - "avg_latency_ms": 5796.511190757155, - "p50_latency_ms": 5842.958671972156, - "p99_latency_ms": 5919.582245871425, - "tpot_ms_per_tok": 45.28567964734975 - }, - "batch_8": { - "batch_size": 8, - "num_input_tokens": 512, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 357.49352020243185, - "avg_latency_ms": 2808.0778209026903, - "p50_latency_ms": 2813.233459368348, - "p99_latency_ms": 2896.668652072549, - "tpot_ms_per_tok": 22.378027986269444 - }, - "batch_32": { - "batch_size": 32, - "num_input_tokens": 512, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 1432.3391490750541, - "avg_latency_ms": 2812.452972715255, - "p50_latency_ms": 2819.5638693869114, - "p99_latency_ms": 2865.5092362314463, - "tpot_ms_per_tok": 22.341077544842847 - }, - "batch_128": { - "batch_size": 128, - "num_input_tokens": 512, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 5643.249306634135, - "avg_latency_ms": 2839.432980850688, - "p50_latency_ms": 2846.7628210783005, - "p99_latency_ms": 2900.5435090512037, - "tpot_ms_per_tok": 22.681967966491356 + "h100": { + "batch_1": { + "batch_size": 1, + "num_input_tokens": 512, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 22.08203581766323, + "avg_latency_ms": 5796.511190757155, + "p50_latency_ms": 5842.958671972156, + "p99_latency_ms": 5919.582245871425, + "tpot_ms_per_tok": 45.28567964734975 + }, + "batch_8": { + "batch_size": 8, + "num_input_tokens": 512, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 357.49352020243185, + "avg_latency_ms": 2808.0778209026903, + "p50_latency_ms": 2813.233459368348, + "p99_latency_ms": 2896.668652072549, + "tpot_ms_per_tok": 22.378027986269444 + }, + "batch_32": { + "batch_size": 32, + "num_input_tokens": 512, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 1432.3391490750541, + "avg_latency_ms": 2812.452972715255, + "p50_latency_ms": 2819.5638693869114, + "p99_latency_ms": 2865.5092362314463, + "tpot_ms_per_tok": 22.341077544842847 + }, + "batch_128": { + "batch_size": 128, + "num_input_tokens": 512, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 5643.249306634135, + "avg_latency_ms": 2839.432980850688, + "p50_latency_ms": 2846.7628210783005, + "p99_latency_ms": 2900.5435090512037, + "tpot_ms_per_tok": 22.681967966491356 + } } -} \ No newline at end of file +} diff --git a/tests/performance_tests/test_cases/gpt/gpt_583m_perf_gb200_4gpu/baseline_values.json b/tests/performance_tests/test_cases/gpt/gpt_583m_perf_gb200_4gpu/baseline_values.json new file mode 100644 index 00000000000..6b73c25593d --- /dev/null +++ b/tests/performance_tests/test_cases/gpt/gpt_583m_perf_gb200_4gpu/baseline_values.json @@ -0,0 +1,52 @@ +{ + "gb200": { + "batch_1": { + "batch_size": 1, + "dataset": "synthetic", + "num_input_tokens_avg": 512.0, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 26.32421878655499, + "avg_latency_ms": 4862.373039405793, + "p50_latency_ms": 3864.351199939847, + "p99_latency_ms": 7142.95435603708, + "tpot_ms_per_tok": 37.987831969803665 + }, + "batch_8": { + "batch_size": 8, + "dataset": "synthetic", + "num_input_tokens_avg": 512.0, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 298.8862382554847, + "avg_latency_ms": 3354.018562519923, + "p50_latency_ms": 3368.747137952596, + "p99_latency_ms": 3462.7067330293357, + "tpot_ms_per_tok": 26.766036625485867 + }, + "batch_32": { + "batch_size": 32, + "dataset": "synthetic", + "num_input_tokens_avg": 512.0, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 1188.7515267430788, + "avg_latency_ms": 3311.2122898193775, + "p50_latency_ms": 3391.8779441155493, + "p99_latency_ms": 3467.822349164635, + "tpot_ms_per_tok": 26.91899802448461 + }, + "batch_128": { + "batch_size": 128, + "dataset": "synthetic", + "num_input_tokens_avg": 512.0, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 4663.364169351257, + "avg_latency_ms": 3324.591547554155, + "p50_latency_ms": 3370.2405970543623, + "p99_latency_ms": 3512.4168978072703, + "tpot_ms_per_tok": 27.447995771217393 + } + } +} diff --git a/tests/performance_tests/test_cases/gpt/gpt_583m_perf_gb200_4gpu/model_config.yaml b/tests/performance_tests/test_cases/gpt/gpt_583m_perf_gb200_4gpu/model_config.yaml new file mode 100644 index 00000000000..31a28715026 --- /dev/null +++ b/tests/performance_tests/test_cases/gpt/gpt_583m_perf_gb200_4gpu/model_config.yaml @@ -0,0 +1,27 @@ +# Inference perf test: 583M mcore-mistral checkpoint, TP=1 PP=1 DP=4 (4 GPUs). +# +# GB200 single-node variant of gpt_583m_perf (which is DP=8 / 2 nodes on GB200). +# GB200 nodes have a 4-GPU/node limit, so the DP=8 single-node configuration +# used on H100 doesn't fit on a single GB200 node. This is a separate test +# (different world size, different baseline) — not a multi-node port of the +# DP=8 case. + +MODEL: gpt_583m +TP: 1 +PP: 1 +DP: 4 +NUM_INPUT_TOKENS: 512 +NUM_OUTPUT_TOKENS: 128 +NUM_WARMUP_ITERS: 2 +NUM_TIMED_ITERS: 5 +BATCH_SIZES: + - 1 + - 8 + - 32 + - 128 +TOLERANCE_PCT: 10 +METRICS: + - throughput_tok_per_sec + - avg_latency_ms + - p50_latency_ms + - tpot_ms_per_tok diff --git a/tests/performance_tests/test_cases/hybrid/hybrid_2b_perf/baseline_values.json b/tests/performance_tests/test_cases/hybrid/hybrid_2b_perf/baseline_values.json index 17deae6e2e8..5422cdb2387 100644 --- a/tests/performance_tests/test_cases/hybrid/hybrid_2b_perf/baseline_values.json +++ b/tests/performance_tests/test_cases/hybrid/hybrid_2b_perf/baseline_values.json @@ -1,50 +1,102 @@ { - "batch_1": { - "batch_size": 1, - "dataset": "gsm8k", - "num_input_tokens_avg": 60.2, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 38.16568346861343, - "avg_latency_ms": 3353.7510838359594, - "p50_latency_ms": 3345.5649770330638, - "p99_latency_ms": 3380.324238911271, - "tpot_ms_per_tok": 26.201548331300728 + "h100": { + "batch_1": { + "batch_size": 1, + "dataset": "gsm8k", + "num_input_tokens_avg": 60.2, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 38.16568346861343, + "avg_latency_ms": 3353.7510838359594, + "p50_latency_ms": 3345.5649770330638, + "p99_latency_ms": 3380.324238911271, + "tpot_ms_per_tok": 26.201548331300728 + }, + "batch_8": { + "batch_size": 8, + "dataset": "gsm8k", + "num_input_tokens_avg": 59.625, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 302.74426211997957, + "avg_latency_ms": 3380.2406550908927, + "p50_latency_ms": 3364.5363999530673, + "p99_latency_ms": 3439.708017045632, + "tpot_ms_per_tok": 26.424943429083214 + }, + "batch_32": { + "batch_size": 32, + "dataset": "gsm8k", + "num_input_tokens_avg": 62.475, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 1211.147008516413, + "avg_latency_ms": 3377.5188652565703, + "p50_latency_ms": 3367.9910679347813, + "p99_latency_ms": 3441.8616080656648, + "tpot_ms_per_tok": 26.421235221641837 + }, + "batch_128": { + "batch_size": 128, + "dataset": "gsm8k", + "num_input_tokens_avg": 61.75, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 4458.541557256171, + "avg_latency_ms": 3652.4659513921506, + "p50_latency_ms": 3593.510515987873, + "p99_latency_ms": 3797.184966970235, + "tpot_ms_per_tok": 28.708939539137646 + } }, - "batch_8": { - "batch_size": 8, - "dataset": "gsm8k", - "num_input_tokens_avg": 59.625, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 302.74426211997957, - "avg_latency_ms": 3380.2406550908927, - "p50_latency_ms": 3364.5363999530673, - "p99_latency_ms": 3439.708017045632, - "tpot_ms_per_tok": 26.424943429083214 - }, - "batch_32": { - "batch_size": 32, - "dataset": "gsm8k", - "num_input_tokens_avg": 62.475, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 1211.147008516413, - "avg_latency_ms": 3377.5188652565703, - "p50_latency_ms": 3367.9910679347813, - "p99_latency_ms": 3441.8616080656648, - "tpot_ms_per_tok": 26.421235221641837 - }, - "batch_128": { - "batch_size": 128, - "dataset": "gsm8k", - "num_input_tokens_avg": 61.75, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 4458.541557256171, - "avg_latency_ms": 3652.4659513921506, - "p50_latency_ms": 3593.510515987873, - "p99_latency_ms": 3797.184966970235, - "tpot_ms_per_tok": 28.708939539137646 + "gb200": { + "batch_1": { + "batch_size": 1, + "dataset": "gsm8k", + "num_input_tokens_avg": 60.2, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 35.173937975487426, + "avg_latency_ms": 3638.992004795, + "p50_latency_ms": 3643.4582789661363, + "p99_latency_ms": 3652.433726005256, + "tpot_ms_per_tok": 28.430140540331195 + }, + "batch_8": { + "batch_size": 8, + "dataset": "gsm8k", + "num_input_tokens_avg": 59.625, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 276.2571793662787, + "avg_latency_ms": 3704.8341338173486, + "p50_latency_ms": 3698.1689609820023, + "p99_latency_ms": 3789.2707429127768, + "tpot_ms_per_tok": 28.958523424989835 + }, + "batch_32": { + "batch_size": 32, + "dataset": "gsm8k", + "num_input_tokens_avg": 62.475, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 1093.1584490536293, + "avg_latency_ms": 3742.3398760358396, + "p50_latency_ms": 3738.3380050305277, + "p99_latency_ms": 3781.2001520069316, + "tpot_ms_per_tok": 29.272975045569183 + }, + "batch_128": { + "batch_size": 128, + "dataset": "gsm8k", + "num_input_tokens_avg": 61.75, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 4147.0849063821415, + "avg_latency_ms": 3924.4852357216587, + "p50_latency_ms": 3952.9280259739608, + "p99_latency_ms": 4002.8255430515856, + "tpot_ms_per_tok": 30.865054101741407 + } } -} \ No newline at end of file +} diff --git a/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf/baseline_values.json b/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf/baseline_values.json index 3c4cf8169e7..5e2155ccfe5 100644 --- a/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf/baseline_values.json +++ b/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf/baseline_values.json @@ -1,50 +1,52 @@ { - "batch_1": { - "batch_size": 1, - "dataset": "gsm8k", - "num_input_tokens_avg": 60.2, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 174.53474487231847, - "avg_latency_ms": 733.3438616711646, - "p50_latency_ms": 725.6480471696705, - "p99_latency_ms": 760.1394890807569, - "tpot_ms_per_tok": 5.729518215593998 - }, - "batch_8": { - "batch_size": 8, - "dataset": "gsm8k", - "num_input_tokens_avg": 59.625, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 1291.0950202594686, - "avg_latency_ms": 791.843310429249, - "p50_latency_ms": 790.5666399747133, - "p99_latency_ms": 800.4119349643588, - "tpot_ms_per_tok": 6.196290648222202 - }, - "batch_32": { - "batch_size": 32, - "dataset": "gsm8k", - "num_input_tokens_avg": 62.475, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 4170.52738241096, - "avg_latency_ms": 977.355056507804, - "p50_latency_ms": 973.6459262203425, - "p99_latency_ms": 993.3186259586364, - "tpot_ms_per_tok": 7.672890516187181 - }, - "batch_128": { - "batch_size": 128, - "dataset": "gsm8k", - "num_input_tokens_avg": 61.75, - "num_output_tokens": 128, - "num_iters": 5, - "throughput_tok_per_sec": 11920.539311597477, - "avg_latency_ms": 1360.820527760734, - "p50_latency_ms": 1379.9351500347257, - "p99_latency_ms": 1393.486256012693, - "tpot_ms_per_tok": 10.737769211118575 + "h100": { + "batch_1": { + "batch_size": 1, + "dataset": "gsm8k", + "num_input_tokens_avg": 60.2, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 174.53474487231847, + "avg_latency_ms": 733.3438616711646, + "p50_latency_ms": 725.6480471696705, + "p99_latency_ms": 760.1394890807569, + "tpot_ms_per_tok": 5.729518215593998 + }, + "batch_8": { + "batch_size": 8, + "dataset": "gsm8k", + "num_input_tokens_avg": 59.625, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 1291.0950202594686, + "avg_latency_ms": 791.843310429249, + "p50_latency_ms": 790.5666399747133, + "p99_latency_ms": 800.4119349643588, + "tpot_ms_per_tok": 6.196290648222202 + }, + "batch_32": { + "batch_size": 32, + "dataset": "gsm8k", + "num_input_tokens_avg": 62.475, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 4170.52738241096, + "avg_latency_ms": 977.355056507804, + "p50_latency_ms": 973.6459262203425, + "p99_latency_ms": 993.3186259586364, + "tpot_ms_per_tok": 7.672890516187181 + }, + "batch_128": { + "batch_size": 128, + "dataset": "gsm8k", + "num_input_tokens_avg": 61.75, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 11920.539311597477, + "avg_latency_ms": 1360.820527760734, + "p50_latency_ms": 1379.9351500347257, + "p99_latency_ms": 1393.486256012693, + "tpot_ms_per_tok": 10.737769211118575 + } } -} \ No newline at end of file +} diff --git a/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf_gb200_4gpu/baseline_values.json b/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf_gb200_4gpu/baseline_values.json new file mode 100644 index 00000000000..eb1eb29cda0 --- /dev/null +++ b/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf_gb200_4gpu/baseline_values.json @@ -0,0 +1,52 @@ +{ + "gb200": { + "batch_1": { + "batch_size": 1, + "dataset": "gsm8k", + "num_input_tokens_avg": 60.2, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 205.28963339383708, + "avg_latency_ms": 623.4671195968986, + "p50_latency_ms": 608.3062621764839, + "p99_latency_ms": 664.121120236814, + "tpot_ms_per_tok": 4.8711665731389076 + }, + "batch_8": { + "batch_size": 8, + "dataset": "gsm8k", + "num_input_tokens_avg": 59.625, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 1479.8669689494195, + "avg_latency_ms": 689.8111295537092, + "p50_latency_ms": 690.9317341633141, + "p99_latency_ms": 697.1075707115233, + "tpot_ms_per_tok": 5.4058913185144775 + }, + "batch_32": { + "batch_size": 32, + "dataset": "gsm8k", + "num_input_tokens_avg": 62.475, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 4169.359459179444, + "avg_latency_ms": 976.2204706174089, + "p50_latency_ms": 951.845585834235, + "p99_latency_ms": 1092.4915480427444, + "tpot_ms_per_tok": 7.675039850437315 + }, + "batch_128": { + "batch_size": 128, + "dataset": "gsm8k", + "num_input_tokens_avg": 61.75, + "num_output_tokens": 128, + "num_iters": 5, + "throughput_tok_per_sec": 12046.630331615843, + "avg_latency_ms": 1346.5355291846208, + "p50_latency_ms": 1354.1678641922772, + "p99_latency_ms": 1424.3129258975387, + "tpot_ms_per_tok": 10.62537792531657 + } + } +} diff --git a/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf_gb200_4gpu/model_config.yaml b/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf_gb200_4gpu/model_config.yaml new file mode 100644 index 00000000000..e63b90a7b73 --- /dev/null +++ b/tests/performance_tests/test_cases/hybrid/hybrid_nanov3_3b_perf_gb200_4gpu/model_config.yaml @@ -0,0 +1,28 @@ +# Inference perf test: nanov3 3B hybrid MoE checkpoint, TP=1 PP=1 EP=4 (4 GPUs). +# +# GB200 single-node variant of hybrid_nanov3_3b_perf (EP=8). GB200 nodes have +# a 4-GPU/node limit, so the EP=8 single-node configuration used on H100 +# doesn't fit on a single GB200 node. EP=4 reshapes the expert sharding — +# this is a separate test with its own baseline, not a port. + +MODEL: hybrid_nanov3_3b +TP: 1 +PP: 1 +DP: 1 +EP: 4 +DATASET: gsm8k +NUM_OUTPUT_TOKENS: 128 +NUM_WARMUP_ITERS: 2 +NUM_TIMED_ITERS: 5 +BATCH_SIZES: + - 1 + - 8 + - 32 + - 128 +TOLERANCE_PCT: 10 +METRICS: + - throughput_tok_per_sec + - avg_latency_ms + - p50_latency_ms + - p99_latency_ms + - tpot_ms_per_tok diff --git a/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml b/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml new file mode 100644 index 00000000000..fb02a6f167d --- /dev/null +++ b/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml @@ -0,0 +1,47 @@ +# Multi-GPU (DP=4) inference perf tests on GB200. Single-node, fills the whole 4-GPU +# allocation that GB200 QOS requires. Exercises the same ZMQ DP-coordinator path the +# H100 gpt-perf-dp8.yaml covers, but at half the DP width (GB200 nodes only have 4 GPUs). +type: basic +format_version: 1 +maintainers: [mcore] +loggers: [stdout] +spec: + name: '{test_case}_{environment}_{platforms}' + model: gpt + build: mcore-pyt-{environment} + nodes: 1 + gpus: 4 + n_repeat: 1 + platforms: dgx_gb200 + time_limit: 3600 + script_setup: | + set -euo pipefail + unset https_proxy + echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc + + cd /opt + rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm + git init + git remote add origin $MCORE_REPO + git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' + git fetch origin $MCORE_MR_COMMIT + git checkout $MCORE_MR_COMMIT + git rev-parse HEAD + script: |- + set -euo pipefail + cd /opt/megatron-lm + + ARGUMENTS=( + "CONFIG_PATH=tests/performance_tests/test_cases/gpt/{test_case}/model_config.yaml" + "CHECKPOINT_LOAD_PATH=/mnt/artifacts/" + "RESULTS_ROOT={assets_dir}/perf_results" + ) + + GPUS_PER_NODE=4 bash ./tests/performance_tests/shell_test_utils/run_perf_test.sh ${{ARGUMENTS[@]}} + +products: + - test_case: [gpt_583m_perf_gb200_4gpu] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/gpt-perf.yaml b/tests/test_utils/recipes/gb200/gpt-perf.yaml new file mode 100644 index 00000000000..9bbd6cf33b7 --- /dev/null +++ b/tests/test_utils/recipes/gb200/gpt-perf.yaml @@ -0,0 +1,49 @@ +# Single-GPU-world-size inference perf tests on GB200. GB200 QOS requires whole-node +# allocations (4 GPUs) so the test runs inside a 4-GPU reservation even though +# WORLD_SIZE is 1 — torchrun only spawns 1 worker. +# +# For multi-GPU GB200 variants, see gpt-perf-dp4.yaml (DP=4 single-node). +type: basic +format_version: 1 +maintainers: [mcore] +loggers: [stdout] +spec: + name: '{test_case}_{environment}_{platforms}' + model: gpt + build: mcore-pyt-{environment} + nodes: 1 + gpus: 4 + n_repeat: 1 + platforms: dgx_gb200 + time_limit: 3600 + script_setup: | + set -euo pipefail + unset https_proxy + echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc + + cd /opt + rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm + git init + git remote add origin $MCORE_REPO + git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' + git fetch origin $MCORE_MR_COMMIT + git checkout $MCORE_MR_COMMIT + git rev-parse HEAD + script: |- + set -euo pipefail + cd /opt/megatron-lm + + ARGUMENTS=( + "CONFIG_PATH=tests/performance_tests/test_cases/gpt/{test_case}/model_config.yaml" + "CHECKPOINT_LOAD_PATH=/mnt/artifacts/" + "RESULTS_ROOT={assets_dir}/perf_results" + ) + + GPUS_PER_NODE=4 bash ./tests/performance_tests/shell_test_utils/run_perf_test.sh ${{ARGUMENTS[@]}} + +products: + - test_case: [gpt_16b_perf] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml b/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml new file mode 100644 index 00000000000..f34b66dae5c --- /dev/null +++ b/tests/test_utils/recipes/gb200/hybrid-perf-ep4.yaml @@ -0,0 +1,47 @@ +# 4-GPU (EP=4) hybrid MoE inference perf tests on GB200. Single-node, fills the whole +# 4-GPU allocation. GB200 variant of the H100 hybrid-perf-ep8.yaml (EP halved from 8 to 4 +# to fit a single GB200 node). +type: basic +format_version: 1 +maintainers: [mcore] +loggers: [stdout] +spec: + name: '{test_case}_{environment}_{platforms}' + model: hybrid + build: mcore-pyt-{environment} + nodes: 1 + gpus: 4 + n_repeat: 1 + platforms: dgx_gb200 + time_limit: 3600 + script_setup: | + set -euo pipefail + unset https_proxy + echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc + + cd /opt + rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm + git init + git remote add origin $MCORE_REPO + git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' + git fetch origin $MCORE_MR_COMMIT + git checkout $MCORE_MR_COMMIT + git rev-parse HEAD + script: |- + set -euo pipefail + cd /opt/megatron-lm + + ARGUMENTS=( + "CONFIG_PATH=tests/performance_tests/test_cases/hybrid/{test_case}/model_config.yaml" + "CHECKPOINT_LOAD_PATH=/mnt/artifacts/" + "RESULTS_ROOT={assets_dir}/perf_results" + ) + + GPUS_PER_NODE=4 bash ./tests/performance_tests/shell_test_utils/run_perf_test.sh ${{ARGUMENTS[@]}} + +products: + - test_case: [hybrid_nanov3_3b_perf_gb200_4gpu] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/hybrid-perf.yaml b/tests/test_utils/recipes/gb200/hybrid-perf.yaml new file mode 100644 index 00000000000..78585557c42 --- /dev/null +++ b/tests/test_utils/recipes/gb200/hybrid-perf.yaml @@ -0,0 +1,48 @@ +# Single-GPU-world-size hybrid inference perf tests on GB200. Runs inside a 4-GPU +# reservation (GB200 QOS minimum) but torchrun only spawns WORLD_SIZE=1. +# +# For multi-GPU GB200 variants, see hybrid-perf-ep4.yaml (EP=4 single-node). +type: basic +format_version: 1 +maintainers: [mcore] +loggers: [stdout] +spec: + name: '{test_case}_{environment}_{platforms}' + model: hybrid + build: mcore-pyt-{environment} + nodes: 1 + gpus: 4 + n_repeat: 1 + platforms: dgx_gb200 + time_limit: 3600 + script_setup: | + set -euo pipefail + unset https_proxy + echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc + + cd /opt + rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm + git init + git remote add origin $MCORE_REPO + git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' + git fetch origin $MCORE_MR_COMMIT + git checkout $MCORE_MR_COMMIT + git rev-parse HEAD + script: |- + set -euo pipefail + cd /opt/megatron-lm + + ARGUMENTS=( + "CONFIG_PATH=tests/performance_tests/test_cases/hybrid/{test_case}/model_config.yaml" + "CHECKPOINT_LOAD_PATH=/mnt/artifacts/" + "RESULTS_ROOT={assets_dir}/perf_results" + ) + + GPUS_PER_NODE=4 bash ./tests/performance_tests/shell_test_utils/run_perf_test.sh ${{ARGUMENTS[@]}} + +products: + - test_case: [hybrid_2b_perf] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200]