diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 6bbe79232c0d..941b361e50de 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -15,6 +15,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.yaml mirror: amd: + automatic_shard: true label: ":amd: (MI300) LM Eval Small Models" dind: false device: mi300_1 @@ -33,6 +34,7 @@ steps: - label: ":amd: (MI300) LM Eval Small Models Harness" key: amd-lm-eval-small-models-harness + automatic_shard: true dind: false device: mi300_1 timeout_in_minutes: 35 @@ -55,6 +57,7 @@ steps: - label: ":nvidia: (H200 MIG 35GB) LM Eval Watermarking" device: h200_35gb key: lm-eval-watermarking + automatic_shard: true # Recorded runs took 80–140 seconds timeout_in_minutes: 10 source_file_dependencies: @@ -69,6 +72,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355 DPX) LM Eval Watermarking" dind: false device: mi355_dpx @@ -267,6 +271,7 @@ steps: - label: ":nvidia: (H100) LM Eval Watermark Feature Combination" key: lm-eval-dspark-watermark-2xh100 + automatic_shard: true device: h100 num_devices: 2 # Local TP2 runs took 84–379 seconds, including startup. @@ -296,6 +301,7 @@ steps: watermarking/test_parallel_compatibility.py mirror: amd: + automatic_shard: true label: ":amd: (MI355) LM Eval Watermark Feature Combination" dind: false device: mi355_2 @@ -678,6 +684,7 @@ steps: - label: ":nvidia: (H200) [Nightly] LM Eval Nemotron 3 Super" key: lm-eval-nemotron-3-super-8xh200 + automatic_shard: true timeout_in_minutes: 35 device: h200 optional: true @@ -686,6 +693,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-nemotron-3-super.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355) LM Eval Nemotron 3 Super" dind: false device: mi355_4 @@ -722,6 +730,7 @@ steps: - label: ":amd: (MI355) [Nightly] LM Eval Large Models" key: amd-lm-eval-large-models + automatic_shard: true dind: false device: mi355_8 optional: true @@ -745,6 +754,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-h100.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355) MoE Refactor Integration" dind: false device: mi355_2 @@ -774,6 +784,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355) MoE Refactor BF16 Integration" dind: false device: mi355_2 @@ -808,6 +819,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355) MoE Refactor BF16 DP Integration" dind: false device: mi355_2 @@ -920,6 +932,7 @@ steps: - label: ":nvidia: (H100) [Nightly] LM Eval Humming DeepEP TEMPORARY" key: lm-eval-humming-h100-deepep + automatic_shard: true optional: true timeout_in_minutes: 60 device: h100 @@ -972,6 +985,7 @@ steps: - label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant k8v4" key: lm-eval-turboquant-k8v4 + automatic_shard: true timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: @@ -983,6 +997,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355 DPX) LM Eval TurboQuant k8v4" dind: false device: mi355_dpx @@ -1133,6 +1148,7 @@ steps: # Octave kernels are built only for gfx942 and gfx1100, so this runs on MI300. - label: ":amd: (MI300) LM Eval Octave KV Cache" key: amd-lm-eval-octave-mi300 + automatic_shard: true dind: false device: mi300_1 num_devices: 1 @@ -1189,6 +1205,7 @@ steps: - label: ":nvidia: (B200) [Nightly] GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-2xb200 + automatic_shard: true timeout_in_minutes: 30 device: b200-k8s optional: true @@ -1201,6 +1218,7 @@ steps: - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355) GPQA Eval (GPT-OSS)" dind: false device: mi355_2 @@ -1279,6 +1297,7 @@ steps: - label: ":nvidia: (DGX Spark) [Nightly] GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-spark + automatic_shard: true timeout_in_minutes: 35 device: dgx-spark optional: true @@ -1293,6 +1312,7 @@ steps: - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355 DPX) GPQA Eval (GPT-OSS TP1)" # The Spark config is a backend-neutral GPT-OSS TP1 baseline. dind: false @@ -1316,6 +1336,7 @@ steps: - label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload" key: kv-offload-small + automatic_shard: true timeout_in_minutes: 30 device: h200_35gb source_file_dependencies: @@ -1328,6 +1349,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "nemotron-h-8b or gemma-4-e4b-it" mirror: amd: + automatic_shard: true label: ":amd: (MI355 DPX) LM Eval KV-Offload" dind: false device: mi355_dpx @@ -1345,6 +1367,7 @@ steps: - label: ":nvidia: (H100) LM Eval KV-Offload Medium" key: kv-offload-medium + automatic_shard: true timeout_in_minutes: 45 device: h100 num_devices: 2 @@ -1358,6 +1381,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "qwen3.5-35b or deepseek-v2-lite" mirror: amd: + automatic_shard: true label: ":amd: (MI355) LM Eval KV-Offload Medium" dind: false device: mi355_2 @@ -1375,6 +1399,7 @@ steps: - label: ":nvidia: (H100) [Nightly] LM Eval KV-Offload Large" key: kv-offload-large + automatic_shard: true # Nightly only: 4-GPU H100 pods wait behind smaller jobs on per-PR runs. optional: true timeout_in_minutes: 40 @@ -1390,6 +1415,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash" mirror: amd: + automatic_shard: true label: ":amd: (MI355) LM Eval KV-Offload Large" dind: false device: mi355_4 @@ -1409,6 +1435,7 @@ steps: - label: ":nvidia: (H200 MIG 35GB) MRCR Eval Small Models" key: mrcr-eval-small-models + automatic_shard: true device: h200_35gb timeout_in_minutes: 25 source_file_dependencies: @@ -1417,6 +1444,7 @@ steps: - pytest -s -v evals/mrcr/test_mrcr_correctness.py --config-list-file=evals/mrcr/configs/models-small.txt mirror: amd: + automatic_shard: true label: ":amd: (MI355 DPX) MRCR Eval Small Models" dind: false device: mi355_dpx @@ -1434,6 +1462,7 @@ steps: - label: ":amd: (MI355) [Nightly] LM Eval Large Models Harness" key: amd-lm-eval-large-models-harness + automatic_shard: true dind: false device: mi355_8 optional: true @@ -1459,6 +1488,7 @@ steps: - label: ":amd: (MI355) [Nightly] LM Eval Large Models ROCm Harness" key: amd-lm-eval-large-models-rocm-harness + automatic_shard: true dind: false device: mi355_8 optional: true