From 6a09ee139071afdedca82a9ebca828bcd8c65139 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Mon, 31 Aug 2026 13:59:28 -0700 Subject: [PATCH 1/5] [TRTLLM-16020][test] Add Kimi K3 GSM8K accuracy tests to GB300 multi-node post-merge CI Register the Kimi K3 GSM8K accuracy legs (added in #17922, previously run only by QA's weekly multinode pipeline) in the GB300 16-GPU 4-node post-merge stage introduced by #18363, so K3 accuracy regressions are caught by CI instead of surfacing a week later: - l0_gb300_multi_nodes_node4_gpu16.yml gains TestKimiK3::test_w4a16_mxfp4[baseline] and [sa], TIMEOUT (120) each (matching the stage's existing 16-GPU entry; tighten once real timings exist). The tests run as-is: the DEP16 configuration and its reference baselines are exactly what QA validated on 4x4 GB300. - The reuse and MMMU legs stay QA-weekly-only; the QA list keeps all four entries (same dual registration the stage's Qwen test uses). - Refresh the test docstring (it claimed no automated stage schedules 16-GPU tests) and note that the CI stage's gb300-only gpu wildcard also enforces the B300 exclusion. scripts/test_to_stage_mapping.py resolves both new entries to GB300-16_GPUs-4_Nodes-PyTorch-SingleNvlinkDomain-Post-Merge-1. Signed-off-by: Michal Guzek --- .../integration/defs/accuracy/test_llm_api_pytorch.py | 11 +++++++---- .../test-db/l0_gb300_multi_nodes_node4_gpu16.yml | 5 +++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 8596f2852a6b..c2146db7e38a 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -4652,16 +4652,19 @@ class TestKimiK3(LlmapiAccuracyTestHarness): # so gate on GB300-class device memory. B300 clears this memory gate but # pairs 8-GPU nodes over InfiniBand (same non-NVL72 topology) — do not # schedule these tests on B300; that exclusion is enforced by QA's - # platform selection, not by this marker. + # platform selection and by the CI stage's gb300-only gpu wildcard, + # not by this marker. @pytest.mark.skip_less_device_memory(200000) @pytest.mark.parametrize("mode", ["baseline", "reuse", "sa"]) def test_w4a16_mxfp4(self, mode: str, monkeypatch: pytest.MonkeyPatch) -> None: """GSM8K on the bf16 + MXFP4-routed-expert checkpoint (16 GPUs, DEP16). - No automated L0 stage schedules 16-GPU functional tests; this case is - registered in qa/llm_function_multinode.txt and run by QA's weekly - multinode pipeline (qualified on 4x4 GB300 nodes). Each mode mirrors + The baseline and sa legs run post-merge in the GB300 16-GPU 4-node CI + stage (test-db list l0_gb300_multi_nodes_node4_gpu16.yml); all three + legs are also registered in qa/llm_function_multinode.txt and run by + QA's weekly multinode pipeline (qualified on 4x4 GB300 nodes; the + reuse leg is QA-weekly-only). Each mode mirrors the corresponding examples/kimi_k3/eval_extra_llm_options*.yaml config - keep them in sync when editing either. The `sa` leg additionally records spec-dec acceptance: AL/AR lines in the eval log (via diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml index aebc6e201db8..f58067359b40 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml @@ -14,3 +14,8 @@ l0_gb300_multi_nodes_node4_gpu16: backend: pytorch tests: - accuracy/test_llm_api_pytorch.py::TestQwen3_8_2_4T_A95B::test_nvfp4_adp16_cutedsl TIMEOUT (90) + # Kimi K3 GSM8K accuracy guard (DEP16). The reuse and MMMU legs stay + # QA-weekly-only via qa/llm_function_multinode.txt, which also keeps + # running the two legs below. + - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[baseline] TIMEOUT (120) + - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[sa] TIMEOUT (120) From 47758759947dc787f3e4079a7ce54be9028cbec4 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Mon, 31 Aug 2026 21:24:22 -0700 Subject: [PATCH 2/5] [TRTLLM-16020][test] Document effective-timeout semantics on the K3 entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback: the TIMEOUT marker in a test-db list is not just a stage-level budget — the list parser attaches it as an item-level pytest timeout marker, which takes precedence over the class-level timeout on TestKimiK3. Say so next to the entries and note that the values should be revisited once the stage has measured K3 runtimes. Signed-off-by: Michal Guzek --- .../test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml index f58067359b40..b2c9209e518c 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml @@ -17,5 +17,9 @@ l0_gb300_multi_nodes_node4_gpu16: # Kimi K3 GSM8K accuracy guard (DEP16). The reuse and MMMU legs stay # QA-weekly-only via qa/llm_function_multinode.txt, which also keeps # running the two legs below. + # The TIMEOUT marker is the effective per-test timeout: the list parser + # attaches it at item level, which takes precedence over TestKimiK3's + # class-level pytest timeout. Revisit the values below once this stage + # has produced measured K3 runtimes. - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[baseline] TIMEOUT (120) - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[sa] TIMEOUT (120) From 7befbb23b1c5fbcb7fd36cce1e2dced7cde6adbc Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Mon, 31 Aug 2026 21:24:53 -0700 Subject: [PATCH 3/5] [TRTLLM-16020][test] Reflow the K3 GSM8K docstring per review Apply the suggested wording: reflow the ragged paragraph and say "modes of this test" instead of "legs" so the docstring no longer reads as inconsistent with the sibling yml comment, which uses "the reuse and MMMU legs" for the QA-weekly-only coverage. Signed-off-by: Michal Guzek --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index c2146db7e38a..1d46f17a8a23 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -4662,9 +4662,9 @@ def test_w4a16_mxfp4(self, mode: str, The baseline and sa legs run post-merge in the GB300 16-GPU 4-node CI stage (test-db list l0_gb300_multi_nodes_node4_gpu16.yml); all three - legs are also registered in qa/llm_function_multinode.txt and run by - QA's weekly multinode pipeline (qualified on 4x4 GB300 nodes; the - reuse leg is QA-weekly-only). Each mode mirrors + modes of this test are also registered in qa/llm_function_multinode.txt + and run by QA's weekly multinode pipeline (qualified on 4x4 GB300 + nodes; the reuse mode is QA-weekly-only). Each mode mirrors the corresponding examples/kimi_k3/eval_extra_llm_options*.yaml config - keep them in sync when editing either. The `sa` leg additionally records spec-dec acceptance: AL/AR lines in the eval log (via From d695d62a664eb4d3a24d6a56180ed1b79ba98540 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Mon, 31 Aug 2026 21:25:30 -0700 Subject: [PATCH 4/5] [TRTLLM-16020][test] Tighten the K3 GSM8K stage timeouts to 90 minutes Follow #18480, which moves the stage's existing 16-GPU entry from TIMEOUT (120) to TIMEOUT (90): a real hang surfaces half an hour sooner, and a K3 GSM8K leg that genuinely needs more than 90 minutes at DEP16 would itself be a severe regression to investigate rather than accommodate with a larger timeout. Revisit against the measured timings from this PR's extra-stage run. The Qwen entry is left untouched here; #18480 owns that change. Signed-off-by: Michal Guzek --- .../test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml index b2c9209e518c..34c2d9209fc1 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml @@ -21,5 +21,5 @@ l0_gb300_multi_nodes_node4_gpu16: # attaches it at item level, which takes precedence over TestKimiK3's # class-level pytest timeout. Revisit the values below once this stage # has produced measured K3 runtimes. - - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[baseline] TIMEOUT (120) - - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[sa] TIMEOUT (120) + - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[baseline] TIMEOUT (90) + - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[sa] TIMEOUT (90) From 9fce386fdcaed2a0869e0da1791920e0ca83e5c4 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Tue, 1 Sep 2026 07:58:20 -0700 Subject: [PATCH 5/5] [TRTLLM-16020][test] Trim the K3 entry comment to three lines per review Drop the TIMEOUT-semantics note from the yml comment; the marker behavior is documented by the list parser itself and the commit history, so the entry comment only needs to state what is covered where. Signed-off-by: Michal Guzek --- .../test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml index 34c2d9209fc1..3a0c3f16fb22 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_node4_gpu16.yml @@ -17,9 +17,5 @@ l0_gb300_multi_nodes_node4_gpu16: # Kimi K3 GSM8K accuracy guard (DEP16). The reuse and MMMU legs stay # QA-weekly-only via qa/llm_function_multinode.txt, which also keeps # running the two legs below. - # The TIMEOUT marker is the effective per-test timeout: the list parser - # attaches it at item level, which takes precedence over TestKimiK3's - # class-level pytest timeout. Revisit the values below once this stage - # has produced measured K3 runtimes. - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[baseline] TIMEOUT (90) - accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[sa] TIMEOUT (90)