[TRTLLM-15036][test] Add Kimi K3 GSM8K/MMMU accuracy tests and register them in QA's weekly multinode list

tests/integration/defs/accuracy/references/acceptance_length.yaml

@@ -1,3 +1,5 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 TestLlama3_1_8BInstruct::test_eagle3:
   ref_al: 1.987311931289228
   min_al: 1.8879463347247665
@@ -34,3 +36,6 @@ TestLlama3_1_8BInstruct::test_draft_target_dynamic_draft_len:
 TestQwen3_5_4B::test_dflash:
   ref_al: 3.96417002949639
   min_al: 3.7659615280215704
+TestKimiK3::test_w4a16_mxfp4:
+  ref_al: 1.318
+  min_al: 1.15

tests/integration/defs/accuracy/references/gsm8k.yaml

@@ -1,3 +1,5 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 meta-llama/Llama-3.1-8B-Instruct:
   - accuracy: 74.20
   - spec_dec_algo: NGram
@@ -296,6 +298,17 @@ moonshotai/Kimi-K2.5:
     accuracy: 93.06
   - quant_algo: NVFP4
     accuracy: 93.06
+# Kimi K3 (KimiLinear KDA/MLA hybrid + latent MoE): bf16 with MXFP4-packed
+# routed experts. Measured 96.51/96.44 (flexible/strict) at the K3 enablement
+# commit, DEP16 on 16 GB300s (examples/kimi_k3/README.md expected scores).
+# No quant_algo key: the checkpoint's quantization lives in the nested
+# text_config.quantization_config (compressed-tensors), which the LLM-args
+# layer does not surface — the reference matcher sees quant_algo=None.
+moonshotai/Kimi-K3:
+  - accuracy: 96.5
+  # SA spec dec is lossless; scores match the baseline within noise.
+  - spec_dec_algo: SA
+    accuracy: 96.5
 nvidia/Llama-3_3-Nemotron-Super-49B-v1:
   - accuracy: 92.57
   - quant_algo: FP8

tests/integration/defs/accuracy/references/mmmu.yaml

@@ -1,3 +1,5 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 google/gemma-3-27b-it:
   - accuracy: 52.0
   - quant_algo: FP8
@@ -78,3 +80,14 @@ moonshotai/Kimi-K2.5:
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 81.56
+# Kimi K3 VL multimodal (MoonViT3d vision tower + KimiLinear KDA/MLA hybrid
+# backbone). bf16 checkpoint with MXFP4-packed routed experts, DEP16 on 4x4
+# GB300; single-run reference (lm-eval stderr +/-1.16 at n=900). A score at or
+# below the K2.5 floor above with clean logs signals the kimi_k3_mmmu
+# extractor silently falling back (truncated CoT) — a harness failure, not a
+# model regression. No quant_algo key: the checkpoint's quantization lives in
+# the nested text_config.quantization_config (compressed-tensors), which the
+# LLM-args layer does not surface — the reference matcher sees
+# quant_algo=None.
+moonshotai/Kimi-K3:
+  - accuracy: 84.89

tests/integration/defs/accuracy/test_llm_api_pytorch.py

@@ -4653,6 +4653,107 @@ def test_nvfp4(self, ep_size, attention_dp):
             task.evaluate(llm)
 
 
+@pytest.mark.timeout(10800)
+class TestKimiK3(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "moonshotai/Kimi-K3"
+    MODEL_PATH = f"{llm_models_root()}/Kimi-K3"
+
+    @skip_pre_blackwell
+    @pytest.mark.skip_less_mpi_world_size(16)
+    # The 16-GPU K3 recipes are qualified on GB300 (one NVL72 domain) only:
+    # on 2-node 180-190 GiB parts (B200/GB200, InfiniBand between nodes) the
+    # EP16 MoE-comm bring-up hangs and the KV-budget assumptions do not hold,
+    # so gate on GB300-class device memory. B300 clears this memory gate but
+    # pairs 8-GPU nodes over InfiniBand (same non-NVL72 topology) — do not
+    # schedule these tests on B300; that exclusion is enforced by QA's
+    # platform selection, not by this marker.
+    @pytest.mark.skip_less_device_memory(200000)
+    @pytest.mark.parametrize("mode", ["baseline", "reuse", "sa"])
+    def test_w4a16_mxfp4(self, mode: str,
+                         monkeypatch: pytest.MonkeyPatch) -> None:
+        """GSM8K on the bf16 + MXFP4-routed-expert checkpoint (16 GPUs, DEP16).
+
+        No automated L0 stage schedules 16-GPU functional tests; this case is
+        registered in qa/llm_function_multinode.txt and run by QA's weekly
+        multinode pipeline (qualified on 4x4 GB300 nodes). Each mode mirrors
+        the corresponding examples/kimi_k3/eval_extra_llm_options*.yaml config
+        - keep them in sync when editing either. The `sa` leg additionally
+        records spec-dec acceptance: AL/AR lines in the eval log (via
+        TLLM_EVAL_SPEC_STATS) and an iteration-stats AL asserted against
+        references/acceptance_length.yaml.
+        """
+        kv_cache_kwargs = dict(
+            enable_block_reuse=False,
+            free_gpu_memory_fraction=0.25,
+            # tokens_per_block=64 keeps the MLA generation path on the
+            # flashinfer trtllm-gen kernel (K3 has 96 query heads).
+            tokens_per_block=64,
+        )
+        llm_kwargs = dict(
+            tensor_parallel_size=16,
+            moe_expert_parallel_size=16,
+            enable_attention_dp=True,
+            max_batch_size=32,
+            max_num_tokens=8192,
+            max_seq_len=8192,
+            trust_remote_code=True,
+            enable_chunked_prefill=True,
+            cuda_graph_config=CudaGraphConfig(enable_padding=True,
+                                              max_batch_size=32),
+            moe_config=MoeConfig(max_num_tokens=33024,
+                                 use_low_precision_moe_combine=True),
+        )
+        if mode == "reuse":
+            kv_cache_kwargs["enable_block_reuse"] = True
+            # Hybrid models expose reusable prefixes only at KDA state
+            # snapshot boundaries; without a snapshot cadence, block reuse
+            # silently never engages.
+            kv_cache_kwargs["mamba_state_config"] = MambaStateConfig(
+                periodic_snapshot_interval=256)
+        elif mode == "sa":
+            llm_kwargs.update(
+                max_batch_size=8,
+                disable_overlap_scheduler=True,
+                enable_chunked_prefill=False,
+                cuda_graph_config=CudaGraphConfig(max_batch_size=8),
+                speculative_config=SADecodingConfig(max_draft_len=2),
+                # AL capture needs per-iteration spec-decoding stats.
+                enable_iter_perf_stats=True,
+                max_stats_len=-1,
+            )
+            # Log corpus-aggregate AL and AR at eval end ("Spec-dec stats:"
+            # lines) — QA records acceptance from the test log (TRTLLM-15036).
+            monkeypatch.setenv("TLLM_EVAL_SPEC_STATS", "1")
+
+        with LLM(self.MODEL_PATH,
+                 kv_cache_config=KvCacheConfig(**kv_cache_kwargs),
+                 **llm_kwargs) as llm:
+            # Reference-key contract: the K3 checkpoint carries its
+            # quantization as nested text_config.quantization_config
+            # (compressed-tensors MXFP4 experts), which the LLM-args layer
+            # does not surface — unlike modelopt-style hf_quant_config.json
+            # checkpoints — so the reference matcher sees quant_algo=None
+            # and the references/gsm8k.yaml entries carry no quant_algo key.
+            # If this fires, the args-level resolution changed: update the
+            # yaml keys together with this assert.
+            assert llm.args.quant_config.quant_algo is None
+            task = GSM8K(self.MODEL_NAME)
+            task.evaluate(llm)
+            if mode == "sa":
+                acceptance_length = _compute_acceptance_length(llm)
+                print(f"[AL] TestKimiK3::test_w4a16_mxfp4[sa] "
+                      f"acceptance_length = {acceptance_length:.3f}")
+                # ref_al/min_al live in references/acceptance_length.yaml.
+                # The reference was measured on the same workload but
+                # through the lm-eval-route estimator, whose weighting may
+                # not match this iteration-stats one exactly — so min_al is
+                # set as a loose acceptance-collapse tripwire rather than
+                # the populate-path default. Tighten both once this test's
+                # own runs establish a baseline.
+                assert_acceptance_length("TestKimiK3::test_w4a16_mxfp4",
+                                         acceptance_length)
+
+
 class TestQwen3_4B(LlmapiAccuracyTestHarness):
     MODEL_NAME = "Qwen3/Qwen3-4B"
 

tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py

@@ -12,10 +12,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing import ClassVar
+
 import pytest
 
 from tensorrt_llm import LLM
-from tensorrt_llm.evaluate.post_processing import strip_thinking_and_extract_mmmu_answer
+from tensorrt_llm.evaluate.post_processing import (
+    extract_kimi_k3_mmmu_answer,
+    strip_thinking_and_extract_mmmu_answer,
+)
 from tensorrt_llm.llmapi import (
     CudaGraphConfig,
     KvCacheConfig,
@@ -513,6 +518,93 @@ def test_nvfp4(self, ep_size, attention_dp):
             )
 
 
+class TestKimiK3(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "moonshotai/Kimi-K3"
+    MODEL_PATH = f"{llm_models_root()}/Kimi-K3"
+    MAX_NUM_TOKENS = 16384
+
+    sampling_params = SamplingParams(
+        max_tokens=MAX_NUM_TOKENS,
+        truncate_prompt_tokens=MMMU.MAX_INPUT_LEN,
+    )
+
+    # K3 reasons inside its <|open|>response<|sep|>...<|close|> channel (no
+    # <think> markup), so the K2.5 strip-thinking extractor cannot see the
+    # answer; extract_kimi_k3_mmmu_answer reads the channel and falls back to
+    # the K2.5 cascade for channel-less outputs. No chat_template_kwargs: K3
+    # thinks by default. preserve_caller_max_tokens keeps max_tokens=16384
+    # over lm-eval's 512-token MMMU default — without it the CoT is truncated
+    # before the response channel opens, the extractor silently falls back,
+    # and the score sinks toward the K2.5 reference floor (see
+    # references/mmmu.yaml): treat such a score with clean logs as a harness
+    # failure, not a model regression.
+    EXTRA_EVALUATOR_KWARGS: ClassVar[dict] = dict(
+        post_process_fn=extract_kimi_k3_mmmu_answer,
+        preserve_caller_max_tokens=True,
+    )
+
+    @skip_pre_blackwell
+    @pytest.mark.timeout(7200)
+    @pytest.mark.skip_less_mpi_world_size(16)
+    # The 16-GPU K3 recipes are qualified on GB300 (one NVL72 domain) only:
+    # on 2-node 180-190 GiB parts (B200/GB200, InfiniBand between nodes) the
+    # EP16 MoE-comm bring-up hangs and the KV-budget assumptions do not hold,
+    # so gate on GB300-class device memory. B300 clears this memory gate but
+    # pairs 8-GPU nodes over InfiniBand (same non-NVL72 topology) — do not
+    # schedule these tests on B300; that exclusion is enforced by QA's
+    # platform selection, not by this marker.
+    @pytest.mark.skip_less_device_memory(200000)
+    def test_w4a16_mxfp4(self) -> None:
+        """MMMU-val on the K3 VL checkpoint (16 GPUs, DEP16).
+
+        No automated L0 stage schedules 16-GPU functional tests; this case is
+        registered in qa/llm_function_multinode.txt and run by QA's weekly
+        multinode pipeline (qualified on 4x4 GB300 nodes; reference in
+        references/mmmu.yaml). Mirrors examples/kimi_k3/run_eval_kimi_k3.sbatch
+        --task mmmu: the base eval_extra_llm_options.yaml serving config with
+        max_seq_len raised to 24576 (8192 input + 16384 output).
+        """
+        with LLM(
+            self.MODEL_PATH,
+            tensor_parallel_size=16,
+            moe_expert_parallel_size=16,
+            enable_attention_dp=True,
+            max_batch_size=32,
+            max_num_tokens=8192,
+            max_seq_len=24576,
+            trust_remote_code=True,
+            enable_chunked_prefill=True,
+            cuda_graph_config=CudaGraphConfig(enable_padding=True, max_batch_size=32),
+            moe_config=MoeConfig(max_num_tokens=33024, use_low_precision_moe_combine=True),
+            # use_kv_cache_manager_v2=False: the VL wrapper inherits K2.5's
+            # V2 cache-manager preference, but the qualified K3 MMMU
+            # configuration ran V1, and under V2 this test's long-generation
+            # shape (max_seq_len 24576, 16k-token outputs) stalled with all
+            # GPUs idle in QA validation (the GSM8K legs' shorter shape runs
+            # fine under V2). Pin V1 until the V2 x KDA-hybrid path is
+            # qualified at this shape.
+            kv_cache_config=KvCacheConfig(
+                free_gpu_memory_fraction=0.25,
+                tokens_per_block=64,
+                use_kv_cache_manager_v2=False,
+            ),
+        ) as llm:
+            # Reference-key contract: the K3 checkpoint carries its
+            # quantization as nested text_config.quantization_config
+            # (compressed-tensors MXFP4 experts), which the LLM-args layer
+            # does not surface, so the reference matcher sees
+            # quant_algo=None and the references/mmmu.yaml entry carries no
+            # quant_algo key. If this fires, the args-level resolution
+            # changed: update the yaml key together with this assert.
+            assert llm.args.quant_config.quant_algo is None
+            task = MMMU(self.MODEL_NAME)
+            task.evaluate(
+                llm,
+                sampling_params=self.sampling_params,
+                extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS,
+            )
+
+
 class TestMistralSmall24B(LlmapiAccuracyTestHarness):
     MODEL_NAME = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
     MODEL_PATH = f"{llm_models_root()}/Mistral-Small-3.1-24B-Instruct-2503"

tests/integration/test_lists/qa/llm_function_multinode.txt

@@ -11,3 +11,7 @@ test_e2e.py::test_multi_nodes_eval[MiniMax-M3-tp16-mmlu]
 test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp2pp1-gen_tp2pp1]
 test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2]
 test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[etcd]
+accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[baseline]
+accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[reuse]
+accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[sa]
+accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK3::test_w4a16_mxfp4