diff --git a/tests/integration/defs/accuracy/references/acceptance_length.yaml b/tests/integration/defs/accuracy/references/acceptance_length.yaml index ce5f8ca35aed..9d19aa27734f 100644 --- a/tests/integration/defs/accuracy/references/acceptance_length.yaml +++ b/tests/integration/defs/accuracy/references/acceptance_length.yaml @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 TestLlama3_1_8BInstruct::test_eagle3: ref_al: 1.987311931289228 min_al: 1.8879463347247665 @@ -34,3 +36,6 @@ TestLlama3_1_8BInstruct::test_draft_target_dynamic_draft_len: TestQwen3_5_4B::test_dflash: ref_al: 3.96417002949639 min_al: 3.7659615280215704 +TestKimiK3::test_w4a16_mxfp4: + ref_al: 1.318 + min_al: 1.15 diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index cd2de8a50867..0d98e20d649b 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 meta-llama/Llama-3.1-8B-Instruct: - accuracy: 74.20 - spec_dec_algo: NGram @@ -296,6 +298,17 @@ moonshotai/Kimi-K2.5: accuracy: 93.06 - quant_algo: NVFP4 accuracy: 93.06 +# Kimi K3 (KimiLinear KDA/MLA hybrid + latent MoE): bf16 with MXFP4-packed +# routed experts. Measured 96.51/96.44 (flexible/strict) at the K3 enablement +# commit, DEP16 on 16 GB300s (examples/kimi_k3/README.md expected scores). +# No quant_algo key: the checkpoint's quantization lives in the nested +# text_config.quantization_config (compressed-tensors), which the LLM-args +# layer does not surface — the reference matcher sees quant_algo=None. +moonshotai/Kimi-K3: + - accuracy: 96.5 + # SA spec dec is lossless; scores match the baseline within noise. + - spec_dec_algo: SA + accuracy: 96.5 nvidia/Llama-3_3-Nemotron-Super-49B-v1: - accuracy: 92.57 - quant_algo: FP8 diff --git a/tests/integration/defs/accuracy/references/mmmu.yaml b/tests/integration/defs/accuracy/references/mmmu.yaml index a9856fdca7bd..9bb299c9e6bb 100644 --- a/tests/integration/defs/accuracy/references/mmmu.yaml +++ b/tests/integration/defs/accuracy/references/mmmu.yaml @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 google/gemma-3-27b-it: - accuracy: 52.0 - quant_algo: FP8 @@ -78,3 +80,14 @@ moonshotai/Kimi-K2.5: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 accuracy: 81.56 +# Kimi K3 VL multimodal (MoonViT3d vision tower + KimiLinear KDA/MLA hybrid +# backbone). bf16 checkpoint with MXFP4-packed routed experts, DEP16 on 4x4 +# GB300; single-run reference (lm-eval stderr +/-1.16 at n=900). A score at or +# below the K2.5 floor above with clean logs signals the kimi_k3_mmmu +# extractor silently falling back (truncated CoT) — a harness failure, not a +# model regression. No quant_algo key: the checkpoint's quantization lives in +# the nested text_config.quantization_config (compressed-tensors), which the +# LLM-args layer does not surface — the reference matcher sees +# quant_algo=None. +moonshotai/Kimi-K3: + - accuracy: 84.89 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 64363508ebc2..b7e1b604939f 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -4653,6 +4653,107 @@ def test_nvfp4(self, ep_size, attention_dp): task.evaluate(llm) +@pytest.mark.timeout(10800) +class TestKimiK3(LlmapiAccuracyTestHarness): + MODEL_NAME = "moonshotai/Kimi-K3" + MODEL_PATH = f"{llm_models_root()}/Kimi-K3" + + @skip_pre_blackwell + @pytest.mark.skip_less_mpi_world_size(16) + # The 16-GPU K3 recipes are qualified on GB300 (one NVL72 domain) only: + # on 2-node 180-190 GiB parts (B200/GB200, InfiniBand between nodes) the + # EP16 MoE-comm bring-up hangs and the KV-budget assumptions do not hold, + # so gate on GB300-class device memory. B300 clears this memory gate but + # pairs 8-GPU nodes over InfiniBand (same non-NVL72 topology) — do not + # schedule these tests on B300; that exclusion is enforced by QA's + # platform selection, not by this marker. + @pytest.mark.skip_less_device_memory(200000) + @pytest.mark.parametrize("mode", ["baseline", "reuse", "sa"]) + def test_w4a16_mxfp4(self, mode: str, + monkeypatch: pytest.MonkeyPatch) -> None: + """GSM8K on the bf16 + MXFP4-routed-expert checkpoint (16 GPUs, DEP16). + + No automated L0 stage schedules 16-GPU functional tests; this case is + registered in qa/llm_function_multinode.txt and run by QA's weekly + multinode pipeline (qualified on 4x4 GB300 nodes). Each mode mirrors + the corresponding examples/kimi_k3/eval_extra_llm_options*.yaml config + - keep them in sync when editing either. The `sa` leg additionally + records spec-dec acceptance: AL/AR lines in the eval log (via + TLLM_EVAL_SPEC_STATS) and an iteration-stats AL asserted against + references/acceptance_length.yaml. + """ + kv_cache_kwargs = dict( + enable_block_reuse=False, + free_gpu_memory_fraction=0.25, + # tokens_per_block=64 keeps the MLA generation path on the + # flashinfer trtllm-gen kernel (K3 has 96 query heads). + tokens_per_block=64, + ) + llm_kwargs = dict( + tensor_parallel_size=16, + moe_expert_parallel_size=16, + enable_attention_dp=True, + max_batch_size=32, + max_num_tokens=8192, + max_seq_len=8192, + trust_remote_code=True, + enable_chunked_prefill=True, + cuda_graph_config=CudaGraphConfig(enable_padding=True, + max_batch_size=32), + moe_config=MoeConfig(max_num_tokens=33024, + use_low_precision_moe_combine=True), + ) + if mode == "reuse": + kv_cache_kwargs["enable_block_reuse"] = True + # Hybrid models expose reusable prefixes only at KDA state + # snapshot boundaries; without a snapshot cadence, block reuse + # silently never engages. + kv_cache_kwargs["mamba_state_config"] = MambaStateConfig( + periodic_snapshot_interval=256) + elif mode == "sa": + llm_kwargs.update( + max_batch_size=8, + disable_overlap_scheduler=True, + enable_chunked_prefill=False, + cuda_graph_config=CudaGraphConfig(max_batch_size=8), + speculative_config=SADecodingConfig(max_draft_len=2), + # AL capture needs per-iteration spec-decoding stats. + enable_iter_perf_stats=True, + max_stats_len=-1, + ) + # Log corpus-aggregate AL and AR at eval end ("Spec-dec stats:" + # lines) — QA records acceptance from the test log (TRTLLM-15036). + monkeypatch.setenv("TLLM_EVAL_SPEC_STATS", "1") + + with LLM(self.MODEL_PATH, + kv_cache_config=KvCacheConfig(**kv_cache_kwargs), + **llm_kwargs) as llm: + # Reference-key contract: the K3 checkpoint carries its + # quantization as nested text_config.quantization_config + # (compressed-tensors MXFP4 experts), which the LLM-args layer + # does not surface — unlike modelopt-style hf_quant_config.json + # checkpoints — so the reference matcher sees quant_algo=None + # and the references/gsm8k.yaml entries carry no quant_algo key. + # If this fires, the args-level resolution changed: update the + # yaml keys together with this assert. + assert llm.args.quant_config.quant_algo is None + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + if mode == "sa": + acceptance_length = _compute_acceptance_length(llm) + print(f"[AL] TestKimiK3::test_w4a16_mxfp4[sa] " + f"acceptance_length = {acceptance_length:.3f}") + # ref_al/min_al live in references/acceptance_length.yaml. + # The reference was measured on the same workload but + # through the lm-eval-route estimator, whose weighting may + # not match this iteration-stats one exactly — so min_al is + # set as a loose acceptance-collapse tripwire rather than + # the populate-path default. Tighten both once this test's + # own runs establish a baseline. + assert_acceptance_length("TestKimiK3::test_w4a16_mxfp4", + acceptance_length) + + class TestQwen3_4B(LlmapiAccuracyTestHarness): MODEL_NAME = "Qwen3/Qwen3-4B" diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 2243af03f22e..ef018713ffa5 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -12,10 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + import pytest from tensorrt_llm import LLM -from tensorrt_llm.evaluate.post_processing import strip_thinking_and_extract_mmmu_answer +from tensorrt_llm.evaluate.post_processing import ( + extract_kimi_k3_mmmu_answer, + strip_thinking_and_extract_mmmu_answer, +) from tensorrt_llm.llmapi import ( CudaGraphConfig, KvCacheConfig, @@ -513,6 +518,93 @@ def test_nvfp4(self, ep_size, attention_dp): ) +class TestKimiK3(LlmapiAccuracyTestHarness): + MODEL_NAME = "moonshotai/Kimi-K3" + MODEL_PATH = f"{llm_models_root()}/Kimi-K3" + MAX_NUM_TOKENS = 16384 + + sampling_params = SamplingParams( + max_tokens=MAX_NUM_TOKENS, + truncate_prompt_tokens=MMMU.MAX_INPUT_LEN, + ) + + # K3 reasons inside its <|open|>response<|sep|>...<|close|> channel (no + # markup), so the K2.5 strip-thinking extractor cannot see the + # answer; extract_kimi_k3_mmmu_answer reads the channel and falls back to + # the K2.5 cascade for channel-less outputs. No chat_template_kwargs: K3 + # thinks by default. preserve_caller_max_tokens keeps max_tokens=16384 + # over lm-eval's 512-token MMMU default — without it the CoT is truncated + # before the response channel opens, the extractor silently falls back, + # and the score sinks toward the K2.5 reference floor (see + # references/mmmu.yaml): treat such a score with clean logs as a harness + # failure, not a model regression. + EXTRA_EVALUATOR_KWARGS: ClassVar[dict] = dict( + post_process_fn=extract_kimi_k3_mmmu_answer, + preserve_caller_max_tokens=True, + ) + + @skip_pre_blackwell + @pytest.mark.timeout(7200) + @pytest.mark.skip_less_mpi_world_size(16) + # The 16-GPU K3 recipes are qualified on GB300 (one NVL72 domain) only: + # on 2-node 180-190 GiB parts (B200/GB200, InfiniBand between nodes) the + # EP16 MoE-comm bring-up hangs and the KV-budget assumptions do not hold, + # so gate on GB300-class device memory. B300 clears this memory gate but + # pairs 8-GPU nodes over InfiniBand (same non-NVL72 topology) — do not + # schedule these tests on B300; that exclusion is enforced by QA's + # platform selection, not by this marker. + @pytest.mark.skip_less_device_memory(200000) + def test_w4a16_mxfp4(self) -> None: + """MMMU-val on the K3 VL checkpoint (16 GPUs, DEP16). + + No automated L0 stage schedules 16-GPU functional tests; this case is + registered in qa/llm_function_multinode.txt and run by QA's weekly + multinode pipeline (qualified on 4x4 GB300 nodes; reference in + references/mmmu.yaml). Mirrors examples/kimi_k3/run_eval_kimi_k3.sbatch + --task mmmu: the base eval_extra_llm_options.yaml serving config with + max_seq_len raised to 24576 (8192 input + 16384 output). + """ + with LLM( + self.MODEL_PATH, + tensor_parallel_size=16, + moe_expert_parallel_size=16, + enable_attention_dp=True, + max_batch_size=32, + max_num_tokens=8192, + max_seq_len=24576, + trust_remote_code=True, + enable_chunked_prefill=True, + cuda_graph_config=CudaGraphConfig(enable_padding=True, max_batch_size=32), + moe_config=MoeConfig(max_num_tokens=33024, use_low_precision_moe_combine=True), + # use_kv_cache_manager_v2=False: the VL wrapper inherits K2.5's + # V2 cache-manager preference, but the qualified K3 MMMU + # configuration ran V1, and under V2 this test's long-generation + # shape (max_seq_len 24576, 16k-token outputs) stalled with all + # GPUs idle in QA validation (the GSM8K legs' shorter shape runs + # fine under V2). Pin V1 until the V2 x KDA-hybrid path is + # qualified at this shape. + kv_cache_config=KvCacheConfig( + free_gpu_memory_fraction=0.25, + tokens_per_block=64, + use_kv_cache_manager_v2=False, + ), + ) as llm: + # Reference-key contract: the K3 checkpoint carries its + # quantization as nested text_config.quantization_config + # (compressed-tensors MXFP4 experts), which the LLM-args layer + # does not surface, so the reference matcher sees + # quant_algo=None and the references/mmmu.yaml entry carries no + # quant_algo key. If this fires, the args-level resolution + # changed: update the yaml key together with this assert. + assert llm.args.quant_config.quant_algo is None + task = MMMU(self.MODEL_NAME) + task.evaluate( + llm, + sampling_params=self.sampling_params, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS, + ) + + class TestMistralSmall24B(LlmapiAccuracyTestHarness): MODEL_NAME = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" MODEL_PATH = f"{llm_models_root()}/Mistral-Small-3.1-24B-Instruct-2503" diff --git a/tests/integration/test_lists/qa/llm_function_multinode.txt b/tests/integration/test_lists/qa/llm_function_multinode.txt index 290c57e8eb6e..3823f8a1c843 100644 --- a/tests/integration/test_lists/qa/llm_function_multinode.txt +++ b/tests/integration/test_lists/qa/llm_function_multinode.txt @@ -11,3 +11,7 @@ test_e2e.py::test_multi_nodes_eval[MiniMax-M3-tp16-mmlu] test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp2pp1-gen_tp2pp1] test_e2e.py::test_openai_disagg_multi_nodes_completion[ctx_tp1pp2-gen_tp1pp2] test_e2e.py::test_openai_disagg_multi_nodes_completion_service_discovery[etcd] +accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[baseline] +accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[reuse] +accuracy/test_llm_api_pytorch.py::TestKimiK3::test_w4a16_mxfp4[sa] +accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK3::test_w4a16_mxfp4