[TRTLLM-15035][test] Wire Kimi K3 spec-dec and suffix-automaton tests into L0 CI

tests/integration/defs/kimi_k3_disagg_parity.py

@@ -39,7 +39,7 @@
 REAL RECIPE (aggregated vs disagg proxy):
 
     # reference: aggregated DEP16 deployment (as in
-    # examples/kimi_k3/run_gsm8k_kimi_k3.sbatch, but served)
+    # examples/kimi_k3/run_eval_kimi_k3.sbatch, but served)
     trtllm-serve $KIMI_K3_CKPT --backend pytorch --port 8000 ...
 
     # candidate: ctx + gen workers behind the disagg proxy

tests/integration/defs/test_kimi_k3_specdec.py

@@ -10,10 +10,14 @@
 logits; see the harness docstring).
 
 Requirements: 4 GPUs and the Kimi K3 checkpoint (env KIMI_K3_CKPT or
-<LLM_MODELS_ROOT>/Kimi-K3). Skips cleanly when the
-checkpoint is absent. The MoE backend defaults to VANILLA (the reference
-dequant path — the bit-parity oracle; slow but fine at 4 layers) so the
-test has no fused-kernel dependency and runs on any arch.
+<LLM_MODELS_ROOT>/Kimi-K3). Fails — deliberately does not skip — when the
+checkpoint is absent: the test is CI-listed (GB300 post-merge), and a
+checkpoint that vanishes from the runners' models mount must surface as a
+regression rather than an indistinguishable green skip. The MoE backend is
+routed to TRTLLM by KimiK3MoERuntime regardless of any
+KIMI_K3_MOE_BACKEND / moe_config.backend override (see the comment on the
+env block below); parity holds because the baseline and spec runs share
+the same backend.
 """
 
 import os
@@ -38,13 +42,48 @@ def _find_checkpoint():
     return None
 
 
+_LFS_MAGIC = b"version https://git-lfs.github.com/spec/v1"
+
+
+def _find_lfs_pointer_files(ckpt):
+    """Top-level checkpoint files that are still git-lfs pointers.
+
+    The checkpoint is staged from a git-lfs clone; a models mirror that has
+    not been hydrated (or has lagged the hydrated source) serves ~130-byte
+    pointer files instead of the real blobs, and the resulting failures are
+    deep and misleading (e.g. tiktoken parsing the pointer text as a vocab).
+    """
+    offenders = []
+    for name in sorted(os.listdir(ckpt)):
+        path = os.path.join(ckpt, name)
+        if not os.path.isfile(path):
+            continue
+        try:
+            with open(path, "rb") as f:
+                head = f.read(len(_LFS_MAGIC))
+        except OSError:
+            continue
+        if head == _LFS_MAGIC:
+            offenders.append(name)
+    return offenders
+
+
 @pytest.mark.skip_less_device(4)
 def test_kimi_k3_sa_specdec_logits_parity():
     ckpt = _find_checkpoint()
-    if ckpt is None:
-        pytest.skip(
-            "Kimi K3 checkpoint not available (set KIMI_K3_CKPT or stage under LLM_MODELS_ROOT)"
-        )
+    # Hard failure, not a skip: on the post-merge stage a skip is
+    # indistinguishable from a pass, so a checkpoint dropped from the
+    # runners' models mount would silently end this coverage.
+    assert ckpt is not None, (
+        "Kimi K3 checkpoint not found (set KIMI_K3_CKPT or stage under LLM_MODELS_ROOT)"
+    )
+    lfs_pointers = _find_lfs_pointer_files(ckpt)
+    assert not lfs_pointers, (
+        f"Kimi K3 checkpoint at {ckpt} is not hydrated on this runner's models "
+        f"mirror — these files are still git-lfs pointers: {lfs_pointers}. "
+        f"This is a checkpoint-staging/mirror-sync problem, not a code failure; "
+        f"re-sync the mirror or 'git lfs pull' the staging copy."
+    )
 
     env = os.environ.copy()
     env.update(
@@ -81,11 +120,9 @@ def test_kimi_k3_disagg_parity_selftest():
     Comparison logic only: canned responses, no servers or GPUs.
     """
     script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "kimi_k3_disagg_parity.py")
-    if not os.path.exists(script):
-        pytest.skip(
-            "kimi_k3_disagg_parity.py harness not present on this branch "
-            "(ships with the disagg parity PR)"
-        )
+    # Hard failure, not a skip: this test is CI-listed (l0_cpu), so a moved or
+    # renamed harness must surface as a regression instead of a silent skip.
+    assert os.path.exists(script), f"kimi_k3_disagg_parity.py harness missing at {script}"
     result = subprocess.run(
         [sys.executable, script, "--self-test"], capture_output=True, text=True, timeout=120
     )

tests/integration/test_lists/test-db/l0_b200.yml

@@ -178,6 +178,10 @@ l0_b200:
   - unittest/_torch/speculative/test_eagle3.py
   - unittest/_torch/speculative/test_dspark_cute_dsl_attention.py
   - unittest/_torch/speculative/test_dspark_cute_dsl_rmsnorm_rope.py
+  # On-device suffix-automaton spec-dec kernel tests (CUDA graph capture;
+  # cuda-availability skipif like the sibling KDA parity tests; relocated
+  # from tests/torch/ which test-db cannot list).
+  - unittest/_torch/speculative/test_suffix_automaton.py
   - unittest/_torch/thop/parallel TIMEOUT (90)
   - unittest/_torch/visual_gen/kernels/parallel
   - unittest/_torch/thop/serial

tests/integration/test_lists/test-db/l0_cpu.yml

@@ -23,6 +23,8 @@ l0_cpu:
   - disaggregated/test_aiperf_gate.py::test_gate_disabled_paths_not_affected
   - disaggregated/test_aiperf_gate.py::test_was_cancelled_metadata_fallback
   - disaggregated/test_aiperf_gate.py::test_non_request_records_excluded_from_denominator
+  # Kimi K3 disagg parity harness self-test (comparison logic only, no GPUs).
+  - test_kimi_k3_specdec.py::test_kimi_k3_disagg_parity_selftest
   - unittest/_torch/attention
   - unittest/_torch/cute_dsl/test_kimi_k3_kda_ptx_patch.py
   - unittest/_torch/distributed

tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml

@@ -36,8 +36,7 @@ l0_gb300_multi_gpus:
   - unittest/_torch/modeling -k "modeling_llama"
   - unittest/_torch/modeling -k "modeling_gpt_oss"
   # Kimi K3 speculative decoding: exact kernel-level verify parity (random
-  # weights, no checkpoint) + truncated-model e2e SA logits parity
-  # (auto-skips when the Kimi K3 checkpoint is not staged).
+  # weights, no checkpoint).
   - unittest/_torch/modules/kimi_kda/test_kimi_kda_verify_parity.py
   # Fused KDA multi-token verify (trtllm::kda_mtp_decode): kernel-level
   # parity vs CPU golden + FLA sequential, and runtime-level fused-vs-
@@ -51,6 +50,11 @@ l0_gb300_multi_gpus:
   # Kimi K3 MoE routing shape on the TRTLLM backend (random weights, no
   # checkpoint; the case moved out of the skipped TestMoeFp4 class).
   - unittest/_torch/moe/test_moe_backend.py -k "kimi_k3_shape"
+  # Kimi K3 truncated-model (first 4 layers) e2e SA spec-dec logits parity;
+  # resolves <LLM_MODELS_ROOT>/Kimi-K3 and FAILS if the checkpoint is not
+  # staged (deliberate: a skipped post-merge entry is indistinguishable
+  # from a pass, so a vanished checkpoint must be a visible regression).
+  - test_kimi_k3_specdec.py::test_kimi_k3_sa_specdec_logits_parity TIMEOUT (40)
   - unittest/_torch/multi_gpu_modeling -k "deepseek"
   - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=False]
   - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=True]