From 874d7b318be1a73d7719cc7979a00b63295151eb Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Fri, 17 Jul 2026 22:37:25 -0700 Subject: [PATCH 1/3] test: leave headroom for SM120 NVFP4 KV cache --- test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py b/test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py index d516ef12158c..f0ad69b9b258 100644 --- a/test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py +++ b/test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py @@ -36,6 +36,8 @@ def test_gsm8k(self): "trtllm_mha", "--page-size", "64", + "--mem-fraction-static", + "0.87", "--cuda-graph-backend-prefill=disabled", ], variant="NVFP4-GEMM+NVFP4-KV+SM120-XQA", From 7e6ea8e515d07a99b1645350ec1408e8c1d62352 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Fri, 17 Jul 2026 23:06:43 -0700 Subject: [PATCH 2/3] test: use GSM8K mixin for NVFP4 KV cache --- .../test_llama8b_nvfp4_kv_cache_sm120.py | 66 ------------------- .../test_llama8b_nvfp4_kv_cache_sm120.py | 46 +++++++++++++ 2 files changed, 46 insertions(+), 66 deletions(-) delete mode 100644 test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py create mode 100644 test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py diff --git a/test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py b/test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py deleted file mode 100644 index f0ad69b9b258..000000000000 --- a/test/registered/models_e2e/test_llama8b_nvfp4_kv_cache_sm120.py +++ /dev/null @@ -1,66 +0,0 @@ -import unittest - -from sglang.srt.utils.common import is_sm120_supported -from sglang.test.accuracy_test_runner import AccuracyTestParams -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.test_utils import CustomTestCase, ModelLaunchSettings - -register_cuda_ci(est_time=300, stage="extra-a", runner_config="1-gpu-small") - -LLAMA8B_NVFP4_MODEL = "nvidia/Llama-3.1-8B-Instruct-NVFP4" -TP_SIZE = 1 - - -@unittest.skipUnless( - is_sm120_supported(), "requires at least 1 SM120 GPU with CUDA 12.8+" -) -class TestLlama8BNVFP4KVCacheSM120(CustomTestCase): - """Llama-3.1-8B-Instruct-NVFP4 with NVFP4 KV cache on SM120.""" - - def test_gsm8k(self): - variants = [ - ModelLaunchSettings( - LLAMA8B_NVFP4_MODEL, - tp_size=TP_SIZE, - extra_args=[ - "--quantization", - "modelopt_fp4", - "--fp4-gemm-backend", - "auto", - "--kv-cache-dtype", - "nvfp4", - "--prefill-attention-backend", - "flashinfer", - "--decode-attention-backend", - "trtllm_mha", - "--page-size", - "64", - "--mem-fraction-static", - "0.87", - "--cuda-graph-backend-prefill=disabled", - ], - variant="NVFP4-GEMM+NVFP4-KV+SM120-XQA", - ) - ] - - run_combined_tests( - models=variants, - test_name="Llama-3.1-8B-Instruct-NVFP4-KV-SM120", - accuracy_params=AccuracyTestParams( - dataset="gsm8k", - # Full GSM8K measured locally with 1319 requested / 1314 scored: - # - FP8 KV: 0.6461187214611872 - # - NVFP4 KV: 0.632420091324201 - # Keep the threshold 0.015 below the NVFP4 KV score. - baseline_accuracy=0.632420091324201 - 0.015, - num_examples=1319, - num_threads=200, - max_tokens=512, - api="completion", - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py b/test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py new file mode 100644 index 000000000000..aadbfabdb198 --- /dev/null +++ b/test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py @@ -0,0 +1,46 @@ +import unittest + +from sglang.srt.utils.common import is_sm120_supported +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.default_fixture import DefaultServerBase + +register_cuda_ci(est_time=300, stage="extra-a", runner_config="1-gpu-small") + + +@unittest.skipUnless( + is_sm120_supported(), "requires at least 1 SM120 GPU with CUDA 12.8+" +) +class TestLlama8BNVFP4KVCacheSM120(GSM8KMixin, DefaultServerBase): + """Llama-3.1-8B-Instruct-NVFP4 with NVFP4 KV cache on SM120.""" + + model = "nvidia/Llama-3.1-8B-Instruct-NVFP4" + # Full GSM8K measured locally with 1319 requested / 1314 scored: + # - FP8 KV: 0.6461187214611872 + # - NVFP4 KV: 0.632420091324201 + # Keep the threshold 0.015 below the NVFP4 KV score. + gsm8k_accuracy_thres = 0.632420091324201 - 0.015 + gsm8k_num_questions = 1319 + gsm8k_num_threads = 200 + + other_args = [ + "--quantization", + "modelopt_fp4", + "--fp4-gemm-backend", + "auto", + "--kv-cache-dtype", + "nvfp4", + "--prefill-attention-backend", + "flashinfer", + "--decode-attention-backend", + "trtllm_mha", + "--page-size", + "64", + "--mem-fraction-static", + "0.87", + "--cuda-graph-backend-prefill=disabled", + ] + + +if __name__ == "__main__": + unittest.main() From c45d712049da8b8b538c1a8e737747dc0fef9841 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Sat, 18 Jul 2026 00:35:49 -0700 Subject: [PATCH 3/3] test: disable inaccurate NVFP4 KV cache test --- test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py b/test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py index aadbfabdb198..cbbf6349e929 100644 --- a/test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py +++ b/test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py @@ -5,7 +5,12 @@ from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase -register_cuda_ci(est_time=300, stage="extra-a", runner_config="1-gpu-small") +register_cuda_ci( + est_time=300, + stage="extra-a", + runner_config="1-gpu-small", + disabled="Temporarily disabled due to failing accuracy", +) @unittest.skipUnless(