diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index 6f77797a8069..15d0ac772cfa 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -515,22 +515,3 @@ zai-org/GLM-5.2: kv_cache_quant_algo: FP8 spec_dec_algo: MTP accuracy: 90 -# Step-3.7-Flash text decoder (MoE) GSM8K, full 1319-sample split, TP4/EP4 with -# TRTLLM attention + MoE backends. FP8 measured on the FP8 block-scale -# checkpoint; NVFP4 measured on the modelopt NVFP4 export (FP8 KV cache). MTP -# (mtp_nextn=3) is lossless for both FP8 and NVFP4 and reuses the non-spec -# baseline accuracy. -stepfun-ai/Step-3.7-Flash: - - accuracy: 88 - - quant_algo: FP8_BLOCK_SCALES - accuracy: 88 - - quant_algo: FP8_BLOCK_SCALES - spec_dec_algo: MTP - accuracy: 88 - - quant_algo: NVFP4 - kv_cache_quant_algo: FP8 - accuracy: 88 - - quant_algo: NVFP4 - kv_cache_quant_algo: FP8 - spec_dec_algo: MTP - accuracy: 88 diff --git a/tests/integration/defs/accuracy/references/mmmu.yaml b/tests/integration/defs/accuracy/references/mmmu.yaml index 17c1469d9ced..480df410b30f 100644 --- a/tests/integration/defs/accuracy/references/mmmu.yaml +++ b/tests/integration/defs/accuracy/references/mmmu.yaml @@ -86,22 +86,3 @@ moonshotai/Kimi-K2.5: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 accuracy: 81.56 -# Step-3.7-Flash multimodal (PerceptionEncoder vision tower + MoE text decoder). -# Reasoning model: ... traces are stripped before MMMU answer -# extraction (see test_llm_api_pytorch_multimodal.py::TestStep3_7). FP8 measured -# on the FP8 block-scale checkpoint over the full 900-sample MMMU val split (TP4 -# / EP4, TRTLLM attention + MoE backends); NVFP4 (modelopt export, FP8 KV cache) -# reuses the FP8 baseline. -stepfun-ai/Step-3.7-Flash: - - quant_algo: FP8_BLOCK_SCALES - accuracy: 60 - - quant_algo: FP8_BLOCK_SCALES - spec_dec_algo: MTP - accuracy: 60 - - quant_algo: NVFP4 - kv_cache_quant_algo: FP8 - accuracy: 60 - - quant_algo: NVFP4 - kv_cache_quant_algo: FP8 - spec_dec_algo: MTP - accuracy: 60 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 49d7789f23e8..456f6895c7a9 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -8424,88 +8424,3 @@ def test_nvfp4_mtp_index_share_mtp_ar(self, tp_size, ep_size): assert aggregate_accept_rate > 0.2, ( f"Aggregate acceptance rate {aggregate_accept_rate:.2%} " f"below threshold 20%") - - -@skip_pre_blackwell -class TestStep3_7(LlmapiAccuracyTestHarness): - # Step-3.7-Flash is a MoE model registered under the multimodal - # architecture (Step3p7ForConditionalGeneration); text-only GSM8K exercises - # the text decoder path. The custom HF config requires trust_remote_code. - MODEL_NAME = "stepfun-ai/Step-3.7-Flash" - - @pytest.mark.skip_less_mpi_world_size(8) - @pytest.mark.skip_less_device_memory(140000) - @parametrize_with_ids("tp_size,ep_size", [(8, 8)]) - def test_auto_dtype(self, tp_size, ep_size): - model_path = f"{llm_models_root()}/Step-3.7-Flash" - kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.7) - with LLM(model_path, - tensor_parallel_size=tp_size, - moe_expert_parallel_size=ep_size, - kv_cache_config=kv_cache_config, - max_seq_len=8192, - trust_remote_code=True) as llm: - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - - @pytest.mark.skip_less_device(4) - @pytest.mark.skip_less_device_memory(80000) - @parametrize_with_ids("mtp_nextn", [0, 3]) - @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) - def test_fp8_block_scales(self, tp_size, ep_size, mtp_nextn): - model_path = f"{llm_models_root()}/Step-3.7-Flash-FP8" - kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.7, - use_kv_cache_manager_v2=True) - pytorch_config = dict( - disable_overlap_scheduler=False, - cuda_graph_config=CudaGraphConfig(enable_padding=True), - moe_config=MoeConfig(backend="TRTLLM"), - ) - - mtp_config = None - if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) - - with LLM(model_path, - tensor_parallel_size=tp_size, - moe_expert_parallel_size=ep_size, - kv_cache_config=kv_cache_config, - max_seq_len=8192, - attn_backend="TRTLLM", - speculative_config=mtp_config, - trust_remote_code=True, - **pytorch_config) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) - - @pytest.mark.skip_less_device(4) - @pytest.mark.skip_less_device_memory(80000) - @parametrize_with_ids("mtp_nextn", [0, 3]) - @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) - def test_nvfp4(self, tp_size, ep_size, mtp_nextn): - model_path = f"{llm_models_root()}/Step-3.7-Flash-NVFP4" - kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.7, - use_kv_cache_manager_v2=True) - pytorch_config = dict( - disable_overlap_scheduler=False, - cuda_graph_config=CudaGraphConfig(enable_padding=True), - moe_config=MoeConfig(backend="TRTLLM"), - ) - - mtp_config = None - if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) - - with LLM(model_path, - tensor_parallel_size=tp_size, - moe_expert_parallel_size=ep_size, - kv_cache_config=kv_cache_config, - max_seq_len=8192, - attn_backend="TRTLLM", - speculative_config=mtp_config, - trust_remote_code=True, - **pytorch_config) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 6ac701374cbd..f9c2f22880e1 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -29,7 +29,6 @@ from ..conftest import ( get_sm_version, llm_models_root, - parametrize_with_ids, skip_post_blackwell_ultra, skip_pre_blackwell, skip_pre_hopper, @@ -821,82 +820,3 @@ def test_auto_dtype( sampling_params=sampling_params, extra_evaluator_kwargs=extra_evaluator_kwargs, ) - - -@skip_pre_blackwell -class TestStep3_7(LlmapiAccuracyTestHarness): - # Step-3.7-Flash is a reasoning VLM: a PerceptionEncoder vision tower plus a - # MoE text decoder, registered under the Step3p7ForConditionalGeneration - # architecture (custom HF config -> trust_remote_code). MMMU exercises the - # vision path end to end. The model emits ... traces before - # its answer, so we strip them and extract the final MMMU letter (same - # handling as Kimi K2.5); preserve_caller_max_tokens keeps our larger - # generation budget instead of lm-eval's 512-token default (too small for - # the chain-of-thought). The text-only GSM8K path lives in - # test_llm_api_pytorch.py::TestStep3_7. - MODEL_NAME = "stepfun-ai/Step-3.7-Flash" - - # Validated with --max_input_length / --max_output_length 4096. - sampling_params = SamplingParams( - max_tokens=4096, - truncate_prompt_tokens=4096, - ) - - EXTRA_EVALUATOR_KWARGS = dict( - post_process_fn=strip_thinking_and_extract_mmmu_answer, - preserve_caller_max_tokens=True, - ) - - kv_cache_config = KvCacheConfig( - free_gpu_memory_fraction=0.7, - use_kv_cache_manager_v2=True, - ) - - def _make_llm(self, model_path: str, mtp_nextn: int = 0): - pytorch_config = dict( - disable_overlap_scheduler=False, - cuda_graph_config=CudaGraphConfig(enable_padding=False), - moe_config=MoeConfig(backend="TRTLLM"), - ) - mtp_config = None - if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) - return LLM( - model_path, - tensor_parallel_size=4, - moe_expert_parallel_size=4, - kv_cache_config=self.kv_cache_config, - max_seq_len=8192, - attn_backend="TRTLLM", - speculative_config=mtp_config, - trust_remote_code=True, - **pytorch_config, - ) - - @pytest.mark.skip_less_device(4) - @pytest.mark.skip_less_device_memory(80000) - @parametrize_with_ids("mtp_nextn", [0, 3]) - def test_fp8_block_scales(self, mtp_nextn): - with self._make_llm(f"{llm_models_root()}/Step-3.7-Flash-FP8", mtp_nextn=mtp_nextn) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES - task = MMMU(self.MODEL_NAME) - task.evaluate( - llm, - sampling_params=self.sampling_params, - extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS, - ) - - @pytest.mark.skip_less_device(4) - @pytest.mark.skip_less_device_memory(80000) - @parametrize_with_ids("mtp_nextn", [0, 3]) - def test_nvfp4(self, mtp_nextn): - with self._make_llm( - f"{llm_models_root()}/Step-3.7-Flash-NVFP4", mtp_nextn=mtp_nextn - ) as llm: - assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 - task = MMMU(self.MODEL_NAME) - task.evaluate( - llm, - sampling_params=self.sampling_params, - extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS, - ) diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 57c39a9acea2..d03a25eadf1c 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -772,11 +772,6 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_firs accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[fp8-latency] accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_w4a8_mxfp4[mxfp8-latency] accuracy/test_llm_api_pytorch.py::TestSeedOss_36B::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestStep3_7::test_auto_dtype[tp_size=8-ep_size=8] TIMEOUT (90) -accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=0] TIMEOUT (90) -accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=3] TIMEOUT (90) -accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=0] TIMEOUT (90) -accuracy/test_llm_api_pytorch.py::TestStep3_7::test_nvfp4[tp_size=4-ep_size=4-mtp_nextn=3] TIMEOUT (90) accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_cuda_graph_matches_eager_logits[tinyllama-1.1b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[gemma-3-1b] accuracy/test_llm_api_pytorch_encode.py::TestDecoderEncode::test_decoder_encode_matches_huggingface[qwen2-7b] @@ -806,10 +801,6 @@ accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL_MOE::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_27B_VL::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_auto_dtype accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3_5_35B_A3B_VL::test_fp8_prequantized -accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_fp8_block_scales[mtp_nextn=0] TIMEOUT (120) -accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_fp8_block_scales[mtp_nextn=3] TIMEOUT (120) -accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=0] TIMEOUT (120) -accuracy/test_llm_api_pytorch_multimodal.py::TestStep3_7::test_nvfp4[mtp_nextn=3] TIMEOUT (120) accuracy/test_llm_api_pytorch_ray.py::TestLlama3_1_8BInstruct::test_pp2_ray unittest/disaggregated/test_cache_transceiver_single_process.py::test_cache_transceiver_v1_masked_dsa_indexer_across_asymmetric_pp disaggregated/test_ad_disagg.py::test_async_eagle3_full_model_handoff diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index c6451316a65e..658962bdd3a6 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -169,7 +169,6 @@ full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=Fa full:B300/accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6529874) full:B300/accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_off-trtllm] SKIP (https://nvbugs/6474894) full:B300/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_dummy_load_format SKIP (https://nvbugs/6525059) -full:B300/accuracy/test_llm_api_pytorch.py::TestStep3_7::test_fp8_block_scales[tp_size=4-ep_size=4-mtp_nextn=3] SKIP (https://nvbugs/6539941) full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6597570) full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6597570) full:B300/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959)