From a72cb29e9da6f3edcb7a793536c6131b03f07e2c Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 13 May 2025 16:56:12 +0800 Subject: [PATCH 1/4] add qwen2_0_5_instruct cp4 test case Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/integration/defs/accuracy/test_cli_flow.py | 5 +++++ tests/integration/test_lists/qa/examples_test_list.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index cae3e823269c..802514973dc1 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -1137,6 +1137,11 @@ class TestQwen2_0_5BInstruct(CliFlowAccuracyTestHarness): def test_auto_dtype(self): self.run(dtype='auto') + @pytest.mark.skip_less_device(4) + def test_auto_dtype_cp4(self): + "RCCA: https://nvbugs/5170106" + self.run(dtype='auto', cp_size=4) + @skip_post_blackwell def test_weight_only(self): self.run(quant_algo=QuantAlgo.W8A16) diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index 4e6504d4cc13..e31ae5ed2153 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -400,6 +400,7 @@ accuracy/test_cli_flow.py::TestGemma2_9BIt::test_weight_only[int4] accuracy/test_cli_flow.py::TestQwen1_5MoeA2_7BChat::test_auto_dtype accuracy/test_cli_flow.py::TestQwen1_5MoeA2_7BChat::test_weight_only accuracy/test_cli_flow.py::TestQwen2_0_5BInstruct::test_auto_dtype +accuracy/test_cli_flow.py::TestQwen2_0_5BInstruct::test_auto_dtype_cp4 accuracy/test_cli_flow.py::TestQwen2_0_5BInstruct::test_weight_only accuracy/test_cli_flow.py::TestQwen2_0_5BInstruct::test_fp8 accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_auto_dtype From a78d8202f703dbf86f03b90ce4ea3a5f19ff6f81 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 14 May 2025 13:45:40 +0800 Subject: [PATCH 2/4] add qwen2.5 fp8 kvcache test case Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../defs/accuracy/references/cnn_dailymail.yaml | 3 +++ .../integration/defs/accuracy/references/mmlu.yaml | 3 +++ tests/integration/defs/accuracy/test_llm_api.py | 13 +++++++++++++ .../test_lists/qa/examples_test_list.txt | 1 + 4 files changed, 20 insertions(+) diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml index 5a69dfd884c9..cbf37eb7e776 100644 --- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml +++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml @@ -275,6 +275,9 @@ Qwen/Qwen2.5-7B-Instruct: - accuracy: 33.014 - quant_algo: FP8 accuracy: 33.248 + - quant_algo: FP8 + kv_cache_quant_algo: FP8 + accuracy: 33.248 nvidia/Nemotron-Mini-4B-Instruct: - quant_algo: FP8 accuracy: 25.247 diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index 999819c328d4..6e2d759a3be7 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -75,6 +75,9 @@ Qwen/Qwen2.5-7B-Instruct: - accuracy: 75.32 - quant_algo: FP8 accuracy: 75.32 + - quant_algo: FP8 + kv_cache_quant_algo: FP8 + accuracy: 75.32 deepseek-ai/DeepSeek-V3-Lite: - accuracy: 71.40 - quant_algo: NVFP4 diff --git a/tests/integration/defs/accuracy/test_llm_api.py b/tests/integration/defs/accuracy/test_llm_api.py index f9387c3fc689..d97f518616ee 100644 --- a/tests/integration/defs/accuracy/test_llm_api.py +++ b/tests/integration/defs/accuracy/test_llm_api.py @@ -277,3 +277,16 @@ def test_fp8(self): extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) task = MMLU(self.MODEL_NAME) task.evaluate(llm) + + @pytest.mark.skip(reason="https://nvbugs/5280461") + @skip_pre_ada + def test_fp8_kvcache(self): + "RCCA: https://nvbugs/5065080" + quant_config = QuantConfig(QuantAlgo.FP8, + kv_cache_quant_algo=QuantAlgo.FP8) + with LLM(self.MODEL_PATH, quant_config=quant_config) as llm: + task = CnnDailymail(self.MODEL_NAME) + task.evaluate(llm, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index e31ae5ed2153..e064cf1cbe5f 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -415,6 +415,7 @@ accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_fp8 accuracy/test_llm_api.py::TestQwen2_5_0_5BInstruct::test_fp8 accuracy/test_llm_api.py::TestQwen2_5_1_5BInstruct::test_fp8 accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8 +accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8_kvcache accuracy/test_llm_api.py::TestMistral7B_0_3::test_quant_tp4[int4] accuracy/test_llm_api.py::TestMistral7B_0_3::test_quant_tp4[int4_awq] accuracy/test_llm_api.py::TestMistral7B_0_3::test_quant_tp4[int8_awq] From 1f723438e4a760f1a0310b9931f259e0192c89e8 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 15 May 2025 13:00:21 +0800 Subject: [PATCH 3/4] add ds distill qwen cpp runner test case Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../defs/accuracy/test_cli_flow.py | 1 + tests/integration/defs/test_e2e.py | 61 +++++++++++++++++++ .../test_lists/qa/examples_test_list.txt | 1 + 3 files changed, 63 insertions(+) diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index 802514973dc1..ed2a5a06e304 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -1137,6 +1137,7 @@ class TestQwen2_0_5BInstruct(CliFlowAccuracyTestHarness): def test_auto_dtype(self): self.run(dtype='auto') + @pytest.mark.skip(reason="https://nvbugs/5280195") @pytest.mark.skip_less_device(4) def test_auto_dtype_cp4(self): "RCCA: https://nvbugs/5170106" diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index e88ec422b8d7..32c676283a03 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -319,6 +319,67 @@ def test_mistral_e2e(llama_example_root, llama_tokenizer_model_root, llm_venv, venv_check_call(llm_venv, run_cmd) +@pytest.mark.parametrize("model_name,model_path", [ + ("DeepSeek-R1-Distill-Qwen-1.5B", "DeepSeek-R1-Distill-Qwen-1.5B"), +]) +def test_qwen_e2e_cpprunner_large_new_tokens(model_name, model_path, llm_venv, + qwen_example_root, cmodel_dir, + engine_dir): + "RCCA: https://nvbugs/5238105" + model_dir = convert_weights( + llm_venv=llm_venv, + example_root=qwen_example_root, + cmodel_dir=cmodel_dir, + model=model_name, + model_path=f"{llm_models_root()}/{model_path}", + ) + + build_cmd = [ + "trtllm-build", f"--checkpoint_dir={model_dir}", + f"--output_dir={engine_dir}", f"--gemm_plugin=float16", + "--max_num_tokens=32768" + ] + + check_call(" ".join(build_cmd), shell=True, env=llm_venv._new_env) + + from transformers import AutoTokenizer + + from tensorrt_llm.runtime import PYTHON_BINDINGS + + if PYTHON_BINDINGS: + from tensorrt_llm.runtime import ModelRunnerCpp + tokenizer = AutoTokenizer.from_pretrained( + f"{llm_models_root()}/{model_path}", + trust_remote_code=True, + use_fast=False) + + message = r"<｜begin▁of▁sentence｜><｜User｜>The operation $\otimes$ is defined for all nonzero numbers by $a \otimes b = \frac{a^{2}}{b}$. Determine $[(1 \otimes 2) \otimes 3] - [1 \otimes (2 \otimes 3)]$. Let's think step by step and output the final answer within \boxed{}.<｜Assistant｜>" + + inputs = tokenizer(message, return_tensors='pt', + add_special_tokens=False)['input_ids'] + + runner = ModelRunnerCpp.from_dir(engine_dir=f"{engine_dir}", + max_input_len=128, + max_output_len=4096, + max_batch_size=8) + + outputs = runner.generate(inputs, + end_id=tokenizer.eos_token_id, + pad_id=tokenizer.pad_token_id, + temperature=0.6, + top_p=1.0, + top_k=1024, + max_new_tokens=1024, + return_dict=True, + min_length=1, + num_return_sequences=4, + output_sequence_lengths=True) + + seq_lengths = outputs['sequence_lengths'] + assert not (seq_lengths == 0).any( + ), f"Found zero length in sequence_lengths tensor: {seq_lengths}" + + def trtllm_bench_prolog( llm_root, llm_venv, diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index e064cf1cbe5f..377d0b6156cc 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -463,6 +463,7 @@ llmapi/test_llm_e2e.py::test_llmapi_load_engine_from_build_command[llama-llama-m test_e2e.py::test_mistral_e2e[use_cpp_session-remove_input_padding--] test_e2e.py::test_mistral_e2e[use_py_session-remove_input_padding--] test_e2e.py::test_mistral_e2e[use_py_session---] +test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] test_e2e.py::test_openai_multi_chat_example test_e2e.py::test_openai_consistent_chat llmapi/test_llm_examples.py::test_llmapi_server_example From b46df780d654f515761486e320c9fde83f4523c6 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Fri, 16 May 2025 12:34:00 +0800 Subject: [PATCH 4/4] trial Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../defs/accuracy/references/cnn_dailymail.yaml | 2 ++ .../integration/defs/accuracy/test_cli_flow.py | 17 +++++++++++------ .../test_lists/qa/examples_test_list.txt | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml index cbf37eb7e776..df5f1d7f3d29 100644 --- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml +++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml @@ -251,6 +251,8 @@ Qwen/Qwen2-0.5B-Instruct: accuracy: 30.930 - quant_algo: FP8 accuracy: 31.140 +Qwen/Qwen2-1.5B: + - accuracy: 32.58 Qwen/Qwen2-7B-Instruct: - accuracy: 36.148 - quant_algo: W8A16 diff --git a/tests/integration/defs/accuracy/test_cli_flow.py b/tests/integration/defs/accuracy/test_cli_flow.py index ed2a5a06e304..881e83f37558 100644 --- a/tests/integration/defs/accuracy/test_cli_flow.py +++ b/tests/integration/defs/accuracy/test_cli_flow.py @@ -1137,12 +1137,6 @@ class TestQwen2_0_5BInstruct(CliFlowAccuracyTestHarness): def test_auto_dtype(self): self.run(dtype='auto') - @pytest.mark.skip(reason="https://nvbugs/5280195") - @pytest.mark.skip_less_device(4) - def test_auto_dtype_cp4(self): - "RCCA: https://nvbugs/5170106" - self.run(dtype='auto', cp_size=4) - @skip_post_blackwell def test_weight_only(self): self.run(quant_algo=QuantAlgo.W8A16) @@ -1154,6 +1148,17 @@ def test_fp8(self): quant_algo=QuantAlgo.FP8) +class TestQwen2_1_5B(CliFlowAccuracyTestHarness): + MODEL_NAME = "Qwen/Qwen2-1.5B" + MODEL_PATH = f"{llm_models_root()}/Qwen2-1.5B" + EXAMPLE_FOLDER = "models/core/qwen" + + @pytest.mark.skip_less_device(4) + def test_auto_dtype_cp4(self): + "RCCA: https://nvbugs/5170106" + self.run(dtype='auto', cp_size=4) + + class TestQwen2_7BInstruct(CliFlowAccuracyTestHarness): MODEL_NAME = "Qwen/Qwen2-7B-Instruct" MODEL_PATH = f"{llm_models_root()}/Qwen2-7B-Instruct" diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt index 377d0b6156cc..e974a19d5eb5 100644 --- a/tests/integration/test_lists/qa/examples_test_list.txt +++ b/tests/integration/test_lists/qa/examples_test_list.txt @@ -400,9 +400,9 @@ accuracy/test_cli_flow.py::TestGemma2_9BIt::test_weight_only[int4] accuracy/test_cli_flow.py::TestQwen1_5MoeA2_7BChat::test_auto_dtype accuracy/test_cli_flow.py::TestQwen1_5MoeA2_7BChat::test_weight_only accuracy/test_cli_flow.py::TestQwen2_0_5BInstruct::test_auto_dtype -accuracy/test_cli_flow.py::TestQwen2_0_5BInstruct::test_auto_dtype_cp4 accuracy/test_cli_flow.py::TestQwen2_0_5BInstruct::test_weight_only accuracy/test_cli_flow.py::TestQwen2_0_5BInstruct::test_fp8 +accuracy/test_cli_flow.py::TestQwen2_1_5B::test_auto_dtype_cp4 accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_auto_dtype accuracy/test_llm_api.py::TestQwen2_7BInstruct::test_weight_only accuracy/test_cli_flow.py::TestQwen2_7BInstruct::test_int4_awq_prequantized