From 0cad007e875c597d99dcb06b0e251cb59d016cee Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Wed, 7 May 2025 13:11:06 +0800
Subject: [PATCH 01/11] add nemotron-h and llama_70b cases

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 .../defs/accuracy/references/cnn_dailymail.yaml            | 1 +
 tests/integration/defs/accuracy/test_llm_api_pytorch.py    | 7 +++++++
 tests/integration/defs/test_e2e.py                         | 2 ++
 tests/integration/test_lists/qa/examples_test_list.txt     | 3 +++
 4 files changed, 13 insertions(+)

diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
index 5a69dfd884c9..8fa235bd5337 100644
--- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
+++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
@@ -147,6 +147,7 @@ meta-llama/Llama-3.2-1B:
   - extra_acc_spec: max_attention_window_size=960;beam_width=4
     accuracy: 0
 meta-llama/Llama-3.3-70B-Instruct:
+  - accuracy: 80
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 34.383
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index ae69d929ec74..99f09671c192 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -186,6 +186,13 @@ def test_fp8_4gpus(self, tp_size, pp_size, fp8kv, attn_backend,
 
 class TestLlama3_3_70BInstruct(LlmapiAccuracyTestHarness):
     MODEL_NAME = "meta-llama/Llama-3.3-70B-Instruct"
+    MODEL_PATH = f"{llm_models_root()}/llama-3.3-models/Llama-3.3-70B-Instruct"
+
+    @pytest.mark.skip_less_device(8)
+    def test_auto_dtype(self):
+        with LLM(self.MODEL_PATH, tensor_parallel_size=8) as llm:
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
 
     @pytest.mark.skip_less_device(4)
     @pytest.mark.skip_device_not_contain(["H100", "H200", "B200"])
diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index cdd909cbb4f1..d0fbd615b8fd 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1476,6 +1476,8 @@ def test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus(
 @pytest.mark.parametrize("model_name,model_path", [
     ("Llama3.1-70B-BF16", "llama-3.1-model/Meta-Llama-3.1-70B"),
     ("Mixtral-8x7B-BF16", "Mixtral-8x7B-v0.1"),
+    ("Nemotron-H-47B", "Nemotron-H-47B-Base-8K"),
+    ("Nemotron-H-56B", "Nemotron-H-56B-Base-8K"),
     pytest.param('Llama3.1-70B-FP8',
                  'llama-3.1-model/Llama-3.1-70B-Instruct-FP8',
                  marks=skip_pre_hopper),
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 4e6504d4cc13..99f91583e4b7 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -423,6 +423,7 @@ accuracy/test_llm_api.py::TestMixtral8x7B::test_tp2
 accuracy/test_llm_api.py::TestMixtral8x7B::test_smooth_quant_tp2pp2
 accuracy/test_llm_api.py::TestMixtral8x7BInstruct::test_awq_tp2
 accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4
+accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4
 accuracy/test_llm_api_pytorch.py::TestMistral7B::test_auto_dtype
@@ -477,6 +478,8 @@ test_e2e.py::test_ptp_quickstart_advanced_8gpus[Llama3.1-70B-FP8-llama-3.1-model
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Llama3.1-405B-FP8-llama-3.1-model/Llama-3.1-405B-Instruct-FP8]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Mixtral-8x7B-BF16-Mixtral-8x7B-v0.1]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Mixtral-8x7B-NVFP4-nvfp4-quantized/Mixtral-8x7B-Instruct-v0.1]
+test_e2e.py::test_ptp_quickstart_advanced_8gpus[Nemotron-H-47B-Nemotron-H-47B-Base-8K]
+test_e2e.py::test_ptp_quickstart_advanced_8gpus[Nemotron-H-56B-Nemotron-H-56B-Base-8K]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Nemotron-Ultra-253B-nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1]
 test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]
 test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]

From 37e6c182ad0fc764d4c6876e00c60ad460e29334 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Wed, 7 May 2025 15:26:11 +0800
Subject: [PATCH 02/11] trial

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 .../accuracy/references/cnn_dailymail.yaml     |  3 ++-
 .../defs/accuracy/references/gsm8k.yaml        |  1 +
 .../defs/accuracy/references/mmlu.yaml         |  1 +
 .../defs/accuracy/test_llm_api_pytorch.py      | 18 +++++++++++++++++-
 .../test_lists/qa/examples_test_list.txt       |  1 +
 5 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
index 8fa235bd5337..14f2e279305d 100644
--- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
+++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
@@ -147,7 +147,6 @@ meta-llama/Llama-3.2-1B:
   - extra_acc_spec: max_attention_window_size=960;beam_width=4
     accuracy: 0
 meta-llama/Llama-3.3-70B-Instruct:
-  - accuracy: 80
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 34.383
@@ -284,6 +283,8 @@ nvidia/Nemotron-Mini-4B-Instruct:
     accuracy: 25.72
 nemotron-nas/Llama-3_1-Nemotron-51B-Instruct:
   - accuracy: 35.672
+nvidia/Nemotron-H-47B-Base-8K:
+  - accuracy: 35.672
 deepseek-ai/DeepSeek-V3-Lite:
   - accuracy: 26.465
   - quant_algo: NVFP4
diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml
index 0f822ab31510..679d2c68265f 100644
--- a/tests/integration/defs/accuracy/references/gsm8k.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k.yaml
@@ -6,6 +6,7 @@ meta-llama/Llama-3.1-8B-Instruct:
     kv_cache_quant_algo: FP8
     accuracy: 72.85
 meta-llama/Llama-3.3-70B-Instruct:
+  - accuracy: 84.07
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 75.61
diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml
index 999819c328d4..10754c8d87d2 100644
--- a/tests/integration/defs/accuracy/references/mmlu.yaml
+++ b/tests/integration/defs/accuracy/references/mmlu.yaml
@@ -26,6 +26,7 @@ meta-llama/Llama-3.1-8B-Instruct:
     kv_cache_quant_algo: FP8
     accuracy: 67.87
 meta-llama/Llama-3.3-70B-Instruct:
+  - accuracy: 81.28
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 79.31
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 99f09671c192..b6659a968b10 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -191,7 +191,9 @@ class TestLlama3_3_70BInstruct(LlmapiAccuracyTestHarness):
     @pytest.mark.skip_less_device(8)
     def test_auto_dtype(self):
         with LLM(self.MODEL_PATH, tensor_parallel_size=8) as llm:
-            task = CnnDailymail(self.MODEL_NAME)
+            task = MMLU(self.MODEL_NAME)
+            task.evaluate(llm)
+            task = GSM8K(self.MODEL_NAME)
             task.evaluate(llm)
 
     @pytest.mark.skip_less_device(4)
@@ -788,6 +790,20 @@ def test_fp8_prequantized(self):
             task.evaluate(llm)
 
 
+class TestMinitronH(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "nvidia/Nemotron-H-47B-Base-8K"
+    MODEL_PATH = f"{llm_models_root()}/Nemotron-H-47B-Base-8K"
+
+    @skip_pre_ada
+    def test_auto_dtype_tp8(self):
+        kv_cache_config = KvCacheConfig(enable_block_reuse=False)
+        with LLM(self.MODEL_PATH,
+                 tensor_parallel_size=8,
+                 kv_cache_config=kv_cache_config) as llm:
+            task = CnnDailymail(self.MODEL_NAME)
+            task.evaluate(llm)
+
+
 class TestNemotronNas(LlmapiAccuracyTestHarness):
     MODEL_NAME = "nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
     MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 99f91583e4b7..49d5dc26b015 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -444,6 +444,7 @@ accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8
 accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2
 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype
+accuracy/test_llm_api_pytorch.py::TestMinitronH::test_auto_dtype_tp8
 accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_8gpus[latency]
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_8gpus[latency_trtllmgen]

From 4c7ea79986843ca4e00ddc7d77543388afdf1e20 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Mon, 12 May 2025 17:14:58 +0800
Subject: [PATCH 03/11] add llm decoder quick_start case

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/test_e2e.py            | 19 +++++++++++++++++++
 .../test_lists/qa/examples_test_list.txt      |  1 +
 2 files changed, 20 insertions(+)

diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index d0fbd615b8fd..cadd0d7dda9b 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1252,6 +1252,25 @@ def test_ptp_quickstart(llm_root, llm_venv):
         _check_mem_usage(running_log, [4.60, 0, 0, 0])
 
 
+@pytest.mark.parametrize("model_name,model_path", [
+    pytest.param('Llama3.1-8B-FP8',
+                 'llama-3.1-model/Llama-3.1-8B-Instruct-FP8',
+                 marks=skip_pre_hopper),
+])
+def test_ptp_quickstart_advanced_llm_decoder(llm_root, llm_venv, model_name,
+                                             model_path):
+    print(f"Testing {model_name}.")
+    example_root = Path(os.path.join(llm_root, "examples", "pytorch"))
+    llm_venv.run_cmd([
+        str(example_root / "quickstart_advanced.py"),
+        "--max_batch_size=4",
+        "--enable_trtllm_decoder",
+        "--temperature=0.8",
+        "--top_p=0.95",
+        f"{llm_models_root()}/{model_path}",
+    ])
+
+
 @pytest.mark.parametrize("model_name,model_path", [
     ("Llama3.1-8B-BF16", "llama-3.1-model/Meta-Llama-3.1-8B"),
     ("Llama3.2-11B-BF16", "llama-3.2-models/Llama-3.2-11B-Vision"),
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 49d5dc26b015..9e26f89a7809 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -468,6 +468,7 @@ test_e2e.py::test_openai_consistent_chat
 llmapi/test_llm_examples.py::test_llmapi_server_example
 # Pivot to Pytorch test cases.
 test_e2e.py::test_ptp_quickstart
+test_e2e.py::test_ptp_quickstart_advanced_llm_decoder[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]
 test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]
 test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]
 test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]

From 6770d733f7f8e584565c62780c209f1097911317 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Tue, 13 May 2025 10:31:57 +0800
Subject: [PATCH 04/11] update nemotron-h test case

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 .../accuracy/references/cnn_dailymail.yaml    |  2 --
 .../defs/accuracy/references/gsm8k.yaml       |  2 ++
 .../defs/accuracy/references/mmlu.yaml        |  2 +-
 .../defs/accuracy/test_llm_api_pytorch.py     | 21 +++++--------------
 .../test_lists/qa/examples_test_list.txt      |  1 -
 5 files changed, 8 insertions(+), 20 deletions(-)

diff --git a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
index 14f2e279305d..5a69dfd884c9 100644
--- a/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
+++ b/tests/integration/defs/accuracy/references/cnn_dailymail.yaml
@@ -283,8 +283,6 @@ nvidia/Nemotron-Mini-4B-Instruct:
     accuracy: 25.72
 nemotron-nas/Llama-3_1-Nemotron-51B-Instruct:
   - accuracy: 35.672
-nvidia/Nemotron-H-47B-Base-8K:
-  - accuracy: 35.672
 deepseek-ai/DeepSeek-V3-Lite:
   - accuracy: 26.465
   - quant_algo: NVFP4
diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml
index 679d2c68265f..0948c2d0d67c 100644
--- a/tests/integration/defs/accuracy/references/gsm8k.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k.yaml
@@ -65,3 +65,5 @@ Qwen3/Qwen3-30B-A3B:
     accuracy: 83.43
 nvidia/Llama-3_3-Nemotron-Super-49B-v1:
   - accuracy: 92.57
+nvidia/Nemotron-H-8B-Base-8K:
+  - accuracy: 46.20
diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml
index 10754c8d87d2..59e67f044e59 100644
--- a/tests/integration/defs/accuracy/references/mmlu.yaml
+++ b/tests/integration/defs/accuracy/references/mmlu.yaml
@@ -118,4 +118,4 @@ nvidia/Llama-3_3-Nemotron-Super-49B-v1:
 nvidia/Llama-3.1-Nemotron-Nano-8B-v1:
   - accuracy: 57.97
 nvidia/Nemotron-H-8B-Base-8K:
-  - accuracy: 87.573
+  - accuracy: 69.590
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index b6659a968b10..c366dae150fd 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -790,20 +790,6 @@ def test_fp8_prequantized(self):
             task.evaluate(llm)
 
 
-class TestMinitronH(LlmapiAccuracyTestHarness):
-    MODEL_NAME = "nvidia/Nemotron-H-47B-Base-8K"
-    MODEL_PATH = f"{llm_models_root()}/Nemotron-H-47B-Base-8K"
-
-    @skip_pre_ada
-    def test_auto_dtype_tp8(self):
-        kv_cache_config = KvCacheConfig(enable_block_reuse=False)
-        with LLM(self.MODEL_PATH,
-                 tensor_parallel_size=8,
-                 kv_cache_config=kv_cache_config) as llm:
-            task = CnnDailymail(self.MODEL_NAME)
-            task.evaluate(llm)
-
-
 class TestNemotronNas(LlmapiAccuracyTestHarness):
     MODEL_NAME = "nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
     MODEL_PATH = f"{llm_models_root()}/nemotron-nas/Llama-3_1-Nemotron-51B-Instruct"
@@ -854,10 +840,13 @@ class TestNemotronH(LlmapiAccuracyTestHarness):
     MODEL_NAME = "nvidia/Nemotron-H-8B-Base-8K"
     MODEL_PATH = f"{llm_models_root()}/Nemotron-H-8B-Base-8K"
 
-    @pytest.mark.skip(reason="https://nvbugspro.nvidia.com/bug/5264431")
     def test_auto_dtype(self):
+        # TODO: remove max_batch_size after mamba cache manager is supported
+        # ToDo: check 47b and 56b model
         kv_cache_config = KvCacheConfig(enable_block_reuse=False)
-        with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config) as llm:
+        with LLM(self.MODEL_PATH,
+                 kv_cache_config=kv_cache_config,
+                 max_batch_size=128) as llm:
             task = MMLU(self.MODEL_NAME)
             task.evaluate(llm)
             task = GSM8K(self.MODEL_NAME)
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 9e26f89a7809..20156da240c5 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -444,7 +444,6 @@ accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8
 accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2
 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype
-accuracy/test_llm_api_pytorch.py::TestMinitronH::test_auto_dtype_tp8
 accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_8gpus[latency]
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_8gpus[latency_trtllmgen]

From 255aeba9f7269e4ba00a69362ea0161281d6c6b9 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Tue, 13 May 2025 12:33:44 +0800
Subject: [PATCH 05/11] add qwen3 quickstart test

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/test_e2e.py                     | 3 +--
 tests/integration/test_lists/qa/examples_test_list.txt | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index cadd0d7dda9b..076936367167 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1276,6 +1276,7 @@ def test_ptp_quickstart_advanced_llm_decoder(llm_root, llm_venv, model_name,
     ("Llama3.2-11B-BF16", "llama-3.2-models/Llama-3.2-11B-Vision"),
     ("Nemotron4_4B-BF16", "nemotron/Minitron-4B-Base"),
     ("Nemotron-H-8B", "Nemotron-H-8B-Base-8K"),
+    ("Qwen3-30B-A3B", "Qwen3/Qwen3-30B-A3B"),
     pytest.param('Llama3.1-8B-NVFP4',
                  'nvfp4-quantized/Meta-Llama-3.1-8B',
                  marks=skip_pre_blackwell),
@@ -1495,8 +1496,6 @@ def test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus(
 @pytest.mark.parametrize("model_name,model_path", [
     ("Llama3.1-70B-BF16", "llama-3.1-model/Meta-Llama-3.1-70B"),
     ("Mixtral-8x7B-BF16", "Mixtral-8x7B-v0.1"),
-    ("Nemotron-H-47B", "Nemotron-H-47B-Base-8K"),
-    ("Nemotron-H-56B", "Nemotron-H-56B-Base-8K"),
     pytest.param('Llama3.1-70B-FP8',
                  'llama-3.1-model/Llama-3.1-70B-Instruct-FP8',
                  marks=skip_pre_hopper),
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 20156da240c5..c746aec880b1 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -474,13 +474,12 @@ test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta
 test_e2e.py::test_ptp_quickstart_advanced[Llama3.2-11B-BF16-llama-3.2-models/Llama-3.2-11B-Vision]
 test_e2e.py::test_ptp_quickstart_advanced[Nemotron4_4B-BF16-nemotron/Minitron-4B-Base]
 test_e2e.py::test_ptp_quickstart_advanced[Nemotron-H-8B-Nemotron-H-8B-Base-8K]
+test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Llama3.1-70B-BF16-llama-3.1-model/Meta-Llama-3.1-70B]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Llama3.1-70B-FP8-llama-3.1-model/Llama-3.1-70B-Instruct-FP8]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Llama3.1-405B-FP8-llama-3.1-model/Llama-3.1-405B-Instruct-FP8]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Mixtral-8x7B-BF16-Mixtral-8x7B-v0.1]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Mixtral-8x7B-NVFP4-nvfp4-quantized/Mixtral-8x7B-Instruct-v0.1]
-test_e2e.py::test_ptp_quickstart_advanced_8gpus[Nemotron-H-47B-Nemotron-H-47B-Base-8K]
-test_e2e.py::test_ptp_quickstart_advanced_8gpus[Nemotron-H-56B-Nemotron-H-56B-Base-8K]
 test_e2e.py::test_ptp_quickstart_advanced_8gpus[Nemotron-Ultra-253B-nemotron-nas/Llama-3_1-Nemotron-Ultra-253B-v1]
 test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]
 test_e2e.py::test_relaxed_acceptance_quickstart_advanced_deepseek_r1_8gpus[DeepSeek-R1-DeepSeek-R1/DeepSeek-R1]

From 70c1a879aef7c56f6746daf6caee462847d57f13 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Tue, 13 May 2025 14:12:57 +0800
Subject: [PATCH 06/11] add trtllm_decoder accuracy test

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 .../defs/accuracy/accuracy_core.py            | 16 ++++++++++----
 .../defs/accuracy/references/mmlu.yaml        |  3 +++
 .../defs/accuracy/test_llm_api_pytorch.py     | 21 +++++++++++++++++++
 .../test_lists/qa/examples_test_list.txt      |  1 +
 4 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py
index 161ca1b0bac6..d82b7fe90d6c 100644
--- a/tests/integration/defs/accuracy/accuracy_core.py
+++ b/tests/integration/defs/accuracy/accuracy_core.py
@@ -31,6 +31,7 @@
 from tensorrt_llm.logger import logger
 from tensorrt_llm.models.modeling_utils import QuantConfig
 from tensorrt_llm.quantization import QuantAlgo
+from tensorrt_llm.sampling_params import SamplingParams
 
 from ..common import venv_check_call, venv_mpi_check_call
 from ..conftest import llm_models_root
@@ -146,7 +147,8 @@ def get_num_samples_and_threshold(self, **acc_specs):
     def evaluate(self,
                  llm: Union[LLM, PyTorchLLM],
                  extra_acc_spec: Optional[str] = None,
-                 extra_evaluator_kwargs: Optional[dict] = None):
+                 extra_evaluator_kwargs: Optional[dict] = None,
+                 sampling_params: Optional[SamplingParams] = None):
         assert self.EVALUATOR_CLS is not None
 
         if llm.args.speculative_config is None:
@@ -175,9 +177,15 @@ def evaluate(self,
                 spec_dec_algo=spec_dec_algo,
                 extra_acc_spec=extra_acc_spec)
 
-        sampling_params = SamplingParams(
-            max_tokens=self.MAX_OUTPUT_LEN,
-            truncate_prompt_tokens=self.MAX_INPUT_LEN)
+        if sampling_params is None:
+            sampling_params = SamplingParams(
+                max_tokens=self.MAX_OUTPUT_LEN,
+                truncate_prompt_tokens=self.MAX_INPUT_LEN)
+        else:
+            if sampling_params.max_tokens is None:
+                sampling_params.max_tokens = self.MAX_OUTPUT_LEN
+            if sampling_params.truncate_prompt_tokens is None:
+                sampling_params.truncate_prompt_tokens = self.MAX_INPUT_LEN
 
         evaluator_kwargs = {}
         if self.EVALUATOR_KWARGS is not None:
diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml
index 59e67f044e59..9a374262659f 100644
--- a/tests/integration/defs/accuracy/references/mmlu.yaml
+++ b/tests/integration/defs/accuracy/references/mmlu.yaml
@@ -22,6 +22,9 @@ meta-llama/Llama-3.1-8B-Instruct:
   - accuracy: 68.17
   - quant_algo: FP8
     accuracy: 67.93
+  - quant_algo: FP8
+    extra_acc_spec: temperature=0.8,top_p=0.95
+    accuracy: 64.62
   - quant_algo: FP8
     kv_cache_quant_algo: FP8
     accuracy: 67.87
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index c366dae150fd..2817248d8558 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -19,6 +19,7 @@
 from tensorrt_llm.llmapi import KvCacheConfig, MTPDecodingConfig
 from tensorrt_llm.models.modeling_utils import QuantConfig
 from tensorrt_llm.quantization import QuantAlgo
+from tensorrt_llm.sampling_params import SamplingParams
 
 from ..conftest import (llm_models_root, parametrize_with_ids, skip_pre_ada,
                         skip_pre_blackwell, skip_pre_hopper)
@@ -183,6 +184,26 @@ def test_fp8_4gpus(self, tp_size, pp_size, fp8kv, attn_backend,
             task = GSM8K(self.MODEL_NAME)
             task.evaluate(llm)
 
+    @skip_pre_hopper
+    def test_fp8_llm_decoder(self):
+        model_path = f"{llm_models_root()}/llama-3.1-model/Llama-3.1-8B-Instruct-FP8"
+        pytorch_config = PyTorchConfig(enable_trtllm_decoder=True)
+        llm = LLM(model_path,
+                  pytorch_backend_config=pytorch_config,
+                  max_batch_size=4)
+        assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
+
+        sampling_params = SamplingParams(
+            temperature=0.8,
+            top_p=0.95,
+        )
+
+        with llm:
+            task = MMLU(self.MODEL_NAME)
+            task.evaluate(llm,
+                          sampling_params=sampling_params,
+                          extra_acc_spec="temperature=0.8,top_p=0.95")
+
 
 class TestLlama3_3_70BInstruct(LlmapiAccuracyTestHarness):
     MODEL_NAME = "meta-llama/Llama-3.3-70B-Instruct"
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index c746aec880b1..2527f3f218ac 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -423,6 +423,7 @@ accuracy/test_llm_api.py::TestMixtral8x7B::test_tp2
 accuracy/test_llm_api.py::TestMixtral8x7B::test_smooth_quant_tp2pp2
 accuracy/test_llm_api.py::TestMixtral8x7BInstruct::test_awq_tp2
 accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4
+accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_llm_decoder
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4

From f37ad1b9018673cabf95b908c8929a7867066d89 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Tue, 13 May 2025 14:57:36 +0800
Subject: [PATCH 07/11] remove quickstart test for llm_decoder

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 .../defs/accuracy/accuracy_core.py            |  1 -
 .../defs/accuracy/test_llm_api_pytorch.py     |  8 +++----
 tests/integration/defs/test_e2e.py            | 21 ++-----------------
 .../test_lists/qa/examples_test_list.txt      |  1 -
 4 files changed, 5 insertions(+), 26 deletions(-)

diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py
index d82b7fe90d6c..d9c9980d5e61 100644
--- a/tests/integration/defs/accuracy/accuracy_core.py
+++ b/tests/integration/defs/accuracy/accuracy_core.py
@@ -31,7 +31,6 @@
 from tensorrt_llm.logger import logger
 from tensorrt_llm.models.modeling_utils import QuantConfig
 from tensorrt_llm.quantization import QuantAlgo
-from tensorrt_llm.sampling_params import SamplingParams
 
 from ..common import venv_check_call, venv_mpi_check_call
 from ..conftest import llm_models_root
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 2817248d8558..cc49adb461f4 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -16,10 +16,10 @@
 
 from tensorrt_llm._torch import LLM
 from tensorrt_llm._torch.pyexecutor.config import PyTorchConfig
-from tensorrt_llm.llmapi import KvCacheConfig, MTPDecodingConfig
+from tensorrt_llm.llmapi import (LLM, KvCacheConfig, MTPDecodingConfig,
+                                 SamplingParams)
 from tensorrt_llm.models.modeling_utils import QuantConfig
 from tensorrt_llm.quantization import QuantAlgo
-from tensorrt_llm.sampling_params import SamplingParams
 
 from ..conftest import (llm_models_root, parametrize_with_ids, skip_pre_ada,
                         skip_pre_blackwell, skip_pre_hopper)
@@ -188,9 +188,7 @@ def test_fp8_4gpus(self, tp_size, pp_size, fp8kv, attn_backend,
     def test_fp8_llm_decoder(self):
         model_path = f"{llm_models_root()}/llama-3.1-model/Llama-3.1-8B-Instruct-FP8"
         pytorch_config = PyTorchConfig(enable_trtllm_decoder=True)
-        llm = LLM(model_path,
-                  pytorch_backend_config=pytorch_config,
-                  max_batch_size=4)
+        llm = LLM(model_path, pytorch_backend_config=pytorch_config)
         assert llm.args.quant_config.quant_algo == QuantAlgo.FP8
 
         sampling_params = SamplingParams(
diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index 076936367167..80acf3a43dff 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1252,25 +1252,6 @@ def test_ptp_quickstart(llm_root, llm_venv):
         _check_mem_usage(running_log, [4.60, 0, 0, 0])
 
 
-@pytest.mark.parametrize("model_name,model_path", [
-    pytest.param('Llama3.1-8B-FP8',
-                 'llama-3.1-model/Llama-3.1-8B-Instruct-FP8',
-                 marks=skip_pre_hopper),
-])
-def test_ptp_quickstart_advanced_llm_decoder(llm_root, llm_venv, model_name,
-                                             model_path):
-    print(f"Testing {model_name}.")
-    example_root = Path(os.path.join(llm_root, "examples", "pytorch"))
-    llm_venv.run_cmd([
-        str(example_root / "quickstart_advanced.py"),
-        "--max_batch_size=4",
-        "--enable_trtllm_decoder",
-        "--temperature=0.8",
-        "--top_p=0.95",
-        f"{llm_models_root()}/{model_path}",
-    ])
-
-
 @pytest.mark.parametrize("model_name,model_path", [
     ("Llama3.1-8B-BF16", "llama-3.1-model/Meta-Llama-3.1-8B"),
     ("Llama3.2-11B-BF16", "llama-3.2-models/Llama-3.2-11B-Vision"),
@@ -1320,9 +1301,11 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path):
                                          dir="./",
                                          delete=True,
                                          delete_on_close=True) as running_log:
+            kv_cache_fraction = 0.6 if "Qwen3" in model_name else None
             llm_venv.run_cmd([
                 str(example_root / "quickstart_advanced.py"),
                 "--enable_chunked_prefill",
+                f"--kv_cache_fraction={kv_cache_fraction}",
                 "--model_dir",
                 f"{llm_models_root()}/{model_path}",
             ],
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 2527f3f218ac..01ba8037a63d 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -468,7 +468,6 @@ test_e2e.py::test_openai_consistent_chat
 llmapi/test_llm_examples.py::test_llmapi_server_example
 # Pivot to Pytorch test cases.
 test_e2e.py::test_ptp_quickstart
-test_e2e.py::test_ptp_quickstart_advanced_llm_decoder[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]
 test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-BF16-llama-3.1-model/Meta-Llama-3.1-8B]
 test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8]
 test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-8B]

From 223c43ae9bf3fbd3b7fd5a1cd2a08b7df643fcd1 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Thu, 15 May 2025 19:55:32 +0800
Subject: [PATCH 08/11] fix import error

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 .../defs/accuracy/references/gsm8k.yaml          |  1 -
 .../defs/accuracy/references/mmlu.yaml           |  1 -
 .../defs/accuracy/test_llm_api_pytorch.py        | 16 +---------------
 tests/integration/defs/test_e2e.py               | 13 ++++++-------
 .../test_lists/qa/examples_test_list.txt         |  1 -
 5 files changed, 7 insertions(+), 25 deletions(-)

diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml
index 0948c2d0d67c..5bb3b56037b1 100644
--- a/tests/integration/defs/accuracy/references/gsm8k.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k.yaml
@@ -6,7 +6,6 @@ meta-llama/Llama-3.1-8B-Instruct:
     kv_cache_quant_algo: FP8
     accuracy: 72.85
 meta-llama/Llama-3.3-70B-Instruct:
-  - accuracy: 84.07
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 75.61
diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml
index 9a374262659f..bfe98705f108 100644
--- a/tests/integration/defs/accuracy/references/mmlu.yaml
+++ b/tests/integration/defs/accuracy/references/mmlu.yaml
@@ -29,7 +29,6 @@ meta-llama/Llama-3.1-8B-Instruct:
     kv_cache_quant_algo: FP8
     accuracy: 67.87
 meta-llama/Llama-3.3-70B-Instruct:
-  - accuracy: 81.28
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 79.31
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index cc49adb461f4..b4514b396a19 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -16,8 +16,7 @@
 
 from tensorrt_llm._torch import LLM
 from tensorrt_llm._torch.pyexecutor.config import PyTorchConfig
-from tensorrt_llm.llmapi import (LLM, KvCacheConfig, MTPDecodingConfig,
-                                 SamplingParams)
+from tensorrt_llm.llmapi import KvCacheConfig, MTPDecodingConfig, SamplingParams
 from tensorrt_llm.models.modeling_utils import QuantConfig
 from tensorrt_llm.quantization import QuantAlgo
 
@@ -202,19 +201,6 @@ def test_fp8_llm_decoder(self):
                           sampling_params=sampling_params,
                           extra_acc_spec="temperature=0.8,top_p=0.95")
 
-
-class TestLlama3_3_70BInstruct(LlmapiAccuracyTestHarness):
-    MODEL_NAME = "meta-llama/Llama-3.3-70B-Instruct"
-    MODEL_PATH = f"{llm_models_root()}/llama-3.3-models/Llama-3.3-70B-Instruct"
-
-    @pytest.mark.skip_less_device(8)
-    def test_auto_dtype(self):
-        with LLM(self.MODEL_PATH, tensor_parallel_size=8) as llm:
-            task = MMLU(self.MODEL_NAME)
-            task.evaluate(llm)
-            task = GSM8K(self.MODEL_NAME)
-            task.evaluate(llm)
-
     @pytest.mark.skip_less_device(4)
     @pytest.mark.skip_device_not_contain(["H100", "H200", "B200"])
     def test_fp8_tp4(self):
diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index 80acf3a43dff..812f983a6a75 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -1301,15 +1301,14 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path):
                                          dir="./",
                                          delete=True,
                                          delete_on_close=True) as running_log:
-            kv_cache_fraction = 0.6 if "Qwen3" in model_name else None
-            llm_venv.run_cmd([
+            cmds = [
                 str(example_root / "quickstart_advanced.py"),
                 "--enable_chunked_prefill",
-                f"--kv_cache_fraction={kv_cache_fraction}",
-                "--model_dir",
-                f"{llm_models_root()}/{model_path}",
-            ],
-                             running_log=running_log)
+                f"--model_dir={llm_models_root()}/{model_path}",
+            ]
+            if "Qwen3" in model_name:
+                cmds.append(f"--kv_cache_fraction=0.6")
+            llm_venv.run_cmd(cmds, running_log=running_log)
             if model_name in mapping:
                 _check_mem_usage(running_log, [mapping[model_name], 0, 0, 0])
 
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index 01ba8037a63d..e3e651b8fcfc 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -424,7 +424,6 @@ accuracy/test_llm_api.py::TestMixtral8x7B::test_smooth_quant_tp2pp2
 accuracy/test_llm_api.py::TestMixtral8x7BInstruct::test_awq_tp2
 accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4
 accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_llm_decoder
-accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4
 accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4
 accuracy/test_llm_api_pytorch.py::TestMistral7B::test_auto_dtype

From 24d0486ca478060d93ce14d2c3a2bb6f366141b9 Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Fri, 16 May 2025 12:27:23 +0800
Subject: [PATCH 09/11] nemotronh fp8 trial

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/accuracy/references/gsm8k.yaml |  3 +++
 tests/integration/defs/accuracy/references/mmlu.yaml  |  5 ++++-
 .../integration/defs/accuracy/test_llm_api_pytorch.py | 11 +++++++++++
 .../integration/test_lists/qa/examples_test_list.txt  |  1 +
 4 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml
index 5bb3b56037b1..3112589d8ce0 100644
--- a/tests/integration/defs/accuracy/references/gsm8k.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k.yaml
@@ -66,3 +66,6 @@ nvidia/Llama-3_3-Nemotron-Super-49B-v1:
   - accuracy: 92.57
 nvidia/Nemotron-H-8B-Base-8K:
   - accuracy: 46.20
+  - quant_algo: FP8
+    kv_cache_quant_algo: FP8
+    accuracy: 46.20
diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml
index bfe98705f108..f552088a2a9d 100644
--- a/tests/integration/defs/accuracy/references/mmlu.yaml
+++ b/tests/integration/defs/accuracy/references/mmlu.yaml
@@ -29,7 +29,7 @@ meta-llama/Llama-3.1-8B-Instruct:
     kv_cache_quant_algo: FP8
     accuracy: 67.87
 meta-llama/Llama-3.3-70B-Instruct:
-  - quant_algo: NVFP4
+  - quant_algo:
     kv_cache_quant_algo: FP8
     accuracy: 79.31
   - quant_algo: FP8
@@ -121,3 +121,6 @@ nvidia/Llama-3.1-Nemotron-Nano-8B-v1:
   - accuracy: 57.97
 nvidia/Nemotron-H-8B-Base-8K:
   - accuracy: 69.590
+  - quant_algo: FP8
+    kv_cache_quant_algo: FP8
+    accuracy: 69.590
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index b4514b396a19..91f25d140931 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -857,6 +857,17 @@ def test_auto_dtype(self):
             task = GSM8K(self.MODEL_NAME)
             task.evaluate(llm)
 
+    def test_fp8_prequantized(self):
+        model_path = f"{llm_models_root()}/nemotron/nemotron-h-8b-128k-step-400-fp8-dynamic"
+        kv_cache_config = KvCacheConfig(enable_block_reuse=False)
+        with LLM(model_path,
+                 kv_cache_config=kv_cache_config,
+                 max_batch_size=128) as llm:
+            task = MMLU(self.MODEL_NAME)
+            task.evaluate(llm)
+            task = GSM8K(self.MODEL_NAME)
+            task.evaluate(llm)
+
 
 class TestQwen2_7BInstruct(LlmapiAccuracyTestHarness):
     MODEL_NAME = "Qwen/Qwen2-7B-Instruct"
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index e3e651b8fcfc..fd525d0ea83f 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -444,6 +444,7 @@ accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8
 accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2
 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype
+accuracy/test_llm_api_pytorch.py::TestNemotronH::test_fp8_prequantized
 accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_8gpus[latency]
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_8gpus[latency_trtllmgen]

From c790f5c1b08c9b4575cb838631d209ba84bf276f Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Fri, 16 May 2025 12:52:14 +0800
Subject: [PATCH 10/11] fix name

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/accuracy/test_llm_api_pytorch.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 91f25d140931..2df79b841eec 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -201,6 +201,10 @@ def test_fp8_llm_decoder(self):
                           sampling_params=sampling_params,
                           extra_acc_spec="temperature=0.8,top_p=0.95")
 
+
+class TestLlama3_3_70BInstruct(LlmapiAccuracyTestHarness):
+    MODEL_NAME = "meta-llama/Llama-3.3-70B-Instruct"
+
     @pytest.mark.skip_less_device(4)
     @pytest.mark.skip_device_not_contain(["H100", "H200", "B200"])
     def test_fp8_tp4(self):

From 85edacf2dfb3d7e2dd0009ab71ce99e5fda9b83c Mon Sep 17 00:00:00 2001
From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
Date: Fri, 16 May 2025 12:59:22 +0800
Subject: [PATCH 11/11] remove nemotronh-fp8

Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com>
---
 tests/integration/defs/accuracy/references/gsm8k.yaml |  3 ---
 tests/integration/defs/accuracy/references/mmlu.yaml  |  5 +----
 .../integration/defs/accuracy/test_llm_api_pytorch.py | 11 -----------
 .../integration/test_lists/qa/examples_test_list.txt  |  1 -
 4 files changed, 1 insertion(+), 19 deletions(-)

diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml
index 3112589d8ce0..5bb3b56037b1 100644
--- a/tests/integration/defs/accuracy/references/gsm8k.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k.yaml
@@ -66,6 +66,3 @@ nvidia/Llama-3_3-Nemotron-Super-49B-v1:
   - accuracy: 92.57
 nvidia/Nemotron-H-8B-Base-8K:
   - accuracy: 46.20
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 46.20
diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml
index f552088a2a9d..bfe98705f108 100644
--- a/tests/integration/defs/accuracy/references/mmlu.yaml
+++ b/tests/integration/defs/accuracy/references/mmlu.yaml
@@ -29,7 +29,7 @@ meta-llama/Llama-3.1-8B-Instruct:
     kv_cache_quant_algo: FP8
     accuracy: 67.87
 meta-llama/Llama-3.3-70B-Instruct:
-  - quant_algo:
+  - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 79.31
   - quant_algo: FP8
@@ -121,6 +121,3 @@ nvidia/Llama-3.1-Nemotron-Nano-8B-v1:
   - accuracy: 57.97
 nvidia/Nemotron-H-8B-Base-8K:
   - accuracy: 69.590
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 69.590
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 2df79b841eec..539f46850c2e 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -861,17 +861,6 @@ def test_auto_dtype(self):
             task = GSM8K(self.MODEL_NAME)
             task.evaluate(llm)
 
-    def test_fp8_prequantized(self):
-        model_path = f"{llm_models_root()}/nemotron/nemotron-h-8b-128k-step-400-fp8-dynamic"
-        kv_cache_config = KvCacheConfig(enable_block_reuse=False)
-        with LLM(model_path,
-                 kv_cache_config=kv_cache_config,
-                 max_batch_size=128) as llm:
-            task = MMLU(self.MODEL_NAME)
-            task.evaluate(llm)
-            task = GSM8K(self.MODEL_NAME)
-            task.evaluate(llm)
-
 
 class TestQwen2_7BInstruct(LlmapiAccuracyTestHarness):
     MODEL_NAME = "Qwen/Qwen2-7B-Instruct"
diff --git a/tests/integration/test_lists/qa/examples_test_list.txt b/tests/integration/test_lists/qa/examples_test_list.txt
index fd525d0ea83f..e3e651b8fcfc 100644
--- a/tests/integration/test_lists/qa/examples_test_list.txt
+++ b/tests/integration/test_lists/qa/examples_test_list.txt
@@ -444,7 +444,6 @@ accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8
 accuracy/test_llm_api_pytorch.py::TestNemotronSuper::test_auto_dtype_tp2
 accuracy/test_llm_api_pytorch.py::TestNemotronNano::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestNemotronH::test_auto_dtype
-accuracy/test_llm_api_pytorch.py::TestNemotronH::test_fp8_prequantized
 accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_8gpus[latency]
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_8gpus[latency_trtllmgen]