[TRTLLM-15078][test] Prune non-Llama-3.1-8B Llama tests

tensorrt_llm/_torch/pyexecutor/sampler/sampler_strategy.py

@@ -328,7 +328,7 @@ def sample(
             )
         case ("greedy", None):
             tokens, softmax = greedy_search_sampling_batch(logits, return_probs=return_probs)
-            temperature = None
+            temperature = None  # type: ignore[assignment]
         case (
             "beam_search",
             beam_width_in,

tests/integration/defs/accuracy/references/SlimPajama-6B.yaml

@@ -1,2 +1 @@
-gradientai/Llama-3-8B-Instruct-Gradient-1048k:
-  - accuracy: 7.663
+{}

tests/integration/defs/accuracy/references/cnn_dailymail.yaml

@@ -70,32 +70,6 @@ lmsys/vicuna-7b-v1.3:
     accuracy: 27.832
   - spec_dec_algo: Eagle3
     accuracy: 27.832
-TinyLlama/TinyLlama-1.1B-Chat-v1.0:
-  - accuracy: 28.328
-  - dtype: float32
-    accuracy: 28.082
-  - quant_algo: W8A16
-    accuracy: 28.003
-  - quant_algo: W8A16
-    kv_cache_quant_algo: INT8
-    accuracy: 27.089
-  - quant_algo: W4A16
-    accuracy: 25.194
-  - quant_algo: W4A16
-    kv_cache_quant_algo: INT8
-    accuracy: 23.987
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 27.882
-  - extra_acc_spec: pp_size=4
-    accuracy: 15.123
-meta-llama/Meta-Llama-3-8B-Instruct:
-  - accuracy: 34.957
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 34.737
-  - quant_algo: W8A16_GPTQ
-    accuracy: 34.858
 meta-llama/Llama-3.1-8B:
   - accuracy: 24.360
   - quant_algo: W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN
@@ -140,48 +114,6 @@ meta-llama/Llama-3.1-8B-Instruct:
   - quant_algo: FP8
     extra_acc_spec: beam_width=2
     accuracy: 31.201
-meta-llama/Llama-3.2-1B:
-  - accuracy: 27.427
-  - quant_algo: W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN
-    accuracy: 27.931
-  - quant_algo: W8A8_SQ_PER_CHANNEL
-    accuracy: 25.631
-  - quant_algo: W4A16_AWQ
-    accuracy: 25.028
-  - quant_algo: W4A16_AWQ
-    kv_cache_quant_algo: INT8
-    accuracy: 24.354
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 27.029
-  - quant_algo: FP8
-    accuracy: 27.029
-  - quant_algo: FP8_PER_CHANNEL_PER_TOKEN
-    accuracy: 27.257
-  - quant_algo: FP8_PER_CHANNEL_PER_TOKEN
-    extra_acc_spec: meta_recipe
-    accuracy: 27.614
-  - extra_acc_spec: max_attention_window_size=960
-    accuracy: 27.259
-  - extra_acc_spec: max_attention_window_size=960;beam_width=4
-    accuracy: 0
-meta-llama/Llama-3.2-3B:
-  - accuracy: 25.495
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 33.629
-meta-llama/Llama-3.3-70B-Instruct:
-  - quant_algo: FP8
-    spec_dec_algo: Eagle
-    accuracy: 33.244
-  - quant_algo: FP8
-    spec_dec_algo: Eagle3
-    accuracy: 33.244
-  - quant_algo: NVFP4
-    kv_cache_quant_algo: FP8
-    accuracy: 34.383
-  - quant_algo: FP8
-    accuracy: 34.927
 mistralai/Mistral-7B-v0.1:
   - accuracy: 25.741
   - extra_acc_spec: beam_width=4
@@ -352,5 +284,3 @@ Qwen3/Qwen3-8B:
   - quant_algo: FP8_BLOCK_SCALES
     accuracy: 30
   - accuracy: 30
-nvidia/Llama-3_3-Nemotron-Super-49B-v1:
-  - accuracy: 34.003

tests/integration/defs/accuracy/references/gpqa_diamond.yaml

@@ -1,16 +1,3 @@
-meta-llama/Llama-3.3-70B-Instruct:
-  - accuracy: 45.96
-  - quant_algo: NVFP4
-    kv_cache_quant_algo: FP8
-    accuracy: 45.55
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 48.03
-  - quant_algo: FP8
-    accuracy: 48.03
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 48.03
 deepseek-ai/DeepSeek-R1:
   - quant_algo: NVFP4
     accuracy: 70.45
@@ -35,30 +22,6 @@ deepseek-ai/DeepSeek-V3.2-Exp:
   - quant_algo: NVFP4
     spec_dec_algo: MTP
     accuracy: 80.0
-nvidia/Llama-3_3-Nemotron-Super-49B-v1:
-  - accuracy: 44.95
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 42.42
-    # GPQA diamond only contains 198 samples, so the score tends to have large variance.
-    # We repeated evaluation 7 times to choose a lower bound score for FP8, 42.42.
-    # random_seed=0: 47.98
-    # random_seed=1: 42.42
-    # random_seed=2: 52.02
-    # random_seed=3: 51.52
-    # random_seed=4: 48.48
-    # random_seed=5: 47.47
-    # random_seed=6: 45.96
-nvidia/Llama-3.1-Nemotron-Nano-8B-v1:
-  - accuracy: 40.40
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 39.39
-nvidia/Llama-3_1-Nemotron-Ultra-253B-v1:
-  - accuracy: 58.08
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 57.07
 GPT-OSS/120B-MXFP4:
   - accuracy: 65.0
   - spec_dec_algo: Eagle

tests/integration/defs/accuracy/references/gsm8k.yaml

@@ -31,24 +31,6 @@ meta-llama/Llama-3.1-8B-Instruct:
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 66.03
-meta-llama/Llama-3.3-70B-Instruct:
-  - accuracy: 83.78
-  - quant_algo: NVFP4
-    kv_cache_quant_algo: FP8
-    accuracy: 87.33
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 90.30
-  - quant_algo: FP8
-    accuracy: 90.30
-meta-llama/Llama-4-Scout-17B-16E-Instruct:
-  - accuracy: 89.70
-  - quant_algo: NVFP4
-    kv_cache_quant_algo: FP8
-    accuracy: 88.61
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 89.45
 deepseek-ai/DeepSeek-V3-Lite:
   - accuracy: 64.74
   - quant_algo: NVFP4
@@ -286,6 +268,11 @@ nvidia/Llama-3_3-Nemotron-Super-49B-v1:
   - quant_algo: FP8
     kv_cache_quant_algo: FP8
     accuracy: 92.42
+nvidia/Nemotron-H-8B-Base-8K:
+  - accuracy: 46.20
+  - quant_algo: FP8
+    kv_cache_quant_algo: FP8
+    accuracy: 85.78
 nvidia/Nemotron-MOE:
   - accuracy: 88.249
   - quant_algo: FP8

tests/integration/defs/accuracy/references/mmlu.yaml

@@ -1,8 +1,3 @@
-meta-llama/Meta-Llama-3-8B-Instruct:
-  - accuracy: 67.74
-  - quant_algo: NVFP4
-    kv_cache_quant_algo: FP8
-    accuracy: 63.47
 meta-llama/Llama-3.1-8B:
   - accuracy: 66.06
   - quant_algo: NVFP4
@@ -35,59 +30,6 @@ meta-llama/Llama-3.1-8B-Instruct:
   - quant_algo: NVFP4
     kv_cache_quant_algo: FP8
     accuracy: 65.11
-meta-llama/Llama-3.2-1B:
-  - quant_algo: W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN
-    accuracy: 32.72
-  - quant_algo: W8A8_SQ_PER_CHANNEL
-    accuracy: 32.07
-  - quant_algo: W4A16_AWQ
-    accuracy: 30.56
-  - quant_algo: W4A16_AWQ
-    kv_cache_quant_algo: INT8
-    accuracy: 31.29
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 31.02
-  - quant_algo: FP8_PER_CHANNEL_PER_TOKEN
-    accuracy: 33.97
-  - quant_algo: FP8_PER_CHANNEL_PER_TOKEN
-    extra_acc_spec: meta_recipe
-    accuracy: 33.87
-  - extra_acc_spec: max_attention_window_size=960
-    accuracy: 32.82
-meta-llama/Llama-3.2-3B:
-  - accuracy: 57.92
-  - spec_dec_algo: Eagle3
-    accuracy: 57.92
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 60.60
-meta-llama/Llama-3.3-70B-Instruct:
-  - accuracy: 81.31
-  - spec_dec_algo: Eagle3
-    accuracy: 81.31
-  - quant_algo: FP8
-    spec_dec_algo: Eagle
-    accuracy: 81.31
-  - quant_algo: FP8
-    spec_dec_algo: Eagle3
-    accuracy: 81.31
-  - quant_algo: NVFP4
-    kv_cache_quant_algo: FP8
-    accuracy: 78.78
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 80.40
-  - quant_algo: FP8
-    accuracy: 80.40
-meta-llama/Llama-4-Scout-17B-16E-Instruct:
-  - accuracy: 80.00
-  - quant_algo: NVFP4
-    kv_cache_quant_algo: FP8
-    accuracy: 79.60
-  - quant_algo: FP8
-    kv_cache_quant_algo: FP8
-    accuracy: 78.58
 mistralai/Mistral-7B-v0.1:
   - accuracy: 66
 mistralai/Mistral-7B-Instruct-v0.3:
@@ -309,6 +251,11 @@ nvidia/Llama-3.1-Nemotron-Nano-8B-v1:
   - quant_algo: FP8
     kv_cache_quant_algo: FP8
     accuracy: 57.12
+nvidia/Nemotron-H-8B-Base-8K:
+  - accuracy: 69.590
+  - quant_algo: FP8
+    kv_cache_quant_algo: FP8
+    accuracy: 69.180
 microsoft/Phi-4-mini-instruct:
   - accuracy: 68.98
   - quant_algo: FP8

tests/integration/defs/accuracy/references/passkey_retrieval_128k.yaml

@@ -1,2 +1 @@
-gradientai/Llama-3-8B-Instruct-Gradient-1048k:
-  - accuracy: 99
+{}