[None][chore] Weekly mass integration of release/1.2

tensorrt_llm/tokenizer/tokenizer.py

@@ -1,4 +1,5 @@
 import os
+import pickle  # nosec B403
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
 
@@ -25,6 +26,12 @@ class TokenizerBase(PreTrainedTokenizerBase):
     ''' This is a protocol for the tokenizer. Users can implement their own tokenizer by inheriting this class.  '''
 
 
+def _reconstruct_transformers_tokenizer(inner_bytes: bytes):
+    '''Reconstruct a TransformersTokenizer from cloudpickle-serialized bytes.'''
+    # nosec B301: inner_bytes is from cloudpickle.dumps() in __reduce__, not untrusted data
+    return TransformersTokenizer(pickle.loads(inner_bytes))  # nosec B301
+
+
 class TransformersTokenizer(TokenizerBase):
     ''' A wrapper for the Transformers' tokenizer.
     This is the default tokenizer for LLM. '''
@@ -33,6 +40,34 @@ def __init__(self, tokenizer):
         self.tokenizer = tokenizer
         self._all_special_tokens_set = set(self.tokenizer.all_special_tokens)
 
+    def __reduce__(self):
+        # In multi-node scenarios, AutoTokenizer.from_pretrained with
+        # trust_remote_code=True may create dynamic Python modules cached at
+        # $HOME/.cache/huggingface/modules/transformers_modules/.
+        # These modules are node-local and won't exist on other nodes.
+        # Standard pickle serializes classes by reference, so deserializing
+        # a tokenizer that uses such a dynamic class fails on other nodes.
+        #
+        # To solve this, we use cloudpickle (if available) to serialize the
+        # inner tokenizer by value, embedding the class definition in the
+        # serialized bytes. This follows vLLM PR #6751.
+        #
+        # See: https://github.com/vllm-project/vllm/pull/6751
+        try:
+            import cloudpickle
+            # Ensure dynamic transformers_modules are registered for by-value
+            # serialization, regardless of how the tokenizer was created.
+            maybe_register_transformers_modules_by_value()
+            inner_bytes = cloudpickle.dumps(self.tokenizer)
+            return (_reconstruct_transformers_tokenizer, (inner_bytes, ))
+        except ImportError:
+            # cloudpickle not installed; fall back to default pickling.
+            # This may fail on other nodes if the tokenizer uses dynamic
+            # modules from trust_remote_code.
+            logger.warning(
+                "cloudpickle is not installed. TransformersTokenizer will not be serializable across nodes in multi-node setups. Install cloudpickle to fix this: pip install cloudpickle")
+            return (TransformersTokenizer, (self.tokenizer, ))
+
     def __call__(self, text: str, *args, **kwargs) -> Any:
         return self.tokenizer(text, *args, **kwargs)
 
@@ -340,6 +375,41 @@ def _llguidance_tokenizer_info(tokenizer):
     return tokenizer_info
 
 
+def maybe_register_transformers_modules_by_value():
+    '''Register transformers dynamic modules for by-value serialization.
+
+    With trust_remote_code, AutoTokenizer.from_pretrained may create dynamic
+    modules cached at $HOME/.cache/huggingface/modules/transformers_modules/.
+    These modules are node-local and won't exist on other nodes in multi-node
+    setups. This function registers them with cloudpickle for by-value
+    serialization, so that objects from these modules can be pickled and
+    sent to other nodes without needing the module files there.
+
+    References:
+        https://github.com/vllm-project/vllm/pull/6751
+        https://github.com/cloudpipe/cloudpickle#overriding-pickles-serialization-mechanism-for-importable-constructs
+    '''
+    try:
+        import transformers_modules
+    except ImportError:
+        # No dynamic modules were created, nothing to register.
+        return
+
+    try:
+        import cloudpickle
+        cloudpickle.register_pickle_by_value(transformers_modules)
+    except ImportError:
+        logger.warning(
+            "cloudpickle is not installed. Objects from trust_remote_code "
+            "dynamic modules may not be serializable across nodes in "
+            "multi-node setups. Install cloudpickle to fix this: "
+            "pip install cloudpickle")
+    except Exception as e:
+        logger.warning(
+            f"Failed to register transformers_modules for by-value "
+            f"serialization: {e}")
+
+
 def load_hf_tokenizer(model_dir: str,
                       trust_remote_code: bool = True,
                       use_fast: bool = True,
@@ -356,7 +426,7 @@ def load_hf_tokenizer(model_dir: str,
     '''
 
     try:
-        return TransformersTokenizer.from_pretrained(
+        tokenizer = TransformersTokenizer.from_pretrained(
             model_dir,
             legacy=False,
             padding_side='left',
@@ -365,6 +435,11 @@ def load_hf_tokenizer(model_dir: str,
             use_fast=use_fast,
             **kwargs)
 
+        if trust_remote_code:
+            maybe_register_transformers_modules_by_value()
+
+        return tokenizer
+
     except Exception as e:
         logger.warning(
             f"Failed to load hf tokenizer from {model_dir}, encounter error: {e}"

tests/integration/defs/accuracy/test_llm_api_pytorch.py

@@ -3983,10 +3983,7 @@ def test_nvfp4(
             task = GSM8K(self.MODEL_NAME)
             task.evaluate(llm)
 
-    @pytest.mark.parametrize(
-        "moe_backend",
-        ["CUTLASS",
-         pytest.param("TRITON", marks=skip_no_hopper), "TRTLLM"])
+    @pytest.mark.parametrize("moe_backend", ["CUTLASS", "TRTLLM"])
     @pytest.mark.parametrize(
         "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler", [
             (1, 1, 1, False, True, True),
@@ -4042,11 +4039,15 @@ def test_w4a8_mxfp4(self, moe_backend, tp_size, pp_size, ep_size,
             task = MMLU(self.MODEL_NAME)
             task.evaluate(llm)
 
-    @skip_pre_blackwell
+    @pytest.mark.parametrize("moe_backend", [
+        pytest.param("TRITON", marks=skip_no_hopper),
+        pytest.param("TRTLLM", marks=skip_pre_blackwell)
+    ])
     @pytest.mark.parametrize(
-        "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler,moe_backend",
-        [(1, 1, 1, False, True, True, "TRTLLM")],
-        ids=["latency-TRTLLM"])
+        "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler", [
+            (1, 1, 1, False, True, True),
+        ],
+        ids=["latency"])
     def test_w4a16_mxfp4(self, tp_size, pp_size, ep_size, attention_dp,
                          cuda_graph, overlap_scheduler, moe_backend):
         pytorch_config = dict(

tests/integration/defs/stress_test/stress_test.py

@@ -384,31 +384,39 @@ def is_port_available(port: int,
     "config",
     [
         # Configuration for TinyLlama model
+        # memory_requirement is in MiB (12 GB = 12288 MiB)
         ModelConfig(model_dir="llama-models-v2/TinyLlama-1.1B-Chat-v1.0",
                     tp_size=1,
-                    memory_requirement=12),
+                    memory_requirement=12288),
         # Configuration for Llama-v3 model
+        # memory_requirement is in MiB (12 GB = 12288 MiB)
         ModelConfig(model_dir="llama-models-v3/llama-v3-8b-instruct-hf",
                     tp_size=1,
-                    memory_requirement=12),
+                    memory_requirement=12288),
         # Configuration for DeepSeek-V3 model
-        ModelConfig(model_dir="DeepSeek-V3", tp_size=8, memory_requirement=96),
+        # memory_requirement is in MiB (96 GB = 98304 MiB)
+        ModelConfig(
+            model_dir="DeepSeek-V3", tp_size=8, memory_requirement=98304),
         # Configuration for DeepSeek-R1 model with FP8 checkpoints (8 GPU setup)
+        # memory_requirement is in MiB (96 GB = 98304 MiB)
         ModelConfig(model_dir="DeepSeek-R1/DeepSeek-R1",
                     tp_size=8,
-                    memory_requirement=96),
+                    memory_requirement=98304),
         # Configuration for DeepSeek-R1 model with FP8 checkpoints (4 GPU setup, requires GB300 288GB)
+        # memory_requirement is in MiB (256 GB = 262144 MiB)
         ModelConfig(model_dir="DeepSeek-R1/DeepSeek-R1",
                     tp_size=4,
-                    memory_requirement=256),
+                    memory_requirement=262144),
         # Configuration for DeepSeek-R1 model with NVFP4 checkpoints (8 GPU setup)
+        # memory_requirement is in MiB (96 GB = 98304 MiB)
         ModelConfig(model_dir="DeepSeek-R1/DeepSeek-R1-0528-FP4",
                     tp_size=8,
-                    memory_requirement=96),
+                    memory_requirement=98304),
         # Configuration for DeepSeek-R1 model with NVFP4 checkpoints (4 GPU setup)
+        # memory_requirement is in MiB (168 GB = 172032 MiB)
         ModelConfig(model_dir="DeepSeek-R1/DeepSeek-R1-0528-FP4",
                     tp_size=4,
-                    memory_requirement=168),
+                    memory_requirement=172032),
     ],
     ids=lambda x: f"{os.path.basename(x.model_dir)}_tp{x.tp_size}")
 def test_run_stress_test(config, stress_time_timeout, backend,
@@ -490,9 +498,12 @@ def stress_test(config,
         return
 
     # Skip if not enough GPU memory
+    # get_device_memory() returns per-GPU memory in MiB
     if get_device_memory() < config.memory_requirement:
         pytest.skip(
-            f"Not enough GPU memory. Required: {config.memory_requirement}GB")
+            f"Not enough GPU memory. Required: {config.memory_requirement} MiB ({config.memory_requirement // 1024} GB), "
+            f"Available: {get_device_memory()} MiB ({get_device_memory() // 1024} GB)"
+        )
 
     # Skip if not enough GPUs for tensor parallelism
     if get_device_count() < config.tp_size:

tests/integration/test_lists/qa/llm_function_core.txt

@@ -148,7 +148,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutl
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=False]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS]
-accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRITON]
+accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRITON]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS]

tests/integration/test_lists/qa/llm_function_core_sanity.txt

@@ -151,7 +151,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtl
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRTLLM]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS]
-accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRITON]
+accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRITON]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS]
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM]

tests/integration/test_lists/test-db/l0_b200.yml

@@ -53,7 +53,7 @@ l0_b200:
   - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=False]
   - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_trtllm-torch_compile=True]
   - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS]
-  - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRITON]
+  - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRITON]
   - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM]
   - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-TRTLLM]
   - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS]

tests/integration/test_lists/waives.txt

@@ -130,7 +130,6 @@ accuracy/test_cli_flow.py::TestSantacoder::test_auto_dtype SKIP (https://nvbugs/
 full:L40S/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5375620)
 full:L20/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_auto_dtype SKIP (https://nvbugs/5375620)
 test_e2e.py::test_ptp_quickstart_advanced_multi_gpus[Llama3.1-405B-FP8-llama-3.1-model/Llama-3.1-405B-Instruct-FP8-8] SKIP (https://nvbugs/5380570)
-test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-120B-gpt_oss/gpt-oss-120b] SKIP (https://nvbugs/5833795)
 examples/test_multimodal.py::test_llm_multimodal_general[Phi-4-multimodal-instruct-pp:1-tp:1-float16-bs:1-cpp_e2e:False-nb:1] SKIP (https://nvbugs/5385992)
 examples/test_recurrentgemma.py::test_llm_recurrentgemma_1gpu[use_cpp_session-recurrentgemma-2b-use_paged_cache-int4_awq-float16-enable_attn_plugin-enable_gemm_plugin] SKIP (https://nvbugs/5401233)
 examples/test_recurrentgemma.py::test_llm_recurrentgemma_2gpu[recurrentgemma-2b] SKIP (https://nvbugs/5401233)
@@ -318,7 +317,7 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtl
 unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_autotune_fp8_fp4[RoutingDSlite-384-1024-1] SKIP (https://nvbugs/5859881)
 accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer] SKIP (https://nvbugs/5859886)
 accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-fp8] SKIP (https://nvbugs/5651865)
-accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRITON] SKIP (https://nvbugs/5864263)
+accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRITON] SKIP (https://nvbugs/5864263)
 accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v1_kv_cache-True-True-triton-auto] SKIP (https://nvbugs/5864187)
 accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_fp8[latency-torch_compile=False] SKIP (https://nvbugs/5863806)
 accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-dp4-trtllm-auto] SKIP (https://nvbugs/5596343)
@@ -334,7 +333,6 @@ accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1
 accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-True-True-False] SKIP (https://nvbugs/5879625)
 accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-False-True-False] SKIP (https://nvbugs/5879625)
 unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_gptoss_style_nvfp4[limitinf-beta0-alpha0.1-RoutingGPTOSS-512-512-1] SKIP (https://nvbugs/5819042)
-disaggregated/test_disaggregated.py::test_disaggregated_benchmark_on_diff_backends[llama-v3-8b-hf] SKIP (https://nvbugs/5839137)
 accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=1-ctx_pp=4] SKIP (https://nvbugs/5845943)
 accuracy/test_cli_flow.py::TestGpt2::test_cuda_graph SKIP (https://nvbugs/5860520)
 unittest/_torch/thop/serial/test_moe.py::TestMoeFp4::test_online_eplb288_topk_input[RoutingDSv3-1024-1024-256] SKIP (https://nvbugs/5859881)

tests/unittest/llmapi/apps/_test_disagg_serving_multi_nodes.py

@@ -197,7 +197,7 @@ def test_completion(client: openai.OpenAI,
         # allow 5ms leniency when comparing the time points from disagg and ctx/gen servers
         validate_timing_metrics(perf_metrics,
                                 "multinode test_completion",
-                                time_leniency_seconds=0.005)
+                                time_tolerance_seconds=0.005)
         # sleep 10 seconds to ensure a successful wait_for_endpoint_ready on rank1
         time.sleep(10)
         disagg_server.terminate()