From a5311f178cc13e87d0d9e03bf4352a0f4934af3c Mon Sep 17 00:00:00 2001 From: Balamurugan Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> Date: Wed, 27 May 2026 23:10:00 -0700 Subject: [PATCH 1/3] unwaive llmc standalone tests Signed-off-by: Balamurugan Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> --- .../llmc/create_standalone_package.py | 19 ++++- .../models/custom/modeling_minimax_m2.py | 3 +- .../auto_deploy/models/quant_config_reader.py | 26 +++++-- .../transform/library/fuse_rope_mla.py | 3 +- .../_torch/auto_deploy/utils/_graph.py | 20 ++++++ .../_torch/auto_deploy/utils/node_utils.py | 22 ++++-- tests/integration/test_lists/waives.txt | 2 - .../standalone/test_standalone_package.py | 70 ++++++++++++++----- 8 files changed, 130 insertions(+), 35 deletions(-) diff --git a/examples/auto_deploy/llmc/create_standalone_package.py b/examples/auto_deploy/llmc/create_standalone_package.py index 67222e371a2e..4a2bc641f81f 100644 --- a/examples/auto_deploy/llmc/create_standalone_package.py +++ b/examples/auto_deploy/llmc/create_standalone_package.py @@ -158,10 +158,17 @@ "test_flashinfer_mamba_cached_op.py", # Require TRT-LLM custom ops (dsv3_router_gemm_op, noaux_tc_op, etc.) "test_deepseek_custom.py", + "test_glm4_moe_modeling.py", "test_glm4_moe_lite_modeling.py", + "test_glm_moe_dsa_modeling.py", + # Full-model tests hit standalone-incompatible HF cache behavior. + "test_granite_moe_hybrid_modeling.py", + # Imports triton_kernels, which is not a standalone dependency. + "test_mxfp4_moe_layout.py", # Require TRT-LLM distributed ops (trtllm_dist_all_gather) "test_gather_logits_before_lm_head.py", - # Multimodal types are None in standalone (MultimodalInput guard) + # Multimodal processors depend on TensorRT-LLM multimodal request types. + "test_gemma4_modeling.py", "test_qwen3_5_moe.py", # Hardware-specific (requires H100+ shared memory) "test_triton_mla_op.py", @@ -377,8 +384,16 @@ def _create_test_conftest(tests_dir: str) -> None: # limitations under the License. \"\"\"Conftest for standalone auto_deploy tests.\"\"\" - import sys + import importlib.util import os + import sys + + _trtllm_spec = importlib.util.find_spec("tensorrt_llm") + if _trtllm_spec is not None: + raise RuntimeError( + "Standalone llmc tests must not be able to import tensorrt_llm; " + f"found {getattr(_trtllm_spec, 'origin', None)!r}" + ) # Add _utils_test to the Python path so test files can import from it sys.path.insert(0, os.path.join(os.path.dirname(__file__), "_utils_test")) diff --git a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py index c14c1a8fedaf..2539ee043396 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py +++ b/tensorrt_llm/_torch/auto_deploy/models/custom/modeling_minimax_m2.py @@ -36,8 +36,7 @@ from transformers.modeling_utils import PreTrainedModel from transformers.utils import ModelOutput -from tensorrt_llm._torch.utils import ActivationType - +from ..._compat import ActivationType from ..hf import AutoModelForCausalLMFactory from .rotary_utils import RotaryEmbeddingBase, build_rope_cos_sin_cache diff --git a/tensorrt_llm/_torch/auto_deploy/models/quant_config_reader.py b/tensorrt_llm/_torch/auto_deploy/models/quant_config_reader.py index 92ca23fd272b..167d09b182b6 100644 --- a/tensorrt_llm/_torch/auto_deploy/models/quant_config_reader.py +++ b/tensorrt_llm/_torch/auto_deploy/models/quant_config_reader.py @@ -25,13 +25,29 @@ from abc import ABC, abstractmethod from typing import Any, Callable, Dict, Optional, Tuple, Type -from tensorrt_llm.quantization.modelopt_config import ( - is_modelopt_quant_config, - read_modelopt_quant_config, -) - +from .._compat import TRTLLM_AVAILABLE from ..utils.logger import ad_logger +# Importing ``tensorrt_llm.quantization.modelopt_config`` triggers +# ``tensorrt_llm.quantization.__init__`` which pulls in ``tensorrt_llm.mapping`` +# and thus ``tensorrt_llm._torch.device_mesh``; none of these exist in the +# standalone llmc package. Gate the import so this module loads cleanly when +# TRT-LLM is absent; the ModelOPT reader path itself is unreachable in that case. +if TRTLLM_AVAILABLE: + from tensorrt_llm.quantization.modelopt_config import ( + is_modelopt_quant_config, + read_modelopt_quant_config, + ) +else: + + def is_modelopt_quant_config(raw: Any) -> bool: + return False + + def read_modelopt_quant_config(raw: Any) -> Dict[str, Any]: + raise NotImplementedError( + "ModelOPT quant config parsing requires TensorRT-LLM; not available in standalone mode." + ) + class QuantConfigReader(ABC): """Base class for reading and parsing quantization config.""" diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py index d3d1563ed444..8d22196b2699 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/fuse_rope_mla.py @@ -35,13 +35,14 @@ import torch from torch.fx import GraphModule, Node -from ...custom_ops.mla.trtllm_mla import _TRTLLM_MLA_ROPE_INFO_KEY from ...models.factory import ModelFactory from ...shim.interface import CachedSequenceInterface from ...utils.logger import ad_logger from ...utils.node_utils import is_op from ..interface import BaseTransform, SharedConfig, TransformInfo, TransformRegistry +_TRTLLM_MLA_ROPE_INFO_KEY = "_trtllm_mla_rope_info" + # At post_load_fusion, only the backend-agnostic torch_rope_* IR ops are # present (optimize_rope has not yet replaced them with flashinfer_rope). _ROPE_OP_TARGETS = [] diff --git a/tensorrt_llm/_torch/auto_deploy/utils/_graph.py b/tensorrt_llm/_torch/auto_deploy/utils/_graph.py index f2d0ab640b51..f4159a502277 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/_graph.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/_graph.py @@ -357,7 +357,27 @@ def lint(gm: GraphModule) -> None: gm.graph.lint() +def _restore_topological_order(gm: GraphModule) -> None: + """Move nodes after their direct inputs if graph rewrites left them out of order.""" + while True: + node_order = {node: idx for idx, node in enumerate(gm.graph.nodes)} + for node in list(gm.graph.nodes): + input_nodes = [ + input_node for input_node in node.all_input_nodes if input_node in node_order + ] + if not input_nodes: + continue + latest_input = max(input_nodes, key=node_order.__getitem__) + if node_order[node] < node_order[latest_input]: + latest_input.append(node) + break + else: + return + + def _canonicalize_single_gm(gm: GraphModule) -> None: + _restore_topological_order(gm) + # clean up graph (needs to be done repeatedly until no more dead code) eliminate_dead_code(gm, is_impure_node=_is_impure_node) diff --git a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py index f89634df51af..286242ce5228 100644 --- a/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py +++ b/tensorrt_llm/_torch/auto_deploy/utils/node_utils.py @@ -46,6 +46,10 @@ OperatorLike = Union[OpOrOverload, Callable] +def _auto_deploy_op(name: str) -> Optional[OpOverloadPacket]: + return getattr(torch.ops.auto_deploy, name, None) + + class LayerType(Enum): """Enum for layer type.""" @@ -670,13 +674,17 @@ def is_any_moe_op(node: Node) -> bool: return is_op( node, ops=[ - torch.ops.auto_deploy.torch_moe, - torch.ops.auto_deploy.torch_quant_fp8_moe, - torch.ops.auto_deploy.torch_quant_nvfp4_moe, - torch.ops.auto_deploy.torch_quant_finegrained_fp8_moe, - torch.ops.auto_deploy.triton_mxfp4_moe, - torch.ops.auto_deploy.torch_moe_fused, - torch.ops.auto_deploy.torch_moe_dense_mlp, + op + for op in [ + _auto_deploy_op("torch_moe"), + _auto_deploy_op("torch_quant_fp8_moe"), + _auto_deploy_op("torch_quant_nvfp4_moe"), + _auto_deploy_op("torch_quant_finegrained_fp8_moe"), + _auto_deploy_op("triton_mxfp4_moe"), + _auto_deploy_op("torch_moe_fused"), + _auto_deploy_op("torch_moe_dense_mlp"), + ] + if op is not None ], ) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 30da247dcaa2..d644b6502533 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -359,8 +359,6 @@ unittest/auto_deploy/singlegpu/models/test_qwen3_5_moe.py::test_vision_attention unittest/auto_deploy/singlegpu/models/test_qwen3_5_moe.py::test_vision_block_matches_reference SKIP (https://nvbugs/6189450) unittest/auto_deploy/singlegpu/models/test_qwen3_5_moe.py::test_vlm_wrapper_delta_is_request_scoped_no_cross_call_leakage SKIP (https://nvbugs/6189450) unittest/auto_deploy/singlegpu/smoke/test_ad_build_small_single.py::test_build_ad[deepseek-ai/DeepSeek-V3-llm_extra_args10] SKIP (https://nvbugs/5888827) -unittest/auto_deploy/standalone SKIP (https://nvbugs/6160629) -unittest/auto_deploy/standalone/test_standalone_package.py::TestStandalonePackage::test_run_unit_tests SKIP (https://nvbugs/6160629) unittest/disaggregated/test_agent_multi_backends.py::test_run_with_different_env[1] SKIP (https://nvbugs/5979673) unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) diff --git a/tests/unittest/auto_deploy/standalone/test_standalone_package.py b/tests/unittest/auto_deploy/standalone/test_standalone_package.py index 9aa11aecfc09..c54a17b3962e 100644 --- a/tests/unittest/auto_deploy/standalone/test_standalone_package.py +++ b/tests/unittest/auto_deploy/standalone/test_standalone_package.py @@ -174,27 +174,65 @@ def test_run_unit_tests(self, standalone_package): if not os.path.isdir(tests_dir): pytest.skip("No tests directory in standalone package") - # Pass through the host env but override PYTHONPATH to use standalone tests. - # The venv's pip install already put llmc in the venv's site-packages, - # so `import llmc` resolves there (not to the host TRT-LLM). + # Keep runtime/compiler environment, but remove host Python import paths. standalone_env = { - **os.environ, - "PYTHONPATH": tests_dir + os.pathsep + os.path.join(tests_dir, "_utils_test"), - # Override PATH to prefer venv's python/pytest - "PATH": os.path.join(standalone_package["venv_dir"], "bin") - + os.pathsep - + os.environ.get("PATH", ""), - # FlashInfer JIT-compiles kernels and caches .so files under - # FLASHINFER_WORKSPACE_BASE (default: $HOME). Ninja records absolute - # source paths from the flashinfer package. Since this venv is - # ephemeral, a subsequent run would find stale cache entries pointing - # at the old (deleted) venv paths, causing all JIT builds to fail. - # Redirect the cache into the venv so it's discarded with it. - "FLASHINFER_WORKSPACE_BASE": standalone_package["venv_dir"], + key: value + for key, value in os.environ.items() + if key not in {"PYTHONHOME", "PYTHONPATH", "PYTHONUSERBASE"} } + standalone_env.update( + { + # Override PATH to prefer venv's python/pytest. + "PATH": os.path.join(standalone_package["venv_dir"], "bin") + + os.pathsep + + os.environ.get("PATH", ""), + # Keep subprocess workers from adding cwd/user paths that can + # expose the TensorRT-LLM checkout to standalone tests. + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", + # FlashInfer JIT-compiles kernels and caches .so files under + # FLASHINFER_WORKSPACE_BASE (default: $HOME). Ninja records absolute + # source paths from the flashinfer package. Since this venv is + # ephemeral, a subsequent run would find stale cache entries pointing + # at the old (deleted) venv paths, causing all JIT builds to fail. + # Redirect the cache into the venv so it's discarded with it. + "FLASHINFER_WORKSPACE_BASE": standalone_package["venv_dir"], + } + ) + + isolation_probe = subprocess.run( + [ + python, + "-I", + "-c", + textwrap.dedent( + """ + import importlib.util + import sys + + spec = importlib.util.find_spec("tensorrt_llm") + if spec is not None: + raise SystemExit( + "tensorrt_llm is importable in standalone test env: " + f"origin={getattr(spec, 'origin', None)!r}, sys.path={sys.path!r}" + ) + """ + ), + ], + capture_output=True, + text=True, + timeout=30, + cwd=pkg_dir, + env=standalone_env, + ) + assert isolation_probe.returncode == 0, ( + f"Standalone env leaked TensorRT-LLM\n" + f"stdout:\n{isolation_probe.stdout}\nstderr:\n{isolation_probe.stderr}" + ) cmd = [ python, + "-I", "-m", "pytest", os.path.join(tests_dir, "singlegpu"), From 7937e3d124d7846fd8f1a474511232826b0e2c3a Mon Sep 17 00:00:00 2001 From: Balamurugan Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> Date: Thu, 28 May 2026 20:37:44 -0700 Subject: [PATCH 2/3] workaround CI analyzer issue Signed-off-by: Balamurugan Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> --- .../unittest/auto_deploy/standalone/test_standalone_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/auto_deploy/standalone/test_standalone_package.py b/tests/unittest/auto_deploy/standalone/test_standalone_package.py index c54a17b3962e..5dff65ae2a00 100644 --- a/tests/unittest/auto_deploy/standalone/test_standalone_package.py +++ b/tests/unittest/auto_deploy/standalone/test_standalone_package.py @@ -214,7 +214,7 @@ def test_run_unit_tests(self, standalone_package): if spec is not None: raise SystemExit( "tensorrt_llm is importable in standalone test env: " - f"origin={getattr(spec, 'origin', None)!r}, sys.path={sys.path!r}" + "origin=%r, sys.path=%r" % (getattr(spec, "origin", None), sys.path) ) """ ), From 1380b43cbae5d03cad9f8293d3bf1e674d13a637 Mon Sep 17 00:00:00 2001 From: Balamurugan Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:07:29 -0700 Subject: [PATCH 3/3] fix llmc standalone optional op imports Signed-off-by: Balamurugan Marimuthu <246387390+bmarimuthu-nv@users.noreply.github.com> --- .../custom_ops/normalization/rms_norm.py | 23 ++++++++++++++++--- .../transform/library/sharding_ir.py | 12 ++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py b/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py index 2d042705423f..4e2b43d78b5c 100644 --- a/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py +++ b/tensorrt_llm/_torch/auto_deploy/custom_ops/normalization/rms_norm.py @@ -21,10 +21,13 @@ import torch.nn.functional as F from einops import rearrange -import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils - from ..quantization.quant import TRTLLM_NVFP4_SCALING_VECTOR_SIZE +try: + import tensorrt_llm.quantization.utils.fp4_utils as fp4_utils +except (ModuleNotFoundError, ImportError): + fp4_utils = None + try: from tensorrt_llm._torch.flashinfer_utils import get_env_enable_pdl except (ModuleNotFoundError, ImportError): @@ -41,8 +44,22 @@ def get_env_enable_pdl() -> bool: from .triton_rms_norm import rms_norm +def _pad_up(x, y: int): + return ((x + y - 1) // y) * y + + def _get_nvfp4_fake_shapes(x: torch.Tensor) -> tuple[tuple[int, ...], int]: - output_shape, sf_size = fp4_utils.get_fp4_shape(x.shape, TRTLLM_NVFP4_SCALING_VECTOR_SIZE) + if fp4_utils is not None: + output_shape, sf_size = fp4_utils.get_fp4_shape(x.shape, TRTLLM_NVFP4_SCALING_VECTOR_SIZE) + return tuple(output_shape), sf_size + + input_shape = tuple(x.shape) + m = 1 + for dim in input_shape[:-1]: + m *= dim + output_shape = list(input_shape) + output_shape[-1] //= 2 + sf_size = _pad_up(m, 128) * _pad_up(input_shape[-1] // TRTLLM_NVFP4_SCALING_VECTOR_SIZE, 4) return tuple(output_shape), sf_size diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/sharding_ir.py b/tensorrt_llm/_torch/auto_deploy/transform/library/sharding_ir.py index 0e49da0b52cf..9bd9c428b171 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/sharding_ir.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/sharding_ir.py @@ -111,6 +111,13 @@ def _shard_scale_and_hook( } ) + +def _auto_deploy_ops(*names: str) -> Tuple[OpOverloadPacket, ...]: + return tuple( + op for name in names if (op := getattr(torch.ops.auto_deploy, name, None)) is not None + ) + + # ============================================================================= # ShardableNode abstract base class # ============================================================================= @@ -524,10 +531,7 @@ def apply(self, gm: GraphModule, dc: DistConfig, max_num_tokens: int = 0) -> int return 1 if count > 0 else 0 -@ShardableNode.register( - torch.ops.auto_deploy.torch_rmsnorm_gated, - torch.ops.auto_deploy.triton_rmsnorm_gated, -) +@ShardableNode.register(*_auto_deploy_ops("torch_rmsnorm_gated", "triton_rmsnorm_gated")) class NormShardableNode(ShardableNode): """Gated RMSNorm op: shard weight parameter."""