From 705fac39d5dd39decb50b4ac85b4fa4a193c06f7 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sun, 5 Jul 2026 23:26:08 -0700 Subject: [PATCH 1/2] [https://nvbugs/6388787][fix] Revert Pass IPC HMAC key through file descriptor (#15654) Reverts NVIDIA/TensorRT-LLM#15654 (48fc7537ba). PR #15654 passes the IPC HMAC key from the parent process to the LLM-API proxy child via a file descriptor. In the QA-pytest Slurm/MPI launch path used by trtllm-bench and trtllm-serve, the fd handshake between Rank0's forked subshell (spawning internal mpirun) and the mgmn_leader_node RemoteMpiCommSessionServer deadlocks silently: - trtllm-bench (nvbugs/6388787): after "start MpiSession with N workers" the bench process emits no further stdout for ~30 min, workers stay alive at 0% GPU / 4 MiB VRAM, no exception, no MPI abort. Perf harness SIGKILLs at _STALL_TIMEOUT=1800s. - trtllm-serve (nvbugs/6405747): serve mode's /health endpoint never binds, and pytest fails with "Server http://localhost:/health did not become ready within 3600s" on both baseline and candidate wheels. This is the third time the "Pass IPC HMAC key via fd" idea has broken bench/serve (previously PR #14378 was reverted by PR #14782 for the same class of failure: BlockingIOError in _read_spawn_proxy_process_ ipc_hmac_key_fd). Reverting until the fd inheritance across fork+exec into the mpirun child is verified end-to-end for both bench and serve launch paths. Signed-off-by: Chenfei Zhang --- tensorrt_llm/commands/serve.py | 26 +----- tensorrt_llm/executor/ipc.py | 8 +- tensorrt_llm/executor/utils.py | 63 ++------------- tensorrt_llm/executor/worker.py | 3 +- tensorrt_llm/llmapi/trtllm-llmapi-launch | 19 +---- tests/unittest/executor/test_ipc.py | 12 --- tests/unittest/executor/test_launcher_envs.py | 79 ------------------- 7 files changed, 16 insertions(+), 194 deletions(-) delete mode 100644 tests/unittest/executor/test_launcher_envs.py diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 6da2d133d8ed..9b648020a7ae 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -26,8 +26,7 @@ from tensorrt_llm.commands._serve_stability import stability_option from tensorrt_llm.commands.utils import (collect_explicit_cli_keys, get_is_diffusion_only_model) -from tensorrt_llm.executor.utils import (LlmLauncherEnvs, - set_spawn_proxy_process_ipc_hmac_key) +from tensorrt_llm.executor.utils import LlmLauncherEnvs from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.llmapi import (BuildConfig, CapacitySchedulerPolicy, DynamicBatchConfig, KvCacheConfig, @@ -1604,13 +1603,11 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, # This mimics the behavior of trtllm-llmapi-launch # TODO: Make the port allocation atomic free_ipc_addr = find_free_ipc_addr() - ipc_hmac_key = secrets.token_hex(32) - set_spawn_proxy_process_ipc_hmac_key(ipc_hmac_key) - os.environ.pop( - LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, None) os.environ[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS] = "1" os.environ[ LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR.value] = free_ipc_addr + os.environ[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY. + value] = secrets.token_hex(32) os.environ[DisaggLauncherEnvs.TLLM_DISAGG_RUN_REMOTE_MPI_SESSION_CLIENT. value] = "1" os.environ[DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX] = str(instance_idx) @@ -1626,6 +1623,7 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS in non_mpi_env assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR in non_mpi_env + assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY in non_mpi_env assert DisaggLauncherEnvs.TLLM_DISAGG_INSTANCE_IDX in non_mpi_env assert DisaggLauncherEnvs.TLLM_DISAGG_RUN_REMOTE_MPI_SESSION_CLIENT in non_mpi_env @@ -1648,24 +1646,13 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, signal.signal(signal.SIGTERM, _signal_handler_cleanup_child) signal.signal(signal.SIGINT, _signal_handler_cleanup_child) - read_fd = -1 - write_fd = -1 try: - read_fd, write_fd = os.pipe() - os.write(write_fd, ipc_hmac_key.encode("ascii")) - os.close(write_fd) - write_fd = -1 - non_mpi_env[LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD. - value] = str(read_fd) _child_p_global = subprocess.Popen( command, env=non_mpi_env, stdout=sys.stdout, # Redirect to parent's stdout stderr=sys.stderr, # Redirect to parent's stderr - pass_fds=(read_fd, ), start_new_session=True) - os.close(read_fd) - read_fd = -1 logger.info( f"Parent process (PID {os.getpid()}) launched child process (PID {_child_p_global.pid})." @@ -1679,11 +1666,6 @@ def _launch_disaggregated_leader(sub_comm, instance_idx: int, config_file: str, launch_remote_mpi_session_server(sub_comm) finally: - if write_fd != -1: - os.close(write_fd) - if read_fd != -1: - os.close(read_fd) - # Restore original signal handlers signal.signal(signal.SIGTERM, original_sigterm_handler) signal.signal(signal.SIGINT, original_sigint_handler) diff --git a/tensorrt_llm/executor/ipc.py b/tensorrt_llm/executor/ipc.py index bea97b944945..a096f9c0433d 100644 --- a/tensorrt_llm/executor/ipc.py +++ b/tensorrt_llm/executor/ipc.py @@ -40,7 +40,7 @@ def __init__(self, use_hmac_encryption: bool = True): ''' Parameters: - address (tuple[str, Optional[bytes]], optional): The address (tcp-ip_port, hmac_auth_key) for the IPC. Defaults to None. Servers generate an HMAC key when none is provided; clients must receive one. + address (tuple[str, Optional[bytes]], optional): The address (tcp-ip_port, hmac_auth_key) for the IPC. Defaults to None. If hmac_auth_key is None and use_hmac_encryption is False, the queue will not use HMAC encryption. socket_type (int): The type of socket to use. Defaults to zmq.PAIR. is_server (bool): Whether the current process is the server or the client. is_async (bool): Whether to use asyncio for the socket. Defaults to False. @@ -48,11 +48,7 @@ def __init__(self, use_hmac_encryption (bool): Whether to use HMAC encryption for pickled data. Defaults to True. ''' - if not use_hmac_encryption: - raise ValueError( - "HMAC encryption is always required. Turning off HMAC " - "encryption risks unauthorized data serialization and " - "deserialization.") + assert use_hmac_encryption, "HMAC encryption is always required. Turning off HMAC encryption risks security vulnerability of unauthorized data serialization and deserialization. " self.socket_type = socket_type self.address_endpoint = address[ diff --git a/tensorrt_llm/executor/utils.py b/tensorrt_llm/executor/utils.py index 611a25a737ef..12eaa9afefa9 100644 --- a/tensorrt_llm/executor/utils.py +++ b/tensorrt_llm/executor/utils.py @@ -23,56 +23,12 @@ class LlmLauncherEnvs(StrEnum): # Spawn a process for the LLM-API Proxy TLLM_SPAWN_PROXY_PROCESS = "TLLM_SPAWN_PROXY_PROCESS" TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR = "TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR" - TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD = ( - "TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD") + TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = "TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY" # Whether to use periodical responses handler in await_responses TLLM_EXECUTOR_PERIODICAL_RESP_IN_AWAIT = "TLLM_EXECUTOR_PERIODICAL_RESP_IN_AWAIT" -_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY: bytes | None = None - - -def _normalize_spawn_proxy_process_ipc_hmac_key(key: str | bytes) -> bytes: - if isinstance(key, bytes): - if len(key) == 32: - return key - key = key.decode("ascii") - - key_bytes = bytes.fromhex(key) - if len(key_bytes) != 32: - raise ValueError("IPC HMAC key must be 32 bytes.") - return key_bytes - - -def set_spawn_proxy_process_ipc_hmac_key(key: str | bytes) -> None: - global _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY - _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = ( - _normalize_spawn_proxy_process_ipc_hmac_key(key)) - - -def _read_spawn_proxy_process_ipc_hmac_key_fd(fd_value: str) -> bytes: - fd = int(fd_value) - chunks: list[bytes] = [] - try: - os.set_blocking(fd, True) - while True: - chunk = os.read(fd, 4096) - if not chunk: - break - chunks.append(chunk) - finally: - os.close(fd) - - try: - key_hex = b"".join(chunks).decode("ascii") - except UnicodeDecodeError as exc: - raise ValueError( - "IPC HMAC key FD must contain an ASCII hex string.") from exc - - return _normalize_spawn_proxy_process_ipc_hmac_key(key_hex) - - def get_spawn_proxy_process_ipc_addr_env() -> str | None: ''' Get the IPC address for the spawn proxy process dynamically. ''' return os.getenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_ADDR) @@ -80,20 +36,11 @@ def get_spawn_proxy_process_ipc_addr_env() -> str | None: def get_spawn_proxy_process_ipc_hmac_key_env() -> bytes: ''' Get the HMAC key for the spawn proxy process dynamically. ''' - global _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY - if _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY is not None: - return _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY - - key_fd = os.environ.pop( - LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD, None) - if key_fd is not None: - _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY = ( - _read_spawn_proxy_process_ipc_hmac_key_fd(key_fd)) - return _SPAWN_PROXY_PROCESS_IPC_HMAC_KEY - - raise RuntimeError( - f"{LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD} is not set. " + key = os.getenv("TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY") + assert key is not None, ( + f"{LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY} is not set. " "HMAC encryption is required for IPC communication.") + return bytes.fromhex(key) def get_spawn_proxy_process_env() -> bool: diff --git a/tensorrt_llm/executor/worker.py b/tensorrt_llm/executor/worker.py index 6b0604764114..c05e44f02cd3 100644 --- a/tensorrt_llm/executor/worker.py +++ b/tensorrt_llm/executor/worker.py @@ -69,8 +69,7 @@ def __init__( # Setup RPC server for stats (skip init_rpc_worker to keep IPC response queue) # Only set up if rpc_addr is provided (for stats RPC support) if rpc_addr is not None: - if not hmac_key: - raise ValueError("hmac_key is required when rpc_addr is set") + assert hmac_key, "hmac_key is required when rpc_addr is set" self.rpc_addr = rpc_addr self.hmac_key = hmac_key self.start_rpc_server() # Reuse from RpcWorkerMixin diff --git a/tensorrt_llm/llmapi/trtllm-llmapi-launch b/tensorrt_llm/llmapi/trtllm-llmapi-launch index d35b60046a26..c11a3bbeb171 100755 --- a/tensorrt_llm/llmapi/trtllm-llmapi-launch +++ b/tensorrt_llm/llmapi/trtllm-llmapi-launch @@ -40,17 +40,7 @@ function maybe_export_free_tcp_addr_for_spawn_proxy_process { export tllm_mpi_size=$(mpi_world_size) log_stderr "tllm_mpi_size: $tllm_mpi_size" -unset TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD -ipc_hmac_key=$(openssl rand -hex 32) - -function run_with_ipc_hmac_key { - local fd - exec {fd}<<<"$ipc_hmac_key" - TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD="$fd" "$@" - local status=$? - exec {fd}<&- - return $status -} +export TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY=$(openssl rand -hex 32) if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then @@ -84,13 +74,12 @@ if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then set +e # Execute the task with cleaned environment - run_with_ipc_hmac_key "${task_with_command[@]}" + "${task_with_command[@]}" task_exit_code=$? log_stderr "Rank${mpi_rank} Task exit code: $task_exit_code" # Stop the MPI Comm server - run_with_ipc_hmac_key python3 -m tensorrt_llm.llmapi.mgmn_leader_node \ - --action stop + python3 -m tensorrt_llm.llmapi.mgmn_leader_node --action stop mpi_exit_code=$? log_stderr "Rank${mpi_rank} MPI Comm server exit code: $mpi_exit_code" @@ -111,7 +100,7 @@ if [ -z "$mpi_rank" ] || [ "$mpi_rank" -eq 0 ]; then log_stderr "Rank${mpi_rank} run mgmn leader node with mpi_world_size: $(mpi_world_size) ..." log_stderr "Rank0 host: $HOSTNAME" - run_with_ipc_hmac_key python3 -m tensorrt_llm.llmapi.mgmn_leader_node + python3 -m tensorrt_llm.llmapi.mgmn_leader_node mgmn_leader_node_exit_code=$? log_stderr "Rank${mpi_rank} MGMN leader node exit code: $mgmn_leader_node_exit_code" diff --git a/tests/unittest/executor/test_ipc.py b/tests/unittest/executor/test_ipc.py index b717d5e05505..d618c3f07c7f 100644 --- a/tests/unittest/executor/test_ipc.py +++ b/tests/unittest/executor/test_ipc.py @@ -257,18 +257,6 @@ def test_hmac_validation_error_client_no_key(self): use_hmac_encryption=True, # But encryption enabled ) - def test_hmac_encryption_cannot_be_disabled(self): - """Test that HMAC cannot be disabled by optimized Python removing asserts.""" - with pytest.raises(ValueError, match="HMAC encryption is always required"): - ZeroMqQueue( - address=None, - socket_type=zmq.PAIR, - is_server=True, - is_async=False, - name="test_hmac_disabled", - use_hmac_encryption=False, - ) - def test_put_noblock_retry(self): """Test put_noblock with retry mechanism.""" server = ZeroMqQueue( diff --git a/tests/unittest/executor/test_launcher_envs.py b/tests/unittest/executor/test_launcher_envs.py deleted file mode 100644 index eea3dda42904..000000000000 --- a/tests/unittest/executor/test_launcher_envs.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import threading - -import pytest - -from tensorrt_llm.executor import utils as executor_utils -from tensorrt_llm.executor.utils import LlmLauncherEnvs - - -def _reset_ipc_hmac_key_env(monkeypatch): - monkeypatch.setattr(executor_utils, "_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY", None) - monkeypatch.delenv( - LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, raising=False - ) - - -def _write_key_fd(key_hex: str) -> int: - read_fd, write_fd = os.pipe() - os.write(write_fd, key_hex.encode("ascii")) - os.close(write_fd) - return read_fd - - -def test_get_spawn_proxy_process_ipc_hmac_key_from_fd(monkeypatch): - _reset_ipc_hmac_key_env(monkeypatch) - key_hex = "01" * 32 - read_fd = _write_key_fd(key_hex) - monkeypatch.setenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, str(read_fd)) - - key = executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() - - assert key == bytes.fromhex(key_hex) - assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD not in os.environ - with pytest.raises(OSError): - os.fstat(read_fd) - - -def test_get_spawn_proxy_process_ipc_hmac_key_caches_fd_key(monkeypatch): - _reset_ipc_hmac_key_env(monkeypatch) - key_hex = "02" * 32 - read_fd = _write_key_fd(key_hex) - monkeypatch.setenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, str(read_fd)) - - key = executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() - - assert key == bytes.fromhex(key_hex) - assert LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD not in os.environ - assert executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() == key - - -def test_get_spawn_proxy_process_ipc_hmac_key_from_nonblocking_fd(monkeypatch): - _reset_ipc_hmac_key_env(monkeypatch) - key_hex = "03" * 32 - read_fd, write_fd = os.pipe() - os.set_blocking(read_fd, False) - - def write_key(): - try: - os.write(write_fd, key_hex.encode("ascii")) - finally: - os.close(write_fd) - - writer = threading.Timer(0.05, write_key) - writer.start() - monkeypatch.setenv(LlmLauncherEnvs.TLLM_SPAWN_PROXY_PROCESS_IPC_HMAC_KEY_FD.value, str(read_fd)) - - try: - key = executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() - finally: - writer.join() - - assert key == bytes.fromhex(key_hex) - - -def test_get_spawn_proxy_process_ipc_hmac_key_missing(monkeypatch): - _reset_ipc_hmac_key_env(monkeypatch) - - with pytest.raises(RuntimeError, match="HMAC encryption is required"): - executor_utils.get_spawn_proxy_process_ipc_hmac_key_env() From c514f81869f09728b449c60223942c4f46abb712 Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Mon, 6 Jul 2026 01:04:07 -0700 Subject: [PATCH 2/2] [https://nvbugs/6388787][test] Unwaive perf sanity cases that passed pre-merge job 46441 Removed 70 perf sanity waives that passed in pre-merge job 46441 (OpenSearch-verified). Kept 12 waives whose test cases did not appear in the job 46441 OpenSearch results (pytest failed before uploading data). Breakdown: - 70 removed (passed in OpenSearch job 46441) - 12 kept (not present in OpenSearch job 46441) Signed-off-by: Chenfei Zhang --- tests/integration/test_lists/waives.txt | 70 ------------------------- 1 file changed, 70 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6b47e5bc4fe5..99bd6e7e9b96 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -446,90 +446,20 @@ perf/test_perf.py::test_perf[t5-bench-float16-input_output_len:128,20-gpus:2] SK perf/test_perf.py::test_perf[t5-bench-float16-maxbs:1-input_output_len:128,20-gpus:2] SKIP perf/test_perf.py::test_perf[t5_base-plugin-float16-bs:8-input_output_len:60,20] SKIP # (https://nvidia.slack.com/archives/C059LSY62BT/p1704525727177449) perf/test_perf.py::test_perf[whisper_large_v3-bench-float16-input_output_len:128,20] SKIP -perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_dep8_mtp1_1k1k] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_dep8_mtp1_8k1k] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_blackwell-r1_fp4_v2_tep8_mtp3] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp8_blackwell-r1_fp8_tp8_mtp3_1k1k] SKIP (https://nvbugs/6402086) perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v32_fp4_grace_blackwell-v32_fp4_dep4_mtp1_1k1k] SKIP (https://nvbugs/6374910) -perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_deepseek_v32_fp4_2_nodes_grace_blackwell-dsv32_fp4_dep8_trtllm_lpc_mnnvl_8k1k] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_gpt_oss_120b_fp4_blackwell-gpt_oss_fp4_tep4_adp_cutlass_8k1k] SKIP (https://nvbugs/6374910) -perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_2_nodes_grace_blackwell-glm5_fp4_dep8_mtp1_8k1k] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_2_nodes_grace_blackwell-glm5_fp4_tep8_mtp3_8k1k] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_blackwell-glm5_fp4_tep8_mtp3_8k1k] SKIP (https://nvbugs/6329155) -perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_dep8_32k8k] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6402069) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6388222) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_1k1k_con4096_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6374893) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6368078) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6402069) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6406265) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6412144) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6374905) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] SKIP (https://nvbugs/6374872) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6374898) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6374893) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6324123) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6324123) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6324123) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_gpt-oss-120b-fp4_8k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6323074) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] SKIP (https://nvbugs/6379406) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6388238) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con64_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6418834) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con1_ctx1_dep2_gen1_tep4_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418834) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_1k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6412144) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con2048_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6402069) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con4096_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6323074) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_1k1k_con4_ctx1_dep4_gen1_tep4_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6323074) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] SKIP (https://nvbugs/6323074) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6374893) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6418510) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_i2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6294413) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_2stage_bf16_t2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6294413) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-ltx2_blackwell-ltx2_nvfp4_i2v_cfg2_ulysses4_compile_on] SKIP (https://nvbugs/6294413) -perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-wan21_t2v_14b_blackwell-wan21_14b_nvfp4_trtllm_cfg2_ulysses4_teacache_on] SKIP (https://nvbugs/6412144) test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/6368053) test_e2e.py::test_draft_token_tree_quickstart_advanced_eagle3_depth_1_tree[Llama-3.1-8b-Instruct-llama-3.1-model/Llama-3.1-8B-Instruct-EAGLE3-LLaMA3.1-Instruct-8B] SKIP (https://nvbugs/6368053)