Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,11 @@
# the `gpu_0` marker.
pytest.mark.gpu_1,
pytest.mark.pre_merge,
pytest.mark.profiled_vram_gib(0),
]

# Intentionally unprofiled: these import-heavy, zero-VRAM tests run in the
# sequential GPU stage so TensorRT-LLM initialization is shared.

_PYTORCH_LLM_CLS_NAME = "dynamo.trtllm.engine.LLM"
_AUTODEPLOY_LLM_CLS_NAME = "tensorrt_llm._torch.auto_deploy.LLM"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@
pytest.mark.multimodal,
pytest.mark.unit,
pytest.mark.gpu_1,
pytest.mark.profiled_vram_gib(0),
]

# Intentionally unprofiled: these import-heavy, zero-VRAM tests run in the
# sequential GPU stage so TensorRT-LLM initialization is shared.


def _config(model: str = "org/model") -> SimpleNamespace:
return SimpleNamespace(model=model, revision=None)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,11 @@
pytest.mark.trtllm,
pytest.mark.pre_merge,
pytest.mark.gpu_1,
pytest.mark.profiled_vram_gib(0),
]

# Intentionally unprofiled: the zero-VRAM tests run in the sequential GPU stage
# so TensorRT-LLM initialization is shared. The CUDA test overrides this below.


# ---------------------------------------------------------------------------
# TRT-LLM attach_logits_processors contract
Expand Down Expand Up @@ -157,10 +159,9 @@ def test_adapter_invokes_or_logs_on_bad_shape(shape, expect_invoke, caplog):
)


# Unlike the rest of this module (CPU-only mocks, module-level
# profiled_vram_gib(0)), this test initializes a real CUDA context, so the
# GPU-parallel scheduler must reserve VRAM for it instead of packing it onto
# an already-full GPU as a zero-VRAM filler.
# Unlike the rest of this module (unprofiled CPU-only mocks), this test
# initializes a real CUDA context, so the GPU-parallel scheduler must reserve
# VRAM for it.
@pytest.mark.profiled_vram_gib(2.0)
@pytest.mark.requested_trtllm_vram_gib(2.0)
def test_adapter_enters_engine_cuda_stream():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
pytest.mark.trtllm,
pytest.mark.gpu_1,
pytest.mark.pre_merge,
pytest.mark.profiled_vram_gib(0),
]

# Intentionally unprofiled: these import-heavy, zero-VRAM tests run in the
# sequential GPU stage so TensorRT-LLM initialization is shared.


def test_tensorrt_llm_metrics_collector_import():
"""Test that we can import MetricsCollector from TensorRT-LLM."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@
pytest.mark.multimodal,
pytest.mark.pre_merge,
pytest.mark.gpu_1,
pytest.mark.profiled_vram_gib(0),
]

# Intentionally unprofiled: these import-heavy, zero-VRAM tests run in the
# sequential GPU stage so TensorRT-LLM initialization is shared.


@pytest.mark.asyncio
@pytest.mark.parametrize(
Expand Down
4 changes: 3 additions & 1 deletion components/src/dynamo/trtllm/tests/test_trtllm_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,11 @@
pytest.mark.trtllm,
pytest.mark.gpu_1,
pytest.mark.pre_merge,
pytest.mark.profiled_vram_gib(0),
]

# Intentionally unprofiled: these import-heavy, zero-VRAM tests run in the
# sequential GPU stage so TensorRT-LLM initialization is shared.

Comment thread
devin-ai-integration[bot] marked this conversation as resolved.

# Create TRTLLM-specific CLI args fixture
# This will use monkeypatch to write to argv
Expand Down
6 changes: 6 additions & 0 deletions examples/backends/trtllm/launch/disagg_multimodal.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,16 @@ fi
HTTP_PORT="${DYN_HTTP_PORT:-8000}"
print_launch_banner --multimodal "Launching Disaggregated Multimodal Serving (2 GPUs)" "$MODEL_PATH" "$HTTP_PORT"

# Multi-worker launchers use numbered status ports; do not pass the single-worker
# alias to the frontend.
unset DYN_SYSTEM_PORT

# run frontend
# dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000)
python3 -m dynamo.frontend &

# run prefill worker
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT1:-8081} \
CUDA_VISIBLE_DEVICES=$PREFILL_CUDA_VISIBLE_DEVICES python3 -m dynamo.trtllm \
--model-path "$MODEL_PATH" \
--served-model-name "$SERVED_MODEL_NAME" \
Expand All @@ -42,6 +47,7 @@ CUDA_VISIBLE_DEVICES=$PREFILL_CUDA_VISIBLE_DEVICES python3 -m dynamo.trtllm \
--disaggregation-mode prefill &

# run decode worker
DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT2:-8082} \
CUDA_VISIBLE_DEVICES=$DECODE_CUDA_VISIBLE_DEVICES python3 -m dynamo.trtllm \
--model-path "$MODEL_PATH" \
--served-model-name "$SERVED_MODEL_NAME" \
Expand Down
3 changes: 2 additions & 1 deletion tests/kvbm_integration/test_consolidator_config_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,13 @@ def test_env_var_disables(self):
assert should_enable_consolidator(arg_map) is False


# Intentionally unprofiled: these import-heavy, zero-VRAM tests run in the
# sequential GPU stage so TensorRT-LLM initialization is shared.
@pytest.mark.unit
@pytest.mark.pre_merge
@pytest.mark.kvbm
@pytest.mark.trtllm
@pytest.mark.gpu_1
@pytest.mark.profiled_vram_gib(0)
class TestShouldEnableConsolidatorTyped:
"""Tests that need trtllm (requires GPU for import)."""

Expand Down
8 changes: 6 additions & 2 deletions tests/serve/test_trtllm.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,11 +352,12 @@ class TRTLLMConfig(EngineConfig):
pytest.mark.pre_merge,
pytest.mark.profiled_vram_gib(15.0),
pytest.mark.requested_trtllm_kv_tokens(1056),
pytest.mark.timeout(360), # 3x measured 118s CI runtime
Comment thread
rmccorm4 marked this conversation as resolved.
],
model="Qwen/Qwen3-VL-2B-Instruct",
frontend_port=DefaultPort.FRONTEND.value,
timeout=900,
delayed_start=120,
timeout=300,
health_check_workers=True,
Comment thread
rmccorm4 marked this conversation as resolved.
Comment thread
rmccorm4 marked this conversation as resolved.
request_payloads=[
multimodal_payload_default(
text="Describe what you see in this image.",
Expand All @@ -366,6 +367,9 @@ class TRTLLMConfig(EngineConfig):
env={
"PREFILL_CUDA_VISIBLE_DEVICES": "0",
"DECODE_CUDA_VISIBLE_DEVICES": "0",
# Make worker /health readiness depend on a successful one-token
# engine canary instead of the system-status server alone.
"DYN_HEALTH_CHECK_ENABLED": "true",
Comment thread
rmccorm4 marked this conversation as resolved.
Comment thread
rmccorm4 marked this conversation as resolved.
},
),
"e_pd_multimodal": TRTLLMConfig(
Expand Down
4 changes: 3 additions & 1 deletion tests/serve/test_trtllm_mm_hashes_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@
pytest.mark.multimodal,
pytest.mark.unit,
pytest.mark.gpu_1,
pytest.mark.profiled_vram_gib(0),
]

# Intentionally unprofiled: these import-heavy, zero-VRAM tests run in the
# sequential GPU stage so TensorRT-LLM initialization is shared.


def test_trtllm_qwen2vl_uses_in_vocab_image_markers() -> None:
from tensorrt_llm._torch.models.modeling_qwen2vl import Qwen2VLInputProcessorBase
Expand Down
Loading