Skip to content

test(trtllm): cover video in shared-runtime sidecar HTTP E2E #17673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
krishung5 wants to merge 1 commit into
base: trtllm-sidecar-image-e2e
Choose a base branch
from
Draft

test(trtllm): cover video in shared-runtime sidecar HTTP E2E #17673

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 24 additions & 4 deletions tests/serve/test_sidecar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@

import pytest

from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS
from tests.router.helper import generate_random_suffix
from tests.serve.common import (
WORKSPACE_DIR,
params_with_model_mark,
run_serve_deployment,
)
from tests.serve.conftest import MULTIMODAL_VIDEO_EXPECTED, MULTIMODAL_VIDEO_URL
from tests.utils.constants import DynamoPortRange
from tests.utils.engine_metrics import (
EngineMetrics,
Expand All @@ -24,6 +26,7 @@
from tests.utils.gpu_args import map_cuda_visible_devices
from tests.utils.payload_builder import (
chat_logprobs_and_structured_output_payloads,
chat_payload,
disaggregated_kv_routing_payload,
disaggregated_token_count_payload,
http_cancellation_payloads,
Expand Down Expand Up @@ -219,8 +222,8 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]:
*http_cancellation_payloads(None, max_tokens=CANCELLATION_MAX_TOKENS),
],
),
"trtllm_aggregated_image": SidecarConfig(
name="trtllm_aggregated_image",
"trtllm_aggregated_multimodal": SidecarConfig(
name="trtllm_aggregated_multimodal",
directory=trtllm_sidecar_dir,
script_name="agg.sh",
marks=[
Expand All @@ -231,16 +234,31 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]:
pytest.mark.requested_trtllm_kv_tokens(32768),
pytest.mark.timeout(960),
pytest.mark.post_merge,
pytest.mark.installs_extra_dependencies,
],
model="Qwen/Qwen3-VL-2B-Instruct",
timeout=900,
delayed_start=60,
env={"PYTHONUNBUFFERED": "1"},
env={
"PYTHONUNBUFFERED": "1",
"DYN_MM_ALLOW_INTERNAL": "1",
"TRTLLM_MEDIA_ALLOW_PRIVATE_URLS": "1",
"DYN_TEST_ONLY_PIP_INSTALL": VALIDATED_SPECS["opencv-python-headless"],
},
request_payloads=[
multimodal_payload_default(
text="Describe what you see in this image.",
expected_response=["mountain", "rock", "trees", "road"],
)
),
chat_payload(
[
{"type": "text", "text": "Describe the video in detail."},
{"type": "video_url", "video_url": {"url": MULTIMODAL_VIDEO_URL}},
],
expected_response=MULTIMODAL_VIDEO_EXPECTED,
max_tokens=100,
temperature=0.0,
),
],
),
"trtllm_disaggregated": SidecarConfig(
Expand Down Expand Up @@ -513,6 +531,8 @@ def sidecar_config_test(request, monkeypatch, discovery_backend):
config = sidecar_configs[request.param]
if config.name.startswith("trtllm_"):
require_trtllm_openengine()
if config.name == "trtllm_aggregated_multimodal":
request.getfixturevalue("image_server")
monkeypatch.setenv("MODEL", config.model)
monkeypatch.setenv("DYN_DISCOVERY_BACKEND", discovery_backend)
monkeypatch.setenv("DYN_NAMESPACE", f"sidecar-{generate_random_suffix()}")
Expand Down
Loading