Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docker/Dockerfile.multi
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ RUN --mount=type=bind,source=docker/common,target=/opt/docker/common \
NCCL_VER=${NCCL_VER} CUBLAS_VER=${CUBLAS_VER} \
TORCH_INSTALL_TYPE=${TORCH_INSTALL_TYPE} \
bash /opt/docker/common/install.sh --base --cmake --ccache --cuda_toolkit \
--tensorrt --polygraphy --mpi4py --pytorch --opencv
Comment thread
schetlur-nv marked this conversation as resolved.
Resolved
--tensorrt --polygraphy --mpi4py --pytorch
Comment thread
tburt-nv marked this conversation as resolved.
Resolved
Comment thread
chzblych marked this conversation as resolved.
Resolved

# Install constraints after install.sh so cleanup() doesn't delete the file mid-RUN
COPY constraints.txt /tmp/constraints.txt
Expand Down
13 changes: 0 additions & 13 deletions docker/common/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ tensorrt=0
polygraphy=0
mpi4py=0
pytorch=0
opencv=0

while [[ $# -gt 0 ]]; do
case $1 in
Expand Down Expand Up @@ -51,10 +50,6 @@ while [[ $# -gt 0 ]]; do
pytorch=1
shift 1
;;
--opencv)
opencv=1
shift 1
;;
--all)
base=1
cmake=1
Expand All @@ -64,7 +59,6 @@ while [[ $# -gt 0 ]]; do
polygraphy=1
mpi4py=1
pytorch=1
opencv=1
shift 1
;;
*)
Expand Down Expand Up @@ -122,10 +116,3 @@ if [ $pytorch -eq 1 ]; then
echo "Installing PyTorch..."
bash $SCRIPT_DIR/install_pytorch.sh $TORCH_INSTALL_TYPE
fi

if [ $opencv -eq 1 ]; then
echo "Installing OpenCV..."
bash -c "pip3 uninstall -y opencv"
rm -rf /usr/local/lib/python3*/dist-packages/cv2/
bash -c "pip3 install opencv-python-headless --force-reinstall --no-deps --no-cache-dir"
fi
10 changes: 10 additions & 0 deletions docs/source/features/multi-modality.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,16 @@ Further optimizations are under development and will be updated as they become a

Please refer to the latest multimodal [support matrix](../models/supported-models.md#multimodal-feature-support-matrix-pytorch-backend).

## Optional dependencies

Video input decoding uses OpenCV (`cv2`) and is not installed by default. Install it only for models with video inputs:

```bash
Comment thread
schetlur-nv marked this conversation as resolved.
Resolved
pip install opencv-python-headless
```

Image-only and text-only workflows do not require this package.

## Examples

The following examples demonstrate how to use TensorRT LLM's multimodal support in various scenarios, including quick run examples, serving endpoints, and performance benchmarking.
Expand Down
1 change: 1 addition & 0 deletions jenkins/L0_Test.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -3594,6 +3594,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
sh "cd ${llmSrc} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt"
}
trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmSrc} && pip3 install -r requirements-dev.txt")
trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install opencv-python-headless")
if (stageName.contains("-Ray-")) {
trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install ray[default]==2.55.1")
}
Expand Down
2 changes: 2 additions & 0 deletions jenkins/scripts/perf/local/slurm_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ slurm_install_setup() {

echo "(Installing TensorRT-LLM and requirements) Install mode: ${INSTALL_MODE:-source}"

retry_command pip install --retries 10 opencv-python-headless

# Support two installation modes: source (default) and wheel
if [ "${INSTALL_MODE:-source}" = "wheel" ]; then
# Wheel installation mode
Expand Down
1 change: 1 addition & 0 deletions jenkins/scripts/slurm_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ slurm_install_setup() {
if [[ $pytestCommand == *--run-ray* ]]; then
retry_command pip3 install --retries 10 "ray[default]==2.55.1"
fi
retry_command pip3 install --retries 10 opencv-python-headless
retry_command bash -c "cd $llmSrcNode && pip3 install --retries 10 -r requirements-dev.txt"
retry_command bash -c "cd $resourcePathNode && pip3 install --retries 10 --force-reinstall --no-deps TensorRT-LLM/tensorrt_llm-*.whl"
gpuUuids=$(nvidia-smi -q | grep "GPU UUID" | awk '{print $4}' | tr '\n' ',' || true)
Expand Down
1 change: 0 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ peft>=0.18.1,<0.19.0
patchelf
einops
flashinfer-python==0.6.14
opencv-python-headless
xgrammar==0.1.32
llguidance==0.7.29
jsonschema
Expand Down
22 changes: 16 additions & 6 deletions tensorrt_llm/inputs/media_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
from urllib.parse import unquote, urljoin, urlparse

import aiohttp
import lazy_loader as lazy
import numpy as np
import requests
import soundfile
Expand All @@ -43,11 +42,6 @@
from tensorrt_llm.inputs.multimodal_data import AudioData, VideoData
from tensorrt_llm.logger import logger

# Lazy import: OpenCV is large and only needed when the cv2-backed video
# decode path is exercised. The proxy triggers the actual `import cv2` on
# first attribute access (e.g. `cv2.VideoCapture`).
cv2 = lazy.load("cv2")


def rgba_to_rgb(
image: Image.Image, background_color: Union[tuple[int, int, int], list[int]] = (255, 255, 255)
Expand Down Expand Up @@ -338,6 +332,18 @@ def extract_audio_from_video(
return audio, target_sr


def _get_cv2():
"""Import OpenCV on demand for the optional cv2-backed video decode path."""
try:
import cv2
except ImportError as exc:
raise ImportError(
"OpenCV (cv2) is required for video decoding but is not installed. "
"Install it with `pip install opencv-python-headless`."
) from exc
return cv2


def _select_cv2_stream_buffered_backend() -> Optional[int]:
"""Return a VideoCapture backend that can read from a Python `BytesIO`.

Expand All @@ -351,6 +357,8 @@ def _select_cv2_stream_buffered_backend() -> Optional[int]:
open. Built-in backends (the FFMPEG path in the PyPI wheels) are always
safe to use.
"""
cv2 = _get_cv2()

# The stream-buffered API was introduced in OpenCV 4.13.0. Older builds
# don't have `cv2.videoio_registry.getStreamBufferedBackends`, so signal
# "no usable backend" and let the caller fall back to the tempfile path.
Expand Down Expand Up @@ -420,6 +428,8 @@ def _load_video_by_cv2(
"""
assert format in ("pt", "np", "pil"), "format must be one of 'pt', 'np', 'pil'"

cv2 = _get_cv2()

# Open the source. Two cases:
# (a) `video` is a file path / URL str -> hand it straight to cv2.
# (b) `video` is mp4 bytes -> feed cv2 from an in-memory BytesIO via
Expand Down
Loading