[None][feat] Cosmos3 video-to-video (V2V) generation

.gitignore

@@ -1,6 +1,7 @@
 __pycache__/
 .mypy_cache/
 .vscode
+.zed
 .cursor
 *.engine
 *.engine.config
@@ -46,14 +47,14 @@ tensorrt_llm/bindings.pyi
 tensorrt_llm/bindings/**/*.pyi
 tensorrt_llm/tensorrt_llm_transfer_agent_binding.*.so
 tensorrt_llm/tensorrt_llm_transfer_agent_binding.pyi
-tensorrt_llm/deep_ep/
+tensorrt_llm/deep_ep
 tensorrt_llm/deep_ep_cpp_tllm.*.so
 tensorrt_llm/deep_ep_cpp_tllm.pyi
-tensorrt_llm/deep_gemm/
+tensorrt_llm/deep_gemm
 tensorrt_llm/deep_gemm_cpp_tllm.*.so
 tensorrt_llm/deep_gemm_cpp_tllm.pyi
 tensorrt_llm/pg_utils_bindings.*.so
-tensorrt_llm/flash_mla/
+tensorrt_llm/flash_mla
 tensorrt_llm/flash_mla_cpp_tllm.*.so
 tensorrt_llm/flash_mla_cpp_tllm.pyi
 tensorrt_llm/runtime/kv_cache_manager_v2/**/*.so
@@ -121,3 +122,5 @@ tests/integration/defs/stress_test/artifacts/
 .claude/agent-tests/perf-test-sync/report.html
 .claude/agent-tests/perf-test-sync/results.json
 .claude/settings.json
+
+.plans/

examples/visual_gen/models/cosmos3/README.md

@@ -1,10 +1,11 @@
 # Cosmos3 Text(+Image)-to-Video(+Audio) generation
 
-Cosmos3 supports four generation modes from a single checkpoint:
+Cosmos3 supports the following generation modes from a single checkpoint:
 
 - **T2V** — text-to-video (`prompts/t2v.json`).
 - **T2I** — text-to-image (`prompts/t2i.json`); emits a still frame (use `--output_type image` / a non-video `--output_path`).
 - **I2V / TI2V** — image-conditioned video (`prompts/i2v.json`). Condition on a reference frame via the prompt file's `vision_path` or `--image_path`. The image may be a local path, a `file://` / `http(s)://` URL, or a `data:` URI.
+- **V2V** — video-conditioned video (`prompts/v2v.json`). Condition on a reference video via `--video_path` (a local MP4/AVI file). Only the first (or last, per `condition_video_keep`) `max(condition_video_latent_indexes) * 4 + 1` input frames condition the output (5 by default); the encoded bytes pass through and each worker decodes just that window on NVDEC (see [Media I/O dependencies](#media-io-dependencies)).
 - **T2AV** — text-to-video with synchronized audio (`prompts/t2av.json` with `enable_audio: true`, or pass `--enable_audio`). Combine with a `vision_path` for image-conditioned audio-video (TI2AV).
 
 ## Checkpoints
@@ -32,6 +33,11 @@ To run without guardrails (you are responsible for safe deployment):
 export TRTLLM_DISABLE_COSMOS3_GUARDRAILS=1
 ```
 
+## Media I/O dependencies
+
+- Saving `.mp4` output requires the `ffmpeg` CLI on `PATH` (`apt-get install -y ffmpeg`); without it the encoder falls back to `.avi`.
+- Decoding MP4/AVI reference videos (V2V) happens in the worker processes on NVDEC via PyNvVideoCodec, a declared TensorRT-LLM dependency — nothing extra to install. Tested combinations: H.264 in MP4 and H.264 in AVI; other containers/codecs/profiles depend on the demuxer and the GPU's NVDEC capabilities and are best-effort.
+
 ## Deployment configs
 
 See `examples/visual_gen/configs/`:
@@ -62,6 +68,15 @@ python cosmos3.py --model nvidia/Cosmos3-Nano \
     --image_path https://example.com/frame.jpg \
     --visual_gen_args ../configs/cosmos3-nano-1gpu.yaml
 
+# V2V: video-conditioned video (continues the first frames of --video_path).
+# Best results when the prompt describes the input video — e.g. continue a
+# T2V output reusing its original prompt. Output size is fixed (1280x720
+# default); inputs are center-cropped, not aspect-matched.
+python cosmos3.py --model /path/to/Cosmos3-Nano \
+    --prompt_file prompts/v2v.json \
+    --video_path /path/to/Cosmos3-Nano/assets/example_i2v_output.mp4 \
+    --visual_gen_args ../configs/cosmos3-nano-1gpu.yaml
+
 # T2AV: text-to-video with synchronized audio
 python cosmos3.py --model nvidia/Cosmos3-Nano \
     --prompt_file prompts/t2av.json \

examples/visual_gen/models/cosmos3/cosmos3.py

@@ -13,6 +13,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""Cosmos3 Text(+Image/Video)-to-Video(+Audio) generation.
+
+One checkpoint serves T2V, T2I, I2V/TI2V, V2V and T2AV; ``prompts/`` holds a
+prompt file per mode and ``--help`` lists the flags. See ``README.md`` in this
+directory for the checkpoints, guardrail setup, deployment configs, and a
+worked command line per mode.
+"""
 
 import argparse
 import json
@@ -141,10 +148,17 @@ def main():
         default=None,
         help=(
             "Prepend the Cosmos3 system prompt (--no-use_system_prompt to disable). "
-            "When omitted, the checkpoint's declared default applies."
+            "When omitted, V2V uses it and every other mode takes the checkpoint's "
+            "declared default."
         ),
     )
     parser.add_argument("--enable_audio", action="store_true", help="Enable audio generation")
+    parser.add_argument(
+        "--video_path",
+        type=str,
+        default=None,
+        help="Reference video for V2V: a local MP4/AVI file (decoded on worker NVDEC)",
+    )
     parser.add_argument(
         "--output_type", type=str, default="video", help="Output type (video, image)"
     )
@@ -193,6 +207,9 @@ def main():
     params.extra_params["use_guardrails"] = not args.disable_guardrails
     params.extra_params["output_type"] = output_type
 
+    if args.video_path is not None:
+        params.extra_params["video"] = Path(args.video_path).read_bytes()
+
     if negative_prompt is None:
         params.negative_prompt = None
     elif isinstance(negative_prompt, str):
@@ -207,6 +224,7 @@ def main():
 
     output.save(args.output_path)
     print(f"Saved: {args.output_path}")
+
     print(output.metrics)
 
 

examples/visual_gen/models/cosmos3/prompts/v2v.json

@@ -0,0 +1,4 @@
+{
+    "model_mode": "video2video",
+    "prompt": "Continue the same scene with smooth natural motion and consistent subjects."
+}

examples/visual_gen/serve/README.md

@@ -286,7 +286,8 @@ You can customize these by:
 - `frame_rate` (canonical) or `fps` (alias): frames per second
 - `num_frames`: when set, wins over the `seconds * frame_rate` derivation
 - `seed`, `num_inference_steps`, `guidance_scale`, `max_sequence_length`, `negative_prompt`: per-request denoise controls
-- `input_reference`: Reference image (TI2V mode); accepted as base64-encoded string in JSON or as a file in multipart form-data
+- `input_reference`: Reference image (I2V/TI2V) or video (V2V), accepted as a base64-encoded string in JSON or as a file in multipart form-data
+  - **Supported formats**: PNG and JPEG images; MP4 and AVI video, with H.264 the tested codec and others best-effort. HEIF/AVIF are not supported.
 - `extra_params`: model-specific overflow (see below)
 - `response_format`: `"b64_json"` or `"url"`
 - `format`: Generation content encoding. Video encoders: `"mp4"`, `"avi"`, `"auto"`. Tensor formats: `"safetensors"`, `"pt"` (carries video + audio + scalar metadata in one payload for LTX-2).
@@ -315,6 +316,7 @@ Examples:
 - **LTX-2**: `stg_scale`, `stg_blocks`, `modality_scale`, `guidance_rescale`, `output_type`, ...
 - **Wan 2.2 A14B**: `guidance_scale_2`, `boundary_ratio`
 - **Wan 2.1 / Flux**: no model-specific `extra_params` declared
+- **Cosmos3**: `condition_video_latent_indexes`, `condition_video_keep` (V2V conditioning), `flow_shift`, `use_system_prompt`, ...
 
 > **Note:** LTX-2 generates video **with audio**. The `ltx2.yml` config must include
 > `text_encoder_path` pointing to a Gemma3 model (e.g., `google/gemma-3-12b-it`).
@@ -357,6 +359,18 @@ curl -X POST "http://localhost:8000/v1/videos" \
   -F "guidance_scale=5.0"
 ```
 
+### Video-to-Video (Multipart with File Upload, Cosmos3)
+```bash
+# The reference is classified by content: image -> I2V, video -> V2V.
+# V2V conditioning knobs ride in extra_params (values below are the defaults).
+curl -X POST "http://localhost:8000/v1/videos" \
+  -F "prompt=Continue the same scene with smooth natural motion and consistent subjects." \
+  -F "input_reference=@./media/reference.mp4" \
+  -F "num_frames=189" \
+  -F "fps=24" \
+  -F 'extra_params={"condition_video_latent_indexes": [0, 1], "condition_video_keep": "first"}'
+```
+
 ### Check Video Status
 ```bash
 curl -X GET "http://localhost:8000/v1/videos/{video_id}"

requirements.txt

@@ -38,6 +38,7 @@ pydantic-settings[yaml]
 msgspec
 omegaconf
 pillow
+PyNvVideoCodec~=2.1.0
 optimum
 # evaluate needs datasets>=2.0.0 which triggers datasets>3.1.0 which is not stable: https://github.com/huggingface/datasets/issues/7467
 datasets==3.1.0

tensorrt_llm/_torch/visual_gen/executor.py

@@ -254,6 +254,11 @@ class DiffusionResponse:
             model-specific fields populated. Set to ``None`` on the error
             path; on the READY signal it carries a ``dict`` instead.
         error_msg: Error message if generation failed.
+        error_type: Failure class when ``error_msg`` is set: ``"client"``
+            (unusable request content → 400 / ``ValueError``), ``"capacity"``
+            (valid request does not fit the deployment → 503 /
+            ``MemoryError``), or ``None`` for unclassified runtime failures
+            (500 / ``RuntimeError``).
         generation: Wall-clock time the executor measured around request
             preparation and the engine's inference call (host
             ``time.perf_counter()``), in seconds. Default ``0.0`` so the
@@ -264,6 +269,7 @@ class DiffusionResponse:
     request_id: int
     output: Optional[PipelineOutput] = None
     error_msg: Optional[str] = None
+    error_type: Optional[str] = None
     generation: float = 0.0
 
 
@@ -462,7 +468,11 @@ def process_request(self, req: DiffusionRequest):
             logger.error(traceback.format_exc())
             if self.rank == 0:
                 self.response_queue.put(
-                    DiffusionResponse(request_id=req.request_id, error_msg=str(e))
+                    DiffusionResponse(
+                        request_id=req.request_id,
+                        error_msg=str(e),
+                        error_type=self.pipeline.classify_request_failure(e),
+                    )
                 )
 
 

tensorrt_llm/_torch/visual_gen/models/cosmos3/defaults.py

@@ -17,9 +17,10 @@
 Shared by the Cosmos3 OmniMoT text-to-video and image-to-video generation paths.
 """
 
-from typing import Dict
+from typing import Dict, Iterable
 
 from tensorrt_llm._torch.visual_gen.pipeline import ExtraParamSchema
+from tensorrt_llm.inputs.media_io import sniff_media_kind
 
 # ---------------------------------------------------------------------------
 # Constant tables
@@ -35,6 +36,71 @@
     "frame_rate": 24.0,
 }
 
+COSMOS3_DEFAULT_CONDITION_VIDEO_LATENT_INDEXES = (0, 1)
+COSMOS3_DEFAULT_CONDITION_VIDEO_KEEP = "first"
+
+
+# ---------------------------------------------------------------------------
+# Conditioning-value normalizers / validators. Declared as the ``validator``
+# of the matching extra-param specs below, so invalid values 400 at preflight;
+# the pipeline reuses them at run time to normalize the same inputs.
+# ---------------------------------------------------------------------------
+
+
+def _normalize_condition_video_latent_indexes(
+    indexes: Iterable[int] | None,
+) -> tuple[int, ...]:
+    if indexes is None:
+        return COSMOS3_DEFAULT_CONDITION_VIDEO_LATENT_INDEXES
+    values = []
+    for index in indexes:
+        # Strict: reject non-integers instead of silently truncating (1.9 -> 1)
+        # or TypeError-ing on None. Integral floats (JSON emitters) coerce.
+        if isinstance(index, bool) or not isinstance(index, (int, float)):
+            raise ValueError(
+                f"Cosmos3 condition_video_latent_indexes must be integers, got {index!r}."
+            )
+        if isinstance(index, float):
+            if not index.is_integer():
+                raise ValueError(
+                    f"Cosmos3 condition_video_latent_indexes must be integers, got {index!r}."
+                )
+            index = int(index)
+        values.append(index)
+    normalized = tuple(values)
+
+    if not normalized:
+        raise ValueError("Cosmos3 condition_video_latent_indexes must not be empty.")
+    if any(index < 0 for index in normalized):
+        raise ValueError(
+            f"Cosmos3 condition_video_latent_indexes must be non-negative, got {normalized}."
+        )
+    return normalized
+
+
+def _normalize_condition_video_keep(keep: str | None) -> str:
+    normalized = str(keep or COSMOS3_DEFAULT_CONDITION_VIDEO_KEEP).strip().lower()
+    if normalized not in {"first", "last"}:
+        raise ValueError("Cosmos3 condition_video_keep must be either first or last.")
+    return normalized
+
+
+def _validate_output_type(output_type: str) -> None:
+    if output_type not in ("video", "image"):
+        raise ValueError(f"Cosmos3 output_type must be 'video' or 'image', got {output_type!r}.")
+
+
+def _validate_video_reference(video) -> None:
+    """Preflight for the ``video`` extra param: encoded MP4/AVI bytes."""
+    if not video:
+        raise ValueError("Cosmos3 video reference bytes are empty.")
+    if sniff_media_kind(video) != "video":
+        raise ValueError(
+            "Cosmos3 video reference bytes are not a recognized video "
+            "container (supported: MP4/AVI)."
+        )
+
+
 # Text-to-image (``output_type="image"``) defaults; resolved in ``infer()``.
 COSMOS3_T2I_PARAMS = {
     "height": 1024,
@@ -70,8 +136,12 @@
     ),
     "use_system_prompt": ExtraParamSchema(
         type="bool",
-        default=False,
-        description="Whether to use the system prompt.",
+        default=None,
+        description=(
+            "Whether to prepend the system prompt. Unset means the model "
+            "decides: V2V uses it, other modes take the checkpoint's "
+            "declared default."
+        ),
     ),
     "use_guardrails": ExtraParamSchema(
         type="bool",
@@ -84,8 +154,43 @@
         description="Whether to enable audio generation.",
     ),
     "output_type": ExtraParamSchema(
-        type="Literal['video', 'image']",
+        type="str",
         default="video",
         description="Output modality: 'video' (T2V/I2V) or 'image' (text-to-image).",
+        validator=_validate_output_type,
+    ),
+    "condition_video_latent_indexes": ExtraParamSchema(
+        type="list",
+        default=list(COSMOS3_DEFAULT_CONDITION_VIDEO_LATENT_INDEXES),
+        description=(
+            "Latent frame indexes OF THE OUTPUT video to pin to the encoded "
+            "reference (not source-frame selection). Each latent frame spans 4 "
+            "pixel frames, so the worker consumes the first (or last, per "
+            "condition_video_keep) max(indexes)*4+1 reference frames."
+        ),
+        validator=_normalize_condition_video_latent_indexes,
+    ),
+    "condition_video_keep": ExtraParamSchema(
+        type="str",
+        default=COSMOS3_DEFAULT_CONDITION_VIDEO_KEEP,
+        description="Which side of the input video to use for conditioning: first or last.",
+        validator=_normalize_condition_video_keep,
+    ),
+    "flow_shift": ExtraParamSchema(
+        type="float",
+        default=None,
+        description="Optional scheduler flow shift override. Uses the Cosmos3 mode default when omitted.",
+    ),
+    "video": ExtraParamSchema(
+        type="bytes",
+        default=None,
+        description=(
+            "V2V reference: encoded MP4/AVI bytes (e.g. "
+            "Path(video).read_bytes()). Each worker rank demuxes them from "
+            "memory and NVDEC-decodes only the conditioning window per "
+            "condition_video_latent_indexes / condition_video_keep, resized "
+            "to the output resolution, then VAE-encodes it."
+        ),
+        validator=_validate_video_reference,
     ),
 }