From 8143c18962c39af15ba1fbe7c01fc512bd236832 Mon Sep 17 00:00:00 2001 From: chen hongwei <1792043268@qq.com> Date: Tue, 15 Sep 2026 16:02:58 +0800 Subject: [PATCH 1/9] [Frontend] MiniMax-H3: Add latent-mask editing to the ComfyUI extension Implements EDIT-02 from #7380: a Latent Mask Editing node that uploads source media and serializes video/audio noise masks to /v1/videos. Signed-off-by: chen hongwei <1792043268@qq.com> --- apps/ComfyUI-vLLM-Omni/README.md | 13 +- apps/ComfyUI-vLLM-Omni/__init__.py | 3 + .../comfyui_vllm_omni/nodes.py | 44 ++ .../comfyui_vllm_omni/utils/api_client.py | 48 ++ .../comfyui_vllm_omni/utils/latent_mask.py | 47 ++ .../comfyui_vllm_omni/utils/types.py | 4 + .../vLLM-Omni Latent Mask Editing.json | 511 ++++++++++++++++++ apps/ComfyUI-vLLM-Omni/tests/mock_server.py | 113 ++++ apps/ComfyUI-vLLM-Omni/tests/test_e2e.py | 151 ++++++ .../tests/test_latent_mask.py | 82 +++ apps/ComfyUI-vLLM-Omni/web/main.js | 1 + 11 files changed, 1016 insertions(+), 1 deletion(-) create mode 100644 apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json create mode 100644 apps/ComfyUI-vLLM-Omni/tests/mock_server.py create mode 100644 apps/ComfyUI-vLLM-Omni/tests/test_e2e.py create mode 100644 apps/ComfyUI-vLLM-Omni/tests/test_latent_mask.py diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index 0d607b30a4c..7b7410d1cbf 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -49,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C - **Generate Image** for text-to-image and image-to-image tasks - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video +- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks) - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks - **TTS** and **TTS Voice Clone** for TTS tasks @@ -66,7 +67,7 @@ Every node carries the vLLM-Omni mark in its title bar and is tinted by what it | Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model | | Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts | | Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded | -| Teal | Video References | Reference media | +| Teal | Video References, Latent Mask Editing | Reference media | Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept. @@ -173,6 +174,16 @@ Open the **vLLM-Omni FastH3 Text to Video** template, then: The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server. +#### Latent-mask editing (MiniMax-H3) + +Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts: + +- `source_video` / `source_audio` — the media to edit. +- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid. +- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend. + +A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3). + ### TTS (e.g., Qwen TTS series) (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**) diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index 55b003ef1d4..a0220a550e3 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -20,6 +20,7 @@ VLLMOmniFastH3Deployment, VLLMOmniGenerateImage, VLLMOmniGenerateVideo, + VLLMOmniLatentMaskEditing, VLLMOmniMiniMaxH3Params, VLLMOmniQwenTTSParams, VLLMOmniRemoteLoRA, @@ -40,6 +41,7 @@ "VLLMOmniTTS": VLLMOmniTTS, "VLLMOmniVoiceClone": VLLMOmniVoiceClone, "VLLMOmniVideoReferences": VLLMOmniVideoReferences, + "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing, # === Params === "VLLMOmniARSampling": VLLMOmniARSampling, "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling, @@ -60,6 +62,7 @@ "VLLMOmniTTS": "TTS (Text to Speech)", "VLLMOmniVoiceClone": "TTS Voice Cloning", "VLLMOmniVideoReferences": "Video References", + "VLLMOmniLatentMaskEditing": "Latent Mask Editing", # === Params === "VLLMOmniARSampling": "AR Sampling Params", "VLLMOmniDiffusionSampling": "Diffusion Sampling Params", diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py index df91ece606f..8431faa0253 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py @@ -14,6 +14,7 @@ AutoregressionSamplingParams, DiffusionSamplingParams, FastH3Deployment, + LatentMaskEditing, MiniMaxH3ModelSpecificParams, QwenTTSModelSpecificParams, VideoReferences, @@ -205,6 +206,7 @@ def INPUT_TYPES(cls): "lora": ("REMOTE_LORA",), "model_params": ("VIDEO_PARAMS",), "fast_h3": ("FASTH3_DEPLOYMENT",), + "latent_edit": ("LATENT_MASK_EDITING",), }, } @@ -237,6 +239,7 @@ async def generate( model_params: dict | None = None, lora: dict | None = None, fast_h3: dict | None = None, + latent_edit: dict | None = None, **kwargs, ): if kwargs: @@ -320,6 +323,7 @@ async def generate( sampling_params=sampling_params, lora=lora, model_params=model_params, + latent_edit=latent_edit, ) return (output,) @@ -952,3 +956,43 @@ def get_references( if video_2 is not None: refs["video_2"] = video_2 return (refs,) + + +class VLLMOmniLatentMaskEditing: + @classmethod + def INPUT_TYPES(cls): + return { + "required": {}, + "optional": { + "source_video": ("VIDEO",), + "source_audio": ("AUDIO",), + "video_mask": ("MASK",), + "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}), + }, + } + + RETURN_TYPES = ("LATENT_MASK_EDITING",) + RETURN_NAMES = ("latent_edit",) + FUNCTION = "get_latent_edit" + CATEGORY = "vLLM-Omni" + + def get_latent_edit( + self, + source_video: VideoInput | None = None, + source_audio: AudioInput | None = None, + video_mask: torch.Tensor | None = None, + audio_mask: float = -1.0, + **kwargs, + ): + if kwargs: + logger.info("Uncaught kwargs: %s", kwargs) + edit = LatentMaskEditing() + if source_video is not None: + edit["source_video"] = source_video + if source_audio is not None: + edit["source_audio"] = source_audio + if video_mask is not None: + edit["video_mask"] = video_mask + if audio_mask >= 0.0: + edit["audio_mask"] = audio_mask + return (edit,) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py index e051e9a09ab..9daa801fc77 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py @@ -20,6 +20,7 @@ from .format import ( audio_to_base64, + audio_to_bytes, base64_to_audio, base64_to_image_tensor, bytes_to_audio, @@ -29,6 +30,7 @@ video_to_base64, video_to_bytes, ) +from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json from .logger import get_logger, pretty_printer from .models import lookup_model_spec from .types import AudioFormat @@ -274,6 +276,7 @@ async def generate_video( sampling_params: dict | None = None, model_params: dict | None = None, lora: dict | None = None, + latent_edit: dict | None = None, spec_model: str | None = None, **extra_params, ) -> VideoInput: @@ -363,6 +366,51 @@ async def generate_video( content_type="video/mp4", ) + # === latent-mask editing (MiniMax H3) === + if latent_edit is not None: + source_video = latent_edit.get("source_video") + source_audio = latent_edit.get("source_audio") + video_mask = latent_edit.get("video_mask") + audio_mask = latent_edit.get("audio_mask") + + if video_mask is None and audio_mask is None: + raise ValueError("Latent-mask editing requires at least one mask.") + + video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item()) + audio_mask_trivial = audio_mask is None or audio_mask == 1.0 + if not video_mask_trivial and source_video is None: + raise ValueError("A non-trivial video mask requires a source video.") + if not audio_mask_trivial and source_audio is None and source_video is None: + raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.") + + if source_video is not None: + form.add_field( + "source_video", + video_to_bytes(source_video, "source.mp4"), + filename="source.mp4", + content_type="video/mp4", + ) + if source_audio is not None: + form.add_field( + "source_audio", + audio_to_bytes(source_audio, "source_audio.mp3"), + filename="source_audio.mp3", + content_type="audio/mpeg", + ) + if video_mask is not None: + mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames) + if len(mask_json) > 1024 * 1024: + form.add_field( + "video_noise_mask", + mask_json.encode("utf-8"), + filename="video-mask.json", + content_type="application/json", + ) + else: + form.add_field("video_noise_mask", mask_json) + if audio_mask is not None: + form.add_field("audio_noise_mask", scalar_mask_to_json(audio_mask)) + # === model specific params. Either use a specialized builder, or add flattened fields as-is === if model_params is not None: model_params = dict(model_params) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py new file mode 100644 index 00000000000..821bc0fb21d --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import json + +import torch +import torch.nn.functional as F + + +def scalar_mask_to_json(value: float) -> str: + if not 0.0 <= value <= 1.0: + raise ValueError(f"mask value must be in [0, 1], got {value}") + return str(value) + + +def _align_frame_count(frame_count: int) -> int: + if frame_count <= 0: + return 1 + current = int(frame_count) + while current % 17 != 5: + current += 1 + return current + + +def _video_latent_t(frame_count: int) -> int: + if frame_count <= 5: + return 2 + return ((int(frame_count) - 5) // 17) * 5 + 2 + + +def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str: + tv = _video_latent_t(_align_frame_count(num_frames)) + height = int(height) // 32 * 32 + width = int(width) // 32 * 32 + gh, gw = height // 16, width // 16 + + if mask.ndim == 2: + mask = mask.unsqueeze(0) + elif mask.ndim != 3: + raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D") + + grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1) + if grid.shape[0] == 1: + grid = grid.expand(tv, gh, gw) + elif grid.shape[0] != tv: + grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0) + return json.dumps(grid.tolist()) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py index bfba4414223..f2668c21a6a 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py @@ -34,6 +34,10 @@ class VideoReferences(dict): pass +class LatentMaskEditing(dict): + pass + + class FastH3Deployment(dict): """Descriptor for a server that fused FastH3 at startup.""" diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json new file mode 100644 index 00000000000..f89bd506dd5 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json @@ -0,0 +1,511 @@ +{ + "id": "latent-mask-editing-7380", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 280 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "SolidMask", + "pos": [ + 40, + 460 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 160, + 120 + ] + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 560, + 260 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 2, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.5 + ] + }, + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 680 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1080, + 200 + ], + "size": [ + 540, + 680 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 4, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 3, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "Partially restyle the clip and soundtrack", + "", + 160, + 120, + 24, + 1.0 + ] + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 1700, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/latent-edit", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 4, + 2, + "MASK" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/tests/mock_server.py b/apps/ComfyUI-vLLM-Omni/tests/mock_server.py new file mode 100644 index 00000000000..66eadb81412 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/tests/mock_server.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Minimal mock of vLLM-Omni's ``/v1/videos`` API for e2e tests. + +Records every received multipart field to ``MOCK_STATE_FILE`` (JSON) so a test +can assert exactly what the client serialized, and returns a synthetic MP4. +""" + +import asyncio +import json +import os +import subprocess +import uuid + +from fastapi import FastAPI, Request +from fastapi.responses import Response +from starlette.datastructures import UploadFile + +app = FastAPI() +_JOBS: dict[str, dict] = {} +_STATE_FILE = os.environ.get("MOCK_STATE_FILE") + + +def _record(fields: dict) -> None: + if _STATE_FILE: + with open(_STATE_FILE, "w") as f: + json.dump(fields, f) + + +def _synthetic_mp4() -> bytes: + return subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=320x240:rate=24:duration=1", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=32000:duration=1", + "-map", + "0:v", + "-map", + "1:a", + "-shortest", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-ar", + "32000", + "-ac", + "2", + "-movflags", + "frag_keyframe+empty_moov", + "-f", + "mp4", + "pipe:1", + ], + check=True, + capture_output=True, + ).stdout + + +@app.post("/v1/videos") +async def create_video(request: Request): + form = await request.form() + fields = {} + for key, value in form.items(): + if isinstance(value, UploadFile): + data = await value.read() + fields[key] = { + "file": True, + "size": len(data), + "filename": value.filename, + "content_type": value.content_type, + } + else: + fields[key] = str(value) + _record(fields) + + job_id = str(uuid.uuid4()) + _JOBS[job_id] = {"status": "queued"} + return {"id": job_id, "status": "queued"} + + +@app.get("/v1/videos/{job_id}") +async def get_video(job_id: str): + if job_id not in _JOBS: + return {"status": "failed", "error": "unknown job"} + return {"id": job_id, "status": "completed"} + + +@app.get("/v1/videos/{job_id}/content") +async def get_video_content(job_id: str): + if job_id not in _JOBS: + return Response(status_code=404) + mp4 = await asyncio.to_thread(_synthetic_mp4) + return Response(content=mp4, media_type="video/mp4") + + +@app.delete("/v1/videos/{job_id}") +async def delete_video(job_id: str): + _JOBS.pop(job_id, None) + return {"deleted": True} diff --git a/apps/ComfyUI-vLLM-Omni/tests/test_e2e.py b/apps/ComfyUI-vLLM-Omni/tests/test_e2e.py new file mode 100644 index 00000000000..eff4e433b3c --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/tests/test_e2e.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""End-to-end test for latent-mask editing serialization. + +Exercises ``VLLMOmniClient.generate_video`` with a ``latent_edit`` payload +against the mock ``/v1/videos`` server and asserts the multipart fields the +client produced. Requires a ComfyUI checkout (``comfy_api``) and CUDA; the +module is skipped when either is unavailable. +""" + +import asyncio +import json +import os +import socket +import subprocess +import sys +import time + +import pytest + +_EXT_ROOT = os.environ.get("VLLM_OMNI_EXT_ROOT") or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_COMFYUI_DIR = os.environ.get("COMFYUI_DIR", "") + +for _p in (_EXT_ROOT, _COMFYUI_DIR): + if _p and _p not in sys.path: + sys.path.insert(0, _p) + +try: + import comfy_api.latest # noqa: F401 # init first to avoid the input/_io circular import + import torch # noqa: F401 + from comfyui_vllm_omni.utils.api_client import VLLMOmniClient + from comfyui_vllm_omni.utils.format import bytes_to_video +except ImportError: + pytest.skip("ComfyUI / extension import unavailable", allow_module_level=True) + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available(), + reason="comfy.model_management requires CUDA", +) + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _make_source_mp4(path: str) -> None: + subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=160x120:rate=24:duration=1", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=32000:duration=1", + "-map", + "0:v", + "-map", + "1:a", + "-shortest", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-ar", + "32000", + "-ac", + "2", + path, + ], + check=True, + capture_output=True, + ) + + +@pytest.fixture +def mock_server(tmp_path): + port = _free_port() + state_file = tmp_path / "state.json" + env = dict(os.environ, MOCK_STATE_FILE=str(state_file)) + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "mock_server:app", "--host", "127.0.0.1", "--port", str(port)], + cwd=_TESTS_DIR, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.time() + 30 + while time.time() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + break + except OSError: + time.sleep(0.2) + yield f"http://127.0.0.1:{port}/v1", state_file + finally: + proc.terminate() + proc.wait(timeout=10) + + +def test_latent_edit_serialization(mock_server, tmp_path): + base_url, state_file = mock_server + source = tmp_path / "source.mp4" + _make_source_mp4(str(source)) + + with open(source, "rb") as f: + source_video = bytes_to_video(f.read()) + + mask = torch.zeros(1, 120, 160) + latent_edit = {"source_video": source_video, "video_mask": mask, "audio_mask": 0.5} + + async def run(): + client = VLLMOmniClient(base_url) + return await client.generate_video( + model="MiniMaxAI/MiniMax-H3", + prompt="restyle the clip", + width=160, + height=120, + num_frames=22, + fps=24, + latent_edit=latent_edit, + ) + + out = asyncio.run(run()) + assert out is not None + + with open(state_file) as f: + fields = json.load(f) + + assert fields["source_video"]["file"] is True + assert fields["source_video"]["content_type"] == "video/mp4" + assert fields["audio_noise_mask"] == "0.5" + + video_mask = json.loads(fields["video_noise_mask"]) + assert len(video_mask) == 7 # num_frames=22 aligns to 22 (17n+5) -> Tv = 7 + assert len(video_mask[0]) == 6 # height 120 floors to 96 (multiple of 32) -> gh 6 + assert len(video_mask[0][0]) == 10 # width 160 -> gw 10 diff --git a/apps/ComfyUI-vLLM-Omni/tests/test_latent_mask.py b/apps/ComfyUI-vLLM-Omni/tests/test_latent_mask.py new file mode 100644 index 00000000000..2bb88eb103f --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/tests/test_latent_mask.py @@ -0,0 +1,82 @@ +import importlib.util +import json +import os + +import pytest +import torch + +_EXT_ROOT = os.environ.get("VLLM_OMNI_EXT_ROOT") or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_LM_PATH = os.path.join(_EXT_ROOT, "comfyui_vllm_omni", "utils", "latent_mask.py") +_spec = importlib.util.spec_from_file_location("latent_mask", _LM_PATH) +_lm = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_lm) +scalar_mask_to_json = _lm.scalar_mask_to_json +video_mask_to_grid_json = _lm.video_mask_to_grid_json + + +def test_scalar_mask_to_json(): + assert scalar_mask_to_json(0.5) == "0.5" + assert scalar_mask_to_json(1.0) == "1.0" + with pytest.raises(ValueError): + scalar_mask_to_json(1.5) + with pytest.raises(ValueError): + scalar_mask_to_json(-0.1) + + +def test_video_mask_grid_shape(): + mask = torch.zeros(256, 448) + grid = json.loads(video_mask_to_grid_json(mask, width=448, height=256, num_frames=107)) + assert len(grid) == 2 + 5 * ((107 - 5) // 17) + assert len(grid[0]) == 256 // 16 + assert len(grid[0][0]) == 448 // 16 + + +def test_video_mask_grid_shape_non_aligned_frames(): + # 96 snaps up to 107 (17n+5), so Tv == 32, matching the server. + grid = json.loads(video_mask_to_grid_json(torch.zeros(256, 448), width=448, height=256, num_frames=96)) + assert len(grid) == 32 + + +def test_video_mask_grid_shape_short_clip(): + # num_frames <= 5 -> Tv == 2. + grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=64, height=64, num_frames=1)) + assert len(grid) == 2 + + +def test_video_mask_grid_shape_unaligned_width(): + # width=500 floors to 480 (multiple of 32) -> gw == 30, not 31. + grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=500, height=256, num_frames=22)) + assert len(grid[0][0]) == 30 + + +def test_video_mask_3d_input(): + grid = json.loads(video_mask_to_grid_json(torch.ones(1, 256, 448), width=448, height=256, num_frames=107)) + assert grid[0][0][0] == 1.0 + + +def test_video_mask_temporal(): + # 3D mask: first half 0 (preserve), second half 1 (regenerate). + mask = torch.zeros(7, 64, 64) + mask[4:] = 1.0 + grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22)) + assert len(grid) == 7 + assert all(v == 0.0 for row in grid[0] for v in row) + assert all(v == 1.0 for row in grid[-1] for v in row) + + +def test_video_mask_temporal_resize(): + # 2 temporal slices resize to tv == 7. + mask = torch.zeros(2, 64, 64) + mask[1] = 1.0 + grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22)) + assert len(grid) == 7 + + +def test_video_mask_uniform_preserved(): + grid = json.loads(video_mask_to_grid_json(torch.full((64, 64), 0.5), width=64, height=64, num_frames=107)) + assert all(v == 0.5 for row in grid for cell in row for v in cell) + + +def test_video_mask_values_in_range(): + grid = json.loads(video_mask_to_grid_json(torch.rand(128, 128), width=128, height=128, num_frames=22)) + assert all(0.0 <= v <= 1.0 for row in grid for cell in row for v in cell) diff --git a/apps/ComfyUI-vLLM-Omni/web/main.js b/apps/ComfyUI-vLLM-Omni/web/main.js index 9aba6260ea8..f8c633e1152 100644 --- a/apps/ComfyUI-vLLM-Omni/web/main.js +++ b/apps/ComfyUI-vLLM-Omni/web/main.js @@ -52,6 +52,7 @@ const FAMILY_BY_OUTPUT = { REMOTE_LORA: "deployment", FASTH3_DEPLOYMENT: "deployment", VIDEO_REFERENCES: "media", + LATENT_MASK_EDITING: "media", }; function familyOf(nodeData) { From d923d61afddba264d0ad91b1f0b8022c3a2e15db Mon Sep 17 00:00:00 2001 From: chen hongwei <1792043268@qq.com> Date: Wed, 16 Sep 2026 13:20:16 +0800 Subject: [PATCH 2/9] [Frontend] Fix H3 example duration and relocate latent-mask tests Use a supported H3 duration (4.458s -> 107 frames, on the 17n+5 lattice) in the latent-mask editing example workflow; 1.0s maps to 24 frames and is rejected. Move the latent-mask tests under tests/e2e/features/comfyui and reuse its CPU fixtures (drop the CUDA skip and ComfyUI checkout requirement). Signed-off-by: chen hongwei <1792043268@qq.com> --- .../vLLM-Omni Latent Mask Editing.json | 2 +- .../features/comfyui/mock_videos_server.py | 0 .../e2e/features/comfyui}/test_latent_mask.py | 16 ++-- .../comfyui/test_latent_mask_editing.py | 79 +++---------------- 4 files changed, 20 insertions(+), 77 deletions(-) rename apps/ComfyUI-vLLM-Omni/tests/mock_server.py => tests/e2e/features/comfyui/mock_videos_server.py (100%) rename {apps/ComfyUI-vLLM-Omni/tests => tests/e2e/features/comfyui}/test_latent_mask.py (84%) rename apps/ComfyUI-vLLM-Omni/tests/test_e2e.py => tests/e2e/features/comfyui/test_latent_mask_editing.py (52%) diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json index f89bd506dd5..4192ef4c5fc 100644 --- a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json @@ -401,7 +401,7 @@ 160, 120, 24, - 1.0 + 4.458 ] }, { diff --git a/apps/ComfyUI-vLLM-Omni/tests/mock_server.py b/tests/e2e/features/comfyui/mock_videos_server.py similarity index 100% rename from apps/ComfyUI-vLLM-Omni/tests/mock_server.py rename to tests/e2e/features/comfyui/mock_videos_server.py diff --git a/apps/ComfyUI-vLLM-Omni/tests/test_latent_mask.py b/tests/e2e/features/comfyui/test_latent_mask.py similarity index 84% rename from apps/ComfyUI-vLLM-Omni/tests/test_latent_mask.py rename to tests/e2e/features/comfyui/test_latent_mask.py index 2bb88eb103f..e208fd5b4f9 100644 --- a/apps/ComfyUI-vLLM-Omni/tests/test_latent_mask.py +++ b/tests/e2e/features/comfyui/test_latent_mask.py @@ -1,17 +1,15 @@ -import importlib.util +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Unit tests for latent-mask serialization helpers.""" + import json -import os import pytest import torch +from comfyui_vllm_omni.utils.latent_mask import scalar_mask_to_json, video_mask_to_grid_json -_EXT_ROOT = os.environ.get("VLLM_OMNI_EXT_ROOT") or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -_LM_PATH = os.path.join(_EXT_ROOT, "comfyui_vllm_omni", "utils", "latent_mask.py") -_spec = importlib.util.spec_from_file_location("latent_mask", _LM_PATH) -_lm = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(_lm) -scalar_mask_to_json = _lm.scalar_mask_to_json -video_mask_to_grid_json = _lm.video_mask_to_grid_json +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def test_scalar_mask_to_json(): diff --git a/apps/ComfyUI-vLLM-Omni/tests/test_e2e.py b/tests/e2e/features/comfyui/test_latent_mask_editing.py similarity index 52% rename from apps/ComfyUI-vLLM-Omni/tests/test_e2e.py rename to tests/e2e/features/comfyui/test_latent_mask_editing.py index eff4e433b3c..c5b6376ccd3 100644 --- a/apps/ComfyUI-vLLM-Omni/tests/test_e2e.py +++ b/tests/e2e/features/comfyui/test_latent_mask_editing.py @@ -4,9 +4,9 @@ """End-to-end test for latent-mask editing serialization. Exercises ``VLLMOmniClient.generate_video`` with a ``latent_edit`` payload -against the mock ``/v1/videos`` server and asserts the multipart fields the -client produced. Requires a ComfyUI checkout (``comfy_api``) and CUDA; the -module is skipped when either is unavailable. +against a mock ``/v1/videos`` server and asserts the multipart fields the +client produced. Runs on CPU: the ComfyUI ``comfy_api`` / ``comfy_extras`` +modules are mocked by this directory's ``conftest.py``. """ import asyncio @@ -18,26 +18,11 @@ import time import pytest +import torch +from comfy_api.input import VideoInput +from comfyui_vllm_omni.utils.api_client import VLLMOmniClient -_EXT_ROOT = os.environ.get("VLLM_OMNI_EXT_ROOT") or os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -_COMFYUI_DIR = os.environ.get("COMFYUI_DIR", "") - -for _p in (_EXT_ROOT, _COMFYUI_DIR): - if _p and _p not in sys.path: - sys.path.insert(0, _p) - -try: - import comfy_api.latest # noqa: F401 # init first to avoid the input/_io circular import - import torch # noqa: F401 - from comfyui_vllm_omni.utils.api_client import VLLMOmniClient - from comfyui_vllm_omni.utils.format import bytes_to_video -except ImportError: - pytest.skip("ComfyUI / extension import unavailable", allow_module_level=True) - -pytestmark = pytest.mark.skipif( - not torch.cuda.is_available(), - reason="comfy.model_management requires CUDA", -) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -48,51 +33,13 @@ def _free_port() -> int: return s.getsockname()[1] -def _make_source_mp4(path: str) -> None: - subprocess.run( - [ - "ffmpeg", - "-hide_banner", - "-loglevel", - "error", - "-y", - "-f", - "lavfi", - "-i", - "testsrc2=size=160x120:rate=24:duration=1", - "-f", - "lavfi", - "-i", - "sine=frequency=440:sample_rate=32000:duration=1", - "-map", - "0:v", - "-map", - "1:a", - "-shortest", - "-c:v", - "libx264", - "-pix_fmt", - "yuv420p", - "-c:a", - "aac", - "-ar", - "32000", - "-ac", - "2", - path, - ], - check=True, - capture_output=True, - ) - - @pytest.fixture def mock_server(tmp_path): port = _free_port() state_file = tmp_path / "state.json" env = dict(os.environ, MOCK_STATE_FILE=str(state_file)) proc = subprocess.Popen( - [sys.executable, "-m", "uvicorn", "mock_server:app", "--host", "127.0.0.1", "--port", str(port)], + [sys.executable, "-m", "uvicorn", "mock_videos_server:app", "--host", "127.0.0.1", "--port", str(port)], cwd=_TESTS_DIR, env=env, stdout=subprocess.DEVNULL, @@ -112,14 +59,12 @@ def mock_server(tmp_path): proc.wait(timeout=10) -def test_latent_edit_serialization(mock_server, tmp_path): +def test_latent_edit_serialization(mock_server): base_url, state_file = mock_server - source = tmp_path / "source.mp4" - _make_source_mp4(str(source)) - - with open(source, "rb") as f: - source_video = bytes_to_video(f.read()) + # A non-trivial video mask requires a source video. The mocked VideoInput + # only needs to provide ``save_to`` for the client's multipart upload. + source_video = VideoInput(b"mock_source_video") mask = torch.zeros(1, 120, 160) latent_edit = {"source_video": source_video, "video_mask": mask, "audio_mask": 0.5} From 8883ac06a47f0a7d58d0e6c76df0b1958799d569 Mon Sep 17 00:00:00 2001 From: chen hongwei <1792043268@qq.com> Date: Wed, 16 Sep 2026 22:44:35 +0800 Subject: [PATCH 3/9] [Frontend] Fix latent-mask editing client: restore import and param The merge resolution dropped the `from .latent_mask import` line and the `latent_edit` parameter from `generate_video`, leaving the latent-mask block referencing undefined names (ruff F821). Restore both. Signed-off-by: chen hongwei <1792043268@qq.com> --- apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py index 25aa723edb3..799705b7abb 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py @@ -30,6 +30,7 @@ video_to_base64, video_to_bytes, ) +from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json from .logger import get_logger, pretty_printer from .models import lookup_model_spec from .types import ( @@ -281,6 +282,7 @@ async def generate_video( sampling_params: dict | None = None, model_params: dict | None = None, lora: dict | None = None, + latent_edit: dict | None = None, spec_model: str | None = None, **extra_params, ) -> VideoInput: From 0646925c25faca042544e4971c1211992f61b85c Mon Sep 17 00:00:00 2001 From: chen hongwei <1792043268@qq.com> Date: Thu, 17 Sep 2026 23:52:43 +0800 Subject: [PATCH 4/9] [Frontend] Add latent-mask editing example workflows Add two example workflows demonstrating the two video_mask modes: - Image Mask: a 2D mask (LoadImageMask) broadcast to every frame - Temporal Mask: a 3D temporal mask (SolidMask + BatchMasksNode) for per-frame control (first half preserved, second half regenerated) Signed-off-by: chen hongwei <1792043268@qq.com> --- ...Omni Latent Mask Editing - Image Mask.json | 499 ++++++++++++++++ ...i Latent Mask Editing - Temporal Mask.json | 560 ++++++++++++++++++ 2 files changed, 1059 insertions(+) create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json new file mode 100644 index 00000000000..8320f424c43 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json @@ -0,0 +1,499 @@ +{ + "id": "latent-mask-editing-7380", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 280 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "LoadImageMask", + "pos": [ + 40, + 460 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "STRING", + "widget": { + "name": "image" + }, + "link": null + }, + { + "name": "channel", + "type": "STRING", + "widget": { + "name": "channel" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImageMask" + }, + "widgets_values": [ + "", + "red" + ] + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 560, + 260 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 2, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.5 + ] + }, + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 680 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1080, + 200 + ], + "size": [ + 540, + 680 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 4, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 3, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "Partially restyle the clip and soundtrack", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 1700, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/latent-edit", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 4, + 2, + "MASK" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json new file mode 100644 index 00000000000..ea05c362137 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json @@ -0,0 +1,560 @@ +{ + "last_node_id": 9, + "last_link_id": 7, + "nodes": [ + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 100 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "file", + "type": "STRING", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "SolidMask", + "pos": [ + 40, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0.0, + 160, + 120 + ] + }, + { + "id": 4, + "type": "SolidMask", + "pos": [ + 40, + 500 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 160, + 120 + ] + }, + { + "id": 5, + "type": "BatchMasksNode", + "pos": [ + 250, + 380 + ], + "size": [ + 200, + 100 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "masks.mask0", + "type": "MASK", + "link": 2, + "slot_index": 0 + }, + { + "name": "masks.mask1", + "type": "MASK", + "link": 3, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "BatchMasksNode" + }, + "widgets_values": [] + }, + { + "id": 6, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 450, + 180 + ], + "size": [ + 400, + 120 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "source_video", + "type": "VIDEO", + "link": 1 + }, + { + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "name": "video_mask", + "type": "MASK", + "link": 4 + }, + { + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + -1.0 + ] + }, + { + "id": 7, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 700 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 8, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 700, + 180 + ], + "size": [ + 450, + 380 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "frame", + "type": "IMAGE", + "link": null + }, + { + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "name": "lora", + "type": "LORA", + "link": null + }, + { + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 6 + }, + { + "name": "fast_h3", + "type": "FAST_H3", + "link": null + }, + { + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 5 + }, + { + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "temporal mask: regenerate the ending", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 9, + "type": "SaveVideo", + "pos": [ + 900, + 180 + ], + "size": [ + 420, + 200 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 7 + }, + { + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "name": "format", + "type": "STRING", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "temporal_mask", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 6, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 5, + 0, + "MASK" + ], + [ + 3, + 4, + 0, + 5, + 1, + "MASK" + ], + [ + 4, + 5, + 0, + 6, + 2, + "MASK" + ], + [ + 5, + 6, + 0, + 8, + 6, + "LATENT_MASK_EDITING" + ], + [ + 6, + 7, + 0, + 8, + 4, + "VIDEO_PARAMS" + ], + [ + 7, + 8, + 0, + 9, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} \ No newline at end of file From f73e8f94bbc6d85b3966650f19aacb2dbdfe54dd Mon Sep 17 00:00:00 2001 From: chen hongwei <1792043268@qq.com> Date: Fri, 18 Sep 2026 00:39:09 +0800 Subject: [PATCH 5/9] Fix example workflow JSON formatting Add trailing newline and normalize to LF for pre-commit. Signed-off-by: chen hongwei <1792043268@qq.com> --- .../vLLM-Omni Latent Mask Editing - Image Mask.json | 2 +- .../vLLM-Omni Latent Mask Editing - Temporal Mask.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json index 8320f424c43..17be731bff7 100644 --- a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json @@ -496,4 +496,4 @@ "config": {}, "extra": {}, "version": 0.4 -} \ No newline at end of file +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json index ea05c362137..b2b3a698085 100644 --- a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json @@ -557,4 +557,4 @@ "config": {}, "extra": {}, "version": 0.4 -} \ No newline at end of file +} From 932ad184df05a00e604b5525eb0d7a03d020aead Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Sun, 20 Sep 2026 19:49:39 +0800 Subject: [PATCH 6/9] [Frontend] Add H3 latent editing workflows and mask grid preview Signed-off-by: Weiming Liao --- apps/ComfyUI-vLLM-Omni/README.md | 2 + apps/ComfyUI-vLLM-Omni/__init__.py | 3 + .../comfyui_vllm_omni/mask_preview.py | 66 + .../comfyui_vllm_omni/utils/latent_mask.py | 8 +- .../docs/wf05-h3-latent-editing.md | 24 + .../vLLM-Omni H3 Latent Editing.json | 2779 +++++++++++++++++ .../features/comfyui/test_wf05_template.py | 73 + 7 files changed, 2953 insertions(+), 2 deletions(-) create mode 100644 apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/mask_preview.py create mode 100644 apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json create mode 100644 tests/e2e/features/comfyui/test_wf05_template.py diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index aca2a0070fb..a38c250828a 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -264,6 +264,8 @@ The node records which server the workflow targets; it does not start one, nor s #### Latent-mask editing (MiniMax-H3) +The [WF-05 template](example_workflows/vLLM-Omni%20H3%20Latent%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and pending real-model validation. + Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts: - `source_video` / `source_audio` — the media to edit. diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index 5df776df8b4..9e58dde646e 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -14,6 +14,7 @@ __email__ = "vllm-omni@vllm.ai" __version__ = "0.0.1" +from .comfyui_vllm_omni.mask_preview import VLLMOmniH3MaskGridPreview from .comfyui_vllm_omni.nodes import ( VLLMOmniARSampling, VLLMOmniDiffusionSampling, @@ -35,6 +36,7 @@ # A dictionary that contains all nodes you want to export with their names NODE_CLASS_MAPPINGS = { + "VLLMOmniH3MaskGridPreview": VLLMOmniH3MaskGridPreview, # === Generation === "VLLMOmniGenerateImage": VLLMOmniGenerateImage, "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo, @@ -57,6 +59,7 @@ # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { + "VLLMOmniH3MaskGridPreview": "H3 Mask Grid Preview", # === Generation === "VLLMOmniGenerateImage": "Generate Image", "VLLMOmniGenerateVideo": "Generate Video", diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/mask_preview.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/mask_preview.py new file mode 100644 index 00000000000..15114fd05f5 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/mask_preview.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import torch +import torch.nn.functional as F + +from .utils.latent_mask import video_mask_to_grid + + +class VLLMOmniH3MaskGridPreview: + """Overlay the request's static spatial mask grid on source frames.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "mask": ("MASK",), + "opacity": ("FLOAT", {"default": 0.48, "min": 0.0, "max": 1.0, "step": 0.01}), + "show_grid": ("BOOLEAN", {"default": True}), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("images",) + FUNCTION = "preview" + CATEGORY = "vLLM-Omni/Video" + DESCRIPTION = ( + "Static H3 spatial mask preview: red means regenerate. Uses input video dimensions " + "rounded down to multiples of 32; match generation dimensions before previewing. " + "Preserves source frame count and FPS. Temporal masks are not supported." + ) + + def preview(self, images, mask, opacity=0.48, show_grid=True): + if images.ndim != 4 or images.shape[0] == 0 or images.shape[-1] < 3: + raise ValueError("Expected a nonempty IMAGE batch with RGB channels.") + if mask.ndim == 3 and mask.shape[0] > 1: + if not torch.equal(mask, mask[:1].expand_as(mask)): + raise ValueError("H3 Mask Grid Preview supports static masks only, not temporal masks.") + mask = mask[:1] + if not torch.isfinite(mask).all() or mask.numel() == 0 or mask.min() < 0 or mask.max() > 1: + raise ValueError("Mask values must be finite and in [0, 1].") + height, width = (int(v) // 32 * 32 for v in images.shape[1:3]) + if min(height, width) < 32: + raise ValueError("H3 preview requires width and height of at least 32.") + grid = video_mask_to_grid(mask, width=width, height=height, num_frames=5)[0] + grid = grid.to(device=images.device, dtype=images.dtype) + cells = grid.repeat_interleave(16, 0).repeat_interleave(16, 1).unsqueeze(-1) + alpha = cells * opacity + lines = torch.zeros_like(cells) + if show_grid: + lines[::16] = 0.16 + lines[:, ::16] = 0.16 + lines = torch.where((lines > 0) & (cells > 0), 0.55, lines) + red = images.new_tensor([1.0, 32 / 255, 32 / 255]) + yellow = images.new_tensor([1.0, 220 / 255, 100 / 255]) + result = images.new_empty((images.shape[0], height, width, 3)) + # Work one frame at a time to avoid multiple full-video temporary tensors. + for index, frame in enumerate(images): + frame = frame[..., :3] + if frame.shape[:2] != (height, width): + frame = F.interpolate( + frame.permute(2, 0, 1)[None], size=(height, width), mode="bilinear", align_corners=False + )[0].permute(1, 2, 0) + result[index] = (frame * (1 - alpha) + red * alpha) * (1 - lines) + yellow * lines + return (result,) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py index 821bc0fb21d..532cf818406 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py @@ -28,7 +28,7 @@ def _video_latent_t(frame_count: int) -> int: return ((int(frame_count) - 5) // 17) * 5 + 2 -def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str: +def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor: tv = _video_latent_t(_align_frame_count(num_frames)) height = int(height) // 32 * 32 width = int(width) // 32 * 32 @@ -44,4 +44,8 @@ def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_ grid = grid.expand(tv, gh, gw) elif grid.shape[0] != tv: grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0) - return json.dumps(grid.tolist()) + return grid + + +def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str: + return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist()) diff --git a/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md new file mode 100644 index 00000000000..4debb85b678 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md @@ -0,0 +1,24 @@ +# H3 latent-mask editing (WF-05) + +Open `vLLM-Omni H3 Latent Editing` from the extension templates. It contains four independent groups: inpainting, object removal, continuation, and extension. Only inpainting is enabled initially. Set the current group's nodes to **Never**, then enable the group you want to run; do not use Bypass for disabling these examples. + +Requires EDIT-01 server support (#7465), EDIT-02 nodes (#7575), and the audio-preserving ComfyUI video output path. Both EDIT PRs are still open as of September 18, 2026. This is a workflow draft, not a claim that stock main supports latent editing. Set the URL and served model name in the selected Generate Video node. The template defaults to `http://127.0.0.1:8001/v1` and `MiniMaxAI/MiniMax-H3`. + +Upload a 1344 × 768 source clip with **107 frames at 24 FPS**. File selectors intentionally start empty and contain no machine-specific paths. All outputs use 24 FPS and a native 1344 × 768 resolution. Keep the Save Video connection to preserve the returned soundtrack. + +- **Inpainting:** upload a black/white mask using the red channel and describe the replacement. White regenerates; black preserves. The spatial mask repeats over time. +- **Object removal:** upload a mask covering the unwanted object's complete motion path and describe the background to reconstruct. A static mask is unsuitable for precise tracking; connect a tracked mask batch if needed. +- **Continuation:** retain the beginning and generate a new ending at the same 107-frame duration. The mask contains 16 preserved and 16 regenerated latent time slices. +- **Extension:** extend the 107-frame source to 209 frames. The mask contains 32 preserved and 30 regenerated latent time slices. EDIT-01 pads the source with its last frame before encoding; the generated tail replaces that padded region. + +At 24 FPS, durations `4.458` and `8.708` round to 107 and 209 frames. Both satisfy `17k+5`. Their latent time lengths are `2 + 5 * ((frames - 5) / 17)`, giving 32 and 62. The temporal mask uses these latent lengths directly to avoid resampling its boundary. If you change the source or output length, update the mask counts too. Preservation operates on encoded latents and does not promise a lossless pixel splice. + +Inpainting and removal use `audio_mask=0` and require source audio. For a silent source, set it to `1` to generate audio. Continuation and extension use `audio_mask=1`, regenerating the entire soundtrack. The current EDIT-02 node exposes only a scalar audio mask; retaining the original audio prefix while generating only its tail is not represented by this template. + +Validation command: + +```bash +python -m pytest --noconftest -o addopts='' tests/e2e/features/comfyui/test_wf05_template.py -q +``` + +This graph-only check does not need engine fixtures or distributed pytest options. The focused tests check wiring, activation defaults, frame constraints, and temporal mask construction. Local result: **4 passed**; node types and connected port names also match the running ComfyUI's `/object_info`. Targeted pre-commit checks passed. Real-model validation is pending for all four groups. Before claiming WF-05 complete, record each input/mask/output, confirm dimensions, frame count and audio presence, and review preserved regions, edited regions, temporal continuity and audio synchronization. diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json new file mode 100644 index 00000000000..4cb65c87a06 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json @@ -0,0 +1,2779 @@ +{ + "id": "d15b1bb1-a6a7-4ea9-8d79-738000000005", + "revision": 0, + "last_node_id": 43, + "last_link_id": 34, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 1740, + 230 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "WF-05 \u2014 H3 latent-mask editing", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Four independent examples. Only Inpainting is enabled. Mute its group before enabling another group (Never mode, not Bypass). Upload your source video and, for spatial edits, a black/white mask. Set the Generate Video URL and model to your EDIT-01-enabled server. Black = preserve; white = regenerate. Preservation is in latent space, not lossless pixel copying.\n\nAll examples: 24 FPS, 1344 x 768. Source clip: 107 frames at 24 FPS. Inpainting/removal/continuation output 107 frames; extension outputs 209. Save Video retains returned audio. See docs/wf05-h3-latent-editing.md for prerequisites and validation status." + ], + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 380 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ], + "title": "Inpainting \u2014 source (107 frames, 24 FPS)" + }, + { + "id": 3, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1030, + 550 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 5, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.0 + ], + "title": "Source and noise masks" + }, + { + "id": 4, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1030, + 820 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ], + "title": "H3 base settings" + }, + { + "id": 5, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1380, + 380 + ], + "size": [ + 420, + 700 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 3, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 2, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "Replace the masked region with a vase of sunflowers. Keep the rest of the scene and camera motion consistent.", + "", + 1344, + 768, + 24, + 4.458 + ], + "title": "Inpainting" + }, + { + "id": 6, + "type": "SaveVideo", + "pos": [ + 1850, + 380 + ], + "size": [ + 360, + 350 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 4, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/WF05-inpainting", + "auto", + "auto" + ], + "title": "Save video with audio" + }, + { + "type": "LoadImageMask", + "size": [ + 350, + 300 + ], + "flags": {}, + "inputs": [ + { + "name": "image", + "type": "COMBO", + "link": null + }, + { + "name": "channel", + "type": "COMBO", + "link": null + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImageMask" + }, + "id": 7, + "title": "White = edit; black = preserve", + "pos": [ + 400, + 380 + ], + "order": 6, + "mode": 0, + "widgets_values": [ + "", + "red" + ] + }, + { + "id": 8, + "type": "MarkdownNote", + "pos": [ + 1850, + 780 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Inpainting guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Upload a black/white RGB mask using the red channel. White covers the region to replace. The same mask applies to every frame; cover the full motion path or use a tracked mask batch instead. Use a source with audio: audio_mask=0 preserves its audio latents. For silent sources set audio_mask=1 to generate an audio track." + ], + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 9, + "type": "LoadVideo", + "pos": [ + 40, + 1320 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 8, + "mode": 2, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ], + "title": "Object removal \u2014 source (107 frames, 24 FPS)" + }, + { + "id": 10, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1030, + 1490 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 9, + "mode": 2, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 6, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 10, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.0 + ], + "title": "Source and noise masks" + }, + { + "id": 11, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1030, + 1760 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 10, + "mode": 2, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 8 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ], + "title": "H3 base settings" + }, + { + "id": 12, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1380, + 1320 + ], + "size": [ + 420, + 700 + ], + "flags": {}, + "order": 11, + "mode": 2, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 8, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 7, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 9 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "Remove the object inside the masked region. Reconstruct the background naturally and maintain consistent camera motion.", + "", + 1344, + 768, + 24, + 4.458 + ], + "title": "Object removal" + }, + { + "id": 13, + "type": "SaveVideo", + "pos": [ + 1850, + 1320 + ], + "size": [ + 360, + 350 + ], + "flags": {}, + "order": 12, + "mode": 2, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 9, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/WF05-removal", + "auto", + "auto" + ], + "title": "Save video with audio" + }, + { + "type": "LoadImageMask", + "size": [ + 350, + 300 + ], + "flags": {}, + "inputs": [ + { + "name": "image", + "type": "COMBO", + "link": null + }, + { + "name": "channel", + "type": "COMBO", + "link": null + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImageMask" + }, + "id": 14, + "title": "White = edit; black = preserve", + "pos": [ + 400, + 1320 + ], + "order": 13, + "mode": 2, + "widgets_values": [ + "", + "red" + ] + }, + { + "id": 15, + "type": "MarkdownNote", + "pos": [ + 1850, + 1720 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 14, + "mode": 2, + "inputs": [], + "outputs": [], + "title": "Object removal guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Upload a black/white RGB mask using the red channel. White covers the region to remove. The same mask applies to every frame; cover the full motion path or use a tracked mask batch instead. Use a source with audio: audio_mask=0 preserves its audio latents. For silent sources set audio_mask=1 to generate an audio track." + ], + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 16, + "type": "LoadVideo", + "pos": [ + 40, + 2260 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 15, + "mode": 2, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 11 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ], + "title": "Continuation \u2014 source (107 frames, 24 FPS)" + }, + { + "id": 17, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1030, + 2430 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 16, + "mode": 2, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 11, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 22, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 12 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 1.0 + ], + "title": "Source and noise masks" + }, + { + "id": 18, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1030, + 2700 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 17, + "mode": 2, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 13 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ], + "title": "H3 base settings" + }, + { + "id": 19, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1380, + 2260 + ], + "size": [ + 420, + 700 + ], + "flags": {}, + "order": 18, + "mode": 2, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 13, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 12, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 14 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "Continue the opening action into a new ending. Maintain the same subject, setting and camera motion, with natural synchronized sound.", + "", + 1344, + 768, + 24, + 4.458 + ], + "title": "Continuation" + }, + { + "id": 20, + "type": "SaveVideo", + "pos": [ + 1850, + 2260 + ], + "size": [ + 360, + 350 + ], + "flags": {}, + "order": 19, + "mode": 2, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 14, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/WF05-continuation", + "auto", + "auto" + ], + "title": "Save video with audio" + }, + { + "id": 21, + "type": "SolidMask", + "pos": [ + 400, + 2260 + ], + "size": [ + 250, + 130 + ], + "flags": {}, + "order": 20, + "mode": 2, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 15 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0.0, + 16, + 16 + ], + "title": "Preserve" + }, + { + "type": "MaskToImage", + "size": [ + 250, + 80 + ], + "flags": {}, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 15 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 16 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage" + }, + "id": 22, + "title": "Mask to image batch", + "pos": [ + 400, + 2425 + ], + "order": 21, + "mode": 2, + "widgets_values": [] + }, + { + "type": "RepeatImageBatch", + "size": [ + 270, + 130 + ], + "flags": {}, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 16 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 19 + ] + } + ], + "properties": { + "Node name for S&R": "RepeatImageBatch" + }, + "id": 23, + "title": "16 latent time slices", + "pos": [ + 700, + 2260 + ], + "order": 22, + "mode": 2, + "widgets_values": [ + 16 + ] + }, + { + "id": 24, + "type": "SolidMask", + "pos": [ + 400, + 2570 + ], + "size": [ + 250, + 130 + ], + "flags": {}, + "order": 23, + "mode": 2, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 17 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 16, + 16 + ], + "title": "Regenerate" + }, + { + "type": "MaskToImage", + "size": [ + 250, + 80 + ], + "flags": {}, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 17 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 18 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage" + }, + "id": 25, + "title": "Mask to image batch", + "pos": [ + 400, + 2735 + ], + "order": 24, + "mode": 2, + "widgets_values": [] + }, + { + "type": "RepeatImageBatch", + "size": [ + 270, + 130 + ], + "flags": {}, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 18 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 20 + ] + } + ], + "properties": { + "Node name for S&R": "RepeatImageBatch" + }, + "id": 26, + "title": "16 latent time slices", + "pos": [ + 700, + 2570 + ], + "order": 25, + "mode": 2, + "widgets_values": [ + 16 + ] + }, + { + "type": "ImageBatch", + "size": [ + 270, + 90 + ], + "flags": {}, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 19 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 20 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 21 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBatch" + }, + "id": 27, + "title": "Preserved prefix then generated suffix", + "pos": [ + 700, + 2860 + ], + "order": 26, + "mode": 2, + "widgets_values": [] + }, + { + "type": "ImageToMask", + "size": [ + 300, + 110 + ], + "flags": {}, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 21 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 22 + ] + } + ], + "properties": { + "Node name for S&R": "ImageToMask" + }, + "id": 28, + "title": "Temporal mask (red channel)", + "pos": [ + 1030, + 2890 + ], + "order": 27, + "mode": 2, + "widgets_values": [ + "red" + ] + }, + { + "id": 29, + "type": "MarkdownNote", + "pos": [ + 1850, + 2660 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 28, + "mode": 2, + "inputs": [], + "outputs": [], + "title": "Continuation guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Regenerate the ending within a 107-frame output: 32 latent time slices, first 16 preserved and last 16 regenerated. This is a latent-time boundary, not a frame-exact splice. Audio mask=1 regenerates the entire soundtrack. EDIT-02 only exposes a scalar audio mask, so this example does not preserve a prefix of the original audio." + ], + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 30, + "type": "LoadVideo", + "pos": [ + 40, + 3200 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 29, + "mode": 2, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 23 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ], + "title": "Extension \u2014 source (107 frames, 24 FPS)" + }, + { + "id": 31, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1030, + 3370 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 30, + "mode": 2, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 23, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 34, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 24 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 1.0 + ], + "title": "Source and noise masks" + }, + { + "id": 32, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1030, + 3640 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 31, + "mode": 2, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 25 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ], + "title": "H3 base settings" + }, + { + "id": 33, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1380, + 3200 + ], + "size": [ + 420, + 700 + ], + "flags": {}, + "order": 32, + "mode": 2, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 25, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 24, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 26 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "Extend the shot beyond the original ending. Continue the motion smoothly with consistent appearance and natural synchronized sound.", + "", + 1344, + 768, + 24, + 8.708 + ], + "title": "Extension" + }, + { + "id": 34, + "type": "SaveVideo", + "pos": [ + 1850, + 3200 + ], + "size": [ + 360, + 350 + ], + "flags": {}, + "order": 33, + "mode": 2, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 26, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/WF05-extension", + "auto", + "auto" + ], + "title": "Save video with audio" + }, + { + "id": 35, + "type": "SolidMask", + "pos": [ + 400, + 3200 + ], + "size": [ + 250, + 130 + ], + "flags": {}, + "order": 34, + "mode": 2, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 27 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0.0, + 16, + 16 + ], + "title": "Preserve" + }, + { + "type": "MaskToImage", + "size": [ + 250, + 80 + ], + "flags": {}, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 27 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 28 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage" + }, + "id": 36, + "title": "Mask to image batch", + "pos": [ + 400, + 3365 + ], + "order": 35, + "mode": 2, + "widgets_values": [] + }, + { + "type": "RepeatImageBatch", + "size": [ + 270, + 130 + ], + "flags": {}, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 28 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 31 + ] + } + ], + "properties": { + "Node name for S&R": "RepeatImageBatch" + }, + "id": 37, + "title": "32 latent time slices", + "pos": [ + 700, + 3200 + ], + "order": 36, + "mode": 2, + "widgets_values": [ + 32 + ] + }, + { + "id": 38, + "type": "SolidMask", + "pos": [ + 400, + 3510 + ], + "size": [ + 250, + 130 + ], + "flags": {}, + "order": 37, + "mode": 2, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 29 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 16, + 16 + ], + "title": "Regenerate" + }, + { + "type": "MaskToImage", + "size": [ + 250, + 80 + ], + "flags": {}, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 29 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 30 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage" + }, + "id": 39, + "title": "Mask to image batch", + "pos": [ + 400, + 3675 + ], + "order": 38, + "mode": 2, + "widgets_values": [] + }, + { + "type": "RepeatImageBatch", + "size": [ + 270, + 130 + ], + "flags": {}, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 30 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 32 + ] + } + ], + "properties": { + "Node name for S&R": "RepeatImageBatch" + }, + "id": 40, + "title": "30 latent time slices", + "pos": [ + 700, + 3510 + ], + "order": 39, + "mode": 2, + "widgets_values": [ + 30 + ] + }, + { + "type": "ImageBatch", + "size": [ + 270, + 90 + ], + "flags": {}, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 31 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 32 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 33 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBatch" + }, + "id": 41, + "title": "Preserved prefix then generated suffix", + "pos": [ + 700, + 3800 + ], + "order": 40, + "mode": 2, + "widgets_values": [] + }, + { + "type": "ImageToMask", + "size": [ + 300, + 110 + ], + "flags": {}, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 33 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 34 + ] + } + ], + "properties": { + "Node name for S&R": "ImageToMask" + }, + "id": 42, + "title": "Temporal mask (red channel)", + "pos": [ + 1030, + 3830 + ], + "order": 41, + "mode": 2, + "widgets_values": [ + "red" + ] + }, + { + "id": 43, + "type": "MarkdownNote", + "pos": [ + 1850, + 3600 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 42, + "mode": 2, + "inputs": [], + "outputs": [], + "title": "Extension guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Extend a 107-frame source to 209 frames: source has 32 latent time slices; output has 62. Preserve the first 32 and regenerate the remaining 30. EDIT-01 pads the short source using its last frame before encoding. Use a 107-frame source at 24 FPS; other lengths require adjusting both mask counts and output duration. Audio mask=1 regenerates the entire soundtrack. EDIT-02 only exposes a scalar audio mask, so this example does not preserve a prefix of the original audio." + ], + "color": "#333", + "bgcolor": "#222" + } + ], + "links": [ + [ + 1, + 2, + 0, + 3, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 5, + 6, + "LATENT_MASK_EDITING" + ], + [ + 3, + 4, + 0, + 5, + 4, + "VIDEO_PARAMS" + ], + [ + 4, + 5, + 0, + 6, + 0, + "VIDEO" + ], + [ + 5, + 7, + 0, + 3, + 2, + "MASK" + ], + [ + 6, + 9, + 0, + 10, + 0, + "VIDEO" + ], + [ + 7, + 10, + 0, + 12, + 6, + "LATENT_MASK_EDITING" + ], + [ + 8, + 11, + 0, + 12, + 4, + "VIDEO_PARAMS" + ], + [ + 9, + 12, + 0, + 13, + 0, + "VIDEO" + ], + [ + 10, + 14, + 0, + 10, + 2, + "MASK" + ], + [ + 11, + 16, + 0, + 17, + 0, + "VIDEO" + ], + [ + 12, + 17, + 0, + 19, + 6, + "LATENT_MASK_EDITING" + ], + [ + 13, + 18, + 0, + 19, + 4, + "VIDEO_PARAMS" + ], + [ + 14, + 19, + 0, + 20, + 0, + "VIDEO" + ], + [ + 15, + 21, + 0, + 22, + 0, + "MASK" + ], + [ + 16, + 22, + 0, + 23, + 0, + "IMAGE" + ], + [ + 17, + 24, + 0, + 25, + 0, + "MASK" + ], + [ + 18, + 25, + 0, + 26, + 0, + "IMAGE" + ], + [ + 19, + 23, + 0, + 27, + 0, + "IMAGE" + ], + [ + 20, + 26, + 0, + 27, + 1, + "IMAGE" + ], + [ + 21, + 27, + 0, + 28, + 0, + "IMAGE" + ], + [ + 22, + 28, + 0, + 17, + 2, + "MASK" + ], + [ + 23, + 30, + 0, + 31, + 0, + "VIDEO" + ], + [ + 24, + 31, + 0, + 33, + 6, + "LATENT_MASK_EDITING" + ], + [ + 25, + 32, + 0, + 33, + 4, + "VIDEO_PARAMS" + ], + [ + 26, + 33, + 0, + 34, + 0, + "VIDEO" + ], + [ + 27, + 35, + 0, + 36, + 0, + "MASK" + ], + [ + 28, + 36, + 0, + 37, + 0, + "IMAGE" + ], + [ + 29, + 38, + 0, + 39, + 0, + "MASK" + ], + [ + 30, + 39, + 0, + 40, + 0, + "IMAGE" + ], + [ + 31, + 37, + 0, + 41, + 0, + "IMAGE" + ], + [ + 32, + 40, + 0, + 41, + 1, + "IMAGE" + ], + [ + 33, + 41, + 0, + 42, + 0, + "IMAGE" + ], + [ + 34, + 42, + 0, + 31, + 2, + "MASK" + ] + ], + "groups": [ + { + "id": 1, + "title": "1. Inpainting \u2014 enabled", + "bounding": [ + 20, + 330, + 2220, + 900 + ], + "color": "#555555", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "2. Object removal \u2014 muted", + "bounding": [ + 20, + 1270, + 2220, + 900 + ], + "color": "#555555", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "3. Continuation \u2014 muted", + "bounding": [ + 20, + 2210, + 2220, + 900 + ], + "color": "#555555", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "4. Extension \u2014 muted", + "bounding": [ + 20, + 3150, + 2220, + 900 + ], + "color": "#555555", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 30, + 30 + ] + } + }, + "version": 0.4 +} diff --git a/tests/e2e/features/comfyui/test_wf05_template.py b/tests/e2e/features/comfyui/test_wf05_template.py new file mode 100644 index 00000000000..fea841d7225 --- /dev/null +++ b/tests/e2e/features/comfyui/test_wf05_template.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Portable graph and temporal-mask contracts for the WF-05 examples.""" + +import json +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +WORKFLOW = ( + Path(__file__).resolve().parents[4] / "apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json" +) + + +def test_wf05_graph_links(): + graph = json.loads(WORKFLOW.read_text()) + nodes = {node["id"]: node for node in graph["nodes"]} + assert len(nodes) == len(graph["nodes"]) + assert len({link[0] for link in graph["links"]}) == len(graph["links"]) + for link_id, source, output, target, slot, kind in graph["links"]: + assert nodes[target]["inputs"][slot]["link"] == link_id + assert link_id in nodes[source]["outputs"][output]["links"] + assert nodes[source]["outputs"][output]["type"] == kind + assert nodes[target]["inputs"][slot]["type"] == kind + assert nodes[source]["mode"] == nodes[target]["mode"] + + +def test_wf05_defaults_and_outputs(): + graph = json.loads(WORKFLOW.read_text()) + generators = [n for n in graph["nodes"] if n["type"] == "VLLMOmniGenerateVideo"] + assert len(generators) == 4 + assert [n["mode"] for n in generators] == [0, 2, 2, 2] + for node in generators: + width, height, fps, duration = node["widgets_values"][-4:] + assert (width, height, fps) == (1344, 768, 24) + frames = round(duration * fps) + assert frames % 17 == 5 + assert frames == (209 if node["title"] == "Extension" else 107) + assert next(i for i in node["inputs"] if i["name"] == "latent_edit")["link"] is not None + saves = [n for n in graph["nodes"] if n["type"] == "SaveVideo"] + assert len(saves) == 4 + assert all(n["inputs"][0]["link"] is not None for n in saves) + + +@pytest.mark.parametrize("title,counts", [("Continuation", (16, 16)), ("Extension", (32, 30))]) +def test_wf05_temporal_mask(title, counts): + graph = json.loads(WORKFLOW.read_text()) + nodes = {n["id"]: n for n in graph["nodes"]} + links = {link[0]: link for link in graph["links"]} + + def upstream(node, name): + link_id = next(i["link"] for i in node["inputs"] if i["name"] == name) + return nodes[links[link_id][1]] + + generator = next(n for n in nodes.values() if n["title"] == title) + edit = upstream(generator, "latent_edit") + assert edit["widgets_values"] == [1.0] + mask = upstream(edit, "video_mask") + assert mask["type"] == "ImageToMask" + batch = upstream(mask, "image") + assert batch["type"] == "ImageBatch" + for slot, value, count in zip(("image1", "image2"), (0.0, 1.0), counts): + repeat = upstream(batch, slot) + assert repeat["type"] == "RepeatImageBatch" + assert repeat["widgets_values"] == [count] + solid = upstream(upstream(repeat, "image"), "mask") + assert solid["type"] == "SolidMask" + assert solid["widgets_values"][0] == value + frames = round(generator["widgets_values"][-1] * 24) + assert sum(counts) == 2 + 5 * ((frames - 5) // 17) From 1ced76c84fb809421fd7e55a909e24db974a2f2a Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Mon, 21 Sep 2026 03:02:36 +0800 Subject: [PATCH 7/9] [Frontend] Refine H3 latent editing workflows and mask previews Signed-off-by: Weiming Liao --- apps/ComfyUI-vLLM-Omni/README.md | 2 +- apps/ComfyUI-vLLM-Omni/__init__.py | 6 +- .../comfyui_vllm_omni/mask_preview.py | 66 - .../comfyui_vllm_omni/temporal_mask.py | 59 + .../comfyui_vllm_omni/utils/api_client.py | 22 +- .../docs/wf05-h3-latent-editing.md | 23 +- .../vLLM-Omni H3 Latent Editing.json | 2779 --------- ...M-Omni MiniMax-H3 Latent Mask Editing.json | 5253 +++++++++++++++++ .../features/comfyui/test_wf05_template.py | 79 +- 9 files changed, 5380 insertions(+), 2909 deletions(-) delete mode 100644 apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/mask_preview.py create mode 100644 apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/temporal_mask.py delete mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index a38c250828a..fe8b895b5a3 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -264,7 +264,7 @@ The node records which server the workflow targets; it does not start one, nor s #### Latent-mask editing (MiniMax-H3) -The [WF-05 template](example_workflows/vLLM-Omni%20H3%20Latent%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and pending real-model validation. +The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations. Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts: diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index 9e58dde646e..e814b91db8e 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -14,7 +14,6 @@ __email__ = "vllm-omni@vllm.ai" __version__ = "0.0.1" -from .comfyui_vllm_omni.mask_preview import VLLMOmniH3MaskGridPreview from .comfyui_vllm_omni.nodes import ( VLLMOmniARSampling, VLLMOmniDiffusionSampling, @@ -33,10 +32,11 @@ VLLMOmniVoiceClone, VLLMOmniWanParams, ) +from .comfyui_vllm_omni.temporal_mask import VLLMOmniH3TemporalMask # A dictionary that contains all nodes you want to export with their names NODE_CLASS_MAPPINGS = { - "VLLMOmniH3MaskGridPreview": VLLMOmniH3MaskGridPreview, + "VLLMOmniH3TemporalMask": VLLMOmniH3TemporalMask, # === Generation === "VLLMOmniGenerateImage": VLLMOmniGenerateImage, "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo, @@ -59,7 +59,7 @@ # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { - "VLLMOmniH3MaskGridPreview": "H3 Mask Grid Preview", + "VLLMOmniH3TemporalMask": "MiniMax-H3 Temporal Mask", # === Generation === "VLLMOmniGenerateImage": "Generate Image", "VLLMOmniGenerateVideo": "Generate Video", diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/mask_preview.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/mask_preview.py deleted file mode 100644 index 15114fd05f5..00000000000 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/mask_preview.py +++ /dev/null @@ -1,66 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project - -import torch -import torch.nn.functional as F - -from .utils.latent_mask import video_mask_to_grid - - -class VLLMOmniH3MaskGridPreview: - """Overlay the request's static spatial mask grid on source frames.""" - - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "images": ("IMAGE",), - "mask": ("MASK",), - "opacity": ("FLOAT", {"default": 0.48, "min": 0.0, "max": 1.0, "step": 0.01}), - "show_grid": ("BOOLEAN", {"default": True}), - } - } - - RETURN_TYPES = ("IMAGE",) - RETURN_NAMES = ("images",) - FUNCTION = "preview" - CATEGORY = "vLLM-Omni/Video" - DESCRIPTION = ( - "Static H3 spatial mask preview: red means regenerate. Uses input video dimensions " - "rounded down to multiples of 32; match generation dimensions before previewing. " - "Preserves source frame count and FPS. Temporal masks are not supported." - ) - - def preview(self, images, mask, opacity=0.48, show_grid=True): - if images.ndim != 4 or images.shape[0] == 0 or images.shape[-1] < 3: - raise ValueError("Expected a nonempty IMAGE batch with RGB channels.") - if mask.ndim == 3 and mask.shape[0] > 1: - if not torch.equal(mask, mask[:1].expand_as(mask)): - raise ValueError("H3 Mask Grid Preview supports static masks only, not temporal masks.") - mask = mask[:1] - if not torch.isfinite(mask).all() or mask.numel() == 0 or mask.min() < 0 or mask.max() > 1: - raise ValueError("Mask values must be finite and in [0, 1].") - height, width = (int(v) // 32 * 32 for v in images.shape[1:3]) - if min(height, width) < 32: - raise ValueError("H3 preview requires width and height of at least 32.") - grid = video_mask_to_grid(mask, width=width, height=height, num_frames=5)[0] - grid = grid.to(device=images.device, dtype=images.dtype) - cells = grid.repeat_interleave(16, 0).repeat_interleave(16, 1).unsqueeze(-1) - alpha = cells * opacity - lines = torch.zeros_like(cells) - if show_grid: - lines[::16] = 0.16 - lines[:, ::16] = 0.16 - lines = torch.where((lines > 0) & (cells > 0), 0.55, lines) - red = images.new_tensor([1.0, 32 / 255, 32 / 255]) - yellow = images.new_tensor([1.0, 220 / 255, 100 / 255]) - result = images.new_empty((images.shape[0], height, width, 3)) - # Work one frame at a time to avoid multiple full-video temporary tensors. - for index, frame in enumerate(images): - frame = frame[..., :3] - if frame.shape[:2] != (height, width): - frame = F.interpolate( - frame.permute(2, 0, 1)[None], size=(height, width), mode="bilinear", align_corners=False - )[0].permute(1, 2, 0) - result[index] = (frame * (1 - alpha) + red * alpha) * (1 - lines) + yellow * lines - return (result,) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/temporal_mask.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/temporal_mask.py new file mode 100644 index 00000000000..f9a5a4b92fd --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/temporal_mask.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import math + +import torch + +from .utils.latent_mask import _align_frame_count, _video_latent_t + + +class VLLMOmniH3TemporalMask: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}), + "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}), + "mode": (["continuation", "extension"],), + "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}), + } + } + + RETURN_TYPES = ("MASK", "STRING", "FLOAT", "IMAGE", "MASK") + RETURN_NAMES = ("mask", "summary", "preview_fps", "preview_images", "preview_mask") + FUNCTION = "build" + CATEGORY = "vLLM-Omni" + + def build(self, images, source_fps, duration, mode, preserve_fraction=0.5): + if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0: + raise ValueError("Source must contain frames and have a positive finite FPS.") + if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1: + raise ValueError("Invalid duration or preserve_fraction.") + frames = _align_frame_count(max(1, round(duration * 24))) + source_seconds = images.shape[0] / source_fps + if mode == "extension": + if frames / 24 <= source_seconds: + raise ValueError("Extension output must be longer than the source; increase duration.") + boundary = source_seconds + elif mode == "continuation": + boundary = min(source_seconds, frames / 24) * preserve_fraction + else: + raise ValueError(f"Unknown temporal mask mode: {mode}") + available = min(frames, math.floor(boundary * 24 + 1e-8)) + prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17) + preserved = _video_latent_t(prefix) if prefix else 0 + total = _video_latent_t(frames) + mask = torch.ones(total, 1, 1) + mask[:preserved] = 0 + indices = torch.arange(frames, device=images.device) + source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1) + preview_images = images.index_select(0, source_indices) + preview_mask = mask.index_select(0, torch.arange(frames) * total // frames) + summary = ( + f"Source: {source_seconds:.3f}s; target: {frames} frames / {total} latent slices; " + f"preserve: {preserved}; generate: {total - preserved}; " + f"requested boundary: {boundary:.3f}s; conservative prefix: {prefix / 24:.3f}s." + ) + return mask, summary, 24.0, preview_images, preview_mask diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py index 799705b7abb..e1825ef2f55 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py @@ -396,17 +396,19 @@ async def generate_video( ) if video_mask is not None: mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames) - if len(mask_json) > 1024 * 1024: - form.add_field( - "video_noise_mask", - mask_json.encode("utf-8"), - filename="video-mask.json", - content_type="application/json", - ) - else: - form.add_field("video_noise_mask", mask_json) + form.add_field( + "video_noise_mask", + mask_json.encode("utf-8"), + filename="video-mask.json", + content_type="application/json", + ) if audio_mask is not None: - form.add_field("audio_noise_mask", scalar_mask_to_json(audio_mask)) + form.add_field( + "audio_noise_mask", + scalar_mask_to_json(audio_mask).encode("utf-8"), + filename="audio-mask.json", + content_type="application/json", + ) # === model specific params. Either use a specialized builder, or add flattened fields as-is === if model_params is not None: diff --git a/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md index 4debb85b678..549d92993ca 100644 --- a/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md +++ b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md @@ -1,24 +1,19 @@ # H3 latent-mask editing (WF-05) -Open `vLLM-Omni H3 Latent Editing` from the extension templates. It contains four independent groups: inpainting, object removal, continuation, and extension. Only inpainting is enabled initially. Set the current group's nodes to **Never**, then enable the group you want to run; do not use Bypass for disabling these examples. +Open `vLLM-Omni MiniMax-H3 Latent Mask Editing`. Upload a source video. Each spatial case has an independent mask that is built with official Solid Mask and Combine Masks nodes; no mask image is required. Configure a compatible H3 server URL and model in each generation node. The workflow uses the EDIT-02 latent-edit interface. -Requires EDIT-01 server support (#7465), EDIT-02 nodes (#7575), and the audio-preserving ComfyUI video output path. Both EDIT PRs are still open as of September 18, 2026. This is a workflow draft, not a claim that stock main supports latent editing. Set the URL and served model name in the selected Generate Video node. The template defaults to `http://127.0.0.1:8001/v1` and `MiniMaxAI/MiniMax-H3`. +Each case has a Mask Preview above its Result in the Output Hub. Select the desired Save Video output and execute only that output to preview locally without running generation. All four cases are enabled; running the whole graph also submits generation requests. -Upload a 1344 × 768 source clip with **107 frames at 24 FPS**. File selectors intentionally start empty and contain no machine-specific paths. All outputs use 24 FPS and a native 1344 × 768 resolution. Keep the Save Video connection to preserve the returned soundtrack. +Set output dimensions and duration directly in Generate Video. For continuation and extension, also update the duration in MiniMax-H3 Temporal Mask. Defaults are 1344 x 768 at 24 FPS, with 5 seconds for removal, inpainting and continuation, and 10 seconds for extension. -- **Inpainting:** upload a black/white mask using the red channel and describe the replacement. White regenerates; black preserves. The spatial mask repeats over time. -- **Object removal:** upload a mask covering the unwanted object's complete motion path and describe the background to reconstruct. A static mask is unsuitable for precise tracking; connect a tracked mask batch if needed. -- **Continuation:** retain the beginning and generate a new ending at the same 107-frame duration. The mask contains 16 preserved and 16 regenerated latent time slices. -- **Extension:** extend the 107-frame source to 209 frames. The mask contains 32 preserved and 30 regenerated latent time slices. EDIT-01 pads the source with its last frame before encoding; the generated tail replaces that padded region. +Preview uses official ComfyUI nodes only. Spatial cases use Empty Image, Image Composite Masked and Image Blend to overlay the original mask in red on source frames, followed by Create Video and Save Video. This preview follows the source resolution, FPS and duration and does not visualize the model's downsampled latent mask. Temporal cases use the same official red-overlay chain. The temporal node samples at 24 FPS and holds the last source frame for extension. With a 24 FPS source, existing frames are selected directly. Preserved frames remain unchanged; generated regions receive a red tint. This is a schematic timeline, not exact decoded-frame boundaries. Neither preview runs inference; previews are silent. -At 24 FPS, durations `4.458` and `8.708` round to 107 and 209 frames. Both satisfy `17k+5`. Their latent time lengths are `2 + 5 * ((frames - 5) / 17)`, giving 32 and 62. The temporal mask uses these latent lengths directly to avoid resampling its boundary. If you change the source or output length, update the mask counts too. Preservation operates on encoded latents and does not promise a lossless pixel splice. +MiniMax-H3 Temporal Mask calculates target frames and latent slice counts from the decoded source video. Continuation preserves the selected fraction of the shorter source/target duration. Extension preserves the source prefix and requires a longer target. The target rounds up to `17k+5` frames; preservation rounds down to complete native chunks. Set the same duration in this node and Generate Video. The node also supplies source frames and a schematic per-frame mask to the official red-overlay preview nodes. -Inpainting and removal use `audio_mask=0` and require source audio. For a silent source, set it to `1` to generate audio. Continuation and extension use `audio_mask=1`, regenerating the entire soundtrack. The current EDIT-02 node exposes only a scalar audio mask; retaining the original audio prefix while generating only its tail is not represented by this template. +Audio masks remain whole-clip scalars. Zero preserves encoded source audio, one regenerates it, and intermediate values partially edit the audio latents. Neither latent video nor audio preservation guarantees lossless reconstruction. -Validation command: +The exported workflow has an empty video selector. The default rectangle is x=1074, y=493, width=180, height=168 on a 1344 x 768 canvas, including 32 pixels of background margin around the original bag mask bounding box. Adjust Mask Canvas to the source dimensions, Mask Rectangle to the object size, and Combine Masks x/y to its position. A filled rectangle may edit more background than the original shaped mask. Provide a matching school clip or adapt the rectangle and prompts to your assets. Reopen the workflow without saving an older canvas over it. -```bash -python -m pytest --noconftest -o addopts='' tests/e2e/features/comfyui/test_wf05_template.py -q -``` +Graph tests check wiring, matching preview/generation defaults, mask sources, vertical output placement and portable inputs. They do not establish real-model visual/audio acceptance. -This graph-only check does not need engine fixtures or distributed pytest options. The focused tests check wiring, activation defaults, frame constraints, and temporal mask construction. Local result: **4 passed**; node types and connected port names also match the running ComfyUI's `/object_info`. Targeted pre-commit checks passed. Real-model validation is pending for all four groups. Before claiming WF-05 complete, record each input/mask/output, confirm dimensions, frame count and audio presence, and review preserved regions, edited regions, temporal continuity and audio synchronization. +The example prompts use wall reconstruction, a curled cat with a swaying tail, heavy rain with distant lightning, and a summer-night firework. Current audio masks are 0, 0, 0.8, and 0.9 respectively; Continuation preserves 0.2 of the shorter source/target duration. diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json deleted file mode 100644 index 4cb65c87a06..00000000000 --- a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json +++ /dev/null @@ -1,2779 +0,0 @@ -{ - "id": "d15b1bb1-a6a7-4ea9-8d79-738000000005", - "revision": 0, - "last_node_id": 43, - "last_link_id": 34, - "nodes": [ - { - "id": 1, - "type": "MarkdownNote", - "pos": [ - 40, - 20 - ], - "size": [ - 1740, - 230 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "WF-05 \u2014 H3 latent-mask editing", - "properties": { - "Node name for S&R": "MarkdownNote" - }, - "widgets_values": [ - "Four independent examples. Only Inpainting is enabled. Mute its group before enabling another group (Never mode, not Bypass). Upload your source video and, for spatial edits, a black/white mask. Set the Generate Video URL and model to your EDIT-01-enabled server. Black = preserve; white = regenerate. Preservation is in latent space, not lossless pixel copying.\n\nAll examples: 24 FPS, 1344 x 768. Source clip: 107 frames at 24 FPS. Inpainting/removal/continuation output 107 frames; extension outputs 209. Save Video retains returned audio. See docs/wf05-h3-latent-editing.md for prerequisites and validation status." - ], - "color": "#333", - "bgcolor": "#222" - }, - { - "id": 2, - "type": "LoadVideo", - "pos": [ - 40, - 380 - ], - "size": [ - 300, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "localized_name": "file", - "name": "file", - "type": "COMBO", - "widget": { - "name": "file" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [ - 1 - ] - } - ], - "properties": { - "Node name for S&R": "LoadVideo" - }, - "widgets_values": [ - "" - ], - "title": "Inpainting \u2014 source (107 frames, 24 FPS)" - }, - { - "id": 3, - "type": "VLLMOmniLatentMaskEditing", - "pos": [ - 1030, - 550 - ], - "size": [ - 300, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "localized_name": "source_video", - "name": "source_video", - "type": "VIDEO", - "link": 1, - "shape": 7 - }, - { - "localized_name": "source_audio", - "name": "source_audio", - "type": "AUDIO", - "link": null - }, - { - "localized_name": "video_mask", - "name": "video_mask", - "type": "MASK", - "link": 5, - "shape": 7 - }, - { - "localized_name": "audio_mask", - "name": "audio_mask", - "type": "FLOAT", - "widget": { - "name": "audio_mask" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "latent_edit", - "name": "latent_edit", - "type": "LATENT_MASK_EDITING", - "links": [ - 2 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniLatentMaskEditing" - }, - "widgets_values": [ - 0.0 - ], - "title": "Source and noise masks" - }, - { - "id": 4, - "type": "VLLMOmniMiniMaxH3Params", - "pos": [ - 1030, - 820 - ], - "size": [ - 300, - 130 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "localized_name": "audio_flow_shift", - "name": "audio_flow_shift", - "type": "FLOAT", - "widget": { - "name": "audio_flow_shift" - }, - "link": null - }, - { - "localized_name": "flow_shift", - "name": "flow_shift", - "type": "FLOAT", - "widget": { - "name": "flow_shift" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video params", - "name": "video params", - "type": "VIDEO_PARAMS", - "links": [ - 3 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniMiniMaxH3Params" - }, - "widgets_values": [ - 3.0, - 12.0 - ], - "title": "H3 base settings" - }, - { - "id": 5, - "type": "VLLMOmniGenerateVideo", - "pos": [ - 1380, - 380 - ], - "size": [ - 420, - 700 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "localized_name": "frame", - "name": "frame", - "type": "IMAGE", - "link": null, - "shape": 7 - }, - { - "localized_name": "references", - "name": "references", - "type": "VIDEO_REFERENCES", - "link": null, - "shape": 7 - }, - { - "localized_name": "sampling_params", - "name": "sampling_params", - "type": "SAMPLING_PARAMS", - "link": null - }, - { - "localized_name": "lora", - "name": "lora", - "type": "REMOTE_LORA", - "link": null - }, - { - "localized_name": "model_params", - "name": "model_params", - "type": "VIDEO_PARAMS", - "link": 3, - "shape": 7 - }, - { - "localized_name": "fast_h3", - "name": "fast_h3", - "type": "FASTH3_DEPLOYMENT", - "link": null - }, - { - "localized_name": "latent_edit", - "name": "latent_edit", - "type": "LATENT_MASK_EDITING", - "link": 2, - "shape": 7 - }, - { - "localized_name": "url", - "name": "url", - "type": "STRING", - "widget": { - "name": "url" - }, - "link": null - }, - { - "localized_name": "model", - "name": "model", - "type": "STRING", - "widget": { - "name": "model" - }, - "link": null - }, - { - "localized_name": "prompt", - "name": "prompt", - "type": "STRING", - "widget": { - "name": "prompt" - }, - "link": null - }, - { - "localized_name": "negative_prompt", - "name": "negative_prompt", - "type": "STRING", - "widget": { - "name": "negative_prompt" - }, - "link": null - }, - { - "localized_name": "width", - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": null - }, - { - "localized_name": "height", - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": null - }, - { - "localized_name": "fps", - "name": "fps", - "type": "INT", - "widget": { - "name": "fps" - }, - "link": null - }, - { - "localized_name": "duration", - "name": "duration", - "type": "FLOAT", - "widget": { - "name": "duration" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [ - 4 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniGenerateVideo" - }, - "widgets_values": [ - "http://127.0.0.1:8001/v1", - "MiniMaxAI/MiniMax-H3", - "Replace the masked region with a vase of sunflowers. Keep the rest of the scene and camera motion consistent.", - "", - 1344, - 768, - 24, - 4.458 - ], - "title": "Inpainting" - }, - { - "id": 6, - "type": "SaveVideo", - "pos": [ - 1850, - 380 - ], - "size": [ - 360, - 350 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "link": 4, - "shape": 7 - }, - { - "localized_name": "filename_prefix", - "name": "filename_prefix", - "type": "STRING", - "widget": { - "name": "filename_prefix" - }, - "link": null - }, - { - "localized_name": "format", - "name": "format", - "type": "COMFY_DYNAMICCOMBO_V3", - "widget": { - "name": "format" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [] - } - ], - "properties": { - "Node name for S&R": "SaveVideo" - }, - "widgets_values": [ - "video/WF05-inpainting", - "auto", - "auto" - ], - "title": "Save video with audio" - }, - { - "type": "LoadImageMask", - "size": [ - 350, - 300 - ], - "flags": {}, - "inputs": [ - { - "name": "image", - "type": "COMBO", - "link": null - }, - { - "name": "channel", - "type": "COMBO", - "link": null - } - ], - "outputs": [ - { - "name": "MASK", - "type": "MASK", - "links": [ - 5 - ] - } - ], - "properties": { - "Node name for S&R": "LoadImageMask" - }, - "id": 7, - "title": "White = edit; black = preserve", - "pos": [ - 400, - 380 - ], - "order": 6, - "mode": 0, - "widgets_values": [ - "", - "red" - ] - }, - { - "id": 8, - "type": "MarkdownNote", - "pos": [ - 1850, - 780 - ], - "size": [ - 360, - 330 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "Inpainting guide", - "properties": { - "Node name for S&R": "MarkdownNote" - }, - "widgets_values": [ - "Upload a black/white RGB mask using the red channel. White covers the region to replace. The same mask applies to every frame; cover the full motion path or use a tracked mask batch instead. Use a source with audio: audio_mask=0 preserves its audio latents. For silent sources set audio_mask=1 to generate an audio track." - ], - "color": "#333", - "bgcolor": "#222" - }, - { - "id": 9, - "type": "LoadVideo", - "pos": [ - 40, - 1320 - ], - "size": [ - 300, - 200 - ], - "flags": {}, - "order": 8, - "mode": 2, - "inputs": [ - { - "localized_name": "file", - "name": "file", - "type": "COMBO", - "widget": { - "name": "file" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [ - 6 - ] - } - ], - "properties": { - "Node name for S&R": "LoadVideo" - }, - "widgets_values": [ - "" - ], - "title": "Object removal \u2014 source (107 frames, 24 FPS)" - }, - { - "id": 10, - "type": "VLLMOmniLatentMaskEditing", - "pos": [ - 1030, - 1490 - ], - "size": [ - 300, - 200 - ], - "flags": {}, - "order": 9, - "mode": 2, - "inputs": [ - { - "localized_name": "source_video", - "name": "source_video", - "type": "VIDEO", - "link": 6, - "shape": 7 - }, - { - "localized_name": "source_audio", - "name": "source_audio", - "type": "AUDIO", - "link": null - }, - { - "localized_name": "video_mask", - "name": "video_mask", - "type": "MASK", - "link": 10, - "shape": 7 - }, - { - "localized_name": "audio_mask", - "name": "audio_mask", - "type": "FLOAT", - "widget": { - "name": "audio_mask" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "latent_edit", - "name": "latent_edit", - "type": "LATENT_MASK_EDITING", - "links": [ - 7 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniLatentMaskEditing" - }, - "widgets_values": [ - 0.0 - ], - "title": "Source and noise masks" - }, - { - "id": 11, - "type": "VLLMOmniMiniMaxH3Params", - "pos": [ - 1030, - 1760 - ], - "size": [ - 300, - 130 - ], - "flags": {}, - "order": 10, - "mode": 2, - "inputs": [ - { - "localized_name": "audio_flow_shift", - "name": "audio_flow_shift", - "type": "FLOAT", - "widget": { - "name": "audio_flow_shift" - }, - "link": null - }, - { - "localized_name": "flow_shift", - "name": "flow_shift", - "type": "FLOAT", - "widget": { - "name": "flow_shift" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video params", - "name": "video params", - "type": "VIDEO_PARAMS", - "links": [ - 8 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniMiniMaxH3Params" - }, - "widgets_values": [ - 3.0, - 12.0 - ], - "title": "H3 base settings" - }, - { - "id": 12, - "type": "VLLMOmniGenerateVideo", - "pos": [ - 1380, - 1320 - ], - "size": [ - 420, - 700 - ], - "flags": {}, - "order": 11, - "mode": 2, - "inputs": [ - { - "localized_name": "frame", - "name": "frame", - "type": "IMAGE", - "link": null, - "shape": 7 - }, - { - "localized_name": "references", - "name": "references", - "type": "VIDEO_REFERENCES", - "link": null, - "shape": 7 - }, - { - "localized_name": "sampling_params", - "name": "sampling_params", - "type": "SAMPLING_PARAMS", - "link": null - }, - { - "localized_name": "lora", - "name": "lora", - "type": "REMOTE_LORA", - "link": null - }, - { - "localized_name": "model_params", - "name": "model_params", - "type": "VIDEO_PARAMS", - "link": 8, - "shape": 7 - }, - { - "localized_name": "fast_h3", - "name": "fast_h3", - "type": "FASTH3_DEPLOYMENT", - "link": null - }, - { - "localized_name": "latent_edit", - "name": "latent_edit", - "type": "LATENT_MASK_EDITING", - "link": 7, - "shape": 7 - }, - { - "localized_name": "url", - "name": "url", - "type": "STRING", - "widget": { - "name": "url" - }, - "link": null - }, - { - "localized_name": "model", - "name": "model", - "type": "STRING", - "widget": { - "name": "model" - }, - "link": null - }, - { - "localized_name": "prompt", - "name": "prompt", - "type": "STRING", - "widget": { - "name": "prompt" - }, - "link": null - }, - { - "localized_name": "negative_prompt", - "name": "negative_prompt", - "type": "STRING", - "widget": { - "name": "negative_prompt" - }, - "link": null - }, - { - "localized_name": "width", - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": null - }, - { - "localized_name": "height", - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": null - }, - { - "localized_name": "fps", - "name": "fps", - "type": "INT", - "widget": { - "name": "fps" - }, - "link": null - }, - { - "localized_name": "duration", - "name": "duration", - "type": "FLOAT", - "widget": { - "name": "duration" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [ - 9 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniGenerateVideo" - }, - "widgets_values": [ - "http://127.0.0.1:8001/v1", - "MiniMaxAI/MiniMax-H3", - "Remove the object inside the masked region. Reconstruct the background naturally and maintain consistent camera motion.", - "", - 1344, - 768, - 24, - 4.458 - ], - "title": "Object removal" - }, - { - "id": 13, - "type": "SaveVideo", - "pos": [ - 1850, - 1320 - ], - "size": [ - 360, - 350 - ], - "flags": {}, - "order": 12, - "mode": 2, - "inputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "link": 9, - "shape": 7 - }, - { - "localized_name": "filename_prefix", - "name": "filename_prefix", - "type": "STRING", - "widget": { - "name": "filename_prefix" - }, - "link": null - }, - { - "localized_name": "format", - "name": "format", - "type": "COMFY_DYNAMICCOMBO_V3", - "widget": { - "name": "format" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [] - } - ], - "properties": { - "Node name for S&R": "SaveVideo" - }, - "widgets_values": [ - "video/WF05-removal", - "auto", - "auto" - ], - "title": "Save video with audio" - }, - { - "type": "LoadImageMask", - "size": [ - 350, - 300 - ], - "flags": {}, - "inputs": [ - { - "name": "image", - "type": "COMBO", - "link": null - }, - { - "name": "channel", - "type": "COMBO", - "link": null - } - ], - "outputs": [ - { - "name": "MASK", - "type": "MASK", - "links": [ - 10 - ] - } - ], - "properties": { - "Node name for S&R": "LoadImageMask" - }, - "id": 14, - "title": "White = edit; black = preserve", - "pos": [ - 400, - 1320 - ], - "order": 13, - "mode": 2, - "widgets_values": [ - "", - "red" - ] - }, - { - "id": 15, - "type": "MarkdownNote", - "pos": [ - 1850, - 1720 - ], - "size": [ - 360, - 330 - ], - "flags": {}, - "order": 14, - "mode": 2, - "inputs": [], - "outputs": [], - "title": "Object removal guide", - "properties": { - "Node name for S&R": "MarkdownNote" - }, - "widgets_values": [ - "Upload a black/white RGB mask using the red channel. White covers the region to remove. The same mask applies to every frame; cover the full motion path or use a tracked mask batch instead. Use a source with audio: audio_mask=0 preserves its audio latents. For silent sources set audio_mask=1 to generate an audio track." - ], - "color": "#333", - "bgcolor": "#222" - }, - { - "id": 16, - "type": "LoadVideo", - "pos": [ - 40, - 2260 - ], - "size": [ - 300, - 200 - ], - "flags": {}, - "order": 15, - "mode": 2, - "inputs": [ - { - "localized_name": "file", - "name": "file", - "type": "COMBO", - "widget": { - "name": "file" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [ - 11 - ] - } - ], - "properties": { - "Node name for S&R": "LoadVideo" - }, - "widgets_values": [ - "" - ], - "title": "Continuation \u2014 source (107 frames, 24 FPS)" - }, - { - "id": 17, - "type": "VLLMOmniLatentMaskEditing", - "pos": [ - 1030, - 2430 - ], - "size": [ - 300, - 200 - ], - "flags": {}, - "order": 16, - "mode": 2, - "inputs": [ - { - "localized_name": "source_video", - "name": "source_video", - "type": "VIDEO", - "link": 11, - "shape": 7 - }, - { - "localized_name": "source_audio", - "name": "source_audio", - "type": "AUDIO", - "link": null - }, - { - "localized_name": "video_mask", - "name": "video_mask", - "type": "MASK", - "link": 22, - "shape": 7 - }, - { - "localized_name": "audio_mask", - "name": "audio_mask", - "type": "FLOAT", - "widget": { - "name": "audio_mask" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "latent_edit", - "name": "latent_edit", - "type": "LATENT_MASK_EDITING", - "links": [ - 12 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniLatentMaskEditing" - }, - "widgets_values": [ - 1.0 - ], - "title": "Source and noise masks" - }, - { - "id": 18, - "type": "VLLMOmniMiniMaxH3Params", - "pos": [ - 1030, - 2700 - ], - "size": [ - 300, - 130 - ], - "flags": {}, - "order": 17, - "mode": 2, - "inputs": [ - { - "localized_name": "audio_flow_shift", - "name": "audio_flow_shift", - "type": "FLOAT", - "widget": { - "name": "audio_flow_shift" - }, - "link": null - }, - { - "localized_name": "flow_shift", - "name": "flow_shift", - "type": "FLOAT", - "widget": { - "name": "flow_shift" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video params", - "name": "video params", - "type": "VIDEO_PARAMS", - "links": [ - 13 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniMiniMaxH3Params" - }, - "widgets_values": [ - 3.0, - 12.0 - ], - "title": "H3 base settings" - }, - { - "id": 19, - "type": "VLLMOmniGenerateVideo", - "pos": [ - 1380, - 2260 - ], - "size": [ - 420, - 700 - ], - "flags": {}, - "order": 18, - "mode": 2, - "inputs": [ - { - "localized_name": "frame", - "name": "frame", - "type": "IMAGE", - "link": null, - "shape": 7 - }, - { - "localized_name": "references", - "name": "references", - "type": "VIDEO_REFERENCES", - "link": null, - "shape": 7 - }, - { - "localized_name": "sampling_params", - "name": "sampling_params", - "type": "SAMPLING_PARAMS", - "link": null - }, - { - "localized_name": "lora", - "name": "lora", - "type": "REMOTE_LORA", - "link": null - }, - { - "localized_name": "model_params", - "name": "model_params", - "type": "VIDEO_PARAMS", - "link": 13, - "shape": 7 - }, - { - "localized_name": "fast_h3", - "name": "fast_h3", - "type": "FASTH3_DEPLOYMENT", - "link": null - }, - { - "localized_name": "latent_edit", - "name": "latent_edit", - "type": "LATENT_MASK_EDITING", - "link": 12, - "shape": 7 - }, - { - "localized_name": "url", - "name": "url", - "type": "STRING", - "widget": { - "name": "url" - }, - "link": null - }, - { - "localized_name": "model", - "name": "model", - "type": "STRING", - "widget": { - "name": "model" - }, - "link": null - }, - { - "localized_name": "prompt", - "name": "prompt", - "type": "STRING", - "widget": { - "name": "prompt" - }, - "link": null - }, - { - "localized_name": "negative_prompt", - "name": "negative_prompt", - "type": "STRING", - "widget": { - "name": "negative_prompt" - }, - "link": null - }, - { - "localized_name": "width", - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": null - }, - { - "localized_name": "height", - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": null - }, - { - "localized_name": "fps", - "name": "fps", - "type": "INT", - "widget": { - "name": "fps" - }, - "link": null - }, - { - "localized_name": "duration", - "name": "duration", - "type": "FLOAT", - "widget": { - "name": "duration" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [ - 14 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniGenerateVideo" - }, - "widgets_values": [ - "http://127.0.0.1:8001/v1", - "MiniMaxAI/MiniMax-H3", - "Continue the opening action into a new ending. Maintain the same subject, setting and camera motion, with natural synchronized sound.", - "", - 1344, - 768, - 24, - 4.458 - ], - "title": "Continuation" - }, - { - "id": 20, - "type": "SaveVideo", - "pos": [ - 1850, - 2260 - ], - "size": [ - 360, - 350 - ], - "flags": {}, - "order": 19, - "mode": 2, - "inputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "link": 14, - "shape": 7 - }, - { - "localized_name": "filename_prefix", - "name": "filename_prefix", - "type": "STRING", - "widget": { - "name": "filename_prefix" - }, - "link": null - }, - { - "localized_name": "format", - "name": "format", - "type": "COMFY_DYNAMICCOMBO_V3", - "widget": { - "name": "format" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [] - } - ], - "properties": { - "Node name for S&R": "SaveVideo" - }, - "widgets_values": [ - "video/WF05-continuation", - "auto", - "auto" - ], - "title": "Save video with audio" - }, - { - "id": 21, - "type": "SolidMask", - "pos": [ - 400, - 2260 - ], - "size": [ - 250, - 130 - ], - "flags": {}, - "order": 20, - "mode": 2, - "inputs": [ - { - "localized_name": "value", - "name": "value", - "type": "FLOAT", - "widget": { - "name": "value" - }, - "link": null - }, - { - "localized_name": "width", - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": null - }, - { - "localized_name": "height", - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "mask", - "name": "mask", - "type": "MASK", - "links": [ - 15 - ] - } - ], - "properties": { - "Node name for S&R": "SolidMask" - }, - "widgets_values": [ - 0.0, - 16, - 16 - ], - "title": "Preserve" - }, - { - "type": "MaskToImage", - "size": [ - 250, - 80 - ], - "flags": {}, - "inputs": [ - { - "name": "mask", - "type": "MASK", - "link": 15 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 16 - ] - } - ], - "properties": { - "Node name for S&R": "MaskToImage" - }, - "id": 22, - "title": "Mask to image batch", - "pos": [ - 400, - 2425 - ], - "order": 21, - "mode": 2, - "widgets_values": [] - }, - { - "type": "RepeatImageBatch", - "size": [ - 270, - 130 - ], - "flags": {}, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 16 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 19 - ] - } - ], - "properties": { - "Node name for S&R": "RepeatImageBatch" - }, - "id": 23, - "title": "16 latent time slices", - "pos": [ - 700, - 2260 - ], - "order": 22, - "mode": 2, - "widgets_values": [ - 16 - ] - }, - { - "id": 24, - "type": "SolidMask", - "pos": [ - 400, - 2570 - ], - "size": [ - 250, - 130 - ], - "flags": {}, - "order": 23, - "mode": 2, - "inputs": [ - { - "localized_name": "value", - "name": "value", - "type": "FLOAT", - "widget": { - "name": "value" - }, - "link": null - }, - { - "localized_name": "width", - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": null - }, - { - "localized_name": "height", - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "mask", - "name": "mask", - "type": "MASK", - "links": [ - 17 - ] - } - ], - "properties": { - "Node name for S&R": "SolidMask" - }, - "widgets_values": [ - 1.0, - 16, - 16 - ], - "title": "Regenerate" - }, - { - "type": "MaskToImage", - "size": [ - 250, - 80 - ], - "flags": {}, - "inputs": [ - { - "name": "mask", - "type": "MASK", - "link": 17 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 18 - ] - } - ], - "properties": { - "Node name for S&R": "MaskToImage" - }, - "id": 25, - "title": "Mask to image batch", - "pos": [ - 400, - 2735 - ], - "order": 24, - "mode": 2, - "widgets_values": [] - }, - { - "type": "RepeatImageBatch", - "size": [ - 270, - 130 - ], - "flags": {}, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 18 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 20 - ] - } - ], - "properties": { - "Node name for S&R": "RepeatImageBatch" - }, - "id": 26, - "title": "16 latent time slices", - "pos": [ - 700, - 2570 - ], - "order": 25, - "mode": 2, - "widgets_values": [ - 16 - ] - }, - { - "type": "ImageBatch", - "size": [ - 270, - 90 - ], - "flags": {}, - "inputs": [ - { - "name": "image1", - "type": "IMAGE", - "link": 19 - }, - { - "name": "image2", - "type": "IMAGE", - "link": 20 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 21 - ] - } - ], - "properties": { - "Node name for S&R": "ImageBatch" - }, - "id": 27, - "title": "Preserved prefix then generated suffix", - "pos": [ - 700, - 2860 - ], - "order": 26, - "mode": 2, - "widgets_values": [] - }, - { - "type": "ImageToMask", - "size": [ - 300, - 110 - ], - "flags": {}, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 21 - } - ], - "outputs": [ - { - "name": "MASK", - "type": "MASK", - "links": [ - 22 - ] - } - ], - "properties": { - "Node name for S&R": "ImageToMask" - }, - "id": 28, - "title": "Temporal mask (red channel)", - "pos": [ - 1030, - 2890 - ], - "order": 27, - "mode": 2, - "widgets_values": [ - "red" - ] - }, - { - "id": 29, - "type": "MarkdownNote", - "pos": [ - 1850, - 2660 - ], - "size": [ - 360, - 330 - ], - "flags": {}, - "order": 28, - "mode": 2, - "inputs": [], - "outputs": [], - "title": "Continuation guide", - "properties": { - "Node name for S&R": "MarkdownNote" - }, - "widgets_values": [ - "Regenerate the ending within a 107-frame output: 32 latent time slices, first 16 preserved and last 16 regenerated. This is a latent-time boundary, not a frame-exact splice. Audio mask=1 regenerates the entire soundtrack. EDIT-02 only exposes a scalar audio mask, so this example does not preserve a prefix of the original audio." - ], - "color": "#333", - "bgcolor": "#222" - }, - { - "id": 30, - "type": "LoadVideo", - "pos": [ - 40, - 3200 - ], - "size": [ - 300, - 200 - ], - "flags": {}, - "order": 29, - "mode": 2, - "inputs": [ - { - "localized_name": "file", - "name": "file", - "type": "COMBO", - "widget": { - "name": "file" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [ - 23 - ] - } - ], - "properties": { - "Node name for S&R": "LoadVideo" - }, - "widgets_values": [ - "" - ], - "title": "Extension \u2014 source (107 frames, 24 FPS)" - }, - { - "id": 31, - "type": "VLLMOmniLatentMaskEditing", - "pos": [ - 1030, - 3370 - ], - "size": [ - 300, - 200 - ], - "flags": {}, - "order": 30, - "mode": 2, - "inputs": [ - { - "localized_name": "source_video", - "name": "source_video", - "type": "VIDEO", - "link": 23, - "shape": 7 - }, - { - "localized_name": "source_audio", - "name": "source_audio", - "type": "AUDIO", - "link": null - }, - { - "localized_name": "video_mask", - "name": "video_mask", - "type": "MASK", - "link": 34, - "shape": 7 - }, - { - "localized_name": "audio_mask", - "name": "audio_mask", - "type": "FLOAT", - "widget": { - "name": "audio_mask" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "latent_edit", - "name": "latent_edit", - "type": "LATENT_MASK_EDITING", - "links": [ - 24 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniLatentMaskEditing" - }, - "widgets_values": [ - 1.0 - ], - "title": "Source and noise masks" - }, - { - "id": 32, - "type": "VLLMOmniMiniMaxH3Params", - "pos": [ - 1030, - 3640 - ], - "size": [ - 300, - 130 - ], - "flags": {}, - "order": 31, - "mode": 2, - "inputs": [ - { - "localized_name": "audio_flow_shift", - "name": "audio_flow_shift", - "type": "FLOAT", - "widget": { - "name": "audio_flow_shift" - }, - "link": null - }, - { - "localized_name": "flow_shift", - "name": "flow_shift", - "type": "FLOAT", - "widget": { - "name": "flow_shift" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video params", - "name": "video params", - "type": "VIDEO_PARAMS", - "links": [ - 25 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniMiniMaxH3Params" - }, - "widgets_values": [ - 3.0, - 12.0 - ], - "title": "H3 base settings" - }, - { - "id": 33, - "type": "VLLMOmniGenerateVideo", - "pos": [ - 1380, - 3200 - ], - "size": [ - 420, - 700 - ], - "flags": {}, - "order": 32, - "mode": 2, - "inputs": [ - { - "localized_name": "frame", - "name": "frame", - "type": "IMAGE", - "link": null, - "shape": 7 - }, - { - "localized_name": "references", - "name": "references", - "type": "VIDEO_REFERENCES", - "link": null, - "shape": 7 - }, - { - "localized_name": "sampling_params", - "name": "sampling_params", - "type": "SAMPLING_PARAMS", - "link": null - }, - { - "localized_name": "lora", - "name": "lora", - "type": "REMOTE_LORA", - "link": null - }, - { - "localized_name": "model_params", - "name": "model_params", - "type": "VIDEO_PARAMS", - "link": 25, - "shape": 7 - }, - { - "localized_name": "fast_h3", - "name": "fast_h3", - "type": "FASTH3_DEPLOYMENT", - "link": null - }, - { - "localized_name": "latent_edit", - "name": "latent_edit", - "type": "LATENT_MASK_EDITING", - "link": 24, - "shape": 7 - }, - { - "localized_name": "url", - "name": "url", - "type": "STRING", - "widget": { - "name": "url" - }, - "link": null - }, - { - "localized_name": "model", - "name": "model", - "type": "STRING", - "widget": { - "name": "model" - }, - "link": null - }, - { - "localized_name": "prompt", - "name": "prompt", - "type": "STRING", - "widget": { - "name": "prompt" - }, - "link": null - }, - { - "localized_name": "negative_prompt", - "name": "negative_prompt", - "type": "STRING", - "widget": { - "name": "negative_prompt" - }, - "link": null - }, - { - "localized_name": "width", - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": null - }, - { - "localized_name": "height", - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": null - }, - { - "localized_name": "fps", - "name": "fps", - "type": "INT", - "widget": { - "name": "fps" - }, - "link": null - }, - { - "localized_name": "duration", - "name": "duration", - "type": "FLOAT", - "widget": { - "name": "duration" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [ - 26 - ] - } - ], - "properties": { - "Node name for S&R": "VLLMOmniGenerateVideo" - }, - "widgets_values": [ - "http://127.0.0.1:8001/v1", - "MiniMaxAI/MiniMax-H3", - "Extend the shot beyond the original ending. Continue the motion smoothly with consistent appearance and natural synchronized sound.", - "", - 1344, - 768, - 24, - 8.708 - ], - "title": "Extension" - }, - { - "id": 34, - "type": "SaveVideo", - "pos": [ - 1850, - 3200 - ], - "size": [ - 360, - 350 - ], - "flags": {}, - "order": 33, - "mode": 2, - "inputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "link": 26, - "shape": 7 - }, - { - "localized_name": "filename_prefix", - "name": "filename_prefix", - "type": "STRING", - "widget": { - "name": "filename_prefix" - }, - "link": null - }, - { - "localized_name": "format", - "name": "format", - "type": "COMFY_DYNAMICCOMBO_V3", - "widget": { - "name": "format" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "video", - "name": "video", - "type": "VIDEO", - "links": [] - } - ], - "properties": { - "Node name for S&R": "SaveVideo" - }, - "widgets_values": [ - "video/WF05-extension", - "auto", - "auto" - ], - "title": "Save video with audio" - }, - { - "id": 35, - "type": "SolidMask", - "pos": [ - 400, - 3200 - ], - "size": [ - 250, - 130 - ], - "flags": {}, - "order": 34, - "mode": 2, - "inputs": [ - { - "localized_name": "value", - "name": "value", - "type": "FLOAT", - "widget": { - "name": "value" - }, - "link": null - }, - { - "localized_name": "width", - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": null - }, - { - "localized_name": "height", - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "mask", - "name": "mask", - "type": "MASK", - "links": [ - 27 - ] - } - ], - "properties": { - "Node name for S&R": "SolidMask" - }, - "widgets_values": [ - 0.0, - 16, - 16 - ], - "title": "Preserve" - }, - { - "type": "MaskToImage", - "size": [ - 250, - 80 - ], - "flags": {}, - "inputs": [ - { - "name": "mask", - "type": "MASK", - "link": 27 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 28 - ] - } - ], - "properties": { - "Node name for S&R": "MaskToImage" - }, - "id": 36, - "title": "Mask to image batch", - "pos": [ - 400, - 3365 - ], - "order": 35, - "mode": 2, - "widgets_values": [] - }, - { - "type": "RepeatImageBatch", - "size": [ - 270, - 130 - ], - "flags": {}, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 28 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 31 - ] - } - ], - "properties": { - "Node name for S&R": "RepeatImageBatch" - }, - "id": 37, - "title": "32 latent time slices", - "pos": [ - 700, - 3200 - ], - "order": 36, - "mode": 2, - "widgets_values": [ - 32 - ] - }, - { - "id": 38, - "type": "SolidMask", - "pos": [ - 400, - 3510 - ], - "size": [ - 250, - 130 - ], - "flags": {}, - "order": 37, - "mode": 2, - "inputs": [ - { - "localized_name": "value", - "name": "value", - "type": "FLOAT", - "widget": { - "name": "value" - }, - "link": null - }, - { - "localized_name": "width", - "name": "width", - "type": "INT", - "widget": { - "name": "width" - }, - "link": null - }, - { - "localized_name": "height", - "name": "height", - "type": "INT", - "widget": { - "name": "height" - }, - "link": null - } - ], - "outputs": [ - { - "localized_name": "mask", - "name": "mask", - "type": "MASK", - "links": [ - 29 - ] - } - ], - "properties": { - "Node name for S&R": "SolidMask" - }, - "widgets_values": [ - 1.0, - 16, - 16 - ], - "title": "Regenerate" - }, - { - "type": "MaskToImage", - "size": [ - 250, - 80 - ], - "flags": {}, - "inputs": [ - { - "name": "mask", - "type": "MASK", - "link": 29 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 30 - ] - } - ], - "properties": { - "Node name for S&R": "MaskToImage" - }, - "id": 39, - "title": "Mask to image batch", - "pos": [ - 400, - 3675 - ], - "order": 38, - "mode": 2, - "widgets_values": [] - }, - { - "type": "RepeatImageBatch", - "size": [ - 270, - 130 - ], - "flags": {}, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 30 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 32 - ] - } - ], - "properties": { - "Node name for S&R": "RepeatImageBatch" - }, - "id": 40, - "title": "30 latent time slices", - "pos": [ - 700, - 3510 - ], - "order": 39, - "mode": 2, - "widgets_values": [ - 30 - ] - }, - { - "type": "ImageBatch", - "size": [ - 270, - 90 - ], - "flags": {}, - "inputs": [ - { - "name": "image1", - "type": "IMAGE", - "link": 31 - }, - { - "name": "image2", - "type": "IMAGE", - "link": 32 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 33 - ] - } - ], - "properties": { - "Node name for S&R": "ImageBatch" - }, - "id": 41, - "title": "Preserved prefix then generated suffix", - "pos": [ - 700, - 3800 - ], - "order": 40, - "mode": 2, - "widgets_values": [] - }, - { - "type": "ImageToMask", - "size": [ - 300, - 110 - ], - "flags": {}, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 33 - } - ], - "outputs": [ - { - "name": "MASK", - "type": "MASK", - "links": [ - 34 - ] - } - ], - "properties": { - "Node name for S&R": "ImageToMask" - }, - "id": 42, - "title": "Temporal mask (red channel)", - "pos": [ - 1030, - 3830 - ], - "order": 41, - "mode": 2, - "widgets_values": [ - "red" - ] - }, - { - "id": 43, - "type": "MarkdownNote", - "pos": [ - 1850, - 3600 - ], - "size": [ - 360, - 330 - ], - "flags": {}, - "order": 42, - "mode": 2, - "inputs": [], - "outputs": [], - "title": "Extension guide", - "properties": { - "Node name for S&R": "MarkdownNote" - }, - "widgets_values": [ - "Extend a 107-frame source to 209 frames: source has 32 latent time slices; output has 62. Preserve the first 32 and regenerate the remaining 30. EDIT-01 pads the short source using its last frame before encoding. Use a 107-frame source at 24 FPS; other lengths require adjusting both mask counts and output duration. Audio mask=1 regenerates the entire soundtrack. EDIT-02 only exposes a scalar audio mask, so this example does not preserve a prefix of the original audio." - ], - "color": "#333", - "bgcolor": "#222" - } - ], - "links": [ - [ - 1, - 2, - 0, - 3, - 0, - "VIDEO" - ], - [ - 2, - 3, - 0, - 5, - 6, - "LATENT_MASK_EDITING" - ], - [ - 3, - 4, - 0, - 5, - 4, - "VIDEO_PARAMS" - ], - [ - 4, - 5, - 0, - 6, - 0, - "VIDEO" - ], - [ - 5, - 7, - 0, - 3, - 2, - "MASK" - ], - [ - 6, - 9, - 0, - 10, - 0, - "VIDEO" - ], - [ - 7, - 10, - 0, - 12, - 6, - "LATENT_MASK_EDITING" - ], - [ - 8, - 11, - 0, - 12, - 4, - "VIDEO_PARAMS" - ], - [ - 9, - 12, - 0, - 13, - 0, - "VIDEO" - ], - [ - 10, - 14, - 0, - 10, - 2, - "MASK" - ], - [ - 11, - 16, - 0, - 17, - 0, - "VIDEO" - ], - [ - 12, - 17, - 0, - 19, - 6, - "LATENT_MASK_EDITING" - ], - [ - 13, - 18, - 0, - 19, - 4, - "VIDEO_PARAMS" - ], - [ - 14, - 19, - 0, - 20, - 0, - "VIDEO" - ], - [ - 15, - 21, - 0, - 22, - 0, - "MASK" - ], - [ - 16, - 22, - 0, - 23, - 0, - "IMAGE" - ], - [ - 17, - 24, - 0, - 25, - 0, - "MASK" - ], - [ - 18, - 25, - 0, - 26, - 0, - "IMAGE" - ], - [ - 19, - 23, - 0, - 27, - 0, - "IMAGE" - ], - [ - 20, - 26, - 0, - 27, - 1, - "IMAGE" - ], - [ - 21, - 27, - 0, - 28, - 0, - "IMAGE" - ], - [ - 22, - 28, - 0, - 17, - 2, - "MASK" - ], - [ - 23, - 30, - 0, - 31, - 0, - "VIDEO" - ], - [ - 24, - 31, - 0, - 33, - 6, - "LATENT_MASK_EDITING" - ], - [ - 25, - 32, - 0, - 33, - 4, - "VIDEO_PARAMS" - ], - [ - 26, - 33, - 0, - 34, - 0, - "VIDEO" - ], - [ - 27, - 35, - 0, - 36, - 0, - "MASK" - ], - [ - 28, - 36, - 0, - 37, - 0, - "IMAGE" - ], - [ - 29, - 38, - 0, - 39, - 0, - "MASK" - ], - [ - 30, - 39, - 0, - 40, - 0, - "IMAGE" - ], - [ - 31, - 37, - 0, - 41, - 0, - "IMAGE" - ], - [ - 32, - 40, - 0, - 41, - 1, - "IMAGE" - ], - [ - 33, - 41, - 0, - 42, - 0, - "IMAGE" - ], - [ - 34, - 42, - 0, - 31, - 2, - "MASK" - ] - ], - "groups": [ - { - "id": 1, - "title": "1. Inpainting \u2014 enabled", - "bounding": [ - 20, - 330, - 2220, - 900 - ], - "color": "#555555", - "font_size": 24, - "flags": {} - }, - { - "id": 2, - "title": "2. Object removal \u2014 muted", - "bounding": [ - 20, - 1270, - 2220, - 900 - ], - "color": "#555555", - "font_size": 24, - "flags": {} - }, - { - "id": 3, - "title": "3. Continuation \u2014 muted", - "bounding": [ - 20, - 2210, - 2220, - 900 - ], - "color": "#555555", - "font_size": 24, - "flags": {} - }, - { - "id": 4, - "title": "4. Extension \u2014 muted", - "bounding": [ - 20, - 3150, - 2220, - 900 - ], - "color": "#555555", - "font_size": 24, - "flags": {} - } - ], - "config": {}, - "extra": { - "ds": { - "scale": 0.65, - "offset": [ - 30, - 30 - ] - } - }, - "version": 0.4 -} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json new file mode 100644 index 00000000000..69b9b5f7aca --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json @@ -0,0 +1,5253 @@ +{ + "id": "eebf673c-c66a-4f20-9e87-b728a82f6e5f", + "revision": 0, + "last_node_id": 99, + "last_link_id": 162, + "nodes": [ + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 2160 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 11, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 1870 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 25, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 3760 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 30 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 29, + "type": "MarkdownNote", + "pos": [ + 1050, + 3470 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Inpainting guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Replace the bag with a small orange tabby cat curled up asleep. Source: 243 frames at 24 FPS. Grow Mask: 32 pixels. Fixed seed 0, 50 steps. Audio mask 0.5 matches the Object Removal baseline. Enable this group and mute other generation groups before running. Quality is not yet validated. Video and spatial mask are linked from Shared Inputs. Grow Mask remains independent for this case." + ], + "widgets_values_named": { + "text": "Replace the bag with a small orange tabby cat curled up asleep. Source: 243 frames at 24 FPS. Grow Mask: 32 pixels. Fixed seed 0, 50 steps. Audio mask 0.5 matches the Object Removal baseline. Enable this group and mute other generation groups before running. Quality is not yet validated. Video and spatial mask are linked from Shared Inputs. Grow Mask remains independent for this case." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 30, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 3470 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 33 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 34, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 5360 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 38 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 45, + "type": "MarkdownNote", + "pos": [ + 1050, + 5070 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Continuation guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Temporal mask follows the actual source frame count/FPS and shared target duration. Preserve fraction selects the available prefix to retain. Boundary rounds down to complete H3 native chunks; a short segment before the requested boundary may regenerate. Preview time mapping is schematic." + ], + "widgets_values_named": { + "text": "Temporal mask follows the actual source frame count/FPS and shared target duration. Preserve fraction selects the available prefix to retain. Boundary rounds down to complete H3 native chunks; a short segment before the requested boundary may regenerate. Preview time mapping is schematic." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 46, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 5070 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 48 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 49, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 6960 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 51 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 59, + "type": "VLLMOmniH3TemporalMask", + "pos": [ + 580, + 6040 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 123 + }, + { + "localized_name": "source_fps", + "name": "source_fps", + "type": "FLOAT", + "widget": { + "name": "source_fps" + }, + "link": 124 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "preserve_fraction", + "name": "preserve_fraction", + "type": "FLOAT", + "widget": { + "name": "preserve_fraction" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 60 + ] + }, + { + "localized_name": "summary", + "name": "summary", + "type": "STRING", + "links": [] + }, + { + "localized_name": "preview_fps", + "name": "preview_fps", + "type": "FLOAT", + "links": [ + 146 + ] + }, + { + "localized_name": "preview_images", + "name": "preview_images", + "type": "IMAGE", + "links": [ + 153, + 156 + ] + }, + { + "localized_name": "preview_mask", + "name": "preview_mask", + "type": "MASK", + "links": [ + 155 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniH3TemporalMask" + }, + "widgets_values": [ + 24, + 10, + "extension", + 0.5 + ], + "widgets_values_named": { + "source_fps": 24, + "duration": 10, + "mode": "extension", + "preserve_fraction": 0.5 + } + }, + { + "id": 60, + "type": "MarkdownNote", + "pos": [ + 1050, + 6670 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Extension guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Temporal mask follows the actual source frame count/FPS and shared target duration. Output duration must exceed the source duration. The original, unpadded source determines the preserved prefix. Boundary rounds down to complete H3 native chunks; a short segment before the requested boundary may regenerate. Preview time mapping is schematic." + ], + "widgets_values_named": { + "text": "Temporal mask follows the actual source frame count/FPS and shared target duration. Output duration must exceed the source duration. The original, unpadded source determines the preserved prefix. Boundary rounds down to complete H3 native chunks; a short segment before the requested boundary may regenerate. Preview time mapping is schematic." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + -268.4516045720269, + 1190 + ], + "size": [ + 420, + 314 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + }, + { + "localized_name": "choose file to upload", + "name": "upload", + "type": "IMAGEUPLOAD", + "widget": { + "name": "upload" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 1, + 17, + 28, + 36, + 49 + ] + } + ], + "title": "Shared Source Video | All Cases", + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "", + "image" + ], + "widgets_values_named": { + "file": "", + "upload": "image" + } + }, + { + "id": 62, + "type": "MarkdownNote", + "pos": [ + 1050, + 1870 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Object Removal guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Object Removal\n\nUse the shared source video and a matching spatial mask. White edits; black preserves.\n\nCover the entire bag, handle, straps, and contact shadow. Check coverage in Mask Preview.\n\nDescribe the area as empty from the first frame to the last, with continuous wall and pavement textures.\n\nKeep frame and references disconnected for this T2VA setup.\n\nTo run only this case, select Save Video (Object Removal) and click Execute to selected output nodes." + ], + "widgets_values_named": { + "text": "## Object Removal\n\nUse the shared source video and a matching spatial mask. White edits; black preserves.\n\nCover the entire bag, handle, straps, and contact shadow. Check coverage in Mask Preview.\n\nDescribe the area as empty from the first frame to the last, with continuous wall and pavement textures.\n\nKeep frame and references disconnected for this T2VA setup.\n\nTo run only this case, select Save Video (Object Removal) and click Execute to selected output nodes." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 61, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 6670 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 61 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 65, + "type": "CreateVideo", + "pos": [ + 1990, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 130 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 131 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 77 + ] + } + ], + "title": "Object Removal | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 69, + "type": "CreateVideo", + "pos": [ + 1990, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 137 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 138 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 91 + ] + } + ], + "title": "Inpainting | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 73, + "type": "CreateVideo", + "pos": [ + 1990, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 152 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 142 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 105 + ] + } + ], + "title": "Continuation | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 77, + "type": "CreateVideo", + "pos": [ + 1990, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 50, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 158 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 146 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 119 + ] + } + ], + "title": "Extension | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 79, + "type": "EmptyImage", + "pos": [ + 1090, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 126 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 80, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 125 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 126 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 127 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 129 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 81, + "type": "ImageBlend", + "pos": [ + 1690, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 128 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 129 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 130 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 82, + "type": "EmptyImage", + "pos": [ + 1090, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 133 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 83, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 132 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 133 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 134 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 136 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 84, + "type": "ImageBlend", + "pos": [ + 1690, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 135 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 136 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 137 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 89, + "type": "EmptyImage", + "pos": [ + 1090, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 148 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 90, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 147 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 148 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 149 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 151 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 91, + "type": "ImageBlend", + "pos": [ + 1690, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 150 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 151 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 152 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 92, + "type": "EmptyImage", + "pos": [ + 1090, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 154 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 93, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 153 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 154 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 155 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 157 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 94, + "type": "ImageBlend", + "pos": [ + 1690, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 156 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 157 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 158 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + -268.4516045720269, + 2138.3724768826028 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "widgets_values_named": { + "text": "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + }, + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 15, + "type": "GetVideoComponents", + "pos": [ + -268.4516045720269, + 2438.3724768826028 + ], + "size": [ + 193.530859375, + 106 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 17 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 120, + 123, + 125, + 128, + 132, + 135 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": [] + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [ + 121, + 124, + 131, + 138 + ] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "COMBO", + "links": [] + }, + { + "localized_name": "color_space", + "name": "color_space", + "type": "COMBO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents" + } + }, + { + "id": 95, + "type": "SolidMask", + "pos": [ + 580, + 1240 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 159 + ] + } + ], + "title": "Mask Canvas", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 1344, + 768 + ], + "widgets_values_named": { + "value": 0, + "width": 1344, + "height": 768 + } + }, + { + "id": 97, + "type": "SolidMask", + "pos": [ + 580, + 2840 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 161 + ] + } + ], + "title": "Mask Canvas", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 1344, + 768 + ], + "widgets_values_named": { + "value": 0, + "width": 1344, + "height": 768 + } + }, + { + "id": 66, + "type": "SaveVideo", + "pos": [ + 2530, + 1240 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 77 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Object Removal)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-object-removal-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-object-removal-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 78, + "type": "SaveVideo", + "pos": [ + 2530, + 6040 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 54, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 119 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Extension)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-extension-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-extension-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 1870 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 10 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 4 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 3 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "title": "Generate Video(Object Removal)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 96, + "type": "SolidMask", + "pos": [ + 580, + 1360 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 160 + ] + } + ], + "title": "Mask Rectangle", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 180, + 168 + ], + "widgets_values_named": { + "value": 1, + "width": 180, + "height": 168 + } + }, + { + "id": 9, + "type": "MaskComposite", + "pos": [ + 580, + 1540 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "MASK", + "link": 159 + }, + { + "localized_name": "source", + "name": "source", + "type": "MASK", + "link": 160 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "operation", + "name": "operation", + "type": "COMBO", + "widget": { + "name": "operation" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 63, + 127 + ] + } + ], + "title": "Spatial Mask | Rectangle Position", + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 1074, + 493, + "add" + ], + "widgets_values_named": { + "x": 1074, + "y": 493, + "operation": "add" + } + }, + { + "id": 98, + "type": "SolidMask", + "pos": [ + 580, + 2960 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 162 + ] + } + ], + "title": "Mask Rectangle", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 180, + 168 + ], + "widgets_values_named": { + "value": 1, + "width": 180, + "height": 168 + } + }, + { + "id": 99, + "type": "MaskComposite", + "pos": [ + 580, + 3140 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "MASK", + "link": 161 + }, + { + "localized_name": "source", + "name": "source", + "type": "MASK", + "link": 162 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "operation", + "name": "operation", + "type": "COMBO", + "widget": { + "name": "operation" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 35, + 134 + ] + } + ], + "title": "Spatial Mask | Rectangle Position", + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 1074, + 493, + "add" + ], + "widgets_values_named": { + "x": 1074, + "y": 493, + "operation": "add" + } + }, + { + "id": 70, + "type": "SaveVideo", + "pos": [ + 2530, + 2840 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 91 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Inpainting)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-inpainting-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-inpainting-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 1870 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 1 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 63 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0 + ], + "widgets_values_named": { + "audio_mask": 0 + } + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 2530, + 1870 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Object Removal)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-object-removal", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-object-removal", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 27, + "type": "SaveVideo", + "pos": [ + 2530, + 3470 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 31 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Inpainting)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-inpainting", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-inpainting", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 24, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 3470 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 28 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 35 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 29 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0 + ], + "widgets_values_named": { + "audio_mask": 0 + } + }, + { + "id": 26, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 3470 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 33 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 30 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 29 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 31 + ] + } + ], + "title": "Generate Video(Inpainting)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 33, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 5070 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 36 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 47 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 37 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.8 + ], + "widgets_values_named": { + "audio_mask": 0.8 + } + }, + { + "id": 35, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 5070 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 48 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 38 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 37 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 39 + ] + } + ], + "title": "Generate Video(Continuation)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 44, + "type": "VLLMOmniH3TemporalMask", + "pos": [ + 580, + 4440 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 120 + }, + { + "localized_name": "source_fps", + "name": "source_fps", + "type": "FLOAT", + "widget": { + "name": "source_fps" + }, + "link": 121 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "preserve_fraction", + "name": "preserve_fraction", + "type": "FLOAT", + "widget": { + "name": "preserve_fraction" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 47 + ] + }, + { + "localized_name": "summary", + "name": "summary", + "type": "STRING", + "links": [] + }, + { + "localized_name": "preview_fps", + "name": "preview_fps", + "type": "FLOAT", + "links": [ + 142 + ] + }, + { + "localized_name": "preview_images", + "name": "preview_images", + "type": "IMAGE", + "links": [ + 147, + 150 + ] + }, + { + "localized_name": "preview_mask", + "name": "preview_mask", + "type": "MASK", + "links": [ + 149 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniH3TemporalMask" + }, + "widgets_values": [ + 24, + 5, + "continuation", + 0.2 + ], + "widgets_values_named": { + "source_fps": 24, + "duration": 5, + "mode": "continuation", + "preserve_fraction": 0.2 + } + }, + { + "id": 36, + "type": "SaveVideo", + "pos": [ + 2530, + 5070 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 47, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 39 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Continuation)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-continuation", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-continuation", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 74, + "type": "SaveVideo", + "pos": [ + 2530, + 4440 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 105 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Continuation)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-continuation-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-continuation-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 50, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 6670 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 61 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 51 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 50 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 52 + ] + } + ], + "title": "Generate Video(Extension)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A", + "", + 1344, + 768, + 24, + 10 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 10 + } + }, + { + "id": 48, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 6670 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 49 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 60 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 50 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.9 + ], + "widgets_values_named": { + "audio_mask": 0.9 + } + }, + { + "id": 51, + "type": "SaveVideo", + "pos": [ + 2530, + 6670 + ], + "size": [ + 420, + 338 + ], + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 52 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Extension)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-extension", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-extension", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ], + [ + 10, + 11, + 0, + 6, + 2, + "SAMPLING_PARAMS" + ], + [ + 17, + 2, + 0, + 15, + 0, + "VIDEO" + ], + [ + 28, + 2, + 0, + 24, + 0, + "VIDEO" + ], + [ + 29, + 24, + 0, + 26, + 6, + "LATENT_MASK_EDITING" + ], + [ + 30, + 25, + 0, + 26, + 4, + "VIDEO_PARAMS" + ], + [ + 31, + 26, + 0, + 27, + 0, + "VIDEO" + ], + [ + 33, + 30, + 0, + 26, + 2, + "SAMPLING_PARAMS" + ], + [ + 35, + 99, + 0, + 24, + 2, + "MASK" + ], + [ + 36, + 2, + 0, + 33, + 0, + "VIDEO" + ], + [ + 37, + 33, + 0, + 35, + 6, + "LATENT_MASK_EDITING" + ], + [ + 38, + 34, + 0, + 35, + 4, + "VIDEO_PARAMS" + ], + [ + 39, + 35, + 0, + 36, + 0, + "VIDEO" + ], + [ + 47, + 44, + 0, + 33, + 2, + "MASK" + ], + [ + 48, + 46, + 0, + 35, + 2, + "SAMPLING_PARAMS" + ], + [ + 49, + 2, + 0, + 48, + 0, + "VIDEO" + ], + [ + 50, + 48, + 0, + 50, + 6, + "LATENT_MASK_EDITING" + ], + [ + 51, + 49, + 0, + 50, + 4, + "VIDEO_PARAMS" + ], + [ + 52, + 50, + 0, + 51, + 0, + "VIDEO" + ], + [ + 60, + 59, + 0, + 48, + 2, + "MASK" + ], + [ + 61, + 61, + 0, + 50, + 2, + "SAMPLING_PARAMS" + ], + [ + 63, + 9, + 0, + 4, + 2, + "MASK" + ], + [ + 77, + 65, + 0, + 66, + 0, + "VIDEO" + ], + [ + 91, + 69, + 0, + 70, + 0, + "VIDEO" + ], + [ + 105, + 73, + 0, + 74, + 0, + "VIDEO" + ], + [ + 119, + 77, + 0, + 78, + 0, + "VIDEO" + ], + [ + 120, + 15, + 0, + 44, + 0, + "IMAGE" + ], + [ + 121, + 15, + 2, + 44, + 1, + "FLOAT" + ], + [ + 123, + 15, + 0, + 59, + 0, + "IMAGE" + ], + [ + 124, + 15, + 2, + 59, + 1, + "FLOAT" + ], + [ + 125, + 15, + 0, + 80, + 0, + "IMAGE" + ], + [ + 126, + 79, + 0, + 80, + 1, + "IMAGE" + ], + [ + 127, + 9, + 0, + 80, + 2, + "MASK" + ], + [ + 128, + 15, + 0, + 81, + 0, + "IMAGE" + ], + [ + 129, + 80, + 0, + 81, + 1, + "IMAGE" + ], + [ + 130, + 81, + 0, + 65, + 0, + "IMAGE" + ], + [ + 131, + 15, + 2, + 65, + 2, + "FLOAT" + ], + [ + 132, + 15, + 0, + 83, + 0, + "IMAGE" + ], + [ + 133, + 82, + 0, + 83, + 1, + "IMAGE" + ], + [ + 134, + 99, + 0, + 83, + 2, + "MASK" + ], + [ + 135, + 15, + 0, + 84, + 0, + "IMAGE" + ], + [ + 136, + 83, + 0, + 84, + 1, + "IMAGE" + ], + [ + 137, + 84, + 0, + 69, + 0, + "IMAGE" + ], + [ + 138, + 15, + 2, + 69, + 2, + "FLOAT" + ], + [ + 142, + 44, + 2, + 73, + 2, + "FLOAT" + ], + [ + 146, + 59, + 2, + 77, + 2, + "FLOAT" + ], + [ + 147, + 44, + 3, + 90, + 0, + "IMAGE" + ], + [ + 148, + 89, + 0, + 90, + 1, + "IMAGE" + ], + [ + 149, + 44, + 4, + 90, + 2, + "MASK" + ], + [ + 150, + 44, + 3, + 91, + 0, + "IMAGE" + ], + [ + 151, + 90, + 0, + 91, + 1, + "IMAGE" + ], + [ + 152, + 91, + 0, + 73, + 0, + "IMAGE" + ], + [ + 153, + 59, + 3, + 93, + 0, + "IMAGE" + ], + [ + 154, + 92, + 0, + 93, + 1, + "IMAGE" + ], + [ + 155, + 59, + 4, + 93, + 2, + "MASK" + ], + [ + 156, + 59, + 3, + 94, + 0, + "IMAGE" + ], + [ + 157, + 93, + 0, + 94, + 1, + "IMAGE" + ], + [ + 158, + 94, + 0, + 77, + 0, + "IMAGE" + ], + [ + 159, + 95, + 0, + 9, + 0, + "MASK" + ], + [ + 160, + 96, + 0, + 9, + 1, + "MASK" + ], + [ + 161, + 97, + 0, + 99, + 0, + "MASK" + ], + [ + 162, + 98, + 0, + 99, + 1, + "MASK" + ] + ], + "groups": [ + { + "id": 1, + "title": "Shared Input", + "bounding": [ + -308.451604572027, + 1080, + 473.7665553206702, + 1508.3724768826028 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 2, + "title": "Output Hub", + "bounding": [ + 2470, + 1080, + 540, + 6400 + ], + "color": "#444", + "flags": {} + }, + { + "id": 10, + "title": "Case 01 | Object Removal", + "bounding": [ + 540, + 1080, + 1880, + 1540 + ], + "color": "#77528d", + "flags": {} + }, + { + "id": 20, + "title": "Case 01 | Object Removal Output", + "bounding": [ + 2490, + 1130, + 500, + 1490 + ], + "color": "#77528d", + "flags": {} + }, + { + "id": 11, + "title": "Case 02 | Inpainting", + "bounding": [ + 540, + 2680, + 1880, + 1540 + ], + "color": "#357e80", + "flags": {} + }, + { + "id": 21, + "title": "Case 02 | Inpainting Output", + "bounding": [ + 2490, + 2730, + 500, + 1490 + ], + "color": "#357e80", + "flags": {} + }, + { + "id": 12, + "title": "Case 03 | Continuation", + "bounding": [ + 540, + 4280, + 1880, + 1540 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 22, + "title": "Case 03 | Continuation Output", + "bounding": [ + 2490, + 4330, + 500, + 1490 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 13, + "title": "Case 04 | Extension", + "bounding": [ + 540, + 5880, + 1880, + 1540 + ], + "color": "#b06634", + "flags": {} + }, + { + "id": 23, + "title": "Case 04 | Extension Output", + "bounding": [ + 2490, + 5930, + 500, + 1490 + ], + "color": "#b06634", + "flags": {} + }, + { + "id": 24, + "title": "Object Removal | Mask Generation", + "bounding": [ + 560, + 1130, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 25, + "title": "Inpainting | Mask Generation", + "bounding": [ + 560, + 2730, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 26, + "title": "Continuation | Mask Generation", + "bounding": [ + 560, + 4330, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 27, + "title": "Extension | Mask Generation", + "bounding": [ + 560, + 5930, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9837545257996714, + "offset": [ + -1807.8247536090078, + -6349.252392678693 + ] + } + }, + "version": 0.4 +} diff --git a/tests/e2e/features/comfyui/test_wf05_template.py b/tests/e2e/features/comfyui/test_wf05_template.py index fea841d7225..717a8f313fd 100644 --- a/tests/e2e/features/comfyui/test_wf05_template.py +++ b/tests/e2e/features/comfyui/test_wf05_template.py @@ -11,7 +11,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] WORKFLOW = ( - Path(__file__).resolve().parents[4] / "apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni H3 Latent Editing.json" + Path(__file__).resolve().parents[4] + / "apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json" ) @@ -30,44 +31,50 @@ def test_wf05_graph_links(): def test_wf05_defaults_and_outputs(): graph = json.loads(WORKFLOW.read_text()) - generators = [n for n in graph["nodes"] if n["type"] == "VLLMOmniGenerateVideo"] + nodes = {n["id"]: n for n in graph["nodes"]} + links = {link[0]: link for link in graph["links"]} + generators = [n for n in nodes.values() if n["type"] == "VLLMOmniGenerateVideo"] assert len(generators) == 4 - assert [n["mode"] for n in generators] == [0, 2, 2, 2] - for node in generators: - width, height, fps, duration = node["widgets_values"][-4:] - assert (width, height, fps) == (1344, 768, 24) - frames = round(duration * fps) - assert frames % 17 == 5 - assert frames == (209 if node["title"] == "Extension" else 107) - assert next(i for i in node["inputs"] if i["name"] == "latent_edit")["link"] is not None - saves = [n for n in graph["nodes"] if n["type"] == "SaveVideo"] - assert len(saves) == 4 - assert all(n["inputs"][0]["link"] is not None for n in saves) + assert len([n for n in nodes.values() if n["type"] == "SaveVideo"]) == 8 + assert not any(n["type"] == "VLLMOmniH3MaskGridPreview" for n in nodes.values()) + for gen in generators: + case = gen["title"].removeprefix("Generate Video(").removesuffix(")") + save = next(n for n in nodes.values() if n.get("title") == f"Mask Preview ({case})") + create = nodes[links[save["inputs"][0]["link"]][1]] + image_node = nodes[links[create["inputs"][0]["link"]][1]] + edit = nodes[links[next(i["link"] for i in gen["inputs"] if i["name"] == "latent_edit")][1]] + mask_link = next(i["link"] for i in edit["inputs"] if i["name"] == "video_mask") + assert image_node["type"] == "ImageBlend" + consumer = nodes[links[next(i["link"] for i in image_node["inputs"] if i["name"] == "image2")][1]] + assert consumer["type"] == "ImageCompositeMasked" + preview_mask = next(i["link"] for i in consumer["inputs"] if i["name"] == "mask") + if case in ("Continuation", "Extension"): + assert nodes[links[mask_link][1]]["type"] == "VLLMOmniH3TemporalMask" + assert links[preview_mask][1:3] == [links[mask_link][1], 4] + fps_link = next(i["link"] for i in create["inputs"] if i["name"] == "fps") + assert links[fps_link][1:3] == [links[mask_link][1], 2] + else: + assert links[mask_link][1:3] == links[preview_mask][1:3] -@pytest.mark.parametrize("title,counts", [("Continuation", (16, 16)), ("Extension", (32, 30))]) -def test_wf05_temporal_mask(title, counts): +def test_wf05_output_layout(): graph = json.loads(WORKFLOW.read_text()) - nodes = {n["id"]: n for n in graph["nodes"]} - links = {link[0]: link for link in graph["links"]} + for case in ("Object Removal", "Inpainting", "Continuation", "Extension"): + preview = next(n for n in graph["nodes"] if n.get("title") == f"Mask Preview ({case})") + result = next(n for n in graph["nodes"] if n.get("title") == f"Save Video ({case})") + assert preview["pos"][0] == result["pos"][0] + assert preview["pos"][1] + preview["size"][1] < result["pos"][1] - def upstream(node, name): - link_id = next(i["link"] for i in node["inputs"] if i["name"] == name) - return nodes[links[link_id][1]] - generator = next(n for n in nodes.values() if n["title"] == title) - edit = upstream(generator, "latent_edit") - assert edit["widgets_values"] == [1.0] - mask = upstream(edit, "video_mask") - assert mask["type"] == "ImageToMask" - batch = upstream(mask, "image") - assert batch["type"] == "ImageBatch" - for slot, value, count in zip(("image1", "image2"), (0.0, 1.0), counts): - repeat = upstream(batch, slot) - assert repeat["type"] == "RepeatImageBatch" - assert repeat["widgets_values"] == [count] - solid = upstream(upstream(repeat, "image"), "mask") - assert solid["type"] == "SolidMask" - assert solid["widgets_values"][0] == value - frames = round(generator["widgets_values"][-1] * 24) - assert sum(counts) == 2 + 5 * ((frames - 5) // 17) +def test_wf05_portable_inputs(): + graph = json.loads(WORKFLOW.read_text()) + for node in graph["nodes"]: + if node["type"] in ("LoadVideo", "LoadImageMask"): + assert node["widgets_values"][0] == "" + assert not any(n["type"] == "ImageBatch" for n in graph["nodes"]) + + +def test_wf05_temporal_nodes(): + graph = json.loads(WORKFLOW.read_text()) + assert sum(n["type"] == "VLLMOmniH3TemporalMask" for n in graph["nodes"]) == 2 + assert not any(n["type"] == "ComfyMathExpression" for n in graph["nodes"]) From f6b0dc18fcbea545a6c27f4cf67ca35e71040fb1 Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Mon, 21 Sep 2026 13:15:36 +0800 Subject: [PATCH 8/9] [Frontend] Simplify MiniMax-H3 temporal mask node and workflows Signed-off-by: Weiming Liao --- apps/ComfyUI-vLLM-Omni/__init__.py | 6 +- .../comfyui_vllm_omni/nodes.py | 48 +++++++++++ .../comfyui_vllm_omni/temporal_mask.py | 59 -------------- ...M-Omni MiniMax-H3 Latent Mask Editing.json | 58 ++++++-------- .../features/comfyui/test_wf05_template.py | 80 ------------------- 5 files changed, 74 insertions(+), 177 deletions(-) delete mode 100644 apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/temporal_mask.py delete mode 100644 tests/e2e/features/comfyui/test_wf05_template.py diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index e814b91db8e..999f85ab26b 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -23,6 +23,7 @@ VLLMOmniGenerateVideo, VLLMOmniLatentMaskEditing, VLLMOmniMiniMaxH3Params, + VLLMOmniMiniMaxH3TemporalMask, VLLMOmniQwenTTSParams, VLLMOmniRemoteLoRA, VLLMOmniSamplingParamsList, @@ -32,11 +33,10 @@ VLLMOmniVoiceClone, VLLMOmniWanParams, ) -from .comfyui_vllm_omni.temporal_mask import VLLMOmniH3TemporalMask # A dictionary that contains all nodes you want to export with their names NODE_CLASS_MAPPINGS = { - "VLLMOmniH3TemporalMask": VLLMOmniH3TemporalMask, + "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask, # === Generation === "VLLMOmniGenerateImage": VLLMOmniGenerateImage, "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo, @@ -59,7 +59,7 @@ # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { - "VLLMOmniH3TemporalMask": "MiniMax-H3 Temporal Mask", + "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask", # === Generation === "VLLMOmniGenerateImage": "Generate Image", "VLLMOmniGenerateVideo": "Generate Video", diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py index ddf95b27b18..5bddb934508 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py @@ -1,12 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +import math from typing import Literal import torch from comfy_api.input import AudioInput, VideoInput from .utils.api_client import VLLMOmniClient +from .utils.latent_mask import _align_frame_count, _video_latent_t from .utils.logger import get_logger from .utils.models import lookup_model_spec from .utils.types import ( @@ -1095,3 +1097,49 @@ def get_latent_edit( if audio_mask >= 0.0: edit["audio_mask"] = audio_mask return (edit,) + + +class VLLMOmniMiniMaxH3TemporalMask: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}), + "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}), + "mode": (["continuation", "extension"],), + "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}), + } + } + + RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK") + RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask") + FUNCTION = "build" + CATEGORY = "vLLM-Omni" + + def build(self, images, source_fps, duration, mode, preserve_fraction=0.5): + if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0: + raise ValueError("Source must contain frames and have a positive finite FPS.") + if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1: + raise ValueError("Invalid duration or preserve_fraction.") + frames = _align_frame_count(max(1, round(duration * 24))) + source_seconds = images.shape[0] / source_fps + if mode == "extension": + if frames / 24 <= source_seconds: + raise ValueError("Extension output must be longer than the source; increase duration.") + boundary = source_seconds + elif mode == "continuation": + boundary = min(source_seconds, frames / 24) * preserve_fraction + else: + raise ValueError(f"Unknown temporal mask mode: {mode}") + available = min(frames, math.floor(boundary * 24 + 1e-8)) + prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17) + preserved = _video_latent_t(prefix) if prefix else 0 + total = _video_latent_t(frames) + mask = torch.ones(total, 1, 1) + mask[:preserved] = 0 + indices = torch.arange(frames, device=images.device) + source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1) + preview_images = images.index_select(0, source_indices) + preview_mask = mask.index_select(0, torch.arange(frames) * total // frames) + return mask, 24.0, preview_images, preview_mask diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/temporal_mask.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/temporal_mask.py deleted file mode 100644 index f9a5a4b92fd..00000000000 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/temporal_mask.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project - -import math - -import torch - -from .utils.latent_mask import _align_frame_count, _video_latent_t - - -class VLLMOmniH3TemporalMask: - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "images": ("IMAGE",), - "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}), - "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}), - "mode": (["continuation", "extension"],), - "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}), - } - } - - RETURN_TYPES = ("MASK", "STRING", "FLOAT", "IMAGE", "MASK") - RETURN_NAMES = ("mask", "summary", "preview_fps", "preview_images", "preview_mask") - FUNCTION = "build" - CATEGORY = "vLLM-Omni" - - def build(self, images, source_fps, duration, mode, preserve_fraction=0.5): - if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0: - raise ValueError("Source must contain frames and have a positive finite FPS.") - if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1: - raise ValueError("Invalid duration or preserve_fraction.") - frames = _align_frame_count(max(1, round(duration * 24))) - source_seconds = images.shape[0] / source_fps - if mode == "extension": - if frames / 24 <= source_seconds: - raise ValueError("Extension output must be longer than the source; increase duration.") - boundary = source_seconds - elif mode == "continuation": - boundary = min(source_seconds, frames / 24) * preserve_fraction - else: - raise ValueError(f"Unknown temporal mask mode: {mode}") - available = min(frames, math.floor(boundary * 24 + 1e-8)) - prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17) - preserved = _video_latent_t(prefix) if prefix else 0 - total = _video_latent_t(frames) - mask = torch.ones(total, 1, 1) - mask[:preserved] = 0 - indices = torch.arange(frames, device=images.device) - source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1) - preview_images = images.index_select(0, source_indices) - preview_mask = mask.index_select(0, torch.arange(frames) * total // frames) - summary = ( - f"Source: {source_seconds:.3f}s; target: {frames} frames / {total} latent slices; " - f"preserve: {preserved}; generate: {total - preserved}; " - f"requested boundary: {boundary:.3f}s; conservative prefix: {prefix / 24:.3f}s." - ) - return mask, summary, 24.0, preview_images, preview_mask diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json index 69b9b5f7aca..fc434173093 100644 --- a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json @@ -250,10 +250,10 @@ "Node name for S&R": "MarkdownNote" }, "widgets_values": [ - "Replace the bag with a small orange tabby cat curled up asleep. Source: 243 frames at 24 FPS. Grow Mask: 32 pixels. Fixed seed 0, 50 steps. Audio mask 0.5 matches the Object Removal baseline. Enable this group and mute other generation groups before running. Quality is not yet validated. Video and spatial mask are linked from Shared Inputs. Grow Mask remains independent for this case." + "Replace the masked region with a curled orange cat gently swaying its tail.\n\nThis case has its own rectangular mask. Leave room for the entire cat and tail motion; check the red overlay before generating.\n\nAudio mask 0 preserves source audio latents. To request new sounds, change the audio prompt and mask together. Fractional audio masks do not selectively retain the school chime while adding meows." ], "widgets_values_named": { - "text": "Replace the bag with a small orange tabby cat curled up asleep. Source: 243 frames at 24 FPS. Grow Mask: 32 pixels. Fixed seed 0, 50 steps. Audio mask 0.5 matches the Object Removal baseline. Enable this group and mute other generation groups before running. Quality is not yet validated. Video and spatial mask are linked from Shared Inputs. Grow Mask remains independent for this case." + "text": "Replace the masked region with a curled orange cat gently swaying its tail.\n\nThis case has its own rectangular mask. Leave room for the entire cat and tail motion; check the red overlay before generating.\n\nAudio mask 0 preserves source audio latents. To request new sounds, change the audio prompt and mask together. Fractional audio masks do not selectively retain the school chime while adding meows." }, "color": "#333", "bgcolor": "#222" @@ -448,10 +448,10 @@ "Node name for S&R": "MarkdownNote" }, "widgets_values": [ - "Temporal mask follows the actual source frame count/FPS and shared target duration. Preserve fraction selects the available prefix to retain. Boundary rounds down to complete H3 native chunks; a short segment before the requested boundary may regenerate. Preview time mapping is schematic." + "Regenerate the ending with an overcast sky, heavy rain, and distant lightning.\n\nPreserve fraction controls how much of the available source prefix is retained. Match Temporal Mask duration to Generate Video duration; these values are not linked.\n\nThe boundary rounds down to complete H3 chunks. White allows regeneration, but does not force existing subjects to disappear.\n\nThe red preview is schematic. Audio mask affects the entire clip, including the preserved video prefix." ], "widgets_values_named": { - "text": "Temporal mask follows the actual source frame count/FPS and shared target duration. Preserve fraction selects the available prefix to retain. Boundary rounds down to complete H3 native chunks; a short segment before the requested boundary may regenerate. Preview time mapping is schematic." + "text": "Regenerate the ending with an overcast sky, heavy rain, and distant lightning.\n\nPreserve fraction controls how much of the available source prefix is retained. Match Temporal Mask duration to Generate Video duration; these values are not linked.\n\nThe boundary rounds down to complete H3 chunks. White allows regeneration, but does not force existing subjects to disappear.\n\nThe red preview is schematic. Audio mask affects the entire clip, including the preserved video prefix." }, "color": "#333", "bgcolor": "#222" @@ -627,7 +627,7 @@ }, { "id": 59, - "type": "VLLMOmniH3TemporalMask", + "type": "VLLMOmniMiniMaxH3TemporalMask", "pos": [ 580, 6040 @@ -692,12 +692,6 @@ 60 ] }, - { - "localized_name": "summary", - "name": "summary", - "type": "STRING", - "links": [] - }, { "localized_name": "preview_fps", "name": "preview_fps", @@ -725,7 +719,7 @@ } ], "properties": { - "Node name for S&R": "VLLMOmniH3TemporalMask" + "Node name for S&R": "VLLMOmniMiniMaxH3TemporalMask" }, "widgets_values": [ 24, @@ -761,10 +755,10 @@ "Node name for S&R": "MarkdownNote" }, "widgets_values": [ - "Temporal mask follows the actual source frame count/FPS and shared target duration. Output duration must exceed the source duration. The original, unpadded source determines the preserved prefix. Boundary rounds down to complete H3 native chunks; a short segment before the requested boundary may regenerate. Preview time mapping is schematic." + "Extend into a summer starry night. Follow a firework rising behind the school with a slow upward camera tilt, ending at its burst.\n\nSet the same duration in Temporal Mask and Generate Video, longer than the source. These values are not linked. The original source length determines the preserved prefix.\n\nThe boundary rounds down to H3 chunks. Preview holds the final source frame under the red overlay; it is not the generated ending.\n\nAudio mask affects the whole clip, not only the extension." ], "widgets_values_named": { - "text": "Temporal mask follows the actual source frame count/FPS and shared target duration. Output duration must exceed the source duration. The original, unpadded source determines the preserved prefix. Boundary rounds down to complete H3 native chunks; a short segment before the requested boundary may regenerate. Preview time mapping is schematic." + "text": "Extend into a summer starry night. Follow a firework rising behind the school with a slow upward camera tilt, ending at its burst.\n\nSet the same duration in Temporal Mask and Generate Video, longer than the source. These values are not linked. The original source length determines the preserved prefix.\n\nThe boundary rounds down to H3 chunks. Preview holds the final source frame under the red overlay; it is not the generated ending.\n\nAudio mask affects the whole clip, not only the extension." }, "color": "#333", "bgcolor": "#222" @@ -851,10 +845,10 @@ "Node name for S&R": "MarkdownNote" }, "widgets_values": [ - "## Object Removal\n\nUse the shared source video and a matching spatial mask. White edits; black preserves.\n\nCover the entire bag, handle, straps, and contact shadow. Check coverage in Mask Preview.\n\nDescribe the area as empty from the first frame to the last, with continuous wall and pavement textures.\n\nKeep frame and references disconnected for this T2VA setup.\n\nTo run only this case, select Save Video (Object Removal) and click Execute to selected output nodes." + "Reconstruct the surrounding wall inside the mask.\n\nSet this case's Mask Canvas, Mask Rectangle, and Rectangle Position to cover the object, handle, and shadow with some background margin. Preview coverage before generating.\n\nSymmetric expansion by d pixels: x/y minus d; width/height plus 2d. A 32-pixel margin improved this school example.\n\nAudio mask 0 preserves source audio latents; reconstruction is not lossless." ], "widgets_values_named": { - "text": "## Object Removal\n\nUse the shared source video and a matching spatial mask. White edits; black preserves.\n\nCover the entire bag, handle, straps, and contact shadow. Check coverage in Mask Preview.\n\nDescribe the area as empty from the first frame to the last, with continuous wall and pavement textures.\n\nKeep frame and references disconnected for this T2VA setup.\n\nTo run only this case, select Save Video (Object Removal) and click Execute to selected output nodes." + "text": "Reconstruct the surrounding wall inside the mask.\n\nSet this case's Mask Canvas, Mask Rectangle, and Rectangle Position to cover the object, handle, and shadow with some background margin. Preview coverage before generating.\n\nSymmetric expansion by d pixels: x/y minus d; width/height plus 2d. A 32-pixel margin improved this school example.\n\nAudio mask 0 preserves source audio latents; reconstruction is not lossless." }, "color": "#333", "bgcolor": "#222" @@ -2296,13 +2290,13 @@ "mode": 0, "inputs": [], "outputs": [], - "title": "Latent-mask editing server prerequisite", + "title": "Usage Guide", "properties": {}, "widgets_values": [ - "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference." ], "widgets_values_named": { - "text": "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + "text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference." }, "color": "#432", "bgcolor": "#000" @@ -3944,7 +3938,7 @@ }, { "id": 44, - "type": "VLLMOmniH3TemporalMask", + "type": "VLLMOmniMiniMaxH3TemporalMask", "pos": [ 580, 4440 @@ -4009,12 +4003,6 @@ 47 ] }, - { - "localized_name": "summary", - "name": "summary", - "type": "STRING", - "links": [] - }, { "localized_name": "preview_fps", "name": "preview_fps", @@ -4042,7 +4030,7 @@ } ], "properties": { - "Node name for S&R": "VLLMOmniH3TemporalMask" + "Node name for S&R": "VLLMOmniMiniMaxH3TemporalMask" }, "widgets_values": [ 24, @@ -4927,7 +4915,7 @@ [ 142, 44, - 2, + 1, 73, 2, "FLOAT" @@ -4935,7 +4923,7 @@ [ 146, 59, - 2, + 1, 77, 2, "FLOAT" @@ -4943,7 +4931,7 @@ [ 147, 44, - 3, + 2, 90, 0, "IMAGE" @@ -4959,7 +4947,7 @@ [ 149, 44, - 4, + 3, 90, 2, "MASK" @@ -4967,7 +4955,7 @@ [ 150, 44, - 3, + 2, 91, 0, "IMAGE" @@ -4991,7 +4979,7 @@ [ 153, 59, - 3, + 2, 93, 0, "IMAGE" @@ -5007,7 +4995,7 @@ [ 155, 59, - 4, + 3, 93, 2, "MASK" @@ -5015,7 +5003,7 @@ [ 156, 59, - 3, + 2, 94, 0, "IMAGE" diff --git a/tests/e2e/features/comfyui/test_wf05_template.py b/tests/e2e/features/comfyui/test_wf05_template.py deleted file mode 100644 index 717a8f313fd..00000000000 --- a/tests/e2e/features/comfyui/test_wf05_template.py +++ /dev/null @@ -1,80 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project - -"""Portable graph and temporal-mask contracts for the WF-05 examples.""" - -import json -from pathlib import Path - -import pytest - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - -WORKFLOW = ( - Path(__file__).resolve().parents[4] - / "apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json" -) - - -def test_wf05_graph_links(): - graph = json.loads(WORKFLOW.read_text()) - nodes = {node["id"]: node for node in graph["nodes"]} - assert len(nodes) == len(graph["nodes"]) - assert len({link[0] for link in graph["links"]}) == len(graph["links"]) - for link_id, source, output, target, slot, kind in graph["links"]: - assert nodes[target]["inputs"][slot]["link"] == link_id - assert link_id in nodes[source]["outputs"][output]["links"] - assert nodes[source]["outputs"][output]["type"] == kind - assert nodes[target]["inputs"][slot]["type"] == kind - assert nodes[source]["mode"] == nodes[target]["mode"] - - -def test_wf05_defaults_and_outputs(): - graph = json.loads(WORKFLOW.read_text()) - nodes = {n["id"]: n for n in graph["nodes"]} - links = {link[0]: link for link in graph["links"]} - generators = [n for n in nodes.values() if n["type"] == "VLLMOmniGenerateVideo"] - assert len(generators) == 4 - assert len([n for n in nodes.values() if n["type"] == "SaveVideo"]) == 8 - assert not any(n["type"] == "VLLMOmniH3MaskGridPreview" for n in nodes.values()) - for gen in generators: - case = gen["title"].removeprefix("Generate Video(").removesuffix(")") - save = next(n for n in nodes.values() if n.get("title") == f"Mask Preview ({case})") - create = nodes[links[save["inputs"][0]["link"]][1]] - image_node = nodes[links[create["inputs"][0]["link"]][1]] - edit = nodes[links[next(i["link"] for i in gen["inputs"] if i["name"] == "latent_edit")][1]] - mask_link = next(i["link"] for i in edit["inputs"] if i["name"] == "video_mask") - assert image_node["type"] == "ImageBlend" - consumer = nodes[links[next(i["link"] for i in image_node["inputs"] if i["name"] == "image2")][1]] - assert consumer["type"] == "ImageCompositeMasked" - preview_mask = next(i["link"] for i in consumer["inputs"] if i["name"] == "mask") - if case in ("Continuation", "Extension"): - assert nodes[links[mask_link][1]]["type"] == "VLLMOmniH3TemporalMask" - assert links[preview_mask][1:3] == [links[mask_link][1], 4] - fps_link = next(i["link"] for i in create["inputs"] if i["name"] == "fps") - assert links[fps_link][1:3] == [links[mask_link][1], 2] - else: - assert links[mask_link][1:3] == links[preview_mask][1:3] - - -def test_wf05_output_layout(): - graph = json.loads(WORKFLOW.read_text()) - for case in ("Object Removal", "Inpainting", "Continuation", "Extension"): - preview = next(n for n in graph["nodes"] if n.get("title") == f"Mask Preview ({case})") - result = next(n for n in graph["nodes"] if n.get("title") == f"Save Video ({case})") - assert preview["pos"][0] == result["pos"][0] - assert preview["pos"][1] + preview["size"][1] < result["pos"][1] - - -def test_wf05_portable_inputs(): - graph = json.loads(WORKFLOW.read_text()) - for node in graph["nodes"]: - if node["type"] in ("LoadVideo", "LoadImageMask"): - assert node["widgets_values"][0] == "" - assert not any(n["type"] == "ImageBatch" for n in graph["nodes"]) - - -def test_wf05_temporal_nodes(): - graph = json.loads(WORKFLOW.read_text()) - assert sum(n["type"] == "VLLMOmniH3TemporalMask" for n in graph["nodes"]) == 2 - assert not any(n["type"] == "ComfyMathExpression" for n in graph["nodes"]) From bb7dc9b6fe982f97f9af9f9743d10a0d38618375 Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Mon, 21 Sep 2026 14:43:24 +0800 Subject: [PATCH 9/9] [Frontend] Update WF-05 docs Signed-off-by: Weiming Liao --- .../docs/wf05-h3-latent-editing.md | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md index 549d92993ca..9ed10273f13 100644 --- a/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md +++ b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md @@ -1,19 +1,17 @@ -# H3 latent-mask editing (WF-05) +# WF-05: MiniMax-H3 latent-mask editing -Open `vLLM-Omni MiniMax-H3 Latent Mask Editing`. Upload a source video. Each spatial case has an independent mask that is built with official Solid Mask and Combine Masks nodes; no mask image is required. Configure a compatible H3 server URL and model in each generation node. The workflow uses the EDIT-02 latent-edit interface. +Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case. -Each case has a Mask Preview above its Result in the Output Hub. Select the desired Save Video output and execute only that output to preview locally without running generation. All four cases are enabled; running the whole graph also submits generation requests. +## Requirements -Set output dimensions and duration directly in Generate Video. For continuation and extension, also update the duration in MiniMax-H3 Temporal Mask. Defaults are 1344 x 768 at 24 FPS, with 5 seconds for removal, inpainting and continuation, and 10 seconds for extension. +- ComfyUI with ComfyUI-vLLM-Omni installed. +- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup. -Preview uses official ComfyUI nodes only. Spatial cases use Empty Image, Image Composite Masked and Image Blend to overlay the original mask in red on source frames, followed by Create Video and Save Video. This preview follows the source resolution, FPS and duration and does not visualize the model's downsampled latent mask. Temporal cases use the same official red-overlay chain. The temporal node samples at 24 FPS and holds the last source frame for extension. With a 24 FPS source, existing frames are selected directly. Preserved frames remain unchanged; generated regions receive a red tint. This is a schematic timeline, not exact decoded-frame boundaries. Neither preview runs inference; previews are silent. +## Use -MiniMax-H3 Temporal Mask calculates target frames and latent slice counts from the decoded source video. Continuation preserves the selected fraction of the shorter source/target duration. Extension preserves the source prefix and requires a longer target. The target rounds up to `17k+5` frames; preservation rounds down to complete native chunks. Set the same duration in this node and Generate Video. The node also supplies source frames and a schematic per-frame mask to the official red-overlay preview nodes. +Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run. -Audio masks remain whole-clip scalars. Zero preserves encoded source audio, one regenerates it, and intermediate values partially edit the audio latents. Neither latent video nor audio preservation guarantees lossless reconstruction. - -The exported workflow has an empty video selector. The default rectangle is x=1074, y=493, width=180, height=168 on a 1344 x 768 canvas, including 32 pixels of background margin around the original bag mask bounding box. Adjust Mask Canvas to the source dimensions, Mask Rectangle to the object size, and Combine Masks x/y to its position. A filled rectangle may edit more background than the original shaped mask. Provide a matching school clip or adapt the rectangle and prompts to your assets. Reopen the workflow without saving an older canvas over it. - -Graph tests check wiring, matching preview/generation defaults, mask sources, vertical output placement and portable inputs. They do not establish real-model visual/audio acceptance. - -The example prompts use wall reconstruction, a curled cat with a swaying tail, heavy rain with distant lightning, and a summer-night firework. Current audio masks are 0, 0, 0.8, and 0.9 respectively; Continuation preserves 0.2 of the shorter source/target duration. +1. Upload your source video in Load Video. +2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate. +3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip). +4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.