diff --git a/configs/dit/minimax_h3_fl2va_embedding.yaml b/configs/dit/minimax_h3_fl2va_embedding.yaml new file mode 100644 index 0000000000..6705bc2d0b --- /dev/null +++ b/configs/dit/minimax_h3_fl2va_embedding.yaml @@ -0,0 +1,98 @@ +# ============================================================================= +# MiniMax H3 FL2VA offline embedding (pre-compute VAE latents) +# ============================================================================= +# +# Entry point: tasks/train_dit.py (plain DiTTrainer; embedding stage picks the transform from data_transform) +# Usage: bash train.sh tasks/train_dit.py configs/dit/minimax_h3_fl2va_embedding.yaml +# +# Encodes raw data → VAE latents + prompt embeddings + packed info → saves parquet. +# Run BEFORE offline_training. The saved parquet is used as input for offline +# training (minimax_h3_fl2va_offline.yaml). +# +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Model +# ----------------------------------------------------------------------------- +model: + # DiT model not loaded during embedding (model_path unused) + condition_model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA + config_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA/transformer/config.json + + condition_model_cfg: + base_model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA + video_vae_subfolder: video_vae/source + skip_encoder_load: false + use_keyframe_condition: true + keyframe_indices: [0, -1] + video_max_frames: 73 + video_max_resolution: 848 + sigma_shift_video: 12.0 + sigma_shift_audio: 3.0 + imgvid_cond_noise_aug: 0.999 + audio_cond_noise_aug: 1.0 +# ----------------------------------------------------------------------------- +# Data +# ----------------------------------------------------------------------------- +data: + train_path: dataset/minimax-h3-demo/minimax_h3/MiniMax-H3-FL2VA/metadata.csv + data_type: diffusion + source_name: minimax_h3 + data_transform: minimax_h3_online + datasets_type: minimax_h3_online + log_sample: false + shuffle: false + text_keys: text + dataloader: + type: native + num_workers: 0 + drop_last: false + mm_configs: + data_dir: dataset/minimax-h3-demo/minimax_h3/MiniMax-H3-FL2VA/ + fps: 24 + min_frames: 124 + max_frames: 124 + height: 480 + width: 832 + offline_embedding_save_dir: output/minimax_h3_fl2va_embedding + +# ----------------------------------------------------------------------------- +# Training +# ----------------------------------------------------------------------------- +train: + training_task: offline_embedding + + global_batch_size: 8 + micro_batch_size: 1 + dyn_bsz: false + + init_device: meta + seed: 42 + + num_train_epochs: 1 + # max_steps unset: process all data (set to 0 would clamp train_steps to 0 + # via VeOmniArguments.train_steps property) + + gradient_checkpointing: + enable: false + + optimizer: + type: adamw + lr: 1.0e-4 + + accelerator: + ulysses_size: 1 + fsdp_config: + fsdp_mode: fsdp2 + offload: false + mixed_precision: + enable: false + + checkpoint: + output_dir: output/minimax_h3_fl2va_embedding + save_steps: 0 + save_hf_weights: false + + wandb: + enable: false + project: minimax_h3_fl2va diff --git a/configs/dit/minimax_h3_fl2va_offline.yaml b/configs/dit/minimax_h3_fl2va_offline.yaml new file mode 100644 index 0000000000..f6be839882 --- /dev/null +++ b/configs/dit/minimax_h3_fl2va_offline.yaml @@ -0,0 +1,107 @@ +# ============================================================================= +# MiniMax H3 FL2VA offline training +# ============================================================================= +# +# Entry point: tasks/train_dit.py (plain DiTTrainer; embedding stage picks the transform from data_transform) +# Usage: bash train.sh tasks/train_dit.py configs/dit/minimax_h3_fl2va_offline.yaml +# +# Uses pre-computed VAE latents from offline_embedding stage. +# offline_embedding: get_condition() encodes raw data → saves parquet +# offline_training: skip encoders, process_condition() adds noise + packs +# +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Model +# ----------------------------------------------------------------------------- +model: + model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA/transformer + condition_model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA + config_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA/transformer/config.json + + condition_model_cfg: + base_model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA + skip_encoder_load: true + use_keyframe_condition: true + keyframe_indices: [0, -1] + video_max_frames: 120 + video_max_resolution: 832 + sigma_shift_video: 12.0 + sigma_shift_audio: 3.0 + imgvid_cond_noise_aug: 0.999 + audio_cond_noise_aug: 1.0 + + ops_implementation: + attn_implementation: eager + rotary_pos_emb_implementation: eager + rms_norm_implementation: eager + swiglu_mlp_implementation: eager + cross_entropy_loss_implementation: eager + moe_implementation: eager + load_balancing_loss_implementation: eager + +# ----------------------------------------------------------------------------- +# Data +# ----------------------------------------------------------------------------- +data: + train_path: output/minimax_h3_fl2va_embedding + data_type: diffusion + source_name: minimax_h3 + data_transform: minimax_h3_offline + datasets_type: minimax_h3_offline + log_sample: false + text_keys: text + mm_configs: + repeat: 100 + dataloader: + type: native + num_workers: 8 + drop_last: true + +# ----------------------------------------------------------------------------- +# Training +# ----------------------------------------------------------------------------- +train: + training_task: offline_training + + global_batch_size: 16 + micro_batch_size: 1 + dyn_bsz: false + + init_device: meta + seed: 42 + + num_train_epochs: 1 + max_steps: 20 + + gradient_checkpointing: + enable: true + + optimizer: + type: adamw + lr: 1.0e-5 + betas: [0.9, 0.999] + lr_decay_style: linear + lr_min: 1.0e-5 + lr_warmup_ratio: 0.0 + weight_decay: 0.01 + + accelerator: + ulysses_size: 1 + fsdp_config: + fsdp_mode: fsdp2 + mixed_precision: + enable: true + param_dtype: bfloat16 + reduce_dtype: float32 + cast_forward_inputs: false + + checkpoint: + output_dir: output/minimax_h3_fl2va_offline + save_steps: 500 + save_epochs: 0 + save_hf_weights: false + + wandb: + enable: false + project: minimax_h3_fl2va diff --git a/docs/examples/minimax_h3.md b/docs/examples/minimax_h3.md new file mode 100644 index 0000000000..c412e3d4a1 --- /dev/null +++ b/docs/examples/minimax_h3.md @@ -0,0 +1,263 @@ +# MiniMax H3 FL2VA Quick Start + +This guide walks through **training** and **inference** for MiniMax H3 FL2VA (first/last frame + text -> video + audio) on an Ascend NPU machine. Every command can be copied and run directly. + +- Verified environment: 4 Ascend NPUs, torch_npu + torchrun +- Verified flow: two-stage offline training (embedding -> offline) for 30 steps + single-card inference + +--- + +## Table of Contents + +1. [Model](#1-model) +2. [Data Format](#2-data-format) +3. [Training (Two Stages, Step by Step)](#3-training-two-stages-step-by-step) +4. [Training Config Notes](#4-training-config-notes) +5. [Inference](#5-inference-step-by-step) +6. [Inference Config Notes](#6-inference-config-notes) + +--- + +## 1. Model + +```shell +modelscope download --model MiniMax/MiniMax-H3 \ + --local_dir pretrained_models/MiniMax-H3 +``` + +--- + +## 2. Data Format + +### 2.1 Directory Layout + +During Stage 1 (offline embedding), if no keyframe images are provided, the first and last frames are extracted from the video itself. + +``` +dataset/my_data/ +├── metadata.csv # index file (must be named metadata.csv, or set via train_path in the config) +├── video.mp4 # training video +├── first.png # (optional) first-frame keyframe for inference +└── last.png # (optional) last-frame keyframe for inference +``` + +### 2.2 metadata.csv Format + +The following 4 columns are **required**, with fixed column names: + +```csv +video,prompt,input_audio,frame_rate +video.mp4,"A girl is very happy, she is speaking in english.",video.mp4,24 +``` + +| Column | Required | Meaning | +|:---|:-----|:-----| +| `video` | Yes | Video file name (relative to the CSV directory) | +| `prompt` | Yes | Text description (matches the video content) | +| `input_audio` | Yes | Audio source; set to `video.mp4` to use the video's own audio track | +| `frame_rate` | Yes | Frame rate; set to `24` | + +### 2.3 Hard Video Constraints (violations raise errors) + +| Constraint | Value | Notes | +|:-----|:---|:-----| +| Frame count | **124** (must satisfy `(num_frames-5) % 17 == 0`) | The Video VAE groups frames by 17; 124 is the demo-config value. 73, 107, 141, etc. are also legal (`(N-5) % 17 == 0`) | +| Resolution | **480x832** (height x width) | Must be divisible by the VAE downsampling factor | +| Frame rate | 24 | `fps: 24` in the config; audio latent length is computed as `num_frames/24*40` | +| Audio | 32kHz stereo | Audio is resampled to 32kHz automatically; videos without audio fail during training | + +--- + +## 3. Training (Two Stages, Step by Step) + +### Step 1: Run Stage 1 (offline embedding) + +```shell +# MiniMax H3 FL2VA +# Offline embedding +bash train.sh tasks/train_dit.py configs/dit/minimax_h3_fl2va_embedding.yaml +``` + +What Stage 1 does: + +- Loads Video VAE / Audio VAE / Text Encoder (**does not load the DiT**) +- Encodes video/audio/text -> VAE latents + prompt embeddings + packed-sequence info +- Writes one parquet per card: `output/minimax_h3_fl2va_embedding/rank__shard_0.parquet` +- Success marker: process exits with no traceback + +### Step 2: Switch to Stage 2 (offline training) + +```shell +bash train.sh tasks/train_dit.py configs/dit/minimax_h3_fl2va_offline.yaml +``` + +What Stage 2 does: + +- Loads the DiT (FSDP2 + gradient checkpointing + bf16), **skips** the VAE/Text Encoder (`skip_encoder_load: true`) +- Reads parquet -> adds noise -> DiT forward/backward -> AdamW update + +## 4. Training Config Notes + +Two config files: + +| Stage | Config | Training task | +|:-----|:-----|:---------| +| Stage 1 | `configs/dit/minimax_h3_fl2va_embedding.yaml` | `training_task: offline_embedding` | +| Stage 2 | `configs/dit/minimax_h3_fl2va_offline.yaml` | `training_task: offline_training` | + +### Stage 1 Config (embedding) + +```yaml +model: + condition_model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA + condition_model_cfg: + base_model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA + video_vae_subfolder: video_vae/source # VAE weights subfolder + skip_encoder_load: false # Stage 1 must be false (encoders are loaded) + use_keyframe_condition: true + keyframe_indices: [0, -1] # first frame + last frame + video_max_frames: 73 # max video latent frame groups + video_max_resolution: 848 + sigma_shift_video: 12.0 + sigma_shift_audio: 3.0 + +data: + train_path: dataset/minimax-h3-demo/minimax_h3/MiniMax-H3-FL2VA/metadata.csv + data_transform: minimax_h3_online # Stage 1 encodes raw video online + datasets_type: minimax_h3_online + dataloader: + num_workers: 0 # Stage 1 is encode-heavy; use 0 workers to avoid memory contention + drop_last: false + mm_configs: + data_dir: dataset/minimax-h3-demo/minimax_h3/MiniMax-H3-FL2VA/ + fps: 24 + min_frames: 124 + max_frames: 124 + height: 480 + width: 832 + offline_embedding_save_dir: output/minimax_h3_fl2va_embedding # Stage 2 reads from here +``` + +**Important**: + +- `train_path` must point to the **metadata.csv file**, not a directory, otherwise you get `png files are not supported` +- When changing data, `fps/min_frames/max_frames/height/width` must match the actual video parameters; the frame count must satisfy `(N-5) % 17 == 0` +- `offline_embedding_save_dir` must match Stage 2's `data.train_path` + +### Stage 2 Config (offline training) + +```yaml +model: + model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA/transformer + condition_model_path: pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA + condition_model_cfg: + skip_encoder_load: true # must be true: do not load VAE/TextEncoder + video_max_frames: 120 + video_max_resolution: 832 + +data: + train_path: output/minimax_h3_fl2va_embedding # output dir of Stage 1 + data_transform: dit_offline + datasets_type: minimax_h3_offline + shuffle: false + mm_configs: + repeat: 100 # dataset repeat count (must be > 1 for small datasets) + +train: + training_task: offline_training + global_batch_size: 8 + micro_batch_size: 1 + init_device: meta + max_steps: 30 + gradient_checkpointing: + enable: true # turning this off OOMs when memory is tight + optimizer: + type: adamw + lr: 1.0e-5 + max_grad_norm: 1.0e9 + accelerator: + fsdp_config: + fsdp_mode: fsdp2 + mixed_precision: + enable: true + param_dtype: bfloat16 + reduce_dtype: float32 + checkpoint: + output_dir: output/minimax_h3_fl2va_offline + save_steps: 10 + save_hf_weights: false +``` + +--- + +## 5. Inference + +```shell +python tasks/infer/infer_minimax_h3.py +``` + +The script runs two tasks sequentially: + +1. **t2va**: text-only -> video + audio (480x832, 124 frames, 50 steps) +2. **fl2va**: first frame + last frame + text -> video + audio (832x480 portrait, 124 frames, 50 steps) + +Output files (repo root): + +- `t2va.mp4` +- `fl2va.mp4` + +--- + +## 6. Inference Config Notes + +All inference config lives in `tasks/infer/infer_minimax_h3.py`: + +```python +pipe = MiniMaxH3Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device=device, + condition_model_path="pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA", + condition_model_cfg={ + "base_model_path": "pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA", + "use_keyframe_condition": True, + "keyframe_indices": [0, -1], # first and last frames + }, + transformer_config_path="pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA/transformer/config.json", + transformer_weights_path="pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA/transformer", + ops_implementation=OpsImplementationConfig( + attn_implementation="eager", + rotary_pos_emb_implementation="eager", + rms_norm_implementation="eager", + swiglu_mlp_implementation="eager", + cross_entropy_loss_implementation="eager", + moe_implementation="eager", + load_balancing_loss_implementation="eager", + ), +) +``` + +Call parameters: + +```python +# t2va +video, audio = pipe( + prompt=prompt, + height=480, width=832, num_frames=124, # frame count must satisfy (N-5) % 17 == 0 + num_inference_steps=50, seed=0, # fewer steps = faster; fixed seed = reproducible +) + +# fl2va +video, audio = pipe( + prompt=prompt, + height=832, width=480, num_frames=124, + num_inference_steps=50, seed=0, + keyframes=[first_frame, last_frame], # images must exist, otherwise FileNotFoundError + keyframe_indices=[0, -1], +) +``` + +**Important**: + +- `num_frames` must satisfy `(N-5) % 17 == 0`, otherwise the Video VAE raises an error + +--- diff --git a/tasks/infer/infer_minimax_h3.py b/tasks/infer/infer_minimax_h3.py new file mode 100644 index 0000000000..4674710c1e --- /dev/null +++ b/tasks/infer/infer_minimax_h3.py @@ -0,0 +1,72 @@ +import torch +from PIL import Image + +from veomni.arguments.arguments_types import OpsImplementationConfig +from veomni.data.multimodal.video_utils import write_video_audio +from veomni.models.diffusers.minimax_h3.inference import MiniMaxH3Pipeline +from veomni.utils.device import get_device_type + + +device = get_device_type() +pipe = MiniMaxH3Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device=device, + condition_model_path="pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA", + condition_model_cfg={ + "base_model_path": "pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA", + "use_keyframe_condition": True, + "keyframe_indices": [0, -1], + }, + transformer_config_path="pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA/transformer/config.json", + transformer_weights_path="pretrained_models/MiniMax-H3/MiniMax/MiniMax-H3/FL2VA/transformer", + ops_implementation=OpsImplementationConfig( + attn_implementation="eager", + rotary_pos_emb_implementation="eager", + rms_norm_implementation="eager", + swiglu_mlp_implementation="eager", + cross_entropy_loss_implementation="eager", + moe_implementation="eager", + load_balancing_loss_implementation="eager", + ), +) + +# Text -> Video + Audio +prompt = "A girl is very happy, she is speaking in english: “I enjoy working with VeOmni, it's a perfect framework.”" +video, audio = pipe( + prompt=prompt, + height=480, + width=832, + num_frames=124, + num_inference_steps=50, + seed=0, +) +write_video_audio( + video=video, + audio=audio, + output_path="t2va.mp4", + fps=24, + audio_sample_rate=32000, +) + +# Text + First Frame + Last Frame -> Video + Audio +# Keyframes are loaded from the local dataset copy. +first_frame = Image.open("dataset/minimax-h3-demo/minimax_h3/MiniMax-H3-FL2VA/first.png") +last_frame = Image.open("dataset/minimax-h3-demo/minimax_h3/MiniMax-H3-FL2VA/last.png") +prompt = "室内家庭争吵短剧场景,竖屏短剧质感,真实真人表演,中式家庭/小饭馆室内环境,暖色灯光,背景有红色装饰和书法字幅,浅景深,情绪强烈,剪辑节奏紧凑。表演要求:真实短剧表演风格,不要夸张舞台腔。男人的语气是愤怒、委屈、急切的反驳,他说“你到底想干什么?”;中老年女性的语气是尖锐、强势、咄咄逼人的质问,她说“你必须赔钱!”。两人之间有强烈对峙感,节奏逐步升级。画面风格:竖屏9:16,手机短剧质感,真人实拍感,浅景深,室内暖光,中近景为主,频繁正反打剪辑,背景保持生活化,不要科幻、不要古装、不要动画感。画面中不要出现任何字幕、文字、平台水印或贴片。 " +video, audio = pipe( + prompt=prompt, + height=832, + width=480, + num_frames=124, + num_inference_steps=50, + seed=0, + keyframes=[first_frame, last_frame], + keyframe_indices=[0, -1], +) +write_video_audio( + video=video, + audio=audio, + output_path="fl2va.mp4", + fps=24, + audio_sample_rate=32000, +) diff --git a/veomni/arguments/arguments_types.py b/veomni/arguments/arguments_types.py index 5226d82dec..3bce4d4106 100644 --- a/veomni/arguments/arguments_types.py +++ b/veomni/arguments/arguments_types.py @@ -16,7 +16,7 @@ import os from dataclasses import MISSING, dataclass, field, fields from pathlib import Path -from typing import Dict, List, Literal, Optional +from typing import Dict, List, Literal, Optional, Tuple from ..utils import logging from ..utils.env import get_env @@ -112,6 +112,10 @@ class OptimizerConfig: default=1.0, metadata={"help": "Clip value for gradient norm."}, ) + betas: Tuple[float, float] = field( + default=(0.9, 0.95), + metadata={"help": "AdamW betas (beta1, beta2). Default (0.9, 0.95)."}, + ) # ---- Muon-specific (only consulted when type == "muon") --------------- muon_lr: Optional[float] = field( default=None, diff --git a/veomni/data/multimodal/audio_utils.py b/veomni/data/multimodal/audio_utils.py index 557a0031fd..72dfc035a5 100644 --- a/veomni/data/multimodal/audio_utils.py +++ b/veomni/data/multimodal/audio_utils.py @@ -13,9 +13,11 @@ # limitations under the License. import io +from fractions import Fraction from io import BytesIO from typing import ByteString, List, Optional, Tuple, Union +import av import numpy as np import torch @@ -200,3 +202,94 @@ def save_audio_tensor_to_file( import soundfile as sf sf.write(output_path, audio, samplerate=sample_rate) + + +def convert_to_stereo(audio_tensor: torch.Tensor) -> torch.Tensor: + """ + Convert audio to stereo. + Supports [C, T] or [B, C, T]. Duplicate mono, keep stereo. + """ + if audio_tensor.size(-2) == 1: + return audio_tensor.repeat(1, 2, 1) if audio_tensor.dim() == 3 else audio_tensor.repeat(2, 1) + return audio_tensor + + +def _resample_audio( + container: av.container.Container, audio_stream: av.audio.AudioStream, frame_in: av.AudioFrame +) -> None: + cc = audio_stream.codec_context + + # Use the encoder's format/layout/rate as the *target* + target_format = cc.format or "fltp" # AAC → usually fltp + target_layout = cc.layout or "stereo" + target_rate = cc.sample_rate or frame_in.sample_rate + + audio_resampler = av.audio.resampler.AudioResampler( + format=target_format, + layout=target_layout, + rate=target_rate, + ) + + audio_next_pts = 0 + for rframe in audio_resampler.resample(frame_in): + if rframe.pts is None: + rframe.pts = audio_next_pts + audio_next_pts += rframe.samples + # Keep the resampler-assigned sample_rate (the target rate), not the + # input frame's rate: the encoder/muxer stamps packets from it. + container.mux(audio_stream.encode(rframe)) + + # Drain the resampler so tail samples are not dropped. + for rframe in audio_resampler.resample(None): + if rframe.pts is None: + rframe.pts = audio_next_pts + audio_next_pts += rframe.samples + container.mux(audio_stream.encode(rframe)) + + # flush audio encoder + for packet in audio_stream.encode(): + container.mux(packet) + + +def _write_audio( + container: av.container.Container, + audio_stream: av.audio.AudioStream, + samples: torch.Tensor, + audio_sample_rate: int, +) -> None: + if samples.ndim == 1: + samples = samples.unsqueeze(0) + samples = convert_to_stereo(samples) + assert samples.ndim == 2 and samples.shape[0] == 2, "audio samples must be [C, S] or [S], C must be 1 or 2" + samples = samples.T + # Convert to int16 packed for ingestion; resampler converts to encoder fmt. + if samples.dtype != torch.int16: + samples = torch.clip(samples, -1.0, 1.0) + samples = (samples * 32767.0).to(torch.int16) + + frame_in = av.AudioFrame.from_ndarray( + samples.contiguous().reshape(1, -1).cpu().numpy(), + format="s16", + layout="stereo", + ) + frame_in.sample_rate = audio_sample_rate + + _resample_audio(container, audio_stream, frame_in) + + +def _prepare_audio_stream(container: av.container.Container, audio_sample_rate: int) -> av.audio.AudioStream: + """ + Prepare the audio stream for writing. + """ + audio_stream = container.add_stream("aac") + supported_sample_rates = audio_stream.codec_context.codec.audio_rates + if supported_sample_rates: + best_rate = min(supported_sample_rates, key=lambda x: abs(x - audio_sample_rate)) + if best_rate != audio_sample_rate: + print(f"Using closest supported audio sample rate: {best_rate}") + else: + best_rate = audio_sample_rate + audio_stream.codec_context.sample_rate = best_rate + audio_stream.codec_context.layout = "stereo" + audio_stream.codec_context.time_base = Fraction(1, best_rate) + return audio_stream diff --git a/veomni/data/multimodal/dit/data_transform.py b/veomni/data/multimodal/dit/data_transform.py index 959b9942f2..51735ab30b 100644 --- a/veomni/data/multimodal/dit/data_transform.py +++ b/veomni/data/multimodal/dit/data_transform.py @@ -1,3 +1,5 @@ +import PIL + from ...data_transform import DATA_TRANSFORM_REGISTRY from ..image_utils import fetch_images from ..preprocess import conv_preprocess @@ -20,6 +22,128 @@ def process_dit_online_example(example, source_name, **kwargs): return [processed_example] +@DATA_TRANSFORM_REGISTRY.register("minimax_h3_online") +def process_minimax_h3_online_example(example, source_name, **kwargs): + """Raw data loading for MiniMax-H3 FL2VA. + + LoadVideo + ImageCropAndResize + LoadAudioWithTorchaudio steps: + - video: imageio reader, fix_frame_rate=True (24fps), num_frames = min_frames + (17n+5), bilinear resize + center crop to (height, width), frames → float32 + [0,1] tensors [3,H,W] (preprocess_video torch_dtype=float32, min_value=0) + - audio: torchaudio.load, trim/pad to int(num_frames/24 * original_sr) at + ORIGINAL sample rate, return (waveform[C,T], sample_rate) + """ + import math + + import imageio + import numpy as np + import torch + import torchvision.transforms.functional as TF + + prompt, audios, _, videos = conv_preprocess(source=source_name, conversations=example, **kwargs) + + num_frames = int(kwargs.get("min_frames", 124)) + height = int(kwargs.get("height", 480)) + width = int(kwargs.get("width", 832)) + frame_rate = float(kwargs.get("fps", 24)) + + frames = [] + n_frames = num_frames + if videos and videos[0]: + reader = imageio.get_reader(videos[0]) + try: + meta = reader.get_meta_data() + raw_fps = meta["fps"] + if "duration" in meta: + available = math.floor(meta["duration"] * frame_rate) + else: + # No duration in meta: fall back to a full frame-count scan. + total_raw_frames = int(reader.count_frames()) + available = math.floor((total_raw_frames / raw_fps) * frame_rate) + if int(available) < num_frames: + n_frames = int(available) + if n_frames < 5: + raise ValueError( + f"Video clip is too short: {n_frames} frames available at " + f"{frame_rate:g}fps, but MiniMax-H3 needs at least 5 frames " + "(frame count must satisfy (N-5) % 17 == 0)." + ) + while n_frames > 1 and n_frames % 17 != 5: + n_frames -= 1 + # Single sequential pass over the stream. raw_idx only moves + # forward, so selected indices are consumed in order; repeated + # indices (round collisions) re-append the current frame, and + # indices past the end of the stream clamp to the last frame — + # exactly the frames the previous get_data(raw_idx) loop selected. + desired = [int(round(i / frame_rate * raw_fps)) for i in range(n_frames)] + ptr = 0 + for j, raw_frame in enumerate(reader.iter_data()): + while ptr < n_frames and desired[ptr] <= j: + img = PIL.Image.fromarray(raw_frame) + # ImageCropAndResize(height, width) + w, h = img.size + scale = max(width / w, height / h) + img = TF.resize( + img, + (round(h * scale), round(w * scale)), + interpolation=TF.InterpolationMode.BILINEAR, + ) + img = TF.center_crop(img, (height, width)) + frames.append(img) + ptr += 1 + last_raw_frame = raw_frame + # Indices beyond the stream end clamp to the last decoded frame. + while frames and ptr < n_frames: + img = PIL.Image.fromarray(last_raw_frame) + w, h = img.size + scale = max(width / w, height / h) + img = TF.resize( + img, + (round(h * scale), round(w * scale)), + interpolation=TF.InterpolationMode.BILINEAR, + ) + img = TF.center_crop(img, (height, width)) + frames.append(img) + ptr += 1 + finally: + reader.close() + # preprocess_video(torch_dtype=float32, min_value=0): [0,1] float32 + frames_t = [torch.tensor(np.array(f, dtype=np.float32)).permute(2, 0, 1) * (1.0 / 255.0) for f in frames] + + audio_out = None + if audios and "audio" in audios and audios["audio"]: + import torchaudio + + waveform, sample_rate = torchaudio.load(audios["audio"]) + target_samples = int((n_frames / frame_rate) * sample_rate) + current_samples = waveform.shape[-1] + if current_samples > target_samples: + waveform = waveform[..., :target_samples] + elif current_samples < target_samples: + waveform = torch.nn.functional.pad(waveform, (0, target_samples - current_samples)) + audio_out = (waveform, sample_rate) + + # FL2VA keyframes selected by keyframe_indices (config, default first + + # last), kept as native uint8 PIL (rebuilding PIL from float tensors later + # would round-trip through *255/uint8 and lose exactness). Source indices + # travel with the sample so the condition model can validate them. + keyframe_indices = list(kwargs.get("keyframe_indices", [0, -1])) + keyframe_images = [] + for idx in keyframe_indices: + if not -len(frames) <= idx < len(frames): + raise ValueError(f"keyframe index {idx} out of range for {len(frames)} frames") + keyframe_images.append(frames[idx]) + + processed_example = { + "inputs": prompt, + "audios": audio_out, + "images": keyframe_images, + "keyframe_indices": keyframe_indices, + "videos": frames_t, + } + return [processed_example] + + @DATA_TRANSFORM_REGISTRY.register("dit_offline") def process_dit_offline_example(example, **kwargs): import pickle as pk diff --git a/veomni/data/multimodal/dit/preprocess.py b/veomni/data/multimodal/dit/preprocess.py index 67aee6bd31..afff0c8af7 100644 --- a/veomni/data/multimodal/dit/preprocess.py +++ b/veomni/data/multimodal/dit/preprocess.py @@ -1,4 +1,6 @@ # dit preprocess should not be used for llm or mllms +import os + from ..preprocess import PREPROCESSOR_REGISTRY @@ -26,3 +28,25 @@ def qwen_image_preprocess(conversations, **kwargs): if image is None: raise ValueError("Qwen-Image data requires one of: image, image_bytes, image_path, target_image.") return prompt, {}, [image], [] + + +@PREPROCESSOR_REGISTRY.register("minimax_h3") +def minimax_h3_preprocess(conversations, **kwargs): + data_dir = kwargs.get("data_dir", "") + prompt = conversations["prompt"] + + # Video path + video_path = conversations.get("video", "") + if video_path and data_dir: + video_path = os.path.join(data_dir, video_path) + + # Audio path (optional) + audio_path = conversations.get("input_audio", "") + if audio_path and data_dir: + audio_path = os.path.join(data_dir, audio_path) + + audios = {"audio": audio_path} if audio_path else {} + videos = [video_path] if video_path else [] + + # FL2VA keyframes extracted from video frames in condition_model.get_condition() + return prompt, audios, [], videos diff --git a/veomni/data/multimodal/video_utils.py b/veomni/data/multimodal/video_utils.py index e5384f3555..d079382286 100644 --- a/veomni/data/multimodal/video_utils.py +++ b/veomni/data/multimodal/video_utils.py @@ -17,14 +17,17 @@ import subprocess from typing import ByteString, Dict, List, Optional, Union +import av import numpy as np import PIL import torch +from PIL import Image from torchvision.transforms import InterpolationMode, functional +from tqdm import tqdm from ...utils import logging from ...utils.import_utils import is_ffmpeg_available -from .audio_utils import extract_audio_from_video +from .audio_utils import _prepare_audio_stream, _write_audio, extract_audio_from_video logger = logging.get_logger(__name__) @@ -873,3 +876,67 @@ def save_video_tensors_to_file( finally: if os.path.exists(tmp_audio_path): os.remove(tmp_audio_path) + + +def write_video_audio( + video: list[Image.Image], + audio: torch.Tensor | None, + output_path: str, + fps: int = 24, + audio_sample_rate: int | None = None, +) -> None: + """ + Writes a sequence of images and an audio tensor to a video file. + + This function utilizes PyAV (or a similar multimedia library) to encode a list of PIL images into a video stream + and multiplex a PyTorch tensor as the audio stream into the output container. + + Args: + video (list[Image.Image]): A list of PIL Image objects representing the video frames. + The length of this list determines the total duration of the video based on the FPS. + audio (torch.Tensor | None): The audio data as a PyTorch tensor. + The shape is typically (channels, samples). If no audio is required, pass None. + channels can be 1 or 2. 1 for mono, 2 for stereo. + output_path (str): The file path (including extension) where the output video will be saved. + fps (int, optional): The frame rate (frames per second) for the video. Defaults to 24. + audio_sample_rate (int | None, optional): The sample rate (e.g., 44100, 48000) for the audio. + If the audio tensor is provided and this is None, the function attempts to infer the rate + based on the audio tensor's length and the video duration. + Raises: + ValueError: If an audio tensor is provided but the sample rate cannot be determined. + """ + duration = len(video) / fps + if audio is not None and audio_sample_rate is None: + audio_sample_rate = int(audio.shape[-1] / duration) + + width, height = video[0].size + # yuv420p (4:2:0) requires even dimensions: crop odd-sized frames to the + # even bound, consistent with save_video_tensors_to_file. + width = width // 2 * 2 + height = height // 2 * 2 + if (width, height) != video[0].size: + video = [frame.crop((0, 0, width, height)) for frame in video] + container = av.open(output_path, mode="w") + stream = container.add_stream("libx264", rate=int(fps)) + stream.width = width + stream.height = height + stream.pix_fmt = "yuv420p" + + if audio is not None: + if audio_sample_rate is None: + raise ValueError("audio_sample_rate is required when audio is provided") + audio_stream = _prepare_audio_stream(container, audio_sample_rate) + + for frame in tqdm(video, total=len(video)): + frame = av.VideoFrame.from_image(frame) + for packet in stream.encode(frame): + container.mux(packet) + + # Flush encoder + for packet in stream.encode(): + container.mux(packet) + + if audio is not None: + _write_audio(container, audio_stream, audio, audio_sample_rate) + + container.close() diff --git a/veomni/models/diffusers/__init__.py b/veomni/models/diffusers/__init__.py index c354606c9a..5ce9849c82 100644 --- a/veomni/models/diffusers/__init__.py +++ b/veomni/models/diffusers/__init__.py @@ -1,4 +1,4 @@ -from . import ltx2_3, qwen_image, wan_t2v +from . import ltx2_3, minimax_h3, qwen_image, wan_t2v -__all__ = ["ltx2_3", "qwen_image", "wan_t2v"] +__all__ = ["ltx2_3", "minimax_h3", "qwen_image", "wan_t2v"] diff --git a/veomni/models/diffusers/minimax_h3/__init__.py b/veomni/models/diffusers/minimax_h3/__init__.py new file mode 100644 index 0000000000..90cb42ae3a --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/__init__.py @@ -0,0 +1,13 @@ +from . import minimax_h3_condition, minimax_h3_transformer +from .minimax_h3_core.offline_loader import ( + build_minimax_h3_offline_dataset, + build_minimax_h3_online_dataset, +) + + +__all__ = [ + "minimax_h3_condition", + "minimax_h3_transformer", + "build_minimax_h3_offline_dataset", + "build_minimax_h3_online_dataset", +] diff --git a/veomni/models/diffusers/minimax_h3/inference.py b/veomni/models/diffusers/minimax_h3/inference.py new file mode 100644 index 0000000000..6cec63297c --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/inference.py @@ -0,0 +1,1304 @@ +"""MiniMax H3 audio/video inference pipeline. + +Ported from the minimax_h3_audio_video pipeline (MiniMaxH3Pipeline) and the +base pipeline machinery (BasePipeline / PipelineUnit / PipelineUnitRunner). +Framework-specific dependencies are replaced by VeOmni equivalents: + +- model loading (ModelPool / ModelConfig): VeOmni registries + (MODEL_CONFIG_REGISTRY / MODELING_REGISTRY) for the condition model + (text encoder + VAEs + schedulers) and build_foundation_model for the DiT. +- get_device_type: veomni.utils.device.get_device_type +- audio utils (convert_to_stereo / resample_waveform): ported below. +- attention / gradient checkpointing: veomni minimax_h3_core (not used here). +- load_models_to_device: VeOmni models have no VRAM-management hooks, so the + pipeline stages models by name explicitly (listed models move to + self.device, all other children move to CPU). + +Known deviations: +- self.dit is the HF wrapper MiniMaxH3DiTModel; model_fn calls its + forward and takes predictions (same raw DiT forward + cond-row slicing + + unpatchify + negation). ref2va audio references are NOT supported by the + wrapper (raises NotImplementedError); t2va and fl2va paths are numerically + equivalent. +""" + +from __future__ import annotations + +import numpy as np +import torch +from einops import reduce, repeat +from PIL import Image +from tqdm import tqdm + +from veomni.models import build_foundation_model +from veomni.models.loader import MODEL_CONFIG_REGISTRY, MODELING_REGISTRY +from veomni.utils.device import get_device_type + +from .minimax_h3_core.flow_match_scheduler import FlowMatchScheduler +from .minimax_h3_core.minimax_h3_audio_vae import MiniMaxH3AudioVAE +from .minimax_h3_core.minimax_h3_dit import pack_audio, patchify_video +from .minimax_h3_core.minimax_h3_text_encoder import ( + MiniMaxH3TextEncoder, + image_token_counts, + presentation_fl2va, + presentation_ref2va, + presentation_t2va, + sample_qwen_video_frames, + video_token_counts, +) +from .minimax_h3_core.minimax_h3_video_vae import MiniMaxH3VideoVAE +from .minimax_h3_core.packed_sequence import build_packed_fl2va +from .minimax_h3_transformer.modeling_minimax_h3_transformer import MiniMaxH3DiTModel + + +# ── Audio utils ────────────────────────────────────────────────────── + + +def convert_to_stereo(audio_tensor: torch.Tensor) -> torch.Tensor: + """ + Convert audio to stereo. + Supports [C, T] or [B, C, T]. Duplicate mono, keep stereo. + """ + if audio_tensor.size(-2) == 1: + return audio_tensor.repeat(1, 2, 1) if audio_tensor.dim() == 3 else audio_tensor.repeat(2, 1) + return audio_tensor + + +def resample_waveform(waveform: torch.Tensor, source_rate: int, target_rate: int) -> torch.Tensor: + """Resample waveform to target sample rate if needed.""" + if source_rate == target_rate: + return waveform + import torchaudio + + resampled = torchaudio.functional.resample(waveform, source_rate, target_rate) + return resampled.to(dtype=waveform.dtype) + + +# ── Base pipeline machinery ────────────────────────────────────────── + + +class PipelineUnit: + def __init__( + self, + seperate_cfg: bool = False, + take_over: bool = False, + input_params: tuple[str] = None, + output_params: tuple[str] = None, + input_params_posi: dict[str, str] = None, + input_params_nega: dict[str, str] = None, + onload_model_names: tuple[str] = None, + ): + self.seperate_cfg = seperate_cfg + self.take_over = take_over + self.input_params = input_params + self.output_params = output_params + self.input_params_posi = input_params_posi + self.input_params_nega = input_params_nega + self.onload_model_names = onload_model_names + + def fetch_input_params(self): + params = [] + if self.input_params is not None: + for param in self.input_params: + params.append(param) + if self.input_params_posi is not None: + for _, param in self.input_params_posi.items(): + params.append(param) + if self.input_params_nega is not None: + for _, param in self.input_params_nega.items(): + params.append(param) + params = sorted(set(params)) + return params + + def fetch_output_params(self): + params = [] + if self.output_params is not None: + for param in self.output_params: + params.append(param) + return params + + def process(self, pipe, **kwargs) -> dict: + return {} + + def post_process(self, pipe, **kwargs) -> dict: + return {} + + +class PipelineUnitRunner: + def __init__(self): + pass + + def __call__( + self, unit: PipelineUnit, pipe: BasePipeline, inputs_shared: dict, inputs_posi: dict, inputs_nega: dict + ) -> tuple[dict, dict]: + if unit.take_over: + # Let the pipeline unit take over this function. + inputs_shared, inputs_posi, inputs_nega = unit.process( + pipe, inputs_shared=inputs_shared, inputs_posi=inputs_posi, inputs_nega=inputs_nega + ) + elif unit.seperate_cfg: + # Positive side + processor_inputs = {name: inputs_posi.get(name_) for name, name_ in unit.input_params_posi.items()} + if unit.input_params is not None: + for name in unit.input_params: + processor_inputs[name] = inputs_shared.get(name) + processor_outputs = unit.process(pipe, **processor_inputs) + inputs_posi.update(processor_outputs) + # Negative side + if inputs_shared["cfg_scale"] != 1: + processor_inputs = {name: inputs_nega.get(name_) for name, name_ in unit.input_params_nega.items()} + if unit.input_params is not None: + for name in unit.input_params: + processor_inputs[name] = inputs_shared.get(name) + processor_outputs = unit.process(pipe, **processor_inputs) + inputs_nega.update(processor_outputs) + else: + inputs_nega.update(processor_outputs) + else: + processor_inputs = {name: inputs_shared.get(name) for name in unit.input_params} + processor_outputs = unit.process(pipe, **processor_inputs) + inputs_shared.update(processor_outputs) + return inputs_shared, inputs_posi, inputs_nega + + +class BasePipeline(torch.nn.Module): + def __init__( + self, + device=None, + torch_dtype=torch.bfloat16, + height_division_factor=32, + width_division_factor=32, + time_division_factor=17, + time_division_remainder=5, + ): + super().__init__() + # The device and torch_dtype is used for the storage of intermediate variables, not models. + self.device = device if device is not None else get_device_type() + self.torch_dtype = torch_dtype + # The following parameters are used for shape check. + self.height_division_factor = height_division_factor + self.width_division_factor = width_division_factor + self.time_division_factor = time_division_factor + self.time_division_remainder = time_division_remainder + # VRAM management (VeOmni models have no onload/offload hooks, + # load_models_to_device stages children explicitly instead). + self.vram_management_enabled = False + # Pipeline Unit Runner + self.unit_runner = PipelineUnitRunner() + + def to(self, *args, **kwargs): + device, dtype, non_blocking, convert_to_format = torch._C._nn._parse_to(*args, **kwargs) + if device is not None: + self.device = device + if dtype is not None: + self.torch_dtype = dtype + super().to(*args, **kwargs) + return self + + def check_resize_height_width(self, height, width, num_frames=None, verbose=1): + # Shape check + if height % self.height_division_factor != 0: + height = ( + (height + self.height_division_factor - 1) // self.height_division_factor * self.height_division_factor + ) + if verbose > 0: + print(f"height % {self.height_division_factor} != 0. We round it up to {height}.") + if width % self.width_division_factor != 0: + width = (width + self.width_division_factor - 1) // self.width_division_factor * self.width_division_factor + if verbose > 0: + print(f"width % {self.width_division_factor} != 0. We round it up to {width}.") + if num_frames is None: + return height, width + else: + if num_frames % self.time_division_factor != self.time_division_remainder: + num_frames = ( + num_frames - self.time_division_remainder + self.time_division_factor - 1 + ) // self.time_division_factor * self.time_division_factor + self.time_division_remainder + if verbose > 0: + print( + f"num_frames % {self.time_division_factor} != {self.time_division_remainder}. We round it up to {num_frames}." + ) + return height, width, num_frames + + def preprocess_image(self, image, torch_dtype=None, device=None, pattern="B C H W", min_value=-1, max_value=1): + # Transform a PIL.Image to torch.Tensor + image = torch.Tensor(np.array(image, dtype=np.float32)) + image = image.to(dtype=torch_dtype or self.torch_dtype, device=device or self.device) + image = image * ((max_value - min_value) / 255) + min_value + image = repeat(image, f"H W C -> {pattern}", **({"B": 1} if "B" in pattern else {})) + return image + + def preprocess_video(self, video, torch_dtype=None, device=None, pattern="B C T H W", min_value=-1, max_value=1): + # Transform a list of PIL.Image to torch.Tensor + video = [ + self.preprocess_image( + image, torch_dtype=torch_dtype, device=device, min_value=min_value, max_value=max_value + ) + for image in video + ] + video = torch.stack(video, dim=pattern.index("T") // 2) + return video + + def vae_output_to_image(self, vae_output, pattern="B C H W", min_value=-1, max_value=1): + # Transform a torch.Tensor to PIL.Image + if pattern != "H W C": + vae_output = reduce(vae_output, f"{pattern} -> H W C", reduction="mean") + image = ((vae_output - min_value) * (255 / (max_value - min_value))).clip(0, 255) + image = image.to(device="cpu", dtype=torch.uint8) + image = Image.fromarray(image.numpy()) + return image + + def vae_output_to_video(self, vae_output, pattern="B C T H W", min_value=-1, max_value=1): + # Transform a torch.Tensor to list of PIL.Image + if pattern != "T H W C": + vae_output = reduce(vae_output, f"{pattern} -> T H W C", reduction="mean") + video = [ + self.vae_output_to_image(image, pattern="H W C", min_value=min_value, max_value=max_value) + for image in vae_output + ] + return video + + def output_audio_format_check(self, audio_output): + # output standard foramt: [C, T], output dtype: float() + # remove batch dim + if audio_output.ndim == 3: + audio_output = audio_output.squeeze(0) + return audio_output.float().cpu() + + def load_models_to_device(self, model_names): + # VeOmni models have no onload/offload VRAM-management hooks; stage + # children by name instead: listed models go to self.device, + # everything else to CPU. "condition_model" is only a handle to the + # encoder container and is skipped (it holds the same text encoder / + # VAE modules as their named children). A self-managed DiT (block + # offload) stages its own weights. + for name, model in self.named_children(): + if name == "condition_model": + continue + if model is not None and getattr(model, "_self_managed", False): + continue + if model is not None: + model.to(self.device if name in model_names else "cpu") + + def generate_noise( + self, shape, seed=None, rand_device="cpu", rand_torch_dtype=torch.float32, device=None, torch_dtype=None + ): + # Initialize Gaussian noise + generator = None if seed is None else torch.Generator(rand_device).manual_seed(seed) + noise = torch.randn(shape, generator=generator, device=rand_device, dtype=rand_torch_dtype) + noise = noise.to(dtype=torch_dtype or self.torch_dtype, device=device or self.device) + return noise + + def step(self, scheduler, latents, progress_id, noise_pred, input_latents=None, inpaint_mask=None, **kwargs): + timestep = scheduler.timesteps[progress_id] + if inpaint_mask is not None: + noise_pred_expected = scheduler.return_to_timestep( + scheduler.timesteps[progress_id], latents, input_latents + ) + noise_pred = self.blend_with_mask(noise_pred_expected, noise_pred, inpaint_mask) + latents_next = scheduler.step(noise_pred, timestep, latents) + return latents_next + + def blend_with_mask(self, base, addition, mask): + return base * (1 - mask) + addition * mask + + def check_vram_management_state(self): + vram_management_enabled = False + for module in self.children(): + if hasattr(module, "vram_management_enabled") and module.vram_management_enabled: + vram_management_enabled = True + return vram_management_enabled + + def cfg_guided_model_fn(self, model_fn, cfg_scale, inputs_shared, inputs_posi, inputs_nega, **inputs_others): + # Positive side forward + noise_pred_posi = model_fn(**inputs_posi, **inputs_shared, **inputs_others) + if cfg_scale != 1.0: + # Negative side forward + noise_pred_nega = model_fn(**inputs_nega, **inputs_shared, **inputs_others) + if isinstance(noise_pred_posi, tuple): + # Separately handling different output types of latents, eg. video and audio latents. + noise_pred = tuple( + n_nega + cfg_scale * (n_posi - n_nega) for n_posi, n_nega in zip(noise_pred_posi, noise_pred_nega) + ) + else: + noise_pred = noise_pred_nega + cfg_scale * (noise_pred_posi - noise_pred_nega) + else: + noise_pred = noise_pred_posi + return noise_pred + + +# ── Pipeline ───────────────────────────────────────────────────────── + + +class MiniMaxH3Pipeline(BasePipeline): + def __init__(self, device=None, torch_dtype=torch.bfloat16): + super().__init__( + device=device, + torch_dtype=torch_dtype, + height_division_factor=32, + width_division_factor=32, + time_division_factor=17, + time_division_remainder=5, + ) + self.scheduler = FlowMatchScheduler(shift=12.0) + self.scheduler_audio = FlowMatchScheduler(shift=3.0) + self.text_encoder: MiniMaxH3TextEncoder = None + self.dit: MiniMaxH3DiTModel = None + self.video_vae: MiniMaxH3VideoVAE = None + self.audio_vae: MiniMaxH3AudioVAE = None + self.tokenizer = None + self.processor = None + self.imgvid_cond_noise_aug = 0.999 + self.audio_cond_noise_aug = 1.0 + self.in_iteration_models = ("dit",) + self.units = [ + MiniMaxH3Unit_ShapeChecker(), + MiniMaxH3Unit_NoiseInitializer(), + MiniMaxH3Unit_InputVideoEmbedder(), + MiniMaxH3Unit_InputAudioEmbedder(), + MiniMaxH3Unit_KeyframeEncoder(), + MiniMaxH3Unit_ReferenceEncoder(), + MiniMaxH3Unit_PromptEmbedder(), + MiniMaxH3Unit_PackedSequenceBuilder(), + ] + self.model_fn = model_fn_minimax_h3 + self.compilable_models = ["dit"] + + @staticmethod + def from_pretrained( + torch_dtype: torch.dtype = torch.bfloat16, + device: str = get_device_type(), + condition_model_path: str = None, + condition_model_cfg: dict = None, + transformer_config_path: str = None, + transformer_weights_path: str = None, + transformer_config_kwargs: dict = None, + ops_implementation=None, + ): + """Load the pipeline through VeOmni model loading paths. + + - condition model (text encoder + video/audio VAE + schedulers + + tokenizer/processor) via MODEL_CONFIG_REGISTRY / MODELING_REGISTRY, + same as DiTTrainer._build_condition_model. + - DiT via build_foundation_model. init_device=: single-process + parallel state reports global_rank=-1, so "cpu" would trigger + empty_init and leave the weights on meta. + """ + pipe = MiniMaxH3Pipeline(device=device, torch_dtype=torch_dtype) + + # Condition model: encoders are always needed at inference time, so + # force load even when the config was written for offline training. + condition_model_cfg = dict(condition_model_cfg or {}) + condition_model_cfg.setdefault("skip_encoder_load", False) + # Video VAE weights live in video_vae/source (checkpoint layout); the + # config default subfolder "video_vae" has no safetensors directly. + condition_model_cfg.setdefault("video_vae_subfolder", "video_vae/source") + config_class = MODEL_CONFIG_REGISTRY["MiniMaxH3ConditionModel"]() + condition_cfg = config_class.from_pretrained(condition_model_path, **condition_model_cfg) + model_class = MODELING_REGISTRY["MiniMaxH3ConditionModel"]() + condition_model = model_class._from_config(condition_cfg) + condition_model.eval() + + # Keep a handle to the condition model, but do NOT register it as an + # nn.Module child: load_models_to_device() stages children by name and + # the condition model contains all encoders (moving it would undo the + # per-encoder staging). + pipe.__dict__["condition_model"] = condition_model + pipe.text_encoder = condition_model._text_encoder + pipe.video_vae = condition_model._video_vae + pipe.audio_vae = condition_model._audio_vae + pipe.tokenizer = condition_model._tokenizer + pipe.processor = condition_model._processor + pipe.scheduler = condition_model._scheduler_video + pipe.scheduler_audio = condition_model._scheduler_audio + pipe.imgvid_cond_noise_aug = condition_cfg.imgvid_cond_noise_aug + pipe.audio_cond_noise_aug = condition_cfg.audio_cond_noise_aug + + # The 50-layer DiT (~60GB bf16) alone nearly fills the NPU. Encoders + # must not sit on it while the DiT allocates; the pipeline units + # re-stage each model per use during __call__. + pipe.load_models_to_device([]) + # Release the allocator pool so the DiT allocation starts from a + # clean state instead of a pool reserved for the offloaded encoders. + if hasattr(torch, "npu"): + torch.npu.empty_cache() + + # DiT (HF wrapper MiniMaxH3DiTModel). The 61.7GB of + # weights do not fit on the NPU (~61.3GB total), so they live on + # CPU and the DiT stages its blocks in two halves per forward. + # With init_device="meta", build_foundation_model forces + # empty_init and skips its internal weight load; the explicit call + # below materializes the weights on CPU instead. + pipe.dit = build_foundation_model( + config_path=transformer_config_path, + weights_path=None, + torch_dtype="bfloat16", + init_device="meta", + ops_implementation=ops_implementation, + config_kwargs=transformer_config_kwargs, + ) + from veomni.models.module_utils import load_model_weights + + load_model_weights(pipe.dit, transformer_weights_path, init_device="cpu") + pipe.dit.dit.enable_block_offload() + pipe.dit._self_managed = True + pipe.dit.eval() + pipe.vram_management_enabled = pipe.check_vram_management_state() + return pipe + + @torch.no_grad() + def __call__( + self, + prompt: str, + negative_prompt: str = " ", + height: int = 768, + width: int = 1344, + num_frames: int = 124, + num_inference_steps: int = 50, + seed: int = 42, + rand_device: str = "cpu", + cfg_scale: float = 1.0, + tiled: bool = True, + tile_size: int = 256, + tile_overlap: int = 64, + use_gradient_checkpointing: bool = False, + use_gradient_checkpointing_offload: bool = False, + # Keyframe to Video + keyframes: list[Image.Image] = None, + keyframe_indices: list[int] = None, + # Reference to Video + references: list[dict] = None, + ref_image_short_edge: int = 2048, + ref_video_short_edge: int = 768, + ref_video_max_pixels: int = 768 * 1344, + progress_bar_cmd=tqdm, + ): + """Generate a joint video + audio sample. + + `num_frames` is snapped up to the nearest 17n+5 and that aligned value + drives every downstream shape (video/audio latent lengths, reference video + capping), so the returned clip may be slightly longer than requested. + + `keyframes` (FL2AV) is a list of PIL images with `keyframe_indices` in + {0, -1}; both are resized onto the target canvas. + + `references` (Ref2AV) is a list of dicts in request order: + + {"type": "image", "image": PIL.Image} + {"type": "video", "video": list[PIL.Image]} # silent + {"type": "audio", "audio": Tensor[C, L], "sample_rate": int} + {"type": "video_audio", "video": list[PIL.Image], + "audio": Tensor[C, L], "sample_rate": int} + + Input contract: `video` frame lists must ALREADY. be 24fps the pipeline never resamples frame rate. + """ + self.scheduler.set_timesteps(num_inference_steps) + self.scheduler_audio.set_timesteps(num_inference_steps) + + inputs_posi = {"prompt": prompt} + inputs_nega = {"negative_prompt": negative_prompt} + inputs_shared = { + "cfg_scale": cfg_scale, + "height": height, + "width": width, + "num_frames": num_frames, + "seed": seed, + "rand_device": rand_device, + "use_gradient_checkpointing": use_gradient_checkpointing, + "use_gradient_checkpointing_offload": use_gradient_checkpointing_offload, + "keyframes": keyframes, + "keyframe_indices": keyframe_indices, + "references": references, + "ref_image_short_edge": ref_image_short_edge, + "ref_video_short_edge": ref_video_short_edge, + "ref_video_max_pixels": ref_video_max_pixels, + "imgvid_cond_noise_aug": self.imgvid_cond_noise_aug, + "audio_cond_noise_aug": self.audio_cond_noise_aug, + } + + # 3. Unit chain + for unit in self.units: + inputs_shared, inputs_posi, inputs_nega = self.unit_runner( + unit, self, inputs_shared, inputs_posi, inputs_nega + ) + + # 4. Denoise loop + self.load_models_to_device(self.in_iteration_models) + models = {name: getattr(self, name) for name in self.in_iteration_models} + for progress_id, _ in enumerate(progress_bar_cmd(self.scheduler.timesteps)): + t_video = float(1.0 - self.scheduler.sigmas[progress_id]) + t_audio = float(1.0 - self.scheduler_audio.sigmas[progress_id]) + noise_pred_video, noise_pred_audio = self.cfg_guided_model_fn( + self.model_fn, + cfg_scale, + inputs_shared, + inputs_posi, + inputs_nega, + **models, + t_video=t_video, + t_audio=t_audio, + device=self.device, + ) + inputs_shared["video_latents"] = self.step( + self.scheduler, inputs_shared["video_latents"], progress_id, noise_pred=noise_pred_video + ) + inputs_shared["audio_latents"] = self.step( + self.scheduler_audio, inputs_shared["audio_latents"], progress_id, noise_pred=noise_pred_audio + ) + + # 5. Decode + self.load_models_to_device(["video_vae"]) + frames = self.video_vae.decode_video( + inputs_shared["video_latents"], + dtype=self.torch_dtype, + tiled=tiled, + tile_size=tile_size, + tile_overlap=tile_overlap, + ) + video = self.vae_output_to_video(frames, min_value=0, max_value=1) + + self.load_models_to_device(["audio_vae"]) + waveform = self.audio_vae.decode_audio(inputs_shared["audio_latents"], dtype=self.torch_dtype) + audio = self.output_audio_format_check(waveform) + return video, audio + + +class MiniMaxH3Unit_ShapeChecker(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("height", "width", "num_frames"), + output_params=("height", "width", "num_frames"), + ) + + def process(self, pipe: MiniMaxH3Pipeline, height, width, num_frames): + height, width, num_frames = pipe.check_resize_height_width(height, width, num_frames) + return {"height": height, "width": width, "num_frames": num_frames} + + +class MiniMaxH3Unit_NoiseInitializer(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("seed", "num_frames", "height", "width", "rand_device"), + output_params=("video_latents", "audio_latents"), + ) + + def process(self, pipe: MiniMaxH3Pipeline, seed, num_frames, height, width, rand_device): + video_latent_t, latent_h, latent_w = ((num_frames - 5) // 17) * 5 + 2, height // 16, width // 16 + video_latents = pipe.generate_noise( + (1, 24, video_latent_t, latent_h, latent_w), + seed=seed, + rand_device=rand_device, + rand_torch_dtype=pipe.torch_dtype, + ) + audio_latent_t = round(num_frames / 24.0 * 40.0) + audio_latents = pipe.generate_noise( + (2, 32, audio_latent_t), seed=seed, rand_device=rand_device, rand_torch_dtype=pipe.torch_dtype + ) + return {"video_latents": video_latents, "audio_latents": audio_latents} + + +class MiniMaxH3Unit_PromptEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + seperate_cfg=True, + input_params_posi={"prompt": "prompt"}, + input_params_nega={"prompt": "negative_prompt"}, + input_params=("keyframes", "ref_blocks", "height", "width"), + output_params=("prompt_embeds", "text_token_tags"), + onload_model_names=("text_encoder",), + ) + + def preprocess_ref_blocks(self, pipe: MiniMaxH3Pipeline, prompt, ref_blocks): + pixel_values = image_grid_thw = pixel_values_videos = video_grid_thw = None + counters = {"image": 0, "audio": 0, "video": 0} + images, videos, timestamps_per_video, condition_labels = [], [], [], [] + for block in ref_blocks: + kind = block["kind"] + if kind == "image": + counters["image"] += 1 + condition_labels.append(("image", counters["image"])) + images.append(block["prepared_image"]) + elif kind == "audio": + counters["audio"] += 1 + condition_labels.append(("audio", counters["audio"])) + elif kind in ("video", "video_audio"): + if int(block["ref_audio_t"]) > 0: + counters["audio"] += 1 + condition_labels.append(("audio", counters["audio"])) + counters["video"] += 1 + condition_labels.append(("video", counters["video"])) + sampled, timestamps = sample_qwen_video_frames(block["prepared_frames"]) + videos.append(np.stack([np.asarray(f) for f in sampled])) + timestamps_per_video.append(timestamps) + else: + raise ValueError(f"unknown reference kind: {kind}") + + image_counts, video_counts, video_timestamps = [], [], [] + if len(images) > 0: + pixel_values, image_grid_thw, image_counts = image_token_counts(pipe.processor, images) + if len(videos) > 0: + pixel_values_videos, video_grid_thw, video_counts, video_timestamps = video_token_counts( + pipe.processor, videos, timestamps_per_video + ) + input_ids, text_token_tags = presentation_ref2va( + pipe.tokenizer, prompt, condition_labels, image_counts, video_counts, video_timestamps + ) + return input_ids, text_token_tags, pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw + + def process(self, pipe: MiniMaxH3Pipeline, prompt, keyframes=None, ref_blocks=None, height=None, width=None): + pipe.load_models_to_device(self.onload_model_names) + pixel_values = image_grid_thw = pixel_values_videos = video_grid_thw = None + if ref_blocks: + input_ids, text_token_tags, pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw = ( + self.preprocess_ref_blocks(pipe, prompt, ref_blocks) + ) + elif keyframes: + keyframes = [img.convert("RGB").resize((width, height), Image.LANCZOS) for img in keyframes] + pixel_values, image_grid_thw, image_counts = image_token_counts(pipe.processor, keyframes) + input_ids, text_token_tags = presentation_fl2va(pipe.tokenizer, prompt, image_counts) + else: + input_ids, text_token_tags = presentation_t2va(pipe.tokenizer, prompt) + + ids = input_ids.unsqueeze(0).to(pipe.device) + kwargs = {"input_ids": ids, "attention_mask": torch.ones_like(ids)} + if pixel_values is not None: + kwargs["pixel_values"] = pixel_values.to(pipe.device, pipe.torch_dtype) + kwargs["image_grid_thw"] = image_grid_thw.to(pipe.device, torch.long) + if pixel_values_videos is not None: + kwargs["pixel_values_videos"] = pixel_values_videos.to(pipe.device, pipe.torch_dtype) + kwargs["video_grid_thw"] = video_grid_thw.to(pipe.device, torch.long) + hidden = pipe.text_encoder(**kwargs) + + return { + "prompt_embeds": hidden.to(pipe.device, pipe.torch_dtype), + "text_token_tags": text_token_tags.view(-1).to(pipe.device, torch.long), + } + + +class MiniMaxH3Unit_InputVideoEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("input_video",), + output_params=("video_latents", "input_latents"), + onload_model_names=("video_vae",), + ) + + def process(self, pipe: MiniMaxH3Pipeline, input_video): + if input_video is None or not pipe.scheduler.training: + return {} + pipe.load_models_to_device(self.onload_model_names) + frames_tensor = pipe.preprocess_video(input_video, torch_dtype=torch.float32, min_value=0, device=pipe.device) + latents = pipe.video_vae.encode_video(frames_tensor, dtype=pipe.torch_dtype).to(pipe.torch_dtype) + return {"video_latents": latents, "input_latents": latents} + + +class MiniMaxH3Unit_InputAudioEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("input_audio",), + output_params=("audio_latents", "audio_input_latents"), + onload_model_names=("audio_vae",), + ) + + def process(self, pipe: MiniMaxH3Pipeline, input_audio): + if input_audio is None or not pipe.scheduler.training: + return {} + pipe.load_models_to_device(self.onload_model_names) + waveform, sample_rate = input_audio + waveform = waveform.squeeze(0) if waveform.dim() == 3 else waveform + assert waveform.dim() == 2, "waveform must be in shape (C, T)" + waveform = resample_waveform( + convert_to_stereo(waveform).float(), sample_rate, pipe.audio_vae.sample_rate + ) # [C, T] + latents = pipe.audio_vae.encode_audio(waveform[:2].to(pipe.device), dtype=pipe.torch_dtype).to( + pipe.torch_dtype + ) # [C, 32, T] + return {"audio_latents": latents, "audio_input_latents": latents} + + +class MiniMaxH3Unit_KeyframeEncoder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("keyframes", "keyframe_indices", "video_latents", "rand_device", "seed", "height", "width"), + output_params=("keyframe_cond_anchor",), + onload_model_names=("video_vae",), + ) + + def process( + self, pipe: MiniMaxH3Pipeline, keyframes, keyframe_indices, video_latents, rand_device, seed, height, width + ): + if keyframes is None: + return {} + assert keyframe_indices is not None and len(keyframes) == len(keyframe_indices), ( + "keyframe_indices must be provided when keyframes is not None" + ) + assert all(idx in (0, -1) for idx in keyframe_indices), ( + "keyframe_indices must be within the range of keyframes (0 or -1)" + ) + pipe.load_models_to_device(self.onload_model_names) + # Encode keyframes + all_cond_rows = [] + for img in keyframes: + img_tensor = pipe.preprocess_image( + img.convert("RGB").resize((width, height), Image.LANCZOS), torch_dtype=torch.float32, min_value=0 + ) + z_norm = pipe.video_vae.encode_video( + img_tensor, dtype=pipe.torch_dtype, process_image=True + ) # [1,24,1,H',W'] + rows = patchify_video(z_norm) + all_cond_rows.append(rows) + clean_cond_rows = torch.cat(all_cond_rows, dim=0).to(device=pipe.device, dtype=pipe.torch_dtype) + if pipe.imgvid_cond_noise_aug == 1.0: + keyframe_cond_anchor = clean_cond_rows + else: + video_latent_t, latent_h, latent_w = (int(x) for x in video_latents.shape[2:]) + ts = torch.tensor(pipe.imgvid_cond_noise_aug, dtype=pipe.torch_dtype, device=pipe.device) + noise = pipe.generate_noise( + (1, 24, video_latent_t + len(keyframes), latent_h, latent_w), seed, rand_device, pipe.torch_dtype + )[:, :, :1] + noise_rows = patchify_video(noise).to(device=pipe.device, dtype=pipe.torch_dtype) + frame_rows = (latent_h // 2) * (latent_w // 2) + parts = [ + ts * clean_cond_rows[i * frame_rows : (i + 1) * frame_rows] + (1.0 - ts) * noise_rows + for i in range(len(keyframes)) + ] + keyframe_cond_anchor = torch.cat(parts, dim=0) + return {"keyframe_cond_anchor": keyframe_cond_anchor.contiguous()} + + +class MiniMaxH3Unit_ReferenceEncoder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=( + "references", + "seed", + "video_latents", + "num_frames", + "ref_image_short_edge", + "ref_video_short_edge", + "ref_video_max_pixels", + ), + output_params=("ref_blocks", "ref_visual_anchor", "ref_audio_anchor"), + onload_model_names=("video_vae", "audio_vae"), + ) + + @staticmethod + def _nearest_multiple(value: float, multiple: int) -> int: + return max(multiple, int(round(float(value) / multiple)) * multiple) + + def _resolve_reference_image_shape(self, pipe: MiniMaxH3Pipeline, width: int, height: int, short_edge: int): + scale = short_edge * 1.0 / min(width, height) + return self._nearest_multiple(width * scale, pipe.width_division_factor), self._nearest_multiple( + height * scale, pipe.height_division_factor + ) + + def _resolve_reference_video_shape( + self, pipe: MiniMaxH3Pipeline, width: int, height: int, short_edge: int, max_pixels: int + ): + scale = min(short_edge * 1.0 / min(width, height), float(np.sqrt(max_pixels / (width * height)))) + return self._nearest_multiple(width * scale, pipe.width_division_factor), self._nearest_multiple( + height * scale, pipe.height_division_factor + ) + + def _trim_reference_video_length(self, pipe: MiniMaxH3Pipeline, frame_count: int) -> int: + frame_count = int(frame_count) + factor, remainder = pipe.time_division_factor, pipe.time_division_remainder + chunks = max(1, (frame_count - remainder) // factor) + used = chunks * factor + remainder + assert used <= frame_count, ( + f"cannot trim {frame_count} reference video frames down to a valid " + f"length: at least {used} frames are required" + ) + return used + + def _encode_image_ref(self, pipe, img: Image.Image, short_edge: int): + target_w, target_h = self._resolve_reference_image_shape(pipe, *img.size, short_edge) + img = img.convert("RGB").resize((target_w, target_h), Image.LANCZOS) + img_tensor = pipe.preprocess_image(img, torch_dtype=torch.float32, min_value=0) + z = pipe.video_vae.encode_video( + img_tensor.to(pipe.device), + dtype=pipe.torch_dtype, + process_image=True, + ) # [1,24,1,H',W'] + return patchify_video(z), z.shape[-2], z.shape[-1], img + + def _encode_video_ref(self, pipe, frames, target_frame_count: int, short_edge: int, max_pixels: int): + target_w, target_h = self._resolve_reference_video_shape(pipe, *frames[0].size, short_edge, max_pixels) + frames = [f.resize((target_w, target_h), Image.LANCZOS).convert("RGB") for f in frames] + frames = frames[: self._trim_reference_video_length(pipe, min(len(frames), target_frame_count))] + frames_tensor = pipe.preprocess_video(frames, torch_dtype=torch.float32, min_value=0, device=pipe.device) + z = pipe.video_vae.encode_video( + frames_tensor, + dtype=pipe.torch_dtype, + process_image=False, + ) # [1,24,T',H',W'] + return patchify_video(z), int(z.shape[2]), int(z.shape[3]), int(z.shape[4]), frames + + def _encode_audio_ref(self, pipe, waveform, sample_rate: int): + waveform = waveform.squeeze(0) if waveform.dim() == 3 else waveform + assert waveform.dim() == 2, "waveform must be in shape (C, T)" + waveform = resample_waveform( + convert_to_stereo(waveform).float(), sample_rate, pipe.audio_vae.sample_rate + ) # [C, T] + latent = pipe.audio_vae.encode_audio(waveform[:2].to(pipe.device), dtype=pipe.torch_dtype) # [C, 32, T] + return pack_audio(latent), latent.shape[-1] + + @staticmethod + def _require(ref, key, kind): + value = ref.get(key) + assert value is not None, f"reference type {kind!r} requires field {key!r}" + return value + + def _build_block( + self, pipe, ref, target_frame_count, ref_image_short_edge, ref_video_short_edge, ref_video_max_pixels + ): + kind = ref["type"] + if kind == "image": + rows, lh, lw, prepared = self._encode_image_ref( + pipe, self._require(ref, "image", kind), ref_image_short_edge + ) + return { + "kind": kind, + "visual_clean": rows, + "latent_t": 1, + "latent_h": lh, + "latent_w": lw, + "prepared_image": prepared, + "ref_audio_t": 0, + } + if kind in ("video", "video_audio"): + if kind == "video" and ref.get("audio") is not None: + raise ValueError("reference type 'video' is silent; use 'video_audio' to pass a soundtrack") + frames = self._require(ref, "video", kind) + if kind == "video_audio": + audio_waveform = self._require(ref, "audio", kind) + audio_sample_rate = int(self._require(ref, "sample_rate", kind)) + rows, lt, lh, lw, prepared = self._encode_video_ref( + pipe, frames, target_frame_count, ref_video_short_edge, ref_video_max_pixels + ) + block = { + "kind": kind, + "visual_clean": rows, + "latent_t": lt, + "latent_h": lh, + "latent_w": lw, + "prepared_frames": prepared, + "ref_audio_t": 0, + } + if kind == "video_audio": + audio_rows, ref_audio_t = self._encode_audio_ref(pipe, audio_waveform, audio_sample_rate) + block["audio_clean"], block["ref_audio_t"] = audio_rows, ref_audio_t + return block + if kind == "audio": + rows, ref_audio_t = self._encode_audio_ref( + pipe, self._require(ref, "audio", kind), int(self._require(ref, "sample_rate", kind)) + ) + return {"kind": kind, "audio_clean": rows, "ref_audio_t": ref_audio_t} + raise ValueError(f"unknown reference type: {kind!r}") + + def process( + self, + pipe: MiniMaxH3Pipeline, + references, + seed, + video_latents, + num_frames, + ref_image_short_edge, + ref_video_short_edge, + ref_video_max_pixels, + ): + if not references: + return {} + pipe.load_models_to_device(self.onload_model_names) + seed = int(seed) if seed is not None else 42 + device = pipe.device + + ref_blocks_out = [ + self._build_block(pipe, ref, num_frames, ref_image_short_edge, ref_video_short_edge, ref_video_max_pixels) + for ref in references + ] + visual_blocks = [b for b in ref_blocks_out if "visual_clean" in b] + imgvid_cond_num_frames = len(visual_blocks) + visual_parts = [] + for block in visual_blocks: + clean = block.pop("visual_clean").to(device=device, dtype=pipe.torch_dtype) + if pipe.imgvid_cond_noise_aug == 1.0: + anchor = clean + else: + full_t = video_latents.shape[2] + imgvid_cond_num_frames + noise_shape = (1, 24, full_t, int(block["latent_h"]), int(block["latent_w"])) + noise = pipe.generate_noise( + noise_shape, + seed=seed, + rand_device="cpu", + rand_torch_dtype=pipe.torch_dtype, + device="cpu", + torch_dtype=pipe.torch_dtype, + ) + noise_rows = patchify_video(noise[:, :, : int(block["latent_t"])]).to( + device=device, dtype=pipe.torch_dtype + ) + ts = torch.tensor(pipe.imgvid_cond_noise_aug, dtype=pipe.torch_dtype, device=device) + anchor = ts * clean + (1.0 - ts) * noise_rows + visual_parts.append(anchor) + + audio_parts = [] + for block in ref_blocks_out: + if "audio_clean" not in block or int(block["ref_audio_t"]) <= 0: + block.pop("audio_clean", None) + continue + clean = block.pop("audio_clean").to(device=device, dtype=pipe.torch_dtype) + if pipe.audio_cond_noise_aug == 1.0: + anchor = clean + else: + noise = pipe.generate_noise( + clean.shape, + seed=seed + 1, + rand_device="cpu", + rand_torch_dtype=pipe.torch_dtype, + device=device, + torch_dtype=pipe.torch_dtype, + ) + ts = torch.tensor(pipe.audio_cond_noise_aug, dtype=pipe.torch_dtype, device=device) + anchor = ts * clean + (1.0 - ts) * noise + audio_parts.append(anchor) + + return { + "ref_blocks": ref_blocks_out, + "ref_visual_anchor": torch.cat(visual_parts, dim=0) if visual_parts else None, + "ref_audio_anchor": torch.cat(audio_parts, dim=0) if audio_parts else None, + } + + +class MiniMaxH3Unit_PackedSequenceBuilder(PipelineUnit): + _INTERP = 32 + _T_GROUP = 5 + _FRAME_PER_TOKEN = (1, 4, 4, 4, 4) + _FRAME_RESCALE = 5.0 / 3.0 + _SEQ_ALIGN = 64 + + def __init__(self): + super().__init__( + seperate_cfg=True, + input_params_posi={"prompt_embeds": "prompt_embeds", "text_token_tags": "text_token_tags"}, + input_params_nega={"prompt_embeds": "prompt_embeds", "text_token_tags": "text_token_tags"}, + input_params=("video_latents", "audio_latents", "keyframe_cond_anchor", "keyframe_indices", "ref_blocks"), + output_params=("packed",), + ) + + @staticmethod + def _to_device(packed: dict, device) -> dict: + return {k: v.to(device) if torch.is_tensor(v) else v for k, v in packed.items()} + + def _aligned_seq_len(self, used: int) -> int: + return ((used + self._SEQ_ALIGN - 1) // self._SEQ_ALIGN) * self._SEQ_ALIGN + + def _axis_from_sqrt_area(self, dim: int, patch: int, sqrt_area: float) -> torch.Tensor: + ratio = dim / sqrt_area + left = (1.0 - ratio) * 0.5 + right = left + ratio + grid = np.linspace(left, right, dim // patch, endpoint=False) * self._INTERP + return torch.from_numpy(grid).to(torch.float64) + + def _video_t_grid(self, n: int, origin: float) -> torch.Tensor: + spans = torch.tensor( + [self._FRAME_RESCALE * self._FRAME_PER_TOKEN[k % self._T_GROUP] for k in range(n)], dtype=torch.float64 + ) + return origin + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)]) + + def _temporal_position_span(self, temporal_length: int) -> float: + spans = np.ones(int(temporal_length), dtype=np.float64) * self._FRAME_RESCALE + for token_index in range(self._T_GROUP): + spans[token_index :: self._T_GROUP] *= self._FRAME_PER_TOKEN[token_index] + return float(spans.sum()) + + def _video_t_span(self, n: int) -> float: + return sum(self._FRAME_RESCALE * self._FRAME_PER_TOKEN[k % self._T_GROUP] for k in range(n)) + + def _frame_grid(self, latent_h: int, latent_w: int, sqrt_area): + h_grid = self._axis_from_sqrt_area(latent_h, 2, sqrt_area) + w_grid = self._axis_from_sqrt_area(latent_w, 2, sqrt_area) + hh, ww = torch.meshgrid(h_grid, w_grid, indexing="ij") + return torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1), w_grid + + def _video_grid(self, latent_t: int, frame: torch.Tensor, origin: float) -> torch.Tensor: + video_g = torch.empty(latent_t, frame.shape[0], 3, dtype=torch.float64) + video_g[:, :, 0] = self._video_t_grid(latent_t, origin)[:, None] + video_g[:, :, 1:] = frame[None] + return video_g.reshape(-1, 3) + + def _audio_w_axis(self, w_grid: torch.Tensor, audio_t: int, audio_rows: int) -> torch.Tensor: + return torch.cat( + [ + torch.full((audio_t,), float(w_grid[0]), dtype=torch.float64), + torch.full((audio_rows - audio_t,), float(w_grid[-1]), dtype=torch.float64), + ] + ) + + def _build_packed_fl2va(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframe_indices, audio_channel=2): + """fl2va layout: [text | cond | audio | video | pad]. + + Delegates to the shared build_packed_fl2va helper (minimax_h3_core. + packed_sequence) so train and inference emit identical packing. + The shared builder also returns metadata keys (update_mask, cond_rows, + text_len, ...) that model_fn_minimax_h3 ignores. + """ + return build_packed_fl2va( + text_len=text_len, + latent_t=latent_t, + latent_h=latent_h, + latent_w=latent_w, + audio_t=audio_t, + keyframe_indices=keyframe_indices, + audio_channel=audio_channel, + ) + + def _build_packed_ref2va(self, text_len, latent_t, latent_h, latent_w, audio_t, ref_blocks, audio_channel=2): + """ref2va layout: [text | ref_0 | ref_1 | ... | target_audio | target_video | pad]""" + ph, pw = latent_h // 2, latent_w // 2 + target_frame_rows = ph * pw + target_video_rows = latent_t * target_frame_rows + target_audio_rows = audio_t * audio_channel + + block_dims, total_ref_visual_rows, total_ref_audio_rows = [], 0, 0 + for b in ref_blocks: + kind = b["kind"] + info = {"kind": kind, "visual_rows": 0, "audio_rows": 0, "ref_audio_t": int(b.get("ref_audio_t", 0))} + if kind in ("image", "video", "video_audio"): + lt_r, lh_r, lw_r = int(b["latent_t"]), int(b["latent_h"]), int(b["latent_w"]) + info.update(visual_rows=lt_r * (lh_r // 2) * (lw_r // 2), latent_t=lt_r, latent_h=lh_r, latent_w=lw_r) + total_ref_visual_rows += info["visual_rows"] + info["audio_rows"] = info["ref_audio_t"] * audio_channel + total_ref_audio_rows += info["audio_rows"] + elif kind == "audio": + info["audio_rows"] = info["ref_audio_t"] * audio_channel + total_ref_audio_rows += info["audio_rows"] + else: + raise ValueError(f"unknown ref kind: {kind}") + block_dims.append(info) + + used = text_len + total_ref_visual_rows + total_ref_audio_rows + target_audio_rows + target_video_rows + seq_len = self._aligned_seq_len(used) + + g = torch.zeros(seq_len, 3, dtype=torch.float64) + g[0:text_len, 0] = torch.arange(text_len, dtype=torch.float64) + token_tags = torch.full((seq_len,), -1, dtype=torch.long) + token_tags[0:text_len] = 1 + + # Iterate through ref blocks, placing them contiguously. + cursor, t_cursor = text_len, float(text_len) + ref_visual_pos_parts, ref_audio_pos_parts = [], [] + # Target w_grid used for audio W-axis (channel separation) + target_sqrt_area = float(np.sqrt(latent_h * latent_w)) + target_w_grid = self._axis_from_sqrt_area(latent_w, 2, target_sqrt_area) + + for info in block_dims: + kind = info["kind"] + if kind == "image": + v_rows, lh_r, lw_r = info["visual_rows"], info["latent_h"], info["latent_w"] + # Own spatial grid + sqrt_area = float(np.sqrt(lh_r * lw_r)) + frame, w_grid = self._frame_grid(lh_r, lw_r, sqrt_area) + sl = slice(cursor, cursor + v_rows) + g[sl, 0] = t_cursor + g[sl, 1:] = frame + token_tags[sl] = 0 + ref_visual_pos_parts.append(torch.arange(sl.start, sl.stop)) + cursor += v_rows + t_cursor += 1.0 + + elif kind in ("video", "video_audio"): + a_rows, v_rows, ref_at = info["audio_rows"], info["visual_rows"], info["ref_audio_t"] + lt_r, lh_r, lw_r = info["latent_t"], info["latent_h"], info["latent_w"] + sqrt_area = float(np.sqrt(lh_r * lw_r)) + frame, rv_w_grid = self._frame_grid(lh_r, lw_r, sqrt_area) + + audio_sl = slice(cursor, cursor + a_rows) + visual_sl = slice(audio_sl.stop, audio_sl.stop + v_rows) + + a_t_grid = t_cursor + torch.arange(ref_at, dtype=torch.float64) + g[audio_sl, 0] = a_t_grid.repeat(audio_channel) + if ref_at: + # W-axis uses THIS reference video's own grid, not the target's. + g[audio_sl, 2] = self._audio_w_axis(rv_w_grid, ref_at, a_rows) + token_tags[audio_sl] = 2 + ref_audio_pos_parts.append(torch.arange(audio_sl.start, audio_sl.stop)) + + g[visual_sl] = self._video_grid(lt_r, frame, t_cursor) + token_tags[visual_sl] = 0 + ref_visual_pos_parts.append(torch.arange(visual_sl.start, visual_sl.stop)) + + cursor = visual_sl.stop + t_cursor += max(float(ref_at), self._video_t_span(lt_r)) + + elif kind == "audio": + a_rows, ref_at = info["audio_rows"], info["ref_audio_t"] + sl = slice(cursor, cursor + a_rows) + a_t_grid = t_cursor + torch.arange(ref_at, dtype=torch.float64) + g[sl, 0] = a_t_grid.repeat(audio_channel) + if ref_at: + g[sl, 2] = self._audio_w_axis(target_w_grid, ref_at, a_rows) + token_tags[sl] = 2 + ref_audio_pos_parts.append(torch.arange(sl.start, sl.stop)) + cursor += a_rows + t_cursor += float(ref_at) + + # Target audio + target video after ref blocks + target_audio_sl = slice(cursor, cursor + target_audio_rows) + target_video_sl = slice(target_audio_sl.stop, target_audio_sl.stop + target_video_rows) + + # Target spatial grid (own) + frame_t, w_grid_t = self._frame_grid(latent_h, latent_w, target_sqrt_area) + g[target_video_sl] = self._video_grid(latent_t, frame_t, t_cursor) + + target_audio_t_grid = t_cursor + torch.arange(audio_t, dtype=torch.float64) + g[target_audio_sl, 0] = target_audio_t_grid.repeat(audio_channel) + g[target_audio_sl, 2] = self._audio_w_axis(w_grid_t, audio_t, target_audio_rows) + + # Build combined img_pos / audio_pos + target_video_pos = torch.arange(target_video_sl.start, target_video_sl.stop) + target_audio_pos = torch.arange(target_audio_sl.start, target_audio_sl.stop) + + if ref_visual_pos_parts: + img_pos = torch.cat(ref_visual_pos_parts + [target_video_pos]) + else: + img_pos = target_video_pos + if ref_audio_pos_parts: + audio_pos = torch.cat(ref_audio_pos_parts + [target_audio_pos]) + else: + audio_pos = target_audio_pos + + token_tags[img_pos] = 0 + token_tags[target_audio_sl] = 2 + + text_pos = torch.arange(0, text_len) + cu = torch.tensor([0, used, seq_len], dtype=torch.int32) + + return { + "img_pos": img_pos, + "audio_pos": audio_pos, + "text_pos": text_pos, + "img_position_ids": g[None], + "token_tags": token_tags, + "cu_seqlens": cu, + "seq_len": seq_len, + } + + def process( + self, + pipe: MiniMaxH3Pipeline, + prompt_embeds, + video_latents, + audio_latents, + text_token_tags=None, + keyframe_cond_anchor=None, + keyframe_indices=None, + ref_blocks=None, + ): + text_len = prompt_embeds.shape[0] + video_latent_t, latent_h, latent_w = video_latents.shape[2:] + audio_latent_t = audio_latents.shape[-1] + if ref_blocks is not None: + packed = self._build_packed_ref2va( + text_len, video_latent_t, latent_h, latent_w, audio_latent_t, ref_blocks + ) + else: + packed = self._build_packed_fl2va( + text_len, + video_latent_t, + latent_h, + latent_w, + audio_latent_t, + keyframe_indices if keyframe_cond_anchor is not None else [], + ) + + packed["token_tags"][packed["text_pos"]] = text_token_tags.cpu() + if pipe.device == "mps": + packed["img_position_ids"] = packed["img_position_ids"].to(torch.float32) + return {"packed": self._to_device(packed, pipe.device)} + + +def model_fn_minimax_h3( + dit, + video_latents, + audio_latents, + packed, + prompt_embeds, + t_video, + t_audio, + keyframe_cond_anchor=None, + ref_visual_anchor=None, + ref_audio_anchor=None, + imgvid_cond_noise_aug=0.999, + audio_cond_noise_aug=1.0, + use_gradient_checkpointing=False, + use_gradient_checkpointing_offload=False, + **kwargs, +): + dtype, device = video_latents.dtype, video_latents.device + f, h, w = video_latents.shape[2:] + audio_channel, audio_t = audio_latents.shape[0], audio_latents.shape[-1] + video_rows = patchify_video(video_latents) + audio_rows = pack_audio(audio_latents) + + img_pos = packed["img_pos"] + audio_pos = packed["audio_pos"] + text_pos = packed["text_pos"] + cu = packed["cu_seqlens"] + seq_len = packed["seq_len"] + text_len = text_pos.shape[0] + # Video Sequence + cond_anchor = ref_visual_anchor if ref_visual_anchor is not None else keyframe_cond_anchor + cond_rows_count = 0 if cond_anchor is None else cond_anchor.shape[0] + x = torch.zeros(1, seq_len, 96, dtype=dtype, device=device) + x[0, img_pos[cond_rows_count:]] = video_rows + if cond_anchor is not None: + x[0, img_pos[:cond_rows_count]] = cond_anchor + # Audio Sequence + ref_audio_rows_count = 0 if ref_audio_anchor is None else ref_audio_anchor.shape[0] + audio_x = torch.zeros(1, seq_len, 32, dtype=dtype, device=device) + audio_x[0, audio_pos[ref_audio_rows_count:]] = audio_rows + if ref_audio_anchor is not None: + audio_x[0, audio_pos[:ref_audio_rows_count]] = ref_audio_anchor + + timesteps = torch.full((seq_len,), float(t_video), dtype=torch.float32, device=device) + timesteps[audio_pos] = float(t_audio) + timesteps[img_pos[:cond_rows_count]] = max(float(t_video), imgvid_cond_noise_aug) + timesteps[audio_pos[:ref_audio_rows_count]] = max(float(t_audio), audio_cond_noise_aug) + unique_timesteps, inverse_indices = torch.unique(timesteps, sorted=True, return_inverse=True) + + refiner_cu = torch.tensor([0, text_len, text_len], dtype=torch.int32, device=device) + + # VeOmni DiT is the HF wrapper MiniMaxH3DiTModel: it runs the + # same raw DiT forward (skip_mask_out_condition=True, update_mask=None), + # slices cond rows, unpatchifies and negates, returning the same + # (video_pred, audio_pred) pair. The wrapper + # does NOT slice ref-audio rows, so ref2va audio references are + # unsupported (t2va / fl2va are unaffected). + if ref_audio_anchor is not None: + raise NotImplementedError( + "ref2va audio references are not supported by the VeOmni wrapper (MiniMaxH3DiTModel)." + ) + outputs = dit( + x=x, + audio_x=audio_x, + img_position_ids=packed["img_position_ids"], + unique_timesteps=unique_timesteps, + inverse_indices=inverse_indices, + token_tags=packed["token_tags"], + prompt_embeds=prompt_embeds, + img_pos_info={"position_ids": img_pos}, + audio_pos_info={"position_ids": audio_pos}, + text_pos_info={"position_ids": text_pos}, + img_pos_for_infer_output_info={"position_ids": img_pos}, + packed_seq_params={"cu_seqlens_q": cu, "max_seqlen_q": int(cu[1])}, + refiner_packed_seq_params={"cu_seqlens_q": refiner_cu, "max_seqlen_q": text_len}, + skip_mask_out_condition=True, + cond_rows=cond_rows_count, + video_latent_shape=(f, h // 2, w // 2), + audio_latent_shape=(audio_channel, audio_t), + use_gradient_checkpointing=use_gradient_checkpointing, + ) + + video_pred, audio_pred = outputs.predictions + return video_pred, audio_pred diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_condition/__init__.py b/veomni/models/diffusers/minimax_h3/minimax_h3_condition/__init__.py new file mode 100644 index 0000000000..f9b03b2137 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_condition/__init__.py @@ -0,0 +1,15 @@ +from ....loader import MODEL_CONFIG_REGISTRY, MODELING_REGISTRY + + +@MODEL_CONFIG_REGISTRY.register("MiniMaxH3ConditionModel") +def register_minimax_h3_condition_config(): + from .configuration_minimax_h3_condition import MiniMaxH3ConditionModelConfig + + return MiniMaxH3ConditionModelConfig + + +@MODELING_REGISTRY.register("MiniMaxH3ConditionModel") +def register_minimax_h3_condition_modeling(architecture: str = None): + from .modeling_minimax_h3_condition import MiniMaxH3ConditionModel + + return MiniMaxH3ConditionModel diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_condition/configuration_minimax_h3_condition.py b/veomni/models/diffusers/minimax_h3/minimax_h3_condition/configuration_minimax_h3_condition.py new file mode 100644 index 0000000000..ca97bedfab --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_condition/configuration_minimax_h3_condition.py @@ -0,0 +1,45 @@ +from typing import Optional + +from transformers import PretrainedConfig + + +class MiniMaxH3ConditionModelConfig(PretrainedConfig): + model_type = "MiniMaxH3ConditionModel" + + def __init__( + self, + base_model_path: Optional[str] = None, + text_encoder_subfolder: str = "text_encoder", + video_vae_subfolder: str = "video_vae", + audio_vae_subfolder: str = "audio_vae", + processor_subfolder: str = "processor", + num_train_timesteps: int = 1000, + sigma_shift_video: float = 12.0, + sigma_shift_audio: float = 3.0, + use_keyframe_condition: bool = True, + keyframe_indices: list = None, + imgvid_cond_noise_aug: float = 0.999, + audio_cond_noise_aug: float = 1.0, + skip_encoder_load: bool = False, + video_max_frames: int = 73, + video_max_resolution: int = 848, + text_encoder_num_retained_layers: int = 50, + **kwargs, + ): + super().__init__(**kwargs) + self.base_model_path = base_model_path + self.text_encoder_subfolder = text_encoder_subfolder + self.video_vae_subfolder = video_vae_subfolder + self.audio_vae_subfolder = audio_vae_subfolder + self.processor_subfolder = processor_subfolder + self.num_train_timesteps = num_train_timesteps + self.sigma_shift_video = sigma_shift_video + self.sigma_shift_audio = sigma_shift_audio + self.use_keyframe_condition = use_keyframe_condition + self.keyframe_indices = keyframe_indices if keyframe_indices is not None else [0, -1] + self.imgvid_cond_noise_aug = imgvid_cond_noise_aug + self.audio_cond_noise_aug = audio_cond_noise_aug + self.skip_encoder_load = skip_encoder_load + self.video_max_frames = video_max_frames + self.video_max_resolution = video_max_resolution + self.text_encoder_num_retained_layers = text_encoder_num_retained_layers diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_condition/modeling_minimax_h3_condition.py b/veomni/models/diffusers/minimax_h3/minimax_h3_condition/modeling_minimax_h3_condition.py new file mode 100644 index 0000000000..a7b06581b1 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_condition/modeling_minimax_h3_condition.py @@ -0,0 +1,635 @@ +"""MiniMaxH3 Condition Model — VAE encoding + text encoding + noise/pack for DiT training. + +Two-stage training flow: + 1. offline_embedding: get_condition() encodes raw data → saves parquet + 2. offline_training: skip_encoder_load=True, process_condition() adds noise + packs + 3. online_training: get_condition() + process_condition() in one step + +Implements the VeOmni ConditionModel contract: + - get_condition(**micro_batch) → dict[str, list] (per-sample lists) + - process_condition(**cached) → dict for model.forward() +""" + +from __future__ import annotations + +import os +from collections import OrderedDict +from typing import Any + +import torch +from transformers import PreTrainedModel + +from .....utils import logging +from .configuration_minimax_h3_condition import MiniMaxH3ConditionModelConfig + + +logger = logging.get_logger(__name__) + +# VAE constraints +_MINIMAX_H3_FRAME_RATE = 24 +_MINIMAX_H3_TIME_DIVISION_FACTOR = 17 +_MINIMAX_H3_TIME_DIVISION_REMAINDER = 5 + + +class MiniMaxH3ConditionModel(PreTrainedModel): + config_class = MiniMaxH3ConditionModelConfig + + def __init__(self, config: MiniMaxH3ConditionModelConfig, **kwargs): + super().__init__(config) + self._video_vae = None + self._audio_vae = None + self._text_encoder = None + self._processor = None + self._tokenizer = None + self._scheduler_video = None + self._scheduler_audio = None + + if not config.skip_encoder_load: + self._init_encoders(config) + + self._init_schedulers(config) + + # ── Initialization ──────────────────────────────────────────────── + + def _init_encoders(self, config: MiniMaxH3ConditionModelConfig): + base = config.base_model_path + + from veomni.models.module_utils import init_empty_weights + + from ..minimax_h3_core.minimax_h3_audio_vae import MiniMaxH3AudioVAE + from ..minimax_h3_core.minimax_h3_text_encoder import MiniMaxH3TextEncoder + from ..minimax_h3_core.minimax_h3_video_vae import MiniMaxH3VideoVAE + + with init_empty_weights(): + self._text_encoder = MiniMaxH3TextEncoder(num_retained_layers=config.text_encoder_num_retained_layers) + self._video_vae = MiniMaxH3VideoVAE() + self._audio_vae = MiniMaxH3AudioVAE() + + # Processor / tokenizer + from transformers import AutoProcessor + + proc_path = f"{base}/{config.processor_subfolder}" if base else None + if proc_path: + try: + self._processor = AutoProcessor.from_pretrained(proc_path) + self._tokenizer = self._processor.tokenizer + except Exception as exc: + raise RuntimeError(f"Failed to load Qwen3-VL processor from {proc_path}") from exc + + # Load weights if base path provided + if base: + self._load_encoder_weights(base, config) + + def _load_encoder_weights(self, base: str, config: MiniMaxH3ConditionModelConfig): + # ── Text Encoder ── + te_path = f"{base}/{config.text_encoder_subfolder}" + if os.path.isdir(te_path): + + def te_filter(k): + n = config.text_encoder_num_retained_layers + return ( + k.startswith("lm_head.") + or k.startswith("model.language_model.norm.") + or any(k.startswith(f"model.language_model.layers.{i}.") for i in range(n, 100)) + ) + + n = self._stream_load_weights(self._text_encoder, te_path, filter_fn=te_filter) + if n == 0: + raise RuntimeError( + f"Text encoder loaded 0 tensors from {te_path}: no .safetensors shards found in the subfolder." + ) + logger.info_rank0("Text encoder loaded (%d tensors).", n) + + # ── Video VAE ── + # HF layout nests the shards under video_vae/source/; fall back to + # the top level for layouts that keep them directly in video_vae/. + vv_path = f"{base}/{config.video_vae_subfolder}" + if os.path.isdir(f"{vv_path}/source"): + vv_path = f"{vv_path}/source" + if os.path.isdir(vv_path): + n = self._stream_load_weights(self._video_vae, vv_path, key_converter=self._convert_video_vae_keys) + if n == 0: + raise RuntimeError( + f"Video VAE loaded 0 tensors from {vv_path}: no .safetensors shards found in the subfolder." + ) + logger.info_rank0("Video VAE loaded (%d tensors).", n) + + # ── Audio VAE ── + av_path = f"{base}/{config.audio_vae_subfolder}" + if os.path.isdir(av_path): + n = self._stream_load_weights(self._audio_vae, av_path, key_converter=self._convert_audio_vae_keys) + if n == 0: + raise RuntimeError( + f"Audio VAE loaded 0 tensors from {av_path}: no .safetensors shards found in the subfolder." + ) + logger.info_rank0("Audio VAE loaded (%d tensors).", n) + + # Cast encoders to bf16: the init context casts buffers too (e.g. + # vision rotary inv_freq). VeOmni meta-init only replaces parameters, + # leaving buffers at their construction dtype (fp32), which changes + # rotary precision. Fail loudly if any encoder parameter is still on + # the meta device (checkpoint tensors missing / strict=False tolerated + # the gap): a meta param would crash or corrupt the forward later. + for enc in [self._text_encoder, self._video_vae, self._audio_vae]: + if enc is not None: + meta_params = [k for k, p in enc.named_parameters() if p.device.type == "meta"] + if meta_params: + raise RuntimeError( + f"{type(enc).__name__}: {len(meta_params)} parameter(s) still on " + f"meta device after loading (first: {meta_params[0]!r}); the " + "checkpoint files are missing these tensors." + ) + enc.to(torch.bfloat16) + enc.requires_grad_(False) + + @staticmethod + def _stream_load_weights(model, path: str, filter_fn=None, key_converter=None) -> int: + """Stream safetensors files into model, bf16, assign=True. + + safe_open per file → get_tensor → cast torch_dtype (bf16) → + state_dict_converter → model.load_state_dict(state_dict, assign=True). + """ + from collections import OrderedDict + + from veomni.models.module_utils import StateDictIterator + + total = 0 + for fname in sorted(os.listdir(path)): + if not fname.endswith(".safetensors"): + continue + state = OrderedDict() + for k, v in StateDictIterator(os.path.join(path, fname)): + if filter_fn is not None and filter_fn(k): + continue + if key_converter is not None: + k2 = key_converter({k: v}) + if not k2: + continue + k, v = next(iter(k2.items())) + state[k] = v.to(torch.bfloat16) + if state: + model.load_state_dict(state, assign=True, strict=False) + total += len(state) + return total + + @staticmethod + def _convert_video_vae_keys(state: OrderedDict) -> OrderedDict: + """Convert VideoVAE state dict keys to VeOmni format.""" + converted = OrderedDict() + for k, v in state.items(): + # decoder.register_tokens -> decoder.register_tokens.weight + if "register_tokens" in k and not k.endswith(".weight"): + k = k + ".weight" + # *.scale1, *.scale2 -> *.scale1.weight, *.scale2.weight + if k.endswith(".scale1") or k.endswith(".scale2"): + k = k + ".weight" + # Skip mask_token (not in model) + if "mask_token" in k: + continue + converted[k] = v + return converted + + @staticmethod + def _convert_audio_vae_keys(state: OrderedDict) -> OrderedDict: + """Convert AudioVAE state dict keys to VeOmni format.""" + converted = OrderedDict() + for k, v in state.items(): + # q_bias, v_bias, zero_k_bias -> q_bias.weight, etc. + if k.endswith(".q_bias") or k.endswith(".v_bias") or k.endswith(".zero_k_bias"): + k = k + ".weight" + converted[k] = v + return converted + + def _init_schedulers(self, config: MiniMaxH3ConditionModelConfig): + from ..minimax_h3_core.flow_match_scheduler import FlowMatchScheduler + + self._scheduler_video = FlowMatchScheduler(shift=config.sigma_shift_video) + self._scheduler_video.set_timesteps(config.num_train_timesteps, training=True) + + self._scheduler_audio = FlowMatchScheduler(shift=config.sigma_shift_audio) + self._scheduler_audio.set_timesteps(config.num_train_timesteps, training=True) + + # ── get_condition (online encoding) ─────────────────────────────── + + @torch.no_grad() + def get_condition( + self, + inputs: list[str], + videos: list[list[torch.Tensor]], + audios: list | None = None, + images: list | None = None, + **kwargs, + ) -> dict[str, list]: + """Encode raw data → per-sample lists of latents / embeddings / packed info. + + Input (from DataCollator via DiTTrainer.preforward): + inputs: list[str] — text prompts + videos: list[list[torch.Tensor]] — each sample: list of frame tensors [C,H,W] + audios: list[(waveform, sr)] | None — each sample: audio (waveform[C,T], sample_rate) + images: list | None — external keyframe images (FL2VA: unused) + + Returns: + prompt_embeds: list[tensor [L, 5120]] + input_latents: list[tensor [1, 24, T_v, H_lat, W_lat]] + audio_input_latents: list[tensor [C, 32, T_a]] + keyframe_cond_anchor: list[tensor [cond_rows, 96]] or None + packed: list[dict] + imgvid_cond_noise_aug: list[float] + audio_cond_noise_aug: list[float] + use_gradient_checkpointing: list[bool] + """ + device = next(self._video_vae.parameters()).device + cfg = self.config + + results = { + "prompt_embeds": [], + "input_latents": [], + "audio_input_latents": [], + "keyframe_cond_anchor": [], + "packed": [], + "imgvid_cond_noise_aug": [], + "audio_cond_noise_aug": [], + "use_gradient_checkpointing": [], + } + + num_samples = len(inputs) + for i in range(num_samples): + prompt = inputs[i] + video_frames = videos[i] + num_frames = len(video_frames) + + # 1. Validate frame count for VAE temporal grouping + if num_frames % _MINIMAX_H3_TIME_DIVISION_FACTOR != _MINIMAX_H3_TIME_DIVISION_REMAINDER: + raise ValueError(f"Sample {i}: num_frames={num_frames}, expected 17n+5 (e.g. 73)") + + # 2. Extract keyframes from video (FL2VA: first + last frame) + # Prefer transform-provided PIL keyframes (native uint8, exact match + # with keyframes = data["video"][0/-1]); fall back to + # rebuilding from frame tensors only when the transform lacks them. + if cfg.use_keyframe_condition and cfg.keyframe_indices: + if images is not None and i < len(images) and images[i]: + keyframe_images = list(images[i]) + else: + from PIL import Image + + keyframe_images = [] + for idx in cfg.keyframe_indices: + frame_tensor = video_frames[idx] + if frame_tensor.dim() == 3: + frame_np = (frame_tensor.permute(1, 2, 0).cpu().numpy() * 255).astype("uint8") + keyframe_images.append(Image.fromarray(frame_np)) + else: + keyframe_images = None + + # 3. Encode text (Qwen3VL) + prompt_embeds, text_token_tags = self._encode_text(prompt, keyframe_images, device) + + # 4. Encode video (VAE) + video_tensor = torch.stack(video_frames).permute(1, 0, 2, 3).unsqueeze(0) # [1,3,T,H,W] + input_latent = self._encode_video(video_tensor, device) + + # 5. Encode audio (VAE) + if audios is not None and i < len(audios) and audios[i] is not None: + audio_latent = self._encode_audio(audios[i], num_frames, device) + else: + # Placeholder: silent audio latent + audio_latent = self._make_silent_audio_latent(num_frames, device) + + # 6. Encode keyframe as condition anchors + if cfg.use_keyframe_condition and keyframe_images: + if len(keyframe_images) != len(cfg.keyframe_indices): + raise ValueError( + f"Sample {i}: got {len(keyframe_images)} keyframe images, " + f"expected {len(cfg.keyframe_indices)} for keyframe_indices={cfg.keyframe_indices}" + ) + # Transform-provided keyframes carry their source indices; + # validate them, not only the count. + sample_keyframe_indices = kwargs.get("keyframe_indices") + if sample_keyframe_indices is not None and i < len(sample_keyframe_indices): + if list(sample_keyframe_indices[i]) != list(cfg.keyframe_indices): + raise ValueError( + f"Sample {i}: keyframes were extracted at indices " + f"{sample_keyframe_indices[i]}, expected {cfg.keyframe_indices}" + ) + keyframe_cond_anchor = self._encode_keyframe_cond(keyframe_images, device, input_latent.shape[2]) + else: + if cfg.use_keyframe_condition and cfg.keyframe_indices: + raise ValueError(f"Sample {i}: keyframe condition is enabled but no keyframe image was produced") + keyframe_cond_anchor = None + + # 7. Build packed sequence (FL2VA layout) + T_v = input_latent.shape[2] + H_lat = input_latent.shape[3] + W_lat = input_latent.shape[4] + audio_ch = audio_latent.shape[0] + T_a = audio_latent.shape[2] + + from ..minimax_h3_core.packed_sequence import build_packed_fl2va + + packed = build_packed_fl2va( + text_len=prompt_embeds.shape[0], + latent_t=T_v, + latent_h=H_lat, + latent_w=W_lat, + audio_t=T_a, + keyframe_indices=cfg.keyframe_indices if cfg.use_keyframe_condition else [], + audio_channel=audio_ch, + text_token_tags=text_token_tags, + ) + + results["prompt_embeds"].append(prompt_embeds) + results["input_latents"].append(input_latent) + results["audio_input_latents"].append(audio_latent) + results["keyframe_cond_anchor"].append(keyframe_cond_anchor) + results["packed"].append(packed) + results["imgvid_cond_noise_aug"].append(cfg.imgvid_cond_noise_aug) + results["audio_cond_noise_aug"].append(cfg.audio_cond_noise_aug) + results["use_gradient_checkpointing"].append(True) + + return results + + def _encode_text(self, prompt: str, keyframe_images, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]: + """Encode prompt + keyframe images via Qwen3VL → (last_hidden_state[0], text_token_tags).""" + from ..minimax_h3_core.minimax_h3_text_encoder import image_token_counts, presentation_fl2va + + if keyframe_images and self._tokenizer: + pixel_values, image_grid_thw, counts = image_token_counts(self._processor, keyframe_images) + input_ids, text_token_tags = presentation_fl2va(self._tokenizer, prompt, counts) + input_ids = input_ids.unsqueeze(0).to(device) + attention_mask = torch.ones_like(input_ids) + pixel_values = pixel_values.to(device=device, dtype=torch.bfloat16) + image_grid_thw = image_grid_thw.to(device=device, dtype=torch.long) + + prompt_embeds = self._text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + ) + else: + # Text-only: use t2va presentation + from ..minimax_h3_core.minimax_h3_text_encoder import presentation_t2va + + input_ids, text_token_tags = presentation_t2va(self._tokenizer, prompt) + input_ids = input_ids.unsqueeze(0).to(device) + prompt_embeds = self._text_encoder( + input_ids=input_ids, + attention_mask=torch.ones_like(input_ids), + ) + + return prompt_embeds.to(device), text_token_tags + + def _encode_video(self, video_tensor: torch.Tensor, device: torch.device) -> torch.Tensor: + """Encode video tensor [1,3,T,H,W] → latent [1,24,T_v,H/16,W/16]. + + Match InputVideoEmbedder: preprocess_video produces float32 + [0,1] frames, encode_video(frames, dtype=bf16) casts inside, output + cast to bf16. + """ + video = video_tensor.to(device=device, dtype=torch.float32) + latent = self._video_vae.encode_video(video, dtype=torch.bfloat16) + return latent.to(torch.bfloat16) + + def _encode_audio(self, audio_input, num_frames: int, device: torch.device) -> torch.Tensor: + """Encode audio → latent [C, 32, T_a]. + + The data loader (minimax_h3_online transform) already + trims/pads the waveform to int(num_frames/24 * original_sr) at the + ORIGINAL sample rate. Here we only do the InputAudioEmbedder steps: + convert_to_stereo(float) → resample to 32kHz → encode_audio(dtype=bf16). + """ + waveform, sample_rate = audio_input + waveform = torch.as_tensor(waveform) + waveform = waveform.squeeze(0) if waveform.dim() == 3 else waveform + assert waveform.dim() == 2, "waveform must be in shape (C, T)" + # convert_to_stereo: duplicate mono, keep stereo + if waveform.size(-2) == 1: + waveform = waveform.repeat(2, 1) + waveform = waveform.float() + if sample_rate != 32000: + import torchaudio + + waveform = torchaudio.functional.resample(waveform, sample_rate, 32000) + latent = self._audio_vae.encode_audio(waveform[:2].to(device), dtype=torch.bfloat16) + return latent.to(torch.bfloat16) + + @staticmethod + def _make_silent_audio_latent(num_frames: int, device: torch.device) -> torch.Tensor: + """Create a zero-filled audio latent tensor as placeholder.""" + T_a = round(num_frames / _MINIMAX_H3_FRAME_RATE * 40) + return torch.zeros(2, 32, T_a, device=device, dtype=torch.bfloat16) + + def _encode_keyframe_cond(self, keyframe_images, device: torch.device, video_latent_t: int) -> torch.Tensor: + """Encode keyframe images → condition anchor [cond_rows, 96] with noise augmentation. + + Matches KeyframeEncoder exactly: + - img float32 [0,1] → encode_video(dtype=bf16, process_image=True) + - clean rows cast to bf16 + - noise = generate_noise((1,24,T_v+len(keyframes),H,W), seed, "cpu", + rand_torch_dtype=bf16)[:, :, :1] → patchify → bf16, ONE noise shared across keyframes + - ts = tensor(0.999, bf16); anchor = ts*clean + (1-ts)*noise per keyframe + """ + from ..minimax_h3_core.minimax_h3_dit import patchify_video + + all_cond_rows = [] + for img in keyframe_images: + img_tensor = self._pil_to_tensor(img).to(device=device, dtype=torch.float32) # [1,3,H,W] + z_norm = self._video_vae.encode_video( + img_tensor, dtype=torch.bfloat16, process_image=True + ) # [1,24,1,H/16,W/16] + rows = patchify_video(z_norm) + all_cond_rows.append(rows) + + clean_cond_rows = torch.cat(all_cond_rows, dim=0).to(device=device, dtype=torch.bfloat16) + + if self.config.imgvid_cond_noise_aug == 1.0: + return clean_cond_rows.contiguous() + + ts = torch.tensor(self.config.imgvid_cond_noise_aug, dtype=torch.bfloat16, device=device) + latent_h, latent_w = int(z_norm.shape[3]), int(z_norm.shape[4]) + noise = torch.randn( + (1, 24, video_latent_t + len(keyframe_images), latent_h, latent_w), + device="cpu", + dtype=torch.bfloat16, + )[:, :, :1] + noise_rows = patchify_video(noise.to(device=device, dtype=torch.bfloat16)) + frame_rows = (latent_h // 2) * (latent_w // 2) + parts = [ + ts * clean_cond_rows[i * frame_rows : (i + 1) * frame_rows] + (1.0 - ts) * noise_rows + for i in range(len(keyframe_images)) + ] + return torch.cat(parts, dim=0).contiguous() + + @staticmethod + def _pil_to_tensor(img) -> torch.Tensor: + """PIL → float32 [0,1] tensor [1,3,H,W] (preprocess_image min_value=0).""" + import numpy as np + + arr = np.array(img, dtype=np.float32) + return torch.tensor(arr).permute(2, 0, 1).unsqueeze(0) * (1.0 / 255.0) + + # ── process_condition (add noise + pack) ────────────────────────── + + @torch.no_grad() + def process_condition( + self, + input_latents: list[torch.Tensor], + audio_input_latents: list[torch.Tensor], + prompt_embeds: list[torch.Tensor], + packed: list[dict], + keyframe_cond_anchor: list[torch.Tensor | None] | None = None, + imgvid_cond_noise_aug: float = 0.999, + audio_cond_noise_aug: float = 1.0, + use_gradient_checkpointing: bool = True, + **kwargs, + ) -> dict[str, Any]: + """Add noise + pack latents into model.forward() inputs. + + Per-sample (batch is list of samples, this processes sample 0): + + 1. Sample shared timestep_id ~ Uniform(0, 999) + 2. Compute sigma_video, sigma_audio from respective schedulers + 3. Add noise: noised = (1-sigma)*clean + sigma*noise + 4. Target = noise - clean (velocity) + 5. Patchify/pack into padded sequence buffers + 6. Per-token timesteps with unique/inverse mapping + + Returns dict with all keys for model.forward() + training_target*. + """ + # DiTDataCollator wraps every value (including scalars) in lists via + # batch[key].append(feature[key]). Unwrap scalars that arrive as + # single-element lists. + if isinstance(imgvid_cond_noise_aug, list): + imgvid_cond_noise_aug = imgvid_cond_noise_aug[0] + if isinstance(audio_cond_noise_aug, list): + audio_cond_noise_aug = audio_cond_noise_aug[0] + if isinstance(use_gradient_checkpointing, list): + use_gradient_checkpointing = use_gradient_checkpointing[0] + + # Process first sample (batch=1 per micro_batch); reject any other + # size for every supplied collection so a mismatched length cannot + # silently truncate to index 0. + supplied = { + "input_latents": input_latents, + "audio_input_latents": audio_input_latents, + "prompt_embeds": prompt_embeds, + "packed": packed, + } + if keyframe_cond_anchor is not None: + supplied["keyframe_cond_anchor"] = keyframe_cond_anchor + for name, coll in supplied.items(): + if len(coll) != 1: + raise ValueError( + f"MiniMaxH3ConditionModel.process_condition supports micro_batch_size=1 only, " + f"got {len(coll)} samples in {name}." + ) + clean_video = input_latents[0] + clean_audio = audio_input_latents[0] + prompt = prompt_embeds[0] + pk = packed[0] + cond_anchor = keyframe_cond_anchor[0] if keyframe_cond_anchor else None + + device = clean_video.device + dtype = clean_video.dtype + + cfg = self.config + seq_len = pk["seq_len"] + audio_ch = pk["audio_channel"] + + # 1. Sample timestep + timestep_id_int = int(torch.randint(0, cfg.num_train_timesteps, (1,), device=device).item()) + # Match the precision path: timestep = scheduler.timesteps[id].to(dtype=model_dtype), + # passed directly to scheduler.add_noise() / scheduler.training_weight(). + # Lower dtype precision (e.g. bfloat16) shifts the argmin index used by add_noise + # and training_weight, changing the effective sigma and weight. + model_dtype = clean_video.dtype + ts_v = self._scheduler_video.timesteps[timestep_id_int].to(dtype=model_dtype) + ts_a = self._scheduler_audio.timesteps[timestep_id_int].to(dtype=model_dtype) + + # t = 1.0 - timestep / num_train_timesteps + t_video = 1.0 - ts_v.float().item() / self._scheduler_video.num_train_timesteps + t_audio = 1.0 - ts_a.float().item() / self._scheduler_audio.num_train_timesteps + + # 2. Add noise + compute velocity target + video_noise = torch.randn_like(clean_video) + video_noised = self._scheduler_video.add_noise(clean_video, video_noise, ts_v) + training_target = self._scheduler_video.training_target(clean_video, video_noise, ts_v) + + audio_noise = torch.randn_like(clean_audio) + audio_noised = self._scheduler_audio.add_noise(clean_audio, audio_noise, ts_a) + training_target_audio = self._scheduler_audio.training_target(clean_audio, audio_noise, ts_a) + + # 3. Patchify + pack + from ..minimax_h3_core.minimax_h3_dit import pack_audio, patchify_video + + video_rows = patchify_video(video_noised) # [Tv*hp*wp, 96] + audio_rows = pack_audio(audio_noised) # [C*Ta, 32] + + # Move packed index tensors to device once (defensive: _to_device + # in preforward should have done this, but NPU indexing requires it). + img_pos = pk["img_pos"].to(device) + cond_rows = pk["cond_rows"] + audio_pos = pk["audio_pos"].to(device) + + # Packed x: [1, seq_len, feats] + x = torch.zeros(1, seq_len, video_rows.shape[-1], device=device, dtype=dtype) + x[0, img_pos[cond_rows:]] = video_rows.to(dtype) + if cond_anchor is not None: + x[0, img_pos[:cond_rows]] = cond_anchor.to(dtype) + + audio_x = torch.zeros(1, seq_len, audio_rows.shape[-1], device=device, dtype=dtype) + audio_x[0, audio_pos] = audio_rows.to(dtype) + + # 4. Per-token timesteps (t = 1 - sigma format) + timesteps = torch.full((seq_len,), t_video, dtype=torch.float32, device=device) + timesteps[audio_pos] = t_audio + if cond_rows > 0: + # Condition tokens: always near-clean (t ≈ 1.0, sigma ≈ 0) + timesteps[img_pos[:cond_rows]] = max(float(t_video), float(cfg.imgvid_cond_noise_aug)) + unique_timesteps, inverse_indices = torch.unique(timesteps, sorted=True, return_inverse=True) + + # 5. Refiner cu_seqlens (text-only, no padding for refiner) + text_len = pk["text_len"] + refiner_cu = torch.tensor([0, text_len, text_len], dtype=torch.int32) + + # 6. Shape info for unpatchify (use keys NOT popped by trainer) + T_v = pk["latent_t"] + T_a = pk["audio_t"] + + video_latent_shape = (T_v, pk["latent_h_patched"], pk["latent_w_patched"]) + audio_latent_shape = (audio_ch, T_a) + + return { + "x": x, + "audio_x": audio_x, + "img_position_ids": pk["img_position_ids"].to(device), + "unique_timesteps": unique_timesteps, + "inverse_indices": inverse_indices, + "update_mask": None, # skip_mask_out_condition=True, handled in modeling.py + "token_tags": pk["token_tags"].to(device), + "prompt_embeds": prompt, + "img_pos_info": {"position_ids": pk["img_pos"].to(device)}, + "audio_pos_info": {"position_ids": pk["audio_pos"].to(device)}, + "text_pos_info": {"position_ids": pk["text_pos"].to(device)}, + "img_pos_for_infer_output_info": {"position_ids": pk["img_pos"].to(device)}, + "packed_seq_params": { + "cu_seqlens_q": pk["cu_seqlens"].to(device), + "max_seqlen_q": int(pk["cu_seqlens"][1]), + }, + "refiner_packed_seq_params": { + "cu_seqlens_q": refiner_cu.to(device), + "max_seqlen_q": text_len, + }, + "skip_mask_out_condition": True, + "cond_rows": cond_rows, + "use_gradient_checkpointing": use_gradient_checkpointing, + "video_latent_shape": video_latent_shape, + "audio_latent_shape": audio_latent_shape, + "training_target": training_target, + "training_target_audio": training_target_audio, + # Pass scheduler references and raw t values so modeling.py can compute training_weight + "scheduler_video": self._scheduler_video, + "scheduler_audio": self._scheduler_audio, + "t_video": t_video, + "t_audio": t_audio, + } diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/core.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/core.py new file mode 100644 index 0000000000..2302a5c567 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/core.py @@ -0,0 +1,383 @@ +"""Core utilities for MiniMax H3. + +Provides attention dispatch and gradient checkpointing compatible with +the original minimax_h3_dit module. +""" + +from __future__ import annotations + +import inspect +import os + +import torch +import torch.nn.functional as F +from einops import rearrange + + +# ── Attention backend detection ────────────────────────────────────── + +try: + import flash_attn_interface + + FLASH_ATTN_3_AVAILABLE = True +except ModuleNotFoundError: + FLASH_ATTN_3_AVAILABLE = False + +try: + import flash_attn + + FLASH_ATTN_2_AVAILABLE = True +except ModuleNotFoundError: + FLASH_ATTN_2_AVAILABLE = False + +try: + from sageattention import sageattn + + SAGE_ATTN_AVAILABLE = True +except ModuleNotFoundError: + SAGE_ATTN_AVAILABLE = False + +try: + import xformers.ops as xops + + XFORMERS_AVAILABLE = True +except ModuleNotFoundError: + XFORMERS_AVAILABLE = False + +try: + if "enable_gqa" in inspect.signature(torch.nn.functional.scaled_dot_product_attention).parameters: + TORCH_SUPPORT_GQA = True + else: + TORCH_SUPPORT_GQA = False +except Exception: + TORCH_SUPPORT_GQA = False + + +def _initialize_attention_priority() -> str: + env_val = os.environ.get("MINIMAX_H3_ATTENTION_IMPLEMENTATION") + if env_val is not None: + return env_val.lower() + if FLASH_ATTN_3_AVAILABLE: + return "flash_attention_3" + if FLASH_ATTN_2_AVAILABLE: + return "flash_attention_2" + if SAGE_ATTN_AVAILABLE: + return "sage_attention" + if XFORMERS_AVAILABLE: + return "xformers" + return "torch" + + +ATTENTION_IMPLEMENTATION = _initialize_attention_priority() + + +# ── Rearrange helpers (einops) ─────────────────────────────────────── + + +def _rearrange_qkv( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_pattern: str = "b n s d", + k_pattern: str = "b n s d", + v_pattern: str = "b n s d", + target: str = "b n s d", + dims: dict | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Rearrange q/k/v to target pattern. + + Einops-based rearrangement, supports any pattern including merged head + dims like "b s (n d)". + """ + dims = {} if dims is None else dims + + def _to_pattern(t: torch.Tensor, pat: str) -> torch.Tensor: + if pat != target: + t = rearrange(t, f"{pat} -> {target}", **dims) + return t + + return _to_pattern(q, q_pattern), _to_pattern(k, k_pattern), _to_pattern(v, v_pattern) + + +def _rearrange_out( + out: torch.Tensor, + out_pattern: str = "b n s d", + target: str = "b n s d", + dims: dict | None = None, +) -> torch.Tensor: + """Reverse of _rearrange_qkv for output.""" + dims = {} if dims is None else dims + if out_pattern != target: + out = rearrange(out, f"{target} -> {out_pattern}", **dims) + return out + + +# ── Backend implementations ────────────────────────────────────────── + + +def _torch_sdpa( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_pattern: str = "b n s d", + k_pattern: str = "b n s d", + v_pattern: str = "b n s d", + out_pattern: str = "b n s d", + dims: dict | None = None, + attn_mask: torch.Tensor | None = None, + scale: float | None = None, + is_causal: bool = False, +) -> torch.Tensor: + q, k, v = _rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, "b n s d", dims) + + if q.shape[1] != k.shape[1] or q.shape[1] != v.shape[1]: + if TORCH_SUPPORT_GQA: + out = F.scaled_dot_product_attention(q, k, v, attn_mask, scale=scale, is_causal=is_causal, enable_gqa=True) + else: + reps = q.shape[1] // k.shape[1] + k = k.repeat_interleave(reps, dim=1) + v = v.repeat_interleave(reps, dim=1) + out = F.scaled_dot_product_attention(q, k, v, attn_mask, scale=scale, is_causal=is_causal) + else: + out = F.scaled_dot_product_attention(q, k, v, attn_mask, scale=scale, is_causal=is_causal) + + return _rearrange_out(out, out_pattern, "b n s d", dims) + + +def _flash_attn_3_forward( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_pattern: str = "b n s d", + k_pattern: str = "b n s d", + v_pattern: str = "b n s d", + out_pattern: str = "b n s d", + dims: dict | None = None, + scale: float | None = None, + is_causal: bool = False, + window_size: int | None = None, +) -> torch.Tensor: + q, k, v = _rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, "b s n d", dims) + ws = (window_size, window_size) if window_size is not None else (-1, -1) + out = flash_attn_interface.flash_attn_func(q, k, v, softmax_scale=scale, causal=is_causal, window_size=ws) + if isinstance(out, tuple): + out = out[0] + return _rearrange_out(out, out_pattern, "b s n d", dims) + + +def _flash_attn_2_forward( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_pattern: str = "b n s d", + k_pattern: str = "b n s d", + v_pattern: str = "b n s d", + out_pattern: str = "b n s d", + dims: dict | None = None, + scale: float | None = None, + is_causal: bool = False, + window_size: int | None = None, +) -> torch.Tensor: + q, k, v = _rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, "b s n d", dims) + ws = (window_size, window_size) if window_size is not None else (-1, -1) + out = flash_attn.flash_attn_func(q, k, v, softmax_scale=scale, causal=is_causal, window_size=ws) + return _rearrange_out(out, out_pattern, "b s n d", dims) + + +def _sage_attn_forward( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_pattern: str = "b n s d", + k_pattern: str = "b n s d", + v_pattern: str = "b n s d", + out_pattern: str = "b n s d", + dims: dict | None = None, + scale: float | None = None, +) -> torch.Tensor: + q, k, v = _rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, "b n s d", dims) + out = sageattn(q, k, v, sm_scale=scale) + return _rearrange_out(out, out_pattern, "b n s d", dims) + + +def _xformers_forward( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_pattern: str = "b n s d", + k_pattern: str = "b n s d", + v_pattern: str = "b n s d", + out_pattern: str = "b n s d", + dims: dict | None = None, + scale: float | None = None, +) -> torch.Tensor: + q, k, v = _rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, "b s n d", dims) + out = xops.memory_efficient_attention(q, k, v, scale=scale) + return _rearrange_out(out, out_pattern, "b s n d", dims) + + +# ── Main dispatch ──────────────────────────────────────────────────── + + +def attention_forward( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q_pattern: str = "b n s d", + k_pattern: str = "b n s d", + v_pattern: str = "b n s d", + out_pattern: str = "b n s d", + dims: dict | None = None, + attn_mask: torch.Tensor | None = None, + scale: float | None = None, + is_causal: bool = False, + compatibility_mode: bool = False, + window_size: int | None = None, +) -> torch.Tensor: + """Dispatch attention to available backend.""" + if compatibility_mode or (attn_mask is not None) or ATTENTION_IMPLEMENTATION == "torch": + return _torch_sdpa( + q, + k, + v, + q_pattern, + k_pattern, + v_pattern, + out_pattern, + dims, + attn_mask=attn_mask, + scale=scale, + is_causal=is_causal, + ) + if ATTENTION_IMPLEMENTATION == "flash_attention_3": + return _flash_attn_3_forward( + q, + k, + v, + q_pattern, + k_pattern, + v_pattern, + out_pattern, + dims, + scale=scale, + is_causal=is_causal, + window_size=window_size, + ) + if ATTENTION_IMPLEMENTATION == "flash_attention_2": + return _flash_attn_2_forward( + q, + k, + v, + q_pattern, + k_pattern, + v_pattern, + out_pattern, + dims, + scale=scale, + is_causal=is_causal, + window_size=window_size, + ) + if ATTENTION_IMPLEMENTATION == "sage_attention": + if window_size is not None or is_causal: + return attention_forward( + q, + k, + v, + q_pattern, + k_pattern, + v_pattern, + out_pattern, + dims, + attn_mask, + scale, + is_causal, + compatibility_mode=True, + ) + return _sage_attn_forward(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, scale=scale) + if ATTENTION_IMPLEMENTATION == "xformers": + if window_size is not None or is_causal: + return attention_forward( + q, + k, + v, + q_pattern, + k_pattern, + v_pattern, + out_pattern, + dims, + attn_mask, + scale, + is_causal, + compatibility_mode=True, + ) + return _xformers_forward(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, scale=scale) + raise NotImplementedError(f"No available attention implementation (current: {ATTENTION_IMPLEMENTATION}).") + + +# ── Gradient checkpointing ──────────────────────────────────────────── + +try: + import deepspeed + + _HAS_DEEPSPEED = True +except ModuleNotFoundError: + _HAS_DEEPSPEED = False + + +def _create_custom_forward(module): + def custom_forward(*inputs, **kwargs): + return module(*inputs, **kwargs) + + return custom_forward + + +def _create_custom_forward_use_reentrant(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + +def _judge_args_requires_grad(*args) -> bool: + for arg in args: + if isinstance(arg, torch.Tensor) and arg.requires_grad: + return True + return False + + +def gradient_checkpoint_forward( + model, + use_gradient_checkpointing: bool, + use_gradient_checkpointing_offload: bool, + *args, + **kwargs, +): + """Gradient checkpoint wrapper. + + Delegates to deepspeed checkpointing when configured, torch checkpointing + otherwise. Falls back to direct call when checkpointing is disabled. + """ + if use_gradient_checkpointing and _HAS_DEEPSPEED and deepspeed.checkpointing.is_configured(): + all_args = args + tuple(kwargs.values()) + if not _judge_args_requires_grad(*all_args): + return model(*args, **kwargs) + return deepspeed.checkpointing.checkpoint( + _create_custom_forward_use_reentrant(model), + *all_args, + ) + if use_gradient_checkpointing_offload: + with torch.autograd.graph.save_on_cpu(): + return torch.utils.checkpoint.checkpoint( + _create_custom_forward(model), + *args, + **kwargs, + use_reentrant=False, + ) + if use_gradient_checkpointing: + return torch.utils.checkpoint.checkpoint( + _create_custom_forward(model), + *args, + **kwargs, + use_reentrant=False, + ) + return model(*args, **kwargs) diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/flow_match_scheduler.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/flow_match_scheduler.py new file mode 100644 index 0000000000..2b4e78c183 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/flow_match_scheduler.py @@ -0,0 +1,88 @@ +"""FlowMatchScheduler — MiniMax-H3 training and inference methods.""" + +import torch + + +class FlowMatchScheduler: + def __init__(self, shift: float = 12.0): + self.num_train_timesteps = 1000 + self.shift = shift + self.sigmas = None + self.timesteps = None + self.linear_timesteps_weights = None + self.training = False + + @staticmethod + def _set_timesteps_minimax_h3(num_inference_steps=50, denoising_strength=1.0, shift=12.0): + num_train_timesteps = 1000 + base = torch.linspace(denoising_strength, 0.0, num_inference_steps + 1, dtype=torch.float32)[:-1] + sigmas = shift * base / (1 + (shift - 1) * base) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + def _set_training_weight(self): + """Compute Gaussian training weights centered at timestep 500/1000.""" + steps = 1000 + x = self.sigmas * self.num_train_timesteps + y = torch.exp(-2 * ((x - steps / 2) / steps) ** 2) + y_shifted = y - y.min() + bsmntw_weighing = y_shifted * (steps / y_shifted.sum()) + if len(self.timesteps) != 1000: + bsmntw_weighing = bsmntw_weighing * (len(self.timesteps) / steps) + bsmntw_weighing = bsmntw_weighing + bsmntw_weighing[1] + self.linear_timesteps_weights = bsmntw_weighing + + def set_timesteps(self, num_inference_steps=100, denoising_strength=1.0, training=False, **kwargs): + self.sigmas, self.timesteps = self._set_timesteps_minimax_h3( + num_inference_steps=num_inference_steps, + denoising_strength=denoising_strength, + shift=self.shift, + ) + if training: + self._set_training_weight() + self.training = True + else: + self.training = False + + def step(self, model_output, timestep, sample, to_final=False, **kwargs): + if isinstance(timestep, torch.Tensor): + timestep = timestep.cpu() + timestep_id = torch.argmin((self.timesteps - timestep).abs()) + sigma = self.sigmas[timestep_id] + if to_final or timestep_id + 1 >= len(self.timesteps): + sigma_ = 0 + else: + sigma_ = self.sigmas[timestep_id + 1] + prev_sample = sample + model_output * (sigma_ - sigma) + return prev_sample + + def return_to_timestep(self, timestep, sample, sample_stablized): + if isinstance(timestep, torch.Tensor): + timestep = timestep.cpu() + timestep_id = torch.argmin((self.timesteps - timestep).abs()) + sigma = self.sigmas[timestep_id] + model_output = (sample - sample_stablized) / sigma + return model_output + + def add_noise(self, original_samples: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + """Add noise to clean latents: (1-sigma)*x0 + sigma*noise.""" + if isinstance(timestep, torch.Tensor): + timestep = timestep.cpu() + timestep_id = torch.argmin((self.timesteps - timestep).abs()) + sigma = self.sigmas[timestep_id] + sample = (1 - sigma) * original_samples + sigma * noise + return sample + + def training_target(self, sample: torch.Tensor, noise: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + """Velocity target: noise - clean. + + Together with model output negation (return -v), the prediction matches target. + """ + target = noise - sample + return target + + def training_weight(self, timestep: torch.Tensor) -> torch.Tensor: + """Gaussian timestep weight (not yet used in VeOmni MSE loss).""" + timestep_id = torch.argmin((self.timesteps - timestep.to(self.timesteps.device)).abs()) + weights = self.linear_timesteps_weights[timestep_id] + return weights diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_vae.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_vae.py new file mode 100644 index 0000000000..e172a9ff2a --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_vae.py @@ -0,0 +1,627 @@ +# SPDX-License-Identifier: Apache-2.0 +import math +from typing import List + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import Parameter + +from .core import attention_forward + + +class WarpedTensor(torch.nn.Module): + def __init__(self, weight=None, shape=None): + super().__init__() + if weight is not None: + self.weight = torch.nn.Parameter(weight) + else: + self.weight = torch.nn.Parameter(torch.empty(shape)) + + def forward(self): + return self.weight + + +class WeightNormedConv1d(torch.nn.Module): + """Weight-normalized Conv1d with float32 norm computation. + + Computes weight = (weight_g.float() * weight_v.float() / ||weight_v.float||).to(dtype) + in float32 for numerical precision, then calls F.conv1d. + """ + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + padding_mode="zeros", + device=None, + dtype=None, + ): + super().__init__() + self.weight_g = torch.nn.Parameter(torch.empty((out_channels, 1, 1))) + self.weight_v = torch.nn.Parameter(torch.empty((out_channels, in_channels, kernel_size))) + self.stride = stride + self.padding = padding + self.dilation = dilation + self.groups = groups + if bias: + self.bias = torch.nn.Parameter(torch.empty((out_channels,))) + else: + self.bias = None + + def forward(self, input): + dtype = self.weight_g.dtype + weight = ( + self.weight_g.float() * self.weight_v.float() / torch.norm(self.weight_v.float(), dim=(1, 2), keepdim=True) + ).to(dtype) + return torch.nn.functional.conv1d( + input, weight, self.bias, self.stride, self.padding, self.dilation, self.groups + ) + + +class WeightNormedConvTranspose1d(torch.nn.Module): + """Weight-normalized ConvTranspose1d with float32 norm computation.""" + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + output_padding=0, + dilation=1, + groups=1, + bias=True, + padding_mode="zeros", + device=None, + dtype=None, + ): + super().__init__() + self.weight_g = torch.nn.Parameter(torch.empty((in_channels, 1, 1))) + self.weight_v = torch.nn.Parameter(torch.empty((in_channels, out_channels, kernel_size))) + self.kernel_size = kernel_size + self.stride = stride + self.padding = (padding,) + self.output_padding = output_padding + self.dilation = dilation + self.groups = groups + if bias: + self.bias = torch.nn.Parameter(torch.empty((out_channels,))) + else: + self.bias = None + + def _output_padding( + self, + input, + output_size: list[int] | None, + stride: list[int], + padding: list[int], + kernel_size: list[int], + num_spatial_dims: int, + dilation: list[int] | None = None, + ) -> list[int]: + if output_size is None: + ret = self.output_padding + else: + has_batch_dim = input.dim() == num_spatial_dims + 2 + num_non_spatial_dims = 2 if has_batch_dim else 1 + if len(output_size) == num_non_spatial_dims + num_spatial_dims: + output_size = output_size[num_non_spatial_dims:] + if len(output_size) != num_spatial_dims: + raise ValueError( + f"ConvTranspose{num_spatial_dims}D: for {input.dim()}D input, output_size must have {num_spatial_dims} " + f"or {num_non_spatial_dims + num_spatial_dims} elements (got {len(output_size)})" + ) + + min_sizes = torch.jit.annotate(list[int], []) + max_sizes = torch.jit.annotate(list[int], []) + for d in range(num_spatial_dims): + dim_size = ( + (input.size(d + num_non_spatial_dims) - 1) * stride[d] + - 2 * padding[d] + + (dilation[d] if dilation is not None else 1) * (kernel_size[d] - 1) + + 1 + ) + min_sizes.append(dim_size) + max_sizes.append(min_sizes[d] + stride[d] - 1) + + for i in range(len(output_size)): + size = output_size[i] + min_size = min_sizes[i] + max_size = max_sizes[i] + if size < min_size or size > max_size: + raise ValueError( + f"requested an output size of {output_size}, but valid sizes range " + f"from {min_sizes} to {max_sizes} (for an input of {input.size()[2:]})" + ) + + res = torch.jit.annotate(list[int], []) + for d in range(num_spatial_dims): + res.append(output_size[d] - min_sizes[d]) + + ret = res + return ret + + def forward(self, input, output_size=None): + dtype = self.weight_g.dtype + weight = ( + self.weight_g.float() * self.weight_v.float() / torch.norm(self.weight_v.float(), dim=(1, 2), keepdim=True) + ).to(dtype) + assert isinstance(self.padding, tuple) + num_spatial_dims = 1 + output_padding = self._output_padding( + input, + output_size, + self.stride, # type: ignore[arg-type] + self.padding, # type: ignore[arg-type] + self.kernel_size, # type: ignore[arg-type] + num_spatial_dims, + self.dilation, # type: ignore[arg-type] + ) + return F.conv_transpose1d( + input, + weight, + self.bias, + self.stride, + self.padding, + output_padding, + self.groups, + self.dilation, + ) + + +class Snake(nn.Module): + def __init__(self, channels, beta=False): + super().__init__() + if not beta: + self.alpha = nn.Parameter(torch.ones(1, channels, 1)) + self.beta = None + else: + self.alpha = nn.Parameter(torch.ones(channels)) + self.beta = Parameter(torch.zeros(channels)) if beta else None + + def forward(self, x): + alpha = self.alpha + if len(alpha.shape) == 1: + alpha = alpha.unsqueeze(0).unsqueeze(-1).exp() + beta = self.alpha if self.beta is None else self.beta + if len(beta.shape) == 1: + beta = beta.unsqueeze(0).unsqueeze(-1).exp() + x = x + (beta + 1e-9).reciprocal() * torch.sin(alpha * x).pow(2) + return x + + +def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): + even = kernel_size % 2 == 0 + half_size = kernel_size // 2 + delta_f = 4 * half_width + A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95 + if A > 50.0: + beta = 0.1102 * (A - 8.7) + elif A >= 21.0: + beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0) + else: + beta = 0.0 + window = torch.kaiser_window(kernel_size, beta=beta, periodic=False) + if even: + time = torch.arange(-half_size, half_size) + 0.5 + else: + time = torch.arange(kernel_size) - half_size + if cutoff == 0: + filter_ = torch.zeros_like(time) + else: + filter_ = 2 * cutoff * window * torch.sinc(2 * cutoff * time) + filter_ /= filter_.sum() + return filter_.view(1, 1, kernel_size) + + +class LowPassFilter1d(nn.Module): + def __init__(self, cutoff=0.25, half_width=0.3, stride: int = 2, kernel_size: int = 12): + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.filter = torch.nn.Parameter(kaiser_sinc_filter1d(cutoff, half_width, kernel_size)) + + def forward(self, x): + x = F.pad(x, (self.kernel_size // 2 - 1, self.kernel_size // 2), mode="replicate") + out = F.conv1d(x, self.filter.expand(x.shape[1], -1, -1), stride=self.stride, groups=x.shape[1]) + return out + + +class UpSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=None): + super().__init__() + self.ratio = ratio + self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size + self.stride = ratio + self.pad = self.kernel_size // ratio - 1 + self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2 + self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2 + filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size) + self.filter = torch.nn.Parameter(filter) + + def forward(self, x): + _, C, _ = x.shape + x = F.pad(x, (self.pad, self.pad), mode="replicate") + x = self.ratio * F.conv_transpose1d( + x, self.filter.expand(C, -1, -1).to(dtype=x.dtype, device=x.device), stride=self.stride, groups=C + ) + x = x[..., self.pad_left : -self.pad_right] + return x + + +class DownSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=None): + super().__init__() + self.ratio = ratio + self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size + self.lowpass = LowPassFilter1d( + cutoff=0.5 / ratio, half_width=0.6 / ratio, stride=ratio, kernel_size=self.kernel_size + ) + + def forward(self, x): + return self.lowpass(x) + + +class Activation1d(nn.Module): + def __init__( + self, + activation, + up_ratio: int = 2, + down_ratio: int = 2, + up_kernel_size: int = 12, + down_kernel_size: int = 12, + ): + super().__init__() + self.act = activation + self.upsample = UpSample1d(up_ratio, up_kernel_size) + self.downsample = DownSample1d(down_ratio, down_kernel_size) + + def forward(self, x): + x = self.upsample(x) + x = self.act(x) + x = self.downsample(x) + return x + + +class GeGluMlp(nn.Module): + def __init__(self, in_features, hidden_features): + super().__init__() + self.norm = nn.LayerNorm(in_features) + self.act = nn.GELU(approximate="tanh") + self.w0 = nn.Linear(in_features, hidden_features) + self.w1 = nn.Linear(in_features, hidden_features) + self.w2 = nn.Linear(hidden_features, in_features) + + def forward(self, x): + x = self.norm(x) + x = self.act(self.w0(x)) * self.w1(x) + x = self.w2(x) + return x + + +class CausalAttention(nn.Module): + """Causal multi-head attention with backend dispatch via attention_forward.""" + + def __init__(self, in_dim, out_dim, num_heads): + super().__init__() + self.qkv = nn.Linear(in_dim, in_dim * 3, bias=False) + self.out_dim = out_dim + self.num_heads = num_heads + self.proj = nn.Linear(out_dim, out_dim) + self.q_bias = WarpedTensor(torch.empty(in_dim)) + self.zero_k_bias = WarpedTensor(torch.empty(in_dim)) + self.v_bias = WarpedTensor(torch.empty(in_dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + q, k, v = self.qkv(x).chunk(3, dim=-1) + q, k, v = q + self.q_bias(), k + self.zero_k_bias(), v + self.v_bias() + x = attention_forward( + q, + k, + v, + q_pattern="b s (n d)", + k_pattern="b s (n d)", + v_pattern="b s (n d)", + out_pattern="b n s d", + dims={"n": self.num_heads}, + is_causal=True, + ) + x = torch.mean(x, dim=1) + x = F.adaptive_avg_pool1d(x, self.out_dim) + x = self.proj(x) + return x + + +class AttnProjection(nn.Module): + def __init__(self, in_dim, out_dim, num_heads, mlp_ratio=2): + super().__init__() + self.norm1 = nn.LayerNorm(in_dim) + self.attn = CausalAttention(in_dim, out_dim, num_heads) + self.proj = nn.Linear(in_dim, out_dim) + self.norm3 = nn.LayerNorm(in_dim) + self.norm2 = nn.LayerNorm(out_dim) + self.mlp = GeGluMlp(in_features=out_dim, hidden_features=out_dim * mlp_ratio) + + def forward(self, x): + x = self.proj(self.norm3(x)) + self.attn(self.norm1(x)) + x = x + self.mlp(self.norm2(x)) + return x + + +class ResidualUnit(nn.Module): + def __init__(self, dim: int = 16, dilation: int = 1): + super().__init__() + pad = ((7 - 1) * dilation) // 2 + self.block = nn.Sequential( + Snake(dim), + WeightNormedConv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad), + Snake(dim), + WeightNormedConv1d(dim, dim, kernel_size=1), + ) + + def forward(self, x): + y = self.block(x) + pad = (x.shape[-1] - y.shape[-1]) // 2 + if pad > 0: + x = x[..., pad:-pad] + return x + y + + +class EncoderBlock(nn.Module): + def __init__(self, dim: int = 16, stride: int = 1): + super().__init__() + self.block = nn.Sequential( + ResidualUnit(dim // 2, dilation=1), + ResidualUnit(dim // 2, dilation=3), + ResidualUnit(dim // 2, dilation=9), + Snake(dim // 2), + WeightNormedConv1d( + dim // 2, + dim, + kernel_size=2 * stride, + stride=stride, + padding=math.ceil(stride / 2), + ), + ) + + def forward(self, x): + return self.block(x) + + +class Encoder(nn.Module): + def __init__(self, d_model: int = 64, strides: list = None, d_latent: int = 64): + if strides is None: + strides = [2, 4, 8, 8] + super().__init__() + self.block = [WeightNormedConv1d(1, d_model, kernel_size=7, padding=3)] + for stride in strides: + d_model *= 2 + self.block += [EncoderBlock(d_model, stride=stride)] + self.block += [ + Snake(d_model), + WeightNormedConv1d(d_model, d_latent, kernel_size=3, padding=1), + ] + self.block = nn.Sequential(*self.block) + + def forward(self, x): + return self.block(x) + + +class AMPBlock1(torch.nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): + super().__init__() + self.convs1 = nn.ModuleList( + [ + WeightNormedConv1d( + channels, channels, kernel_size, stride=1, dilation=d, padding=int((kernel_size * d - d) / 2) + ) + for d in dilation + ] + ) + self.convs2 = nn.ModuleList( + [ + WeightNormedConv1d( + channels, channels, kernel_size, stride=1, dilation=1, padding=int((kernel_size - 1) / 2) + ) + for _ in range(len(dilation)) + ] + ) + self.activations = nn.ModuleList( + [Activation1d(activation=Snake(channels, beta=True)) for _ in range(2 * len(dilation))] + ) + + def forward(self, x): + acts1, acts2 = self.activations[::2], self.activations[1::2] + for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2): + xt = a1(x) + xt = c1(xt) + xt = a2(xt) + xt = c2(xt) + x = xt + x + return x + + +class BigVGAN(torch.nn.Module): + def __init__(self): + super().__init__() + self.conv_pre = WeightNormedConv1d(2048, 1024, 7, 1, padding=3) + self.ups = nn.ModuleList( + [ + nn.ModuleList( + [WeightNormedConvTranspose1d(1024 // (2**i), 1024 // (2 ** (i + 1)), k, u, padding=(k - u) // 2)] + ) + for i, (u, k) in enumerate(zip([5, 5, 2, 2, 2, 2, 2], [9, 9, 4, 4, 4, 4, 4])) + ] + ) + self.resblocks = nn.ModuleList() + for i in range(7): + ch = 1024 // (2 ** (i + 1)) + for k in [3, 7, 11]: + self.resblocks.append(AMPBlock1(ch, k, (1, 3, 5))) + self.activation_post = Activation1d(activation=Snake(ch, beta=True)) + self.conv_post = WeightNormedConv1d(ch, 1, 7, 1, padding=3, bias=False) + + def forward(self, x): + x = self.conv_pre(x) + for i, up in enumerate(self.ups): + x = up[0](x) + xs = self.resblocks[i * 3](x) + for j in range(1, 3): + xs += self.resblocks[i * 3 + j](x) + x = xs / 3 + x = self.activation_post(x) + x = self.conv_post(x) + x = torch.clamp(x, min=-1.0, max=1.0) + return x + + +_AUDIO_LATENTS_MEAN = [ + -0.020211687488382354, + 0.3876466479950502, + -0.04398279799186767, + -0.28591514936373, + 0.08179686214561671, + -0.35782641352446604, + 0.040623809960919084, + -0.01552534501956604, + -0.223362481667332, + 0.1821006842509091, + 0.2941778783780663, + -0.07901167601970885, + -0.056815072777201, + -0.3699028221860095, + -0.31616315591624855, + 0.5905951377425391, + -0.052139568068853864, + 0.013673160263486295, + -0.03691647864630577, + 0.09732660653298163, + -0.3394662328788498, + -0.30685677538541667, + -0.24504598907458763, + -0.034698524462007344, + 0.02868032184767538, + -0.21217779266454084, + -0.1678263169941987, + 0.3221287889040614, + -0.1223055851554907, + 0.4356604928128464, + -0.0502599202236253, + 0.3979258376211797, +] +_AUDIO_LATENTS_STD = [ + 1.6895524230479284, + 2.76263727217653, + 1.7945344281264435, + 1.6801681847309828, + 1.6390226546605453, + 2.7788298348882177, + 1.7659090095747236, + 1.6199757612137327, + 2.6336525640336896, + 1.8539356672817833, + 2.5056497896915633, + 1.811019237886178, + 1.9579657790720237, + 1.6685498243529284, + 1.4922469314453364, + 3.298670198067373, + 1.9491804496832168, + 1.8720003270431442, + 1.8334080103291832, + 1.6488070416529093, + 1.6176957696319716, + 1.9131449234774398, + 1.5695245398428617, + 1.6943659940415912, + 1.8318420762504692, + 1.5540637421583379, + 1.9344930328968526, + 1.599198216109855, + 1.718045989838149, + 1.6307219190837705, + 1.8661226051202384, + 1.5613768203168363, +] + + +class MiniMaxH3AudioVAE(nn.Module): + def __init__( + self, + encoder_dim: int = 64, + encoder_rates: List[int] = None, + decoder_dim: int = 1024, + decoder_rates: List[int] = None, + sample_rate: int = 32000, + vae_latent_channels: int = 32, + attn_proj: bool = True, + decoder_type: str = "bigvgan", + ): + if decoder_rates is None: + decoder_rates = [5, 5, 2, 2, 2, 2, 2] + if encoder_rates is None: + encoder_rates = [2, 4, 4, 5, 5] + super().__init__() + self.sample_rate = sample_rate + self.attn_proj = attn_proj + latent_dim = encoder_dim * (2 ** len(encoder_rates)) + self.hop_length = int(np.prod(encoder_rates)) + self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim) + self.mean_proj = nn.Conv1d(vae_latent_channels, vae_latent_channels, 1) + self.logs_proj = nn.Conv1d(vae_latent_channels, vae_latent_channels, 1) + self.dec_in_proj = nn.Conv1d(vae_latent_channels, latent_dim, 1) + self.decoder = BigVGAN() + self.pre_block = AttnProjection(latent_dim, vae_latent_channels, num_heads=8) + + def _latent_stats(self, device): + mean = torch.tensor(_AUDIO_LATENTS_MEAN, device=device, dtype=torch.float32).view(1, -1, 1) + std = torch.tensor(_AUDIO_LATENTS_STD, device=device, dtype=torch.float32).view(1, -1, 1) + return mean, std + + def preprocess(self, audio_data): + length = audio_data.shape[-1] + right_pad = math.ceil(length / self.hop_length) * self.hop_length - length + return nn.functional.pad(audio_data, (0, right_pad)) + + def encode(self, audio_data: torch.Tensor) -> torch.Tensor: + z = self.encoder(audio_data) + if self.attn_proj: + z = self.pre_block(z.transpose(1, 2)).transpose(1, 2) + mean = self.mean_proj(z) + return mean + + def decode(self, z: torch.Tensor) -> torch.Tensor: + z = self.dec_in_proj(z) + return self.decoder(z) + + @torch.no_grad() + def encode_audio(self, waveform: torch.Tensor, dtype=None) -> torch.Tensor: + if waveform.dim() != 2: + raise ValueError(f"expected waveform [C, L], got {list(waveform.shape)}") + out_dtype = waveform.dtype + audio_data = waveform.unsqueeze(1).to(dtype if dtype is not None else out_dtype) + latent = self.encode(self.preprocess(audio_data)) + latent_channels = len(_AUDIO_LATENTS_MEAN) + if latent.dim() != 3 or latent.shape[1] != latent_channels: + raise ValueError(f"expected audio latent [C, {latent_channels}, T], got {list(latent.shape)}") + mean, std = self._latent_stats(latent.device) + return ((latent.to(torch.float32) - mean) / std).to(out_dtype) + + @torch.no_grad() + def decode_audio(self, latents: torch.Tensor, dtype=None) -> torch.Tensor: + mean, std = self._latent_stats(latents.device) + z = (latents.to(latents.device, torch.float32) * std + mean).to(dtype if dtype is not None else latents.dtype) + waveform = self.decode(z) + return waveform.transpose(0, 1).to(latents.dtype) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + return self.decode(z) diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_dit.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_dit.py new file mode 100644 index 0000000000..f3593addd2 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_dit.py @@ -0,0 +1,549 @@ +from __future__ import annotations + +import math + +import torch +import torch.nn as nn + +from veomni.utils.device import IS_NPU_AVAILABLE + +from .core import attention_forward, gradient_checkpoint_forward + + +if IS_NPU_AVAILABLE: + from torch_npu import npu_rms_norm, npu_rotary_mul + + +MINIMAX_H3_ADALN_MODALITY_NUM = 3 +_PATCH_T, _PATCH_H, _PATCH_W = 1, 2, 2 + + +def patchify_video(latent: torch.Tensor) -> torch.Tensor: + # [1,24,T,H,W] -> [T*(H/2)*(W/2), 96] + b, c, ft, fh, fw = (int(x) for x in latent.shape) + t, h, w = ft // _PATCH_T, fh // _PATCH_H, fw // _PATCH_W + packed = latent.reshape(b, c, t, _PATCH_T, h, _PATCH_H, w, _PATCH_W) + packed = torch.einsum("nctrhpwq->nthwcrpq", packed) + return packed.reshape(b * t * h * w, c * _PATCH_T * _PATCH_H * _PATCH_W).contiguous() + + +def unpatchify_video(rows: torch.Tensor, ft: int, fh: int, fw: int, channel: int = 24) -> torch.Tensor: + # [T*(H/2)*(W/2), 96] -> [1,24,T,H,W] (inverse of patchify_video; ft/fh/fw match its input) + t, h, w = ft // _PATCH_T, fh // _PATCH_H, fw // _PATCH_W + packed = rows.reshape(-1, t, h, w, channel, _PATCH_T, _PATCH_H, _PATCH_W) + latent = torch.einsum("nthwcrpq->nctrhpwq", packed) + return latent.reshape(-1, channel, ft, fh, fw).contiguous() + + +def pack_audio(latent: torch.Tensor) -> torch.Tensor: + # [audio_channel, 32, T] -> [audio_channel*T, 32] (channel-major) + ac, ld, steps = (int(x) for x in latent.shape) + return latent.permute(0, 2, 1).reshape(ac * steps, ld).contiguous() + + +def unpack_audio(rows: torch.Tensor, audio_channel: int, steps: int, latent_dim: int = 32) -> torch.Tensor: + # [audio_channel*T, 32] -> [audio_channel, 32, T] + return rows.reshape(audio_channel, steps, latent_dim).permute(0, 2, 1).contiguous() + + +class _ASCEND_RMSNorm(nn.RMSNorm): + def forward(self, x): + return npu_rms_norm(x, self.weight, epsilon=self.eps)[0] + + +def _norm(size: int, *, eps: float) -> nn.RMSNorm: + if IS_NPU_AVAILABLE: + return _ASCEND_RMSNorm(size, eps=eps) + return nn.RMSNorm(size, eps=eps) + + +def _rotate_half(x: torch.Tensor) -> torch.Tensor: + x1, x2 = torch.chunk(x, 2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + + +def _apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + # cos/sin are precomputed once per forward and shared across all blocks. + rot_dim = cos.shape[-1] + x_rot, x_pass = x[..., :rot_dim], x[..., rot_dim:] + if IS_NPU_AVAILABLE: + x_rot = npu_rotary_mul(x_rot, cos, sin, rotary_mode="half") + else: + x_rot = (x_rot * cos) + (_rotate_half(x_rot) * sin) + return torch.cat((x_rot, x_pass), dim=-1) + + +def _modulate_scale_shift(x, shift, scale, indices): + # Cast back to x: index_select on the AdaLN params can promote the expression. + return (x * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)).to(x.dtype) + + +def _modulate_gate(x, gate, other, indices): + return (x + gate.index_select(0, indices) * other).to(x.dtype) + + +def _sdpa_varlen_attention(q, k, v, cu_seqlens, softmax_scale): + out = torch.empty_like(q) + # Host-side segment bounds: the DiT / token-refiner entries convert the + # cu_seqlens tensor once per forward and pass a tuple. Tensor fallback + # keeps any external caller working (paying one sync). + bounds = tuple(cu_seqlens) if isinstance(cu_seqlens, torch.Tensor) else cu_seqlens + for start, stop in zip(bounds[:-1], bounds[1:]): + if stop == start: + continue + seg_q = q[start:stop].transpose(0, 1).unsqueeze(0) + seg_k = k[start:stop].transpose(0, 1).unsqueeze(0) + seg_v = v[start:stop].transpose(0, 1).unsqueeze(0) + seg_out = attention_forward(seg_q, seg_k, seg_v, scale=softmax_scale) + out[start:stop] = seg_out.squeeze(0).transpose(0, 1) + return out + + +class MiniMaxH3Rope(nn.Module): + def __init__(self, inv_freq_len: int) -> None: + super().__init__() + self.inv_freq_len = inv_freq_len + self.register_buffer("inv_freq", self._build_inv_freq(), persistent=False) + + def _build_inv_freq(self, device=None) -> torch.Tensor: + steps = torch.arange(0, self.inv_freq_len, dtype=torch.float32, device=device) + return 1.0 / (10000.0 ** (steps / self.inv_freq_len)) + + def forward(self, img_position_ids: torch.Tensor) -> torch.Tensor: + if img_position_ids.dim() != 3 or img_position_ids.shape[0] != 1: + raise ValueError(f"img_position_ids must be [1, S, 3], got {list(img_position_ids.shape)}") + pos = img_position_ids[0].to(torch.float32) + inv_freq = self._build_inv_freq(pos.device) + per_axis = pos.unsqueeze(-1) * inv_freq.view(1, 1, -1) + t_f, h_f, w_f = per_axis.unbind(dim=1) + half = torch.cat((t_f, h_f, w_f), dim=-1) + return torch.cat((half, half), dim=-1) + + +class MiniMaxH3TimeEmbedder(nn.Module): + def __init__(self, timestep_input_dim, time_embed_hidden_size, time_embed_dim): + super().__init__() + self.frequency_embedding_size = timestep_input_dim + self.proj_in = nn.Linear(timestep_input_dim, time_embed_hidden_size, bias=True) + self.proj_out = nn.Linear(time_embed_hidden_size, time_embed_dim, bias=True) + + def forward(self, t: torch.Tensor, *, dtype: torch.dtype) -> torch.Tensor: + half = self.frequency_embedding_size // 2 + freqs = torch.exp(-math.log(10000.0) * torch.arange(half, dtype=torch.float32, device=t.device) / half) + args = t.to(torch.float32)[:, None] * freqs[None] + t_freq = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + hidden = self.proj_in(t_freq.to(dtype)) + hidden = nn.functional.silu(hidden) + return self.proj_out(hidden) + + +class MiniMaxH3Attention(nn.Module): + def __init__(self, hidden_size, num_attention_heads, attention_head_dim, qk_norm_eps): + super().__init__() + self.num_heads = num_attention_heads + self.head_dim = attention_head_dim + inner_dim = self.num_heads * self.head_dim + self.softmax_scale = self.head_dim**-0.5 + self.qkv_proj = nn.Linear(hidden_size, inner_dim * 3, bias=False) + self.q_norm = _norm(attention_head_dim, eps=qk_norm_eps) + self.k_norm = _norm(attention_head_dim, eps=qk_norm_eps) + self.out_proj = nn.Linear(inner_dim, hidden_size, bias=False) + + def forward(self, x, *, rope_cos, rope_sin, cu_seqlens, max_seqlen=None): + total = x.shape[0] + qkv = self.qkv_proj(x) + qkv = qkv.view(total, self.num_heads, 3, self.head_dim) + q = qkv[:, :, 0, :] + k = qkv[:, :, 1, :] + v = qkv[:, :, 2, :] + q = self.q_norm(q) + k = self.k_norm(k) + if rope_cos is not None: + q = _apply_rope(q, rope_cos, rope_sin) + k = _apply_rope(k, rope_cos, rope_sin) + out = _sdpa_varlen_attention(q, k, v, cu_seqlens=cu_seqlens, softmax_scale=self.softmax_scale) + out = out.reshape(total, self.num_heads * self.head_dim) + return self.out_proj(out) + + +class MiniMaxH3MLP(nn.Module): + def __init__(self, hidden_size, ffn_hidden_size): + super().__init__() + self.fc1 = nn.Linear(hidden_size, ffn_hidden_size * 2, bias=False) + self.fc2 = nn.Linear(ffn_hidden_size, hidden_size, bias=False) + + def forward(self, x): + hidden = self.fc1(x) + gate, up = hidden.chunk(2, dim=-1) + hidden = nn.functional.silu(gate) * up + return self.fc2(hidden) + + +class MiniMaxH3AdalnProj(nn.Module): + def __init__(self, hidden_size, time_embed_dim, out_features, *, expand_ratio, modality_num): + super().__init__() + if out_features != expand_ratio * hidden_size * modality_num: + raise ValueError( + f"adaln out_features mismatch: {out_features} != {expand_ratio}*{hidden_size}*{modality_num}" + ) + self.expand_ratio = expand_ratio + self.modality_num = modality_num + self.hidden_size = hidden_size + self.linear = nn.Linear(time_embed_dim, out_features, bias=True) + + def forward(self, t_emb): + x = nn.functional.silu(t_emb) + x = self.linear(x) + m = x.shape[0] + x = x.view(m * self.modality_num, self.expand_ratio * self.hidden_size) + return tuple(x.chunk(self.expand_ratio, dim=-1)) + + +class MiniMaxH3TokenRefinerBlock(nn.Module): + def __init__(self, hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps): + super().__init__() + self.norm1 = _norm(hidden_size, eps=norm_eps) + self.norm2 = _norm(hidden_size, eps=norm_eps) + self.attn = MiniMaxH3Attention(hidden_size, num_attention_heads, attention_head_dim, qk_norm_eps) + self.mlp = MiniMaxH3MLP(hidden_size, ffn_hidden_size) + + def forward(self, x, *, cu_seqlens, max_seqlen): + x = x + self.attn(self.norm1(x), rope_cos=None, rope_sin=None, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) + x = x + self.mlp(self.norm2(x)) + return x + + +class MiniMaxH3TokenRefiner(nn.Module): + def __init__( + self, + num_layers, + hidden_size, + num_attention_heads, + attention_head_dim, + ffn_hidden_size, + norm_eps, + qk_norm_eps, + final_norm_eps, + ): + super().__init__() + self.blocks = nn.ModuleList( + [ + MiniMaxH3TokenRefinerBlock( + hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps + ) + for _ in range(num_layers) + ] + ) + self.final_norm = _norm(hidden_size, eps=final_norm_eps) + + def forward(self, x, *, cu_seqlens, max_seqlen): + for block in self.blocks: + x = block(x, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) + return self.final_norm(x) + + +class MiniMaxH3DiTBlock(nn.Module): + def __init__( + self, + hidden_size, + num_attention_heads, + attention_head_dim, + ffn_hidden_size, + time_embed_dim, + adaln_out_features, + norm_eps, + qk_norm_eps, + ): + super().__init__() + self.norm1 = _norm(hidden_size, eps=norm_eps) + self.norm2 = _norm(hidden_size, eps=norm_eps) + self.attn = MiniMaxH3Attention(hidden_size, num_attention_heads, attention_head_dim, qk_norm_eps) + self.mlp = MiniMaxH3MLP(hidden_size, ffn_hidden_size) + self.adaln_proj = MiniMaxH3AdalnProj( + hidden_size, time_embed_dim, adaln_out_features, expand_ratio=6, modality_num=MINIMAX_H3_ADALN_MODALITY_NUM + ) + + def forward(self, x, *, t_emb, combined_indices, rope_cos, rope_sin, cu_seqlens, max_seqlen): + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(t_emb) + residual = x + h = self.norm1(x) + h = _modulate_scale_shift(h, shift_msa, scale_msa, combined_indices) + h = self.attn(h, rope_cos=rope_cos, rope_sin=rope_sin, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) + x = _modulate_gate(residual, gate_msa, h, combined_indices) + residual = x + h = self.norm2(x) + h = _modulate_scale_shift(h, shift_mlp, scale_mlp, combined_indices) + h = self.mlp(h) + out = _modulate_gate(residual, gate_mlp, h, combined_indices) + return out + + +class MiniMaxH3FinalLayer(nn.Module): + def __init__( + self, + hidden_size, + time_embed_dim, + final_adaln_out_features, + latents_dim, + audio_latents_dim, + patch_size, + final_norm_eps, + ): + super().__init__() + video_patch_dim = latents_dim * patch_size[0] * patch_size[1] * patch_size[2] + self.norm = _norm(hidden_size, eps=final_norm_eps) + self.adaln_proj = MiniMaxH3AdalnProj( + hidden_size, time_embed_dim, final_adaln_out_features, expand_ratio=2, modality_num=1 + ) + self.video_out = nn.Linear(hidden_size, video_patch_dim, bias=True) + self.audio_out = nn.Linear(hidden_size, audio_latents_dim, bias=True) + + def forward(self, x, *, t_emb, inverse_indices): + shift, scale = self.adaln_proj(t_emb) + h = self.norm(x) + h = _modulate_scale_shift(h, shift, scale, inverse_indices) + video = self.video_out(h) + audio = self.audio_out(h) + return video, audio + + +class MiniMaxH3DiT(nn.Module): + _repeated_blocks = ["MiniMaxH3DiTBlock"] + + def __init__( + self, + num_layers: int = 8, + token_refiner_num_layers: int = 2, + hidden_size: int = 5376, + num_attention_heads: int = 56, + attention_head_dim: int = 128, + ffn_hidden_size: int = 14336, + latents_dim: int = 24, + audio_latents_dim: int = 32, + patch_size: tuple = (1, 2, 2), + text_dim: int = 5120, + timestep_input_dim: int = 256, + time_embed_hidden_size: int = 5376, + time_embed_dim: int = 2688, + adaln_out_features: int = 96768, + final_adaln_out_features: int = 10752, + rope_inv_freq_len: int = 16, + norm_eps: float = 1e-5, + qk_norm_eps: float = 1e-5, + final_norm_eps: float = 1e-5, + **kwargs, + ): + super().__init__() + self._block_offload_enabled = False + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.num_channels_latents = latents_dim + video_patch_dim = latents_dim * patch_size[0] * patch_size[1] * patch_size[2] + + self.video_patch_proj = nn.Linear(video_patch_dim, hidden_size, bias=True) + self.audio_patch_proj = nn.Linear(audio_latents_dim, hidden_size, bias=True) + self.condition_proj = nn.Linear(text_dim, hidden_size, bias=True) + self.time_embedder = MiniMaxH3TimeEmbedder(timestep_input_dim, time_embed_hidden_size, time_embed_dim) + self.rope = MiniMaxH3Rope(rope_inv_freq_len) + self.token_refiner = MiniMaxH3TokenRefiner( + token_refiner_num_layers, + hidden_size, + num_attention_heads, + attention_head_dim, + ffn_hidden_size, + norm_eps, + qk_norm_eps, + final_norm_eps, + ) + self.blocks = nn.ModuleList( + [ + MiniMaxH3DiTBlock( + hidden_size, + num_attention_heads, + attention_head_dim, + ffn_hidden_size, + time_embed_dim, + adaln_out_features, + norm_eps, + qk_norm_eps, + ) + for _ in range(num_layers) + ] + ) + self.final_layer = MiniMaxH3FinalLayer( + hidden_size, + time_embed_dim, + final_adaln_out_features, + latents_dim, + audio_latents_dim, + patch_size, + final_norm_eps, + ) + + def enable_block_offload(self, split: int = None): + """Inference-only VRAM management: run the main blocks in two + device-resident halves (block-level offloading). The first + half stays on the compute device between steps; the second half is + staged in per step and moved back to CPU afterwards.""" + self._block_offload_enabled = True + self._block_swap = (len(self.blocks) // 2) if split is None else split + + def _stage_side_modules(self, device): + # Small fixed modules outside the main block loop (patch projs, + # token refiner, final layer, ~4.5GB total): keep them on the + # compute device for the whole forward. + for module in ( + self.video_patch_proj, + self.audio_patch_proj, + self.condition_proj, + self.time_embedder, + self.rope, + self.token_refiner, + self.final_layer, + ): + module.to(device) + + def _embed( + self, + *, + x, + audio_x, + text_embeddings_selected, + unique_timesteps, + img_pos, + audio_pos, + text_pos, + refiner_cu_seqlens, + refiner_max_seqlen, + seq_len, + device, + ): + dtype = text_embeddings_selected.dtype + x_rows = x.view(-1, x.shape[-1]).index_select(0, img_pos).to(dtype) + video_embed = self.video_patch_proj(x_rows) + audio_rows = audio_x.view(-1, audio_x.shape[-1]).index_select(0, audio_pos).to(dtype) + audio_embed = self.audio_patch_proj(audio_rows) + text_rows = text_embeddings_selected.to(device=device) + text_embed = self.condition_proj(text_rows) + text_embed = self.token_refiner(text_embed, cu_seqlens=refiner_cu_seqlens, max_seqlen=refiner_max_seqlen) + + embeddings = torch.zeros((seq_len, self.hidden_size), device=device, dtype=dtype) + embeddings[text_pos] = text_embed.to(dtype)[: text_pos.shape[0]] + embeddings[img_pos] = video_embed.to(dtype)[: img_pos.shape[0]] + embeddings[audio_pos] = audio_embed.to(dtype)[: audio_pos.shape[0]] + + t_emb = self.time_embedder(unique_timesteps, dtype=dtype) + return embeddings, t_emb + + def forward( + self, + x, + audio_x, + img_position_ids, + unique_timesteps, + inverse_indices, + update_mask, + token_tags, + prompt_embeds, + img_pos_info, + audio_pos_info, + text_pos_info, + img_pos_for_infer_output_info, + packed_seq_params, + refiner_packed_seq_params, + use_gradient_checkpointing=False, + use_gradient_checkpointing_offload=False, + update_audio_mask=None, + skip_mask_out_condition=False, + ) -> tuple[torch.Tensor, torch.Tensor]: + inverse_indices = inverse_indices.view(-1).to(torch.long) + token_tags = token_tags.view(-1).to(torch.long) + text_selected = prompt_embeds + + img_pos = img_pos_info["position_ids"].view(-1).to(torch.long) + audio_pos = audio_pos_info["position_ids"].view(-1).to(torch.long) + text_pos = text_pos_info["position_ids"].view(-1).to(torch.long) + infer_out_pos = img_pos_for_infer_output_info["position_ids"].view(-1).to(torch.long) + + cu_seqlens = packed_seq_params["cu_seqlens_q"].to(torch.int32) + max_seqlen = int(packed_seq_params["max_seqlen_q"]) + refiner_cu = refiner_packed_seq_params["cu_seqlens_q"].to(torch.int32) + refiner_max = int(refiner_packed_seq_params["max_seqlen_q"]) + + if x.dim() != 3 or x.shape[0] != 1: + raise ValueError(f"x must be [1, S, C], got {list(x.shape)}") + seq_len = int(x.shape[1]) + device = x.device + + if self._block_offload_enabled: + self._stage_side_modules(device) + + rope_freqs = self.rope(img_position_ids).to(device) + + decoder_input, t_emb = self._embed( + x=x, + audio_x=audio_x, + text_embeddings_selected=text_selected, + unique_timesteps=unique_timesteps.view(-1).to(device), + img_pos=img_pos.to(device), + audio_pos=audio_pos.to(device), + text_pos=text_pos.to(device), + refiner_cu_seqlens=tuple(refiner_cu.to(device).tolist()), + refiner_max_seqlen=refiner_max, + seq_len=seq_len, + device=device, + ) + + combined_indices = (inverse_indices * MINIMAX_H3_ADALN_MODALITY_NUM + token_tags.clamp(min=0)).to(device) + inverse_indices = inverse_indices.to(device) + + # cos/sin are shared across all blocks; compute once instead of per q/k + # in _apply_rope. Same fp32 inputs, same ops: bit-identical results. + rope_cos = torch.cos(rope_freqs).to(decoder_input.dtype).unsqueeze(1) + rope_sin = torch.sin(rope_freqs).to(decoder_input.dtype).unsqueeze(1) + + hidden = decoder_input + cu_seqlens = cu_seqlens.to(device) + # Single device→host sync per forward: segment bounds shared across + # every block instead of one tolist() per attention module. + cu_bounds = tuple(cu_seqlens.tolist()) + block_swap = self._block_swap if self._block_offload_enabled else 0 + for i, block in enumerate(self.blocks): + if self._block_offload_enabled: + if i == 0: + for b in self.blocks[:block_swap]: + b.to(device) + elif i == block_swap: + for b in self.blocks[:block_swap]: + b.to("cpu") + for b in self.blocks[block_swap:]: + b.to(device) + hidden = gradient_checkpoint_forward( + block, + use_gradient_checkpointing, + use_gradient_checkpointing_offload, + hidden, + t_emb=t_emb, + combined_indices=combined_indices, + rope_cos=rope_cos, + rope_sin=rope_sin, + cu_seqlens=cu_bounds, + max_seqlen=max_seqlen, + ) + + if self._block_offload_enabled: + for b in self.blocks[block_swap:]: + b.to("cpu") + + video_logits, audio_logits = self.final_layer(hidden, t_emb=t_emb, inverse_indices=inverse_indices) + + video_logits = video_logits.index_select(0, infer_out_pos.to(device)) + audio_logits = audio_logits.index_select(0, audio_pos.to(device)) + if not skip_mask_out_condition: + update_mask = update_mask.view(-1).to(device) + if update_mask.shape[0] != video_logits.shape[0]: + raise ValueError(f"update_mask length mismatch: {update_mask.shape[0]} != {video_logits.shape[0]}") + video_logits = video_logits * update_mask.unsqueeze(-1) + if update_audio_mask is not None: + audio_logits = audio_logits * update_audio_mask.view(-1).unsqueeze(-1) + return video_logits, audio_logits diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_text_encoder.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_text_encoder.py new file mode 100644 index 0000000000..adfd192027 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_text_encoder.py @@ -0,0 +1,238 @@ +import torch +from transformers import Qwen3VLConfig, Qwen3VLModel + + +VISION_START = "<|vision_start|>" +VISION_END = "<|vision_end|>" +IMAGE_PAD = "<|image_pad|>" +VIDEO_PAD = "<|video_pad|>" + +PRESENTATION_TEXT_TAG = 1 +PRESENTATION_VIDEO_TAG = 0 + +QWEN_VIDEO_SAMPLE_FPS = 2.0 +QWEN_TEMPORAL_PATCH = 2 +MINIMAX_SUPPORTED_FPS = 24 + + +class MiniMaxH3TextEncoder(torch.nn.Module): + def __init__(self, num_retained_layers: int = 64): + super().__init__() + self.num_retained_layers = num_retained_layers + config = Qwen3VLConfig( + **{ + "architectures": ["Qwen3VLForConditionalGeneration"], + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": False, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 25600, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 64, + "num_hidden_layers": num_retained_layers, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": True, + "mrope_section": [24, 20, 20], + "rope_type": "default", + }, + "rope_theta": 5000000, + "use_cache": True, + "vocab_size": 151936, + }, + "tie_word_embeddings": False, + "transformers_version": "4.57.0.dev0", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [8, 16, 24], + "depth": 27, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 5120, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2, + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652, + } + ) + self.model = Qwen3VLModel(config) + self.model.language_model.norm = torch.nn.Identity() + self.config = config + self.image_token_id = config.image_token_id + self.video_token_id = config.video_token_id + + def forward( + self, + input_ids=None, + attention_mask=None, + pixel_values=None, + image_grid_thw=None, + pixel_values_videos=None, + video_grid_thw=None, + **kwargs, + ): + mm_token_type_ids = None + if pixel_values is not None or pixel_values_videos is not None: + mm_token_type_ids = torch.zeros_like(input_ids, dtype=torch.int32) + mm_token_type_ids[input_ids == self.image_token_id] = 1 + mm_token_type_ids[input_ids == self.video_token_id] = 2 + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + mm_token_type_ids=mm_token_type_ids, + output_hidden_states=False, + use_cache=False, + return_dict=True, + **kwargs, + ) + return outputs.last_hidden_state[0] + + +def image_token_counts(processor, images): + vision = processor.image_processor(images=images, return_tensors="pt") + merge = int(processor.image_processor.merge_size) ** 2 + counts = [int(vision["image_grid_thw"][i].prod().item()) // merge for i in range(len(images))] + return vision["pixel_values"], vision["image_grid_thw"], counts + + +def video_token_counts(processor, videos, timestamps_per_video): + vout = processor.video_processor(videos=videos, do_sample_frames=False, return_tensors="pt") + grid = vout["video_grid_thw"] + merge = int(processor.image_processor.merge_size) ** 2 + block_counts, block_timestamps = [], [] + for index, timestamps in enumerate(timestamps_per_video): + n_blocks = int(grid[index, 0]) + per_block = int(grid[index, 1]) * int(grid[index, 2]) // merge + block_counts.append([per_block] * n_blocks) + block_timestamps.append([float(t) for t in timestamps]) + return vout["pixel_values_videos"], grid, block_counts, block_timestamps + + +def _text_ids(tokenizer, text: str) -> list[int]: + return list(tokenizer(text, add_special_tokens=False)["input_ids"]) + + +def _vision_block_ids(tokenizer, pad_token: str, count: int) -> list[int]: + return ( + [tokenizer.convert_tokens_to_ids(VISION_START)] + + [tokenizer.convert_tokens_to_ids(pad_token)] * int(count) + + [tokenizer.convert_tokens_to_ids(VISION_END)] + ) + + +class _Presentation: + def __init__(self): + self.ids: list[int] = [] + self.tags: list[int] = [] + + def text(self, token_ids: list[int]): + self.ids += token_ids + self.tags += [PRESENTATION_TEXT_TAG] * len(token_ids) + + def vision(self, token_ids: list[int]): + self.ids += token_ids + self.tags += [PRESENTATION_VIDEO_TAG] * len(token_ids) + + def build(self): + return (torch.tensor(self.ids, dtype=torch.long), torch.tensor(self.tags, dtype=torch.long)) + + +def presentation_t2va(tokenizer, prompt: str): + presentation = _Presentation() + presentation.text(_text_ids(tokenizer, prompt)) + return presentation.build() + + +def presentation_fl2va(tokenizer, prompt: str, image_token_counts): + presentation = _Presentation() + for index, count in enumerate(image_token_counts, start=1): + presentation.text(_text_ids(tokenizer, f": ")) + presentation.vision(_vision_block_ids(tokenizer, IMAGE_PAD, count)) + presentation.text(_text_ids(tokenizer, prompt)) + return presentation.build() + + +def presentation_ref2va( + tokenizer, prompt: str, condition_labels, image_token_counts, video_block_token_counts, video_block_timestamps +): + if not prompt: + raise ValueError("prompt must be non-empty") + presentation = _Presentation() + image_seen = 0 + video_seen = 0 + for cond_type, ordinal in condition_labels: + if cond_type == "image": + image_seen += 1 + if image_seen > len(image_token_counts): + raise ValueError("image_token_count required for an image reference") + count = int(image_token_counts[image_seen - 1]) + if count <= 0: + raise ValueError("image_token_count required for an image reference") + presentation.text(_text_ids(tokenizer, f": ")) + presentation.vision(_vision_block_ids(tokenizer, IMAGE_PAD, count)) + elif cond_type == "audio": + presentation.text(_text_ids(tokenizer, f"