From 21afeefd1a7bd53bb311c5be399dfddd7c4dd4cd Mon Sep 17 00:00:00 2001 From: vvyuervv <2794113628@qq.com> Date: Tue, 18 Aug 2026 16:50:32 +0800 Subject: [PATCH 01/11] [model, config] feat: add MiniMax H3 core model implementation --- .../minimax_h3/minimax_h3_core/flow_match.py | 411 +++++++++ .../minimax_h3_core/minimax_h3_audio_vae.py | 481 +++++++++++ .../minimax_h3_core/minimax_h3_audio_video.py | 784 ++++++++++++++++++ .../minimax_h3_core/minimax_h3_dit.py | 398 +++++++++ .../minimax_h3_text_encoder.py | 231 ++++++ .../minimax_h3_core/minimax_h3_video_vae.py | 552 ++++++++++++ veomni/utils/audio.py | 109 +++ veomni/utils/audio_video.py | 185 +++++ veomni/utils/video_data.py | 217 +++++ 9 files changed, 3368 insertions(+) create mode 100644 veomni/models/diffusers/minimax_h3/minimax_h3_core/flow_match.py create mode 100644 veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_vae.py create mode 100644 veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_video.py create mode 100644 veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_dit.py create mode 100644 veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_text_encoder.py create mode 100644 veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_video_vae.py create mode 100644 veomni/utils/audio.py create mode 100644 veomni/utils/audio_video.py create mode 100644 veomni/utils/video_data.py diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/flow_match.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/flow_match.py new file mode 100644 index 0000000000..2547d19927 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/flow_match.py @@ -0,0 +1,411 @@ +import torch, math +import numpy as np +from typing_extensions import Literal + + +class FlowMatchScheduler(): + + def __init__(self, template: Literal["FLUX.1", "Wan", "Qwen-Image", "FLUX.2", "Z-Image", "LTX-2", "Qwen-Image-Lightning", "ERNIE-Image", "ACE-Step", "Ideogram4", "Krea-2", "Boogu", "MiniMax-H3"] = "FLUX.1"): + self.set_timesteps_fn = { + "FLUX.1": FlowMatchScheduler.set_timesteps_flux, + "Wan": FlowMatchScheduler.set_timesteps_wan, + "Qwen-Image": FlowMatchScheduler.set_timesteps_qwen_image, + "FLUX.2": FlowMatchScheduler.set_timesteps_flux2, + "Z-Image": FlowMatchScheduler.set_timesteps_z_image, + "LTX-2": FlowMatchScheduler.set_timesteps_ltx2, + "Qwen-Image-Lightning": FlowMatchScheduler.set_timesteps_qwen_image_lightning, + "ERNIE-Image": FlowMatchScheduler.set_timesteps_ernie_image, + "ACE-Step": FlowMatchScheduler.set_timesteps_ace_step, + "HiDream-O1-Image": FlowMatchScheduler.set_timesteps_hidream_o1_image, + "Ideogram4": FlowMatchScheduler.set_timesteps_ideogram4, + "Krea-2": FlowMatchScheduler.set_timesteps_krea2, + "Boogu": FlowMatchScheduler.set_timesteps_boogu, + "MiniMax-H3": FlowMatchScheduler.set_timesteps_minimax_h3, + }.get(template, FlowMatchScheduler.set_timesteps_flux) + self.num_train_timesteps = 1000 + + @staticmethod + def set_timesteps_flux(num_inference_steps=100, denoising_strength=1.0, shift=None): + sigma_min = 0.003/1.002 + sigma_max = 1.0 + shift = 3 if shift is None else shift + num_train_timesteps = 1000 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps) + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_wan(num_inference_steps=100, denoising_strength=1.0, shift=None): + sigma_min = 0.0 + sigma_max = 1.0 + shift = 5 if shift is None else shift + num_train_timesteps = 1000 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def _calculate_shift_qwen_image(image_seq_len, base_seq_len=256, max_seq_len=8192, base_shift=0.5, max_shift=0.9): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + @staticmethod + def set_timesteps_qwen_image(num_inference_steps=100, denoising_strength=1.0, exponential_shift_mu=None, dynamic_shift_len=None): + sigma_min = 0.0 + sigma_max = 1.0 + num_train_timesteps = 1000 + shift_terminal = 0.02 + # Sigmas + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + # Mu + if exponential_shift_mu is not None: + mu = exponential_shift_mu + elif dynamic_shift_len is not None: + mu = FlowMatchScheduler._calculate_shift_qwen_image(dynamic_shift_len) + else: + mu = 0.8 + sigmas = math.exp(mu) / (math.exp(mu) + (1 / sigmas - 1)) + # Shift terminal + one_minus_z = 1 - sigmas + scale_factor = one_minus_z[-1] / (1 - shift_terminal) + sigmas = 1 - (one_minus_z / scale_factor) + # Timesteps + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_qwen_image_lightning(num_inference_steps=100, denoising_strength=1.0, exponential_shift_mu=None, dynamic_shift_len=None): + sigma_min = 0.0 + sigma_max = 1.0 + num_train_timesteps = 1000 + base_shift = math.log(3) + max_shift = math.log(3) + # Sigmas + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + # Mu + if exponential_shift_mu is not None: + mu = exponential_shift_mu + elif dynamic_shift_len is not None: + mu = FlowMatchScheduler._calculate_shift_qwen_image(dynamic_shift_len, base_shift=base_shift, max_shift=max_shift) + else: + mu = 0.8 + sigmas = math.exp(mu) / (math.exp(mu) + (1 / sigmas - 1)) + # Timesteps + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def compute_empirical_mu(image_seq_len, num_steps): + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + + @staticmethod + def set_timesteps_flux2(num_inference_steps=100, denoising_strength=1.0, dynamic_shift_len=None): + sigma_min = 1 / num_inference_steps + sigma_max = 1.0 + num_train_timesteps = 1000 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps) + if dynamic_shift_len is None: + # If you ask me why I set mu=0.8, + # I can only say that it yields better training results. + mu = 0.8 + else: + mu = FlowMatchScheduler.compute_empirical_mu(dynamic_shift_len, num_inference_steps) + sigmas = math.exp(mu) / (math.exp(mu) + (1 / sigmas - 1)) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_ernie_image(num_inference_steps=50, denoising_strength=1.0, shift=3.0): + sigma_min = 0.0 + sigma_max = 1.0 + num_train_timesteps = 1000 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + if shift is not None and shift != 1.0: + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_ace_step(num_inference_steps=8, denoising_strength=1.0, shift=3.0): + num_train_timesteps = 1000 + sigma_start = denoising_strength + sigmas = torch.linspace(sigma_start, 0.0, num_inference_steps + 1)[:-1] + if shift is not None and shift != 1.0: + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_z_image(num_inference_steps=100, denoising_strength=1.0, shift=None, target_timesteps=None): + sigma_min = 0.0 + sigma_max = 1.0 + shift = 3 if shift is None else shift + num_train_timesteps = 1000 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + timesteps = sigmas * num_train_timesteps + if target_timesteps is not None: + target_timesteps = target_timesteps.to(dtype=timesteps.dtype, device=timesteps.device) + for timestep in target_timesteps: + timestep_id = torch.argmin((timesteps - timestep).abs()) + timesteps[timestep_id] = timestep + return sigmas, timesteps + + @staticmethod + def set_timesteps_joyai_image(num_inference_steps=100, denoising_strength=1.0, shift=None): + sigma_min = 0.0 + sigma_max = 1.0 + shift = 4.0 if shift is None else shift + num_train_timesteps = 1000 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_hidream_o1_image(num_inference_steps=28, denoising_strength=1.0, shift=None, special_case=None, **kwargs): + num_train_timesteps = 1000 + shift = 3.0 if shift is None else shift + if special_case == "dev": + timesteps_list = [ + 999, 987, 974, 960, 945, 929, 913, 895, 877, 857, 836, 814, 790, 764, 737, + 707, 675, 640, 602, 560, 515, 464, 409, 347, 278, 199, 110, 8, + ] + sigmas = torch.tensor([t / 1000.0 for t in timesteps_list], dtype=torch.float32) + timesteps = torch.tensor(timesteps_list, dtype=torch.float32) + return sigmas, timesteps + else: + sigma_min = 0.0 + sigma_max = 1.0 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_ideogram4(num_inference_steps=50, denoising_strength=1.0, image_resolution=(1024, 1024), mu=0.0, std=1.5): + num_pixels = image_resolution[0] * image_resolution[1] + known_pixels = 512 * 512 + mean = mu + 0.5 * math.log(num_pixels / known_pixels) + logsnr_min = -15.0 + logsnr_max = 18.0 + t_min = 1.0 / (1 + math.exp(0.5 * logsnr_max)) + t_max = 1.0 / (1 + math.exp(0.5 * logsnr_min)) + step_intervals = torch.linspace(0.0, denoising_strength, num_inference_steps + 1, dtype=torch.float64) + sigmas = [] + for i in range(num_inference_steps + 1): + z = torch.special.ndtri(step_intervals[i]) + y = mean + std * z + t_ = torch.special.expit(y) + t_ = 1 - t_ + t_ = t_.clamp(t_min, t_max) + sigmas.append(float(t_.to(torch.float32))) + sigmas = torch.tensor(sigmas, dtype=torch.float32) + one_minus_t = (1 - sigmas)[:-1].flip(0) + sigma_start = one_minus_t[0] * denoising_strength + if one_minus_t[0] > 0: + one_minus_t = one_minus_t * (sigma_start / one_minus_t[0]) + sigmas = sigmas.flip(dims=(0,)) + timesteps = sigmas[:-1] + sigmas = (1 - sigmas)[:-1] + return sigmas, timesteps + + def set_timesteps_boogu(num_inference_steps=50, denoising_strength=1.0, sigmas=None): + if sigmas is not None: + sigmas = torch.tensor(sigmas, dtype=torch.float32) + timesteps = 1 - sigmas + return sigmas, timesteps + t_arr = np.linspace(1-denoising_strength, 1, num_inference_steps + 1, dtype=np.float32) + mu = 1.15 + sigma = 1 + eps = 1e-8 + t1 = 1.0 - t_arr + t1 = np.clip(t1, eps, 1.0 - eps) + num = math.exp(mu) + denom = num + np.power(1.0 / t1 - 1.0, sigma) + y = num / denom + t_arr = 1.0 - y + timesteps = torch.from_numpy(t_arr).float()[:-1] + sigmas = 1 - timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_krea2(num_inference_steps=28, denoising_strength=1.0, dynamic_shift_len=None, y1=0.5, y2=1.15, mu=None): + x1 = 256 + x2 = 6400 + sigma = 1 + ts = torch.linspace(denoising_strength, 0, num_inference_steps + 1) + if mu is None and dynamic_shift_len is None: + # Training + mu = 0.8 + elif mu is None: + # Raw + slope = (y2 - y1) / (x2 - x1) + mu = slope * dynamic_shift_len + (y1 - slope * x1) + ts = math.exp(mu) / (math.exp(mu) + (1.0 / ts - 1.0) ** sigma) + sigmas, timesteps = ts[:-1], ts[:-1] + return sigmas, timesteps + + @staticmethod + def set_timesteps_ltx2(num_inference_steps=100, denoising_strength=1.0, dynamic_shift_len=None, terminal=0.1, special_case=None): + num_train_timesteps = 1000 + if special_case == "stage2": + sigmas = torch.Tensor([0.909375, 0.725, 0.421875]) + elif special_case == "ditilled_stage1": + sigmas = torch.Tensor([1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875]) + else: + dynamic_shift_len = dynamic_shift_len or 4096 + sigma_shift = FlowMatchScheduler._calculate_shift_qwen_image( + image_seq_len=dynamic_shift_len, + base_seq_len=1024, + max_seq_len=4096, + base_shift=0.95, + max_shift=2.05, + ) + sigma_min = 0.0 + sigma_max = 1.0 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + sigmas = math.exp(sigma_shift) / (math.exp(sigma_shift) + (1 / sigmas - 1)) + # Shift terminal + one_minus_z = 1.0 - sigmas + scale_factor = one_minus_z[-1] / (1 - terminal) + sigmas = 1.0 - (one_minus_z / scale_factor) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + @staticmethod + def set_timesteps_minimax_h3(num_inference_steps=50, denoising_strength=1.0, shift=12.0): + num_train_timesteps = 1000 + base = torch.linspace(denoising_strength, 0.0, num_inference_steps+1, dtype=torch.float32)[:-1] + sigmas = shift * base / (1 + (shift - 1) * base) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + + def set_training_weight(self): + steps = 1000 + x = self.sigmas * self.num_train_timesteps + y = torch.exp(-2 * ((x - steps / 2) / steps) ** 2) + y_shifted = y - y.min() + bsmntw_weighing = y_shifted * (steps / y_shifted.sum()) + if len(self.timesteps) != 1000: + # This is an empirical formula. + bsmntw_weighing = bsmntw_weighing * (len(self.timesteps) / steps) + bsmntw_weighing = bsmntw_weighing + bsmntw_weighing[1] + self.linear_timesteps_weights = bsmntw_weighing + + def set_timesteps(self, num_inference_steps=100, denoising_strength=1.0, training=False, **kwargs): + self.sigmas, self.timesteps = self.set_timesteps_fn( + num_inference_steps=num_inference_steps, + denoising_strength=denoising_strength, + **kwargs, + ) + if training: + self.set_training_weight() + self.training = True + else: + self.training = False + + def step(self, model_output, timestep, sample, to_final=False, **kwargs): + if isinstance(timestep, torch.Tensor): + timestep = timestep.cpu() + timestep_id = torch.argmin((self.timesteps - timestep).abs()) + sigma = self.sigmas[timestep_id] + if to_final or timestep_id + 1 >= len(self.timesteps): + sigma_ = 0 + else: + sigma_ = self.sigmas[timestep_id + 1] + prev_sample = sample + model_output * (sigma_ - sigma) + return prev_sample + + def return_to_timestep(self, timestep, sample, sample_stablized): + if isinstance(timestep, torch.Tensor): + timestep = timestep.cpu() + timestep_id = torch.argmin((self.timesteps - timestep).abs()) + sigma = self.sigmas[timestep_id] + model_output = (sample - sample_stablized) / sigma + return model_output + + def add_noise(self, original_samples, noise, timestep): + if isinstance(timestep, torch.Tensor): + timestep = timestep.cpu() + timestep_id = torch.argmin((self.timesteps - timestep).abs()) + sigma = self.sigmas[timestep_id] + sample = (1 - sigma) * original_samples + sigma * noise + return sample + + def training_target(self, sample, noise, timestep): + target = noise - sample + return target + + def training_weight(self, timestep): + timestep_id = torch.argmin((self.timesteps - timestep.to(self.timesteps.device)).abs()) + weights = self.linear_timesteps_weights[timestep_id] + return weights + + +class HiDreamO1FlashScheduler(FlowMatchScheduler): + + def __init__(self, noise_scale_start=7.5, noise_scale_end=7.5, noise_clip_std=2.5): + self.set_timesteps_fn = HiDreamO1FlashScheduler.set_timesteps_hidream_o1_image_dev + self.num_train_timesteps = 1000 + self.noise_clip_std = noise_clip_std + num_steps = 28 + self.noise_scale_schedule = [ + noise_scale_start + (noise_scale_end - noise_scale_start) * i / (num_steps - 1) + for i in range(num_steps) + ] + + @staticmethod + def set_timesteps_hidream_o1_image_dev(**kwargs): + timesteps_list = [ + 999, 987, 974, 960, 945, 929, 913, 895, 877, 857, 836, 814, 790, 764, 737, + 707, 675, 640, 602, 560, 515, 464, 409, 347, 278, 199, 110, 8, + ] + sigmas = torch.tensor([t / 1000.0 for t in timesteps_list], dtype=torch.float32) + timesteps = torch.tensor(timesteps_list, dtype=torch.float32) + return sigmas, timesteps + + def clip_noise(self, noise): + if self.noise_clip_std > 0: + noise_std = noise.std().item() + clip_val = self.noise_clip_std * noise_std + noise = noise.clamp(min=-clip_val, max=clip_val) + return noise + + def step(self, model_output, timestep, sample): + timestep_id = torch.argmin((self.timesteps - timestep).abs()) + sigma = self.sigmas[timestep_id] + sigma_ = self.sigmas[timestep_id + 1] if timestep_id + 1 < len(self.sigmas) else 0 + denoised = sample - model_output * sigma + + noise = self.clip_noise(torch.randn(denoised.shape, device=denoised.device, dtype=denoised.dtype)) + sample = sigma_ * noise * self.noise_scale_schedule[timestep_id] + (1.0 - sigma_) * denoised + return sample diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_vae.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_vae.py new file mode 100644 index 0000000000..cf9a114b62 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_vae.py @@ -0,0 +1,481 @@ +# SPDX-License-Identifier: Apache-2.0 +import math +from typing import List + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import Parameter +from ..core.attention import attention_forward + + +class WarpedTensor(torch.nn.Module): + def __init__(self, weight=None, shape=None): + super().__init__() + if weight is not None: + self.weight = torch.nn.Parameter(weight) + else: + self.weight = torch.nn.Parameter(torch.empty(shape)) + + def forward(self): + return self.weight + + +class WeightNormedConv1d(torch.nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, stride = 1, padding = 0, dilation = 1, groups = 1, bias = True, padding_mode = "zeros", device=None, dtype=None): + super().__init__() + self.weight_g = torch.nn.Parameter(torch.empty((out_channels, 1, 1))) + self.weight_v = torch.nn.Parameter(torch.empty((out_channels, in_channels, kernel_size))) + self.stride = stride + self.padding = padding + self.dilation = dilation + self.groups = groups + if bias: + self.bias = torch.nn.Parameter(torch.empty((out_channels,))) + else: + self.bias = None + + def forward(self, input): + dtype = self.weight_g.dtype + weight = (self.weight_g.float() * self.weight_v.float() / torch.norm(self.weight_v.float(), dim=(1, 2), keepdim=True)).to(dtype) + return torch.nn.functional.conv1d( + input, weight, self.bias, self.stride, self.padding, self.dilation, self.groups + ) + + +class WeightNormedConvTranspose1d(torch.nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, stride = 1, padding = 0, output_padding = 0, dilation = 1, groups = 1, bias = True, padding_mode = "zeros", device=None, dtype=None): + super().__init__() + self.weight_g = torch.nn.Parameter(torch.empty((in_channels, 1, 1))) + self.weight_v = torch.nn.Parameter(torch.empty((in_channels, out_channels, kernel_size))) + self.kernel_size = kernel_size + self.stride = stride + self.padding = (padding,) + self.output_padding = output_padding + self.dilation = dilation + self.groups = groups + if bias: + self.bias = torch.nn.Parameter(torch.empty((out_channels,))) + else: + self.bias = None + + def _output_padding( + self, + input, + output_size: list[int] | None, + stride: list[int], + padding: list[int], + kernel_size: list[int], + num_spatial_dims: int, + dilation: list[int] | None = None, + ) -> list[int]: + if output_size is None: + ret = self.output_padding + else: + has_batch_dim = input.dim() == num_spatial_dims + 2 + num_non_spatial_dims = 2 if has_batch_dim else 1 + if len(output_size) == num_non_spatial_dims + num_spatial_dims: + output_size = output_size[num_non_spatial_dims:] + if len(output_size) != num_spatial_dims: + raise ValueError( + f"ConvTranspose{num_spatial_dims}D: for {input.dim()}D input, output_size must have {num_spatial_dims} " + f"or {num_non_spatial_dims + num_spatial_dims} elements (got {len(output_size)})" + ) + + min_sizes = torch.jit.annotate(list[int], []) + max_sizes = torch.jit.annotate(list[int], []) + for d in range(num_spatial_dims): + dim_size = ( + (input.size(d + num_non_spatial_dims) - 1) * stride[d] + - 2 * padding[d] + + (dilation[d] if dilation is not None else 1) + * (kernel_size[d] - 1) + + 1 + ) + min_sizes.append(dim_size) + max_sizes.append(min_sizes[d] + stride[d] - 1) + + for i in range(len(output_size)): + size = output_size[i] + min_size = min_sizes[i] + max_size = max_sizes[i] + if size < min_size or size > max_size: + raise ValueError( + f"requested an output size of {output_size}, but valid sizes range " + f"from {min_sizes} to {max_sizes} (for an input of {input.size()[2:]})" + ) + + res = torch.jit.annotate(list[int], []) + for d in range(num_spatial_dims): + res.append(output_size[d] - min_sizes[d]) + + ret = res + return ret + + def forward(self, input, output_size=None): + dtype = self.weight_g.dtype + weight = (self.weight_g.float() * self.weight_v.float() / torch.norm(self.weight_v.float(), dim=(1, 2), keepdim=True)).to(dtype) + assert isinstance(self.padding, tuple) + num_spatial_dims = 1 + output_padding = self._output_padding( + input, + output_size, + self.stride, # type: ignore[arg-type] + self.padding, # type: ignore[arg-type] + self.kernel_size, # type: ignore[arg-type] + num_spatial_dims, + self.dilation, # type: ignore[arg-type] + ) + return F.conv_transpose1d( + input, + weight, + self.bias, + self.stride, + self.padding, + output_padding, + self.groups, + self.dilation, + ) + + +class Snake(nn.Module): + def __init__(self, channels, beta=False): + super().__init__() + if not beta: + self.alpha = nn.Parameter(torch.ones(1, channels, 1)) + self.beta = None + else: + self.alpha = nn.Parameter(torch.ones(channels)) + self.beta = Parameter(torch.zeros(channels)) if beta else None + + def forward(self, x): + alpha = self.alpha + if len(alpha.shape) == 1: alpha = alpha.unsqueeze(0).unsqueeze(-1).exp() + beta = self.alpha if self.beta is None else self.beta + if len(beta.shape) == 1: beta = beta.unsqueeze(0).unsqueeze(-1).exp() + x = x + (beta + 1e-9).reciprocal() * torch.sin(alpha * x).pow(2) + return x + + +def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): + even = kernel_size % 2 == 0 + half_size = kernel_size // 2 + delta_f = 4 * half_width + A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95 + if A > 50.0: + beta = 0.1102 * (A - 8.7) + elif A >= 21.0: + beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0) + else: + beta = 0.0 + window = torch.kaiser_window(kernel_size, beta=beta, periodic=False) + if even: + time = torch.arange(-half_size, half_size) + 0.5 + else: + time = torch.arange(kernel_size) - half_size + if cutoff == 0: + filter_ = torch.zeros_like(time) + else: + filter_ = 2 * cutoff * window * torch.sinc(2 * cutoff * time) + filter_ /= filter_.sum() + return filter_.view(1, 1, kernel_size) + + +class LowPassFilter1d(nn.Module): + def __init__(self, cutoff=0.25, half_width=0.3, stride: int = 2, kernel_size: int = 12): + super().__init__() + self.kernel_size = kernel_size + self.stride = stride + self.filter = torch.nn.Parameter(kaiser_sinc_filter1d(cutoff, half_width, kernel_size)) + + def forward(self, x): + x = F.pad(x, (self.kernel_size // 2 - 1, self.kernel_size // 2), mode="replicate") + out = F.conv1d(x, self.filter.expand(x.shape[1], -1, -1), stride=self.stride, groups=x.shape[1]) + return out + + +class UpSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=None): + super().__init__() + self.ratio = ratio + self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size + self.stride = ratio + self.pad = self.kernel_size // ratio - 1 + self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2 + self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2 + filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size) + self.filter = torch.nn.Parameter(filter) + + def forward(self, x): + _, C, _ = x.shape + x = F.pad(x, (self.pad, self.pad), mode="replicate") + x = self.ratio * F.conv_transpose1d(x, self.filter.expand(C, -1, -1).to(dtype=x.dtype, device=x.device), stride=self.stride, groups=C) + x = x[..., self.pad_left : -self.pad_right] + return x + + +class DownSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=None): + super().__init__() + self.ratio = ratio + self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size + self.lowpass = LowPassFilter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, stride=ratio, kernel_size=self.kernel_size) + + def forward(self, x): + return self.lowpass(x) + + +class Activation1d(nn.Module): + def __init__( + self, + activation, + up_ratio: int = 2, + down_ratio: int = 2, + up_kernel_size: int = 12, + down_kernel_size: int = 12, + ): + super().__init__() + self.act = activation + self.upsample = UpSample1d(up_ratio, up_kernel_size) + self.downsample = DownSample1d(down_ratio, down_kernel_size) + + def forward(self, x): + x = self.upsample(x) + x = self.act(x) + x = self.downsample(x) + return x + + +class GeGluMlp(nn.Module): + def __init__(self, in_features, hidden_features): + super().__init__() + self.norm = nn.LayerNorm(in_features) + self.act = nn.GELU(approximate="tanh") + self.w0 = nn.Linear(in_features, hidden_features) + self.w1 = nn.Linear(in_features, hidden_features) + self.w2 = nn.Linear(hidden_features, in_features) + + def forward(self, x): + x = self.norm(x) + x = self.act(self.w0(x)) * self.w1(x) + x = self.w2(x) + return x + + +class CausalAttention(nn.Module): + def __init__(self, in_dim, out_dim, num_heads): + super().__init__() + self.qkv = nn.Linear(in_dim, in_dim * 3, bias=False) + self.out_dim = out_dim + self.num_heads = num_heads + self.proj = nn.Linear(out_dim, out_dim) + self.q_bias = WarpedTensor(torch.empty(in_dim)) + self.zero_k_bias = WarpedTensor(torch.empty(in_dim)) + self.v_bias = WarpedTensor(torch.empty(in_dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + q, k, v = self.qkv(x).chunk(3, dim=-1) + q, k, v = q + self.q_bias(), k + self.zero_k_bias(), v + self.v_bias() + x = attention_forward(q, k, v, q_pattern="b s (n d)", k_pattern="b s (n d)", v_pattern="b s (n d)", out_pattern="b n s d", dims={"n": self.num_heads}, is_causal=True) + x = torch.mean(x, dim=1) + x = F.adaptive_avg_pool1d(x, self.out_dim) + x = self.proj(x) + return x + + +class AttnProjection(nn.Module): + def __init__(self, in_dim, out_dim, num_heads, mlp_ratio=2): + super().__init__() + self.norm1 = nn.LayerNorm(in_dim) + self.attn = CausalAttention(in_dim, out_dim, num_heads) + self.proj = nn.Linear(in_dim, out_dim) + self.norm3 = nn.LayerNorm(in_dim) + self.norm2 = nn.LayerNorm(out_dim) + self.mlp = GeGluMlp(in_features=out_dim, hidden_features=out_dim * mlp_ratio) + + def forward(self, x): + x = self.proj(self.norm3(x)) + self.attn(self.norm1(x)) + x = x + self.mlp(self.norm2(x)) + return x + + +class ResidualUnit(nn.Module): + def __init__(self, dim: int = 16, dilation: int = 1): + super().__init__() + pad = ((7 - 1) * dilation) // 2 + self.block = nn.Sequential( + Snake(dim), + WeightNormedConv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad), + Snake(dim), + WeightNormedConv1d(dim, dim, kernel_size=1), + ) + + def forward(self, x): + y = self.block(x) + pad = (x.shape[-1] - y.shape[-1]) // 2 + if pad > 0: + x = x[..., pad:-pad] + return x + y + + +class EncoderBlock(nn.Module): + def __init__(self, dim: int = 16, stride: int = 1): + super().__init__() + self.block = nn.Sequential( + ResidualUnit(dim // 2, dilation=1), + ResidualUnit(dim // 2, dilation=3), + ResidualUnit(dim // 2, dilation=9), + Snake(dim // 2), + WeightNormedConv1d( + dim // 2, + dim, + kernel_size=2 * stride, + stride=stride, + padding=math.ceil(stride / 2), + ), + ) + + def forward(self, x): + return self.block(x) + + +class Encoder(nn.Module): + def __init__(self, d_model: int = 64, strides: list = [2, 4, 8, 8], d_latent: int = 64): + super().__init__() + self.block = [WeightNormedConv1d(1, d_model, kernel_size=7, padding=3)] + for stride in strides: + d_model *= 2 + self.block += [EncoderBlock(d_model, stride=stride)] + self.block += [ + Snake(d_model), + WeightNormedConv1d(d_model, d_latent, kernel_size=3, padding=1), + ] + self.block = nn.Sequential(*self.block) + + def forward(self, x): + return self.block(x) + + +class AMPBlock1(torch.nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): + super().__init__() + self.convs1 = nn.ModuleList([WeightNormedConv1d(channels, channels, kernel_size, stride=1, dilation=d, padding=int((kernel_size * d - d) / 2)) for d in dilation]) + self.convs2 = nn.ModuleList([WeightNormedConv1d(channels, channels, kernel_size, stride=1, dilation=1, padding=int((kernel_size - 1) / 2)) for _ in range(len(dilation))]) + self.activations = nn.ModuleList([Activation1d(activation=Snake(channels, beta=True)) for _ in range(2 * len(dilation))]) + + def forward(self, x): + acts1, acts2 = self.activations[::2], self.activations[1::2] + for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2): + xt = a1(x) + xt = c1(xt) + xt = a2(xt) + xt = c2(xt) + x = xt + x + return x + + +class BigVGAN(torch.nn.Module): + def __init__(self): + super().__init__() + self.conv_pre = WeightNormedConv1d(2048, 1024, 7, 1, padding=3) + self.ups = nn.ModuleList([ + nn.ModuleList([WeightNormedConvTranspose1d(1024 // (2**i), 1024 // (2**(i+1)), k, u, padding=(k-u)//2)]) + for i, (u, k) in enumerate(zip([5, 5, 2, 2, 2, 2, 2], [9, 9, 4, 4, 4, 4, 4])) + ]) + self.resblocks = nn.ModuleList() + for i in range(7): + ch = 1024 // (2 ** (i + 1)) + for k in [3, 7, 11]: + self.resblocks.append(AMPBlock1(ch, k, (1, 3, 5))) + self.activation_post = Activation1d(activation=Snake(ch, beta=True)) + self.conv_post = WeightNormedConv1d(ch, 1, 7, 1, padding=3, bias=False) + + def forward(self, x): + x = self.conv_pre(x) + for i, up in enumerate(self.ups): + x = up[0](x) + xs = self.resblocks[i * 3](x) + for j in range(1, 3): + xs += self.resblocks[i * 3 + j](x) + x = xs / 3 + x = self.activation_post(x) + x = self.conv_post(x) + x = torch.clamp(x, min=-1.0, max=1.0) + return x + + +_AUDIO_LATENTS_MEAN = [-0.020211687488382354, 0.3876466479950502, -0.04398279799186767, -0.28591514936373, 0.08179686214561671, -0.35782641352446604, 0.040623809960919084, -0.01552534501956604, -0.223362481667332, 0.1821006842509091, 0.2941778783780663, -0.07901167601970885, -0.056815072777201, -0.3699028221860095, -0.31616315591624855, 0.5905951377425391, -0.052139568068853864, 0.013673160263486295, -0.03691647864630577, 0.09732660653298163, -0.3394662328788498, -0.30685677538541667, -0.24504598907458763, -0.034698524462007344, 0.02868032184767538, -0.21217779266454084, -0.1678263169941987, 0.3221287889040614, -0.1223055851554907, 0.4356604928128464, -0.0502599202236253, 0.3979258376211797] +_AUDIO_LATENTS_STD = [1.6895524230479284, 2.76263727217653, 1.7945344281264435, 1.6801681847309828, 1.6390226546605453, 2.7788298348882177, 1.7659090095747236, 1.6199757612137327, 2.6336525640336896, 1.8539356672817833, 2.5056497896915633, 1.811019237886178, 1.9579657790720237, 1.6685498243529284, 1.4922469314453364, 3.298670198067373, 1.9491804496832168, 1.8720003270431442, 1.8334080103291832, 1.6488070416529093, 1.6176957696319716, 1.9131449234774398, 1.5695245398428617, 1.6943659940415912, 1.8318420762504692, 1.5540637421583379, 1.9344930328968526, 1.599198216109855, 1.718045989838149, 1.6307219190837705, 1.8661226051202384, 1.5613768203168363] + + +class MiniMaxH3AudioVAE(nn.Module): + + def __init__( + self, + encoder_dim: int = 64, + encoder_rates: List[int] = [2, 4, 4, 5, 5], + decoder_dim: int = 1024, + decoder_rates: List[int] = [5, 5, 2, 2, 2, 2, 2], + sample_rate: int = 32000, + vae_latent_channels: int = 32, + attn_proj: bool = True, + decoder_type: str = "bigvgan", + ): + super().__init__() + self.sample_rate = sample_rate + self.attn_proj = attn_proj + latent_dim = encoder_dim * (2 ** len(encoder_rates)) + self.hop_length = int(np.prod(encoder_rates)) + self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim) + self.mean_proj = nn.Conv1d(vae_latent_channels, vae_latent_channels, 1) + self.logs_proj = nn.Conv1d(vae_latent_channels, vae_latent_channels, 1) + self.dec_in_proj = nn.Conv1d(vae_latent_channels, latent_dim, 1) + self.decoder = BigVGAN() + self.pre_block = AttnProjection(latent_dim, vae_latent_channels, num_heads=8) + + def _latent_stats(self, device): + mean = torch.tensor(_AUDIO_LATENTS_MEAN, device=device, dtype=torch.float32).view(1, -1, 1) + std = torch.tensor(_AUDIO_LATENTS_STD, device=device, dtype=torch.float32).view(1, -1, 1) + return mean, std + + def preprocess(self, audio_data): + length = audio_data.shape[-1] + right_pad = math.ceil(length / self.hop_length) * self.hop_length - length + return nn.functional.pad(audio_data, (0, right_pad)) + + def encode(self, audio_data: torch.Tensor) -> torch.Tensor: + z = self.encoder(audio_data) + if self.attn_proj: + z = self.pre_block(z.transpose(1, 2)).transpose(1, 2) + mean = self.mean_proj(z) + return mean + + def decode(self, z: torch.Tensor) -> torch.Tensor: + z = self.dec_in_proj(z) + return self.decoder(z) + + @torch.no_grad() + def encode_audio(self, waveform: torch.Tensor, dtype=None) -> torch.Tensor: + if waveform.dim() != 2: + raise ValueError(f"expected waveform [C, L], got {list(waveform.shape)}") + out_dtype = waveform.dtype + audio_data = waveform.unsqueeze(1).to(dtype if dtype is not None else out_dtype) + latent = self.encode(self.preprocess(audio_data)) + latent_channels = len(_AUDIO_LATENTS_MEAN) + if latent.dim() != 3 or latent.shape[1] != latent_channels: + raise ValueError( + f"expected audio latent [C, {latent_channels}, T], got {list(latent.shape)}" + ) + mean, std = self._latent_stats(latent.device) + return ((latent.to(torch.float32) - mean) / std).to(out_dtype) + + @torch.no_grad() + def decode_audio(self, latents: torch.Tensor, dtype=None) -> torch.Tensor: + mean, std = self._latent_stats(latents.device) + z = (latents.to(latents.device, torch.float32) * std + mean).to(dtype if dtype is not None else latents.dtype) + waveform = self.decode(z) + return waveform.transpose(0, 1).to(latents.dtype) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + return self.decode(z) diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_video.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_video.py new file mode 100644 index 0000000000..c49e67d169 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_audio_video.py @@ -0,0 +1,784 @@ +import numpy as np +import torch +from PIL import Image +from tqdm import tqdm +from transformers import AutoProcessor + +from ..core import ModelConfig +from ..core.device.npu_compatible_device import get_device_type +from ..diffusion import FlowMatchScheduler +from ..diffusion.base_pipeline import BasePipeline, PipelineUnit +from ..models.minimax_h3_dit import MiniMaxH3DiT, patchify_video, unpatchify_video, pack_audio, unpack_audio +from ..models.minimax_h3_text_encoder import ( + MiniMaxH3TextEncoder, presentation_t2va, presentation_fl2va, presentation_ref2va, + sample_qwen_video_frames, image_token_counts, video_token_counts, +) +from ..models.minimax_h3_video_vae import MiniMaxH3VideoVAE +from ..models.minimax_h3_audio_vae import MiniMaxH3AudioVAE +from ..utils.data.audio import convert_to_stereo, resample_waveform + +class MiniMaxH3Pipeline(BasePipeline): + + def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16): + super().__init__(device=device, torch_dtype=torch_dtype, height_division_factor=32, width_division_factor=32, time_division_factor=17, time_division_remainder=5) + self.scheduler = FlowMatchScheduler("MiniMax-H3") + self.scheduler_audio = FlowMatchScheduler("MiniMax-H3") + self.text_encoder: MiniMaxH3TextEncoder = None + self.dit: MiniMaxH3DiT = None + self.video_vae: MiniMaxH3VideoVAE = None + self.audio_vae: MiniMaxH3AudioVAE = None + self.tokenizer = None + self.processor = None + self.imgvid_cond_noise_aug = 0.999 + self.audio_cond_noise_aug = 1.0 + self.in_iteration_models = ("dit",) + self.units = [ + MiniMaxH3Unit_ShapeChecker(), + MiniMaxH3Unit_NoiseInitializer(), + MiniMaxH3Unit_InputVideoEmbedder(), + MiniMaxH3Unit_InputAudioEmbedder(), + MiniMaxH3Unit_KeyframeEncoder(), + MiniMaxH3Unit_ReferenceEncoder(), + MiniMaxH3Unit_PromptEmbedder(), + MiniMaxH3Unit_PackedSequenceBuilder(), + ] + self.model_fn = model_fn_minimax_h3 + self.compilable_models = ["dit"] + + @staticmethod + def from_pretrained( + torch_dtype: torch.dtype = torch.bfloat16, + device: str = get_device_type(), + model_configs: list[ModelConfig] = [], + processor_config: ModelConfig = ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/processor/"), + vram_limit: float = None, + ): + pipe = MiniMaxH3Pipeline(device=device, torch_dtype=torch_dtype) + model_pool = pipe.download_and_load_models(model_configs, vram_limit) + pipe.text_encoder = model_pool.fetch_model("minimax_h3_text_encoder") + pipe.dit = model_pool.fetch_model("minimax_h3_dit") + pipe.video_vae = model_pool.fetch_model("minimax_h3_video_vae") + pipe.audio_vae = model_pool.fetch_model("minimax_h3_audio_vae") + if processor_config is not None: + processor_config.download_if_necessary() + pipe.processor = AutoProcessor.from_pretrained(processor_config.path) + pipe.tokenizer = pipe.processor.tokenizer + pipe.vram_management_enabled = pipe.check_vram_management_state() + return pipe + + @torch.no_grad() + def __call__( + self, + prompt: str, + negative_prompt: str = " ", + height: int = 768, + width: int = 1344, + num_frames: int = 124, + num_inference_steps: int = 50, + seed: int = 42, + rand_device: str = "cpu", + cfg_scale: float = 1.0, + flow_shift: float = 12.0, + audio_flow_shift: float = 3.0, + tiled: bool = True, + tile_size: int = 256, + tile_overlap: int = 64, + use_gradient_checkpointing: bool = False, + use_gradient_checkpointing_offload: bool = False, + # Keyframe to Video + keyframes: list[Image.Image] = None, + keyframe_indices: list[int] = None, + # Reference to Video + references: list[dict] = None, + ref_image_short_edge: int = 2048, + ref_video_short_edge: int = 768, + ref_video_max_pixels: int = 768 * 1344, + progress_bar_cmd=tqdm, + ): + """Generate a joint video + audio sample. + + `num_frames` is snapped up to the nearest 17n+5 and that aligned value + drives every downstream shape (video/audio latent lengths, reference video + capping), so the returned clip may be slightly longer than requested. + + `keyframes` (FL2AV) is a list of PIL images with `keyframe_indices` in + {0, -1}; both are resized onto the target canvas. + + `references` (Ref2AV) is a list of dicts in request order: + + {"type": "image", "image": PIL.Image} + {"type": "video", "video": list[PIL.Image]} # silent + {"type": "audio", "audio": Tensor[C, L], "sample_rate": int} + {"type": "video_audio", "video": list[PIL.Image], + "audio": Tensor[C, L], "sample_rate": int} + + Input contract: `video` frame lists must ALREADY. be 24fps the pipeline never resamples frame rate. + """ + self.scheduler.set_timesteps(num_inference_steps, shift=flow_shift) + self.scheduler_audio.set_timesteps(num_inference_steps, shift=audio_flow_shift) + + inputs_posi = {"prompt": prompt} + inputs_nega = {"negative_prompt": negative_prompt} + inputs_shared = { + "cfg_scale": cfg_scale, + "height": height, "width": width, "num_frames": num_frames, + "seed": seed, "rand_device": rand_device, + "use_gradient_checkpointing": use_gradient_checkpointing, + "use_gradient_checkpointing_offload": use_gradient_checkpointing_offload, + "keyframes": keyframes, "keyframe_indices": keyframe_indices, + "references": references, "ref_image_short_edge": ref_image_short_edge, "ref_video_short_edge": ref_video_short_edge, "ref_video_max_pixels": ref_video_max_pixels, + "imgvid_cond_noise_aug": self.imgvid_cond_noise_aug, "audio_cond_noise_aug": self.audio_cond_noise_aug, + } + + # 3. Unit chain + for unit in self.units: + inputs_shared, inputs_posi, inputs_nega = self.unit_runner(unit, self, inputs_shared, inputs_posi, inputs_nega) + + # 4. Denoise loop + self.load_models_to_device(self.in_iteration_models) + models = {name: getattr(self, name) for name in self.in_iteration_models} + for progress_id, _ in enumerate(progress_bar_cmd(self.scheduler.timesteps)): + t_video = float(1.0 - self.scheduler.sigmas[progress_id]) + t_audio = float(1.0 - self.scheduler_audio.sigmas[progress_id]) + noise_pred_video, noise_pred_audio = self.cfg_guided_model_fn( + self.model_fn, cfg_scale, inputs_shared, inputs_posi, inputs_nega, + **models, t_video=t_video, t_audio=t_audio, device=self.device, + ) + inputs_shared["video_latents"] = self.step(self.scheduler, inputs_shared["video_latents"], progress_id, noise_pred=noise_pred_video) + inputs_shared["audio_latents"] = self.step(self.scheduler_audio, inputs_shared["audio_latents"], progress_id, noise_pred=noise_pred_audio) + + # 5. Decode + self.load_models_to_device(["video_vae"]) + frames = self.video_vae.decode_video(inputs_shared["video_latents"], dtype=self.torch_dtype, tiled=tiled, tile_size=tile_size, tile_overlap=tile_overlap) + video = self.vae_output_to_video(frames, min_value=0, max_value=1) + + self.load_models_to_device(["audio_vae"]) + waveform = self.audio_vae.decode_audio(inputs_shared["audio_latents"], dtype=self.torch_dtype) + audio = self.output_audio_format_check(waveform) + return video, audio + + +class MiniMaxH3Unit_ShapeChecker(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("height", "width", "num_frames"), + output_params=("height", "width", "num_frames"), + ) + + def process(self, pipe: MiniMaxH3Pipeline, height, width, num_frames): + height, width, num_frames = pipe.check_resize_height_width(height, width, num_frames) + return {"height": height, "width": width, "num_frames": num_frames} + + +class MiniMaxH3Unit_NoiseInitializer(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("seed", "num_frames", "height", "width", "rand_device"), + output_params=("video_latents", "audio_latents"), + ) + + def process(self, pipe: MiniMaxH3Pipeline, seed, num_frames, height, width, rand_device): + video_latent_t, latent_h, latent_w = ((num_frames - 5) // 17) * 5 + 2, height // 16, width // 16 + video_latents = pipe.generate_noise((1, 24, video_latent_t, latent_h, latent_w), seed=seed, rand_device=rand_device, rand_torch_dtype=pipe.torch_dtype) + audio_latent_t = round(num_frames / 24.0 * 40.0) + audio_latents = pipe.generate_noise((2, 32, audio_latent_t), seed=seed, rand_device=rand_device, rand_torch_dtype=pipe.torch_dtype) + return {"video_latents": video_latents, "audio_latents": audio_latents} + + +class MiniMaxH3Unit_PromptEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + seperate_cfg=True, + input_params_posi={"prompt": "prompt"}, + input_params_nega={"prompt": "negative_prompt"}, + input_params=("keyframes", "ref_blocks", "height", "width"), + output_params=("prompt_embeds", "text_token_tags"), + onload_model_names=("text_encoder",), + ) + + def preprocess_ref_blocks(self, pipe: MiniMaxH3Pipeline, prompt, ref_blocks): + pixel_values = image_grid_thw = pixel_values_videos = video_grid_thw = None + counters = {"image": 0, "audio": 0, "video": 0} + images, videos, timestamps_per_video, condition_labels = [], [], [], [] + for block in ref_blocks: + kind = block["kind"] + if kind == "image": + counters["image"] += 1 + condition_labels.append(("image", counters["image"])) + images.append(block["prepared_image"]) + elif kind == "audio": + counters["audio"] += 1 + condition_labels.append(("audio", counters["audio"])) + elif kind in ("video", "video_audio"): + if int(block["ref_audio_t"]) > 0: + counters["audio"] += 1 + condition_labels.append(("audio", counters["audio"])) + counters["video"] += 1 + condition_labels.append(("video", counters["video"])) + sampled, timestamps = sample_qwen_video_frames(block["prepared_frames"]) + videos.append(np.stack([np.asarray(f) for f in sampled])) + timestamps_per_video.append(timestamps) + else: + raise ValueError(f"unknown reference kind: {kind}") + + image_counts, video_counts, video_timestamps = [], [], [] + if len(images) > 0: + pixel_values, image_grid_thw, image_counts = image_token_counts(pipe.processor, images) + if len(videos) > 0: + pixel_values_videos, video_grid_thw, video_counts, video_timestamps = video_token_counts(pipe.processor, videos, timestamps_per_video) + input_ids, text_token_tags = presentation_ref2va(pipe.tokenizer, prompt, condition_labels, image_counts, video_counts, video_timestamps) + return input_ids, text_token_tags, pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw + + def process(self, pipe: MiniMaxH3Pipeline, prompt, keyframes=None, ref_blocks=None, height=None, width=None): + pipe.load_models_to_device(self.onload_model_names) + pixel_values = image_grid_thw = pixel_values_videos = video_grid_thw = None + if ref_blocks: + input_ids, text_token_tags, pixel_values, image_grid_thw, pixel_values_videos, video_grid_thw = self.preprocess_ref_blocks(pipe, prompt, ref_blocks) + elif keyframes: + keyframes = [img.convert("RGB").resize((width, height), Image.LANCZOS) for img in keyframes] + pixel_values, image_grid_thw, image_counts = image_token_counts(pipe.processor, keyframes) + input_ids, text_token_tags = presentation_fl2va(pipe.tokenizer, prompt, image_counts) + else: + input_ids, text_token_tags = presentation_t2va(pipe.tokenizer, prompt) + + ids = input_ids.unsqueeze(0).to(pipe.device) + kwargs = {"input_ids": ids, "attention_mask": torch.ones_like(ids)} + if pixel_values is not None: + kwargs["pixel_values"] = pixel_values.to(pipe.device, pipe.torch_dtype) + kwargs["image_grid_thw"] = image_grid_thw.to(pipe.device, torch.long) + if pixel_values_videos is not None: + kwargs["pixel_values_videos"] = pixel_values_videos.to(pipe.device, pipe.torch_dtype) + kwargs["video_grid_thw"] = video_grid_thw.to(pipe.device, torch.long) + hidden = pipe.text_encoder(**kwargs) + + return {"prompt_embeds": hidden.to(pipe.device, pipe.torch_dtype), "text_token_tags": text_token_tags.view(-1).to(pipe.device, torch.long)} + + +class MiniMaxH3Unit_InputVideoEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("input_video",), + output_params=("video_latents", "input_latents"), + onload_model_names=("video_vae",) + ) + + def process(self, pipe: MiniMaxH3Pipeline, input_video): + if input_video is None or not pipe.scheduler.training: + return {} + pipe.load_models_to_device(self.onload_model_names) + frames_tensor = pipe.preprocess_video(input_video, torch_dtype=torch.float32, min_value=0, device=pipe.device) + latents = pipe.video_vae.encode_video(frames_tensor, dtype=pipe.torch_dtype).to(pipe.torch_dtype) + return {"video_latents": latents, "input_latents": latents} + + +class MiniMaxH3Unit_InputAudioEmbedder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("input_audio",), + output_params=("audio_latents", "audio_input_latents"), + onload_model_names=("audio_vae",) + ) + + def process(self, pipe: MiniMaxH3Pipeline, input_audio): + if input_audio is None or not pipe.scheduler.training: + return {} + pipe.load_models_to_device(self.onload_model_names) + waveform, sample_rate = input_audio + waveform = waveform.squeeze(0) if waveform.dim() == 3 else waveform + assert waveform.dim() == 2, "waveform must be in shape (C, T)" + waveform = resample_waveform(convert_to_stereo(waveform).float(), sample_rate, pipe.audio_vae.sample_rate) # [C, T] + latents = pipe.audio_vae.encode_audio(waveform[:2].to(pipe.device), dtype=pipe.torch_dtype).to(pipe.torch_dtype) # [C, 32, T] + return {"audio_latents": latents, "audio_input_latents": latents} + + +class MiniMaxH3Unit_KeyframeEncoder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("keyframes", "keyframe_indices", "video_latents", "rand_device", "seed", "height", "width"), + output_params=("keyframe_cond_anchor",), + onload_model_names=("video_vae",) + ) + + def process(self, pipe: MiniMaxH3Pipeline, keyframes, keyframe_indices, video_latents, rand_device, seed, height, width): + if keyframes is None: + return {} + assert keyframe_indices is not None and len(keyframes) == len(keyframe_indices), "keyframe_indices must be provided when keyframes is not None" + assert all(idx in (0, -1) for idx in keyframe_indices), "keyframe_indices must be within the range of keyframes (0 or -1)" + pipe.load_models_to_device(self.onload_model_names) + # Encode keyframes + all_cond_rows = [] + for img in keyframes: + img_tensor = pipe.preprocess_image(img.convert("RGB").resize((width, height), Image.LANCZOS), torch_dtype=torch.float32, min_value=0) + z_norm = pipe.video_vae.encode_video(img_tensor, dtype=pipe.torch_dtype, process_image=True) # [1,24,1,H',W'] + rows = patchify_video(z_norm) + all_cond_rows.append(rows) + clean_cond_rows = torch.cat(all_cond_rows, dim=0).to(device=pipe.device, dtype=pipe.torch_dtype) + if pipe.imgvid_cond_noise_aug == 1.0: + keyframe_cond_anchor = clean_cond_rows + else: + video_latent_t, latent_h, latent_w = (int(x) for x in video_latents.shape[2:]) + ts = torch.tensor(pipe.imgvid_cond_noise_aug, dtype=pipe.torch_dtype, device=pipe.device) + noise = pipe.generate_noise((1, 24, video_latent_t + len(keyframes), latent_h, latent_w), seed, rand_device, pipe.torch_dtype)[:,:,:1] + noise_rows = patchify_video(noise).to(device=pipe.device, dtype=pipe.torch_dtype) + frame_rows = (latent_h // 2) * (latent_w // 2) + parts = [ + ts * clean_cond_rows[i * frame_rows:(i + 1) * frame_rows] + (1.0 - ts) * noise_rows + for i in range(len(keyframes)) + ] + keyframe_cond_anchor = torch.cat(parts, dim=0) + return {"keyframe_cond_anchor": keyframe_cond_anchor.contiguous()} + + +class MiniMaxH3Unit_ReferenceEncoder(PipelineUnit): + def __init__(self): + super().__init__( + input_params=("references", "seed", "video_latents", "num_frames", "ref_image_short_edge", "ref_video_short_edge", "ref_video_max_pixels"), + output_params=("ref_blocks", "ref_visual_anchor", "ref_audio_anchor"), + onload_model_names=("video_vae", "audio_vae") + ) + + @staticmethod + def _nearest_multiple(value: float, multiple: int) -> int: + return max(multiple, int(round(float(value) / multiple)) * multiple) + + def _resolve_reference_image_shape(self, pipe: MiniMaxH3Pipeline, width: int, height: int, short_edge: int): + scale = short_edge * 1.0 / min(width, height) + return self._nearest_multiple(width * scale, pipe.width_division_factor), self._nearest_multiple(height * scale, pipe.height_division_factor) + + def _resolve_reference_video_shape(self, pipe: MiniMaxH3Pipeline, width: int, height: int, short_edge: int, max_pixels: int): + scale = min(short_edge * 1.0 / min(width, height), float(np.sqrt(max_pixels / (width * height)))) + return self._nearest_multiple(width * scale, pipe.width_division_factor), self._nearest_multiple(height * scale, pipe.height_division_factor) + + def _trim_reference_video_length(self, pipe: MiniMaxH3Pipeline, frame_count: int) -> int: + frame_count = int(frame_count) + factor, remainder = pipe.time_division_factor, pipe.time_division_remainder + chunks = max(1, (frame_count - remainder) // factor) + used = chunks * factor + remainder + assert used <= frame_count, f"cannot trim {frame_count} reference video frames down to a valid " f"length: at least {used} frames are required" + return used + + def _encode_image_ref(self, pipe, img: Image.Image, short_edge: int): + target_w, target_h = self._resolve_reference_image_shape(pipe, *img.size, short_edge) + img = img.convert("RGB").resize((target_w, target_h), Image.LANCZOS) + img_tensor = pipe.preprocess_image(img, torch_dtype=torch.float32, min_value=0) + z = pipe.video_vae.encode_video(img_tensor.to(pipe.device), dtype=pipe.torch_dtype, process_image=True,) # [1,24,1,H',W'] + return patchify_video(z), z.shape[-2], z.shape[-1], img + + def _encode_video_ref(self, pipe, frames, target_frame_count: int, short_edge: int, max_pixels: int): + target_w, target_h = self._resolve_reference_video_shape(pipe, *frames[0].size, short_edge, max_pixels) + frames = [f.resize((target_w, target_h), Image.LANCZOS).convert("RGB") for f in frames] + frames = frames[: self._trim_reference_video_length(pipe, min(len(frames), target_frame_count))] + frames_tensor = pipe.preprocess_video(frames, torch_dtype=torch.float32, min_value=0, device=pipe.device) + z = pipe.video_vae.encode_video(frames_tensor, dtype=pipe.torch_dtype, process_image=False,) # [1,24,T',H',W'] + return patchify_video(z), int(z.shape[2]), int(z.shape[3]), int(z.shape[4]), frames + + def _encode_audio_ref(self, pipe, waveform, sample_rate: int): + waveform = waveform.squeeze(0) if waveform.dim() == 3 else waveform + assert waveform.dim() == 2, "waveform must be in shape (C, T)" + waveform = resample_waveform(convert_to_stereo(waveform).float(), sample_rate, pipe.audio_vae.sample_rate) # [C, T] + latent = pipe.audio_vae.encode_audio(waveform[:2].to(pipe.device), dtype=pipe.torch_dtype) # [C, 32, T] + return pack_audio(latent), latent.shape[-1] + + @staticmethod + def _require(ref, key, kind): + value = ref.get(key) + assert value is not None, f"reference type {kind!r} requires field {key!r}" + return value + + def _build_block(self, pipe, ref, target_frame_count, ref_image_short_edge, ref_video_short_edge, ref_video_max_pixels): + kind = ref["type"] + if kind == "image": + rows, lh, lw, prepared = self._encode_image_ref(pipe, self._require(ref, "image", kind), ref_image_short_edge) + return {"kind": kind, "visual_clean": rows, "latent_t": 1, "latent_h": lh, "latent_w": lw, "prepared_image": prepared, "ref_audio_t": 0} + if kind in ("video", "video_audio"): + if kind == "video" and ref.get("audio") is not None: + raise ValueError("reference type 'video' is silent; use 'video_audio' to pass a soundtrack") + frames = self._require(ref, "video", kind) + if kind == "video_audio": + audio_waveform = self._require(ref, "audio", kind) + audio_sample_rate = int(self._require(ref, "sample_rate", kind)) + rows, lt, lh, lw, prepared = self._encode_video_ref(pipe, frames, target_frame_count, ref_video_short_edge, ref_video_max_pixels) + block = {"kind": kind, "visual_clean": rows, "latent_t": lt, "latent_h": lh, "latent_w": lw, "prepared_frames": prepared, "ref_audio_t": 0} + if kind == "video_audio": + audio_rows, ref_audio_t = self._encode_audio_ref(pipe, audio_waveform, audio_sample_rate) + block["audio_clean"], block["ref_audio_t"] = audio_rows, ref_audio_t + return block + if kind == "audio": + rows, ref_audio_t = self._encode_audio_ref(pipe, self._require(ref, "audio", kind), int(self._require(ref, "sample_rate", kind))) + return {"kind": kind, "audio_clean": rows, "ref_audio_t": ref_audio_t} + + def process(self, pipe: MiniMaxH3Pipeline, references, seed, video_latents, num_frames, ref_image_short_edge, ref_video_short_edge, ref_video_max_pixels): + if not references: + return {} + pipe.load_models_to_device(self.onload_model_names) + seed = int(seed) if seed is not None else 42 + device = pipe.device + + ref_blocks_out = [self._build_block(pipe, ref, num_frames, ref_image_short_edge, ref_video_short_edge, ref_video_max_pixels) for ref in references] + visual_blocks = [b for b in ref_blocks_out if "visual_clean" in b] + imgvid_cond_num_frames = len(visual_blocks) + visual_parts = [] + for block in visual_blocks: + clean = block.pop("visual_clean").to(device=device, dtype=pipe.torch_dtype) + if pipe.imgvid_cond_noise_aug == 1.0: + anchor = clean + else: + full_t = video_latents.shape[2] + imgvid_cond_num_frames + noise_shape = (1, 24, full_t, int(block["latent_h"]), int(block["latent_w"])) + noise = pipe.generate_noise(noise_shape, seed=seed, rand_device="cpu", rand_torch_dtype=pipe.torch_dtype, device="cpu", torch_dtype=pipe.torch_dtype) + noise_rows = patchify_video(noise[:, :, : int(block["latent_t"])]).to(device=device, dtype=pipe.torch_dtype) + ts = torch.tensor(pipe.imgvid_cond_noise_aug, dtype=pipe.torch_dtype, device=device) + anchor = ts * clean + (1.0 - ts) * noise_rows + visual_parts.append(anchor) + + audio_parts = [] + for block in ref_blocks_out: + if "audio_clean" not in block or int(block["ref_audio_t"]) <= 0: + block.pop("audio_clean", None) + continue + clean = block.pop("audio_clean").to(device=device, dtype=pipe.torch_dtype) + if pipe.audio_cond_noise_aug == 1.0: + anchor = clean + else: + noise = pipe.generate_noise(clean.shape, seed=seed + 1, rand_device="cpu", rand_torch_dtype=pipe.torch_dtype, device=device, torch_dtype=pipe.torch_dtype) + ts = torch.tensor(pipe.audio_cond_noise_aug, dtype=pipe.torch_dtype, device=device) + anchor = ts * clean + (1.0 - ts) * noise + audio_parts.append(anchor) + + return { + "ref_blocks": ref_blocks_out, + "ref_visual_anchor": torch.cat(visual_parts, dim=0) if visual_parts else None, + "ref_audio_anchor": torch.cat(audio_parts, dim=0) if audio_parts else None, + } + + +class MiniMaxH3Unit_PackedSequenceBuilder(PipelineUnit): + _INTERP = 32 + _T_GROUP = 5 + _FRAME_PER_TOKEN = (1, 4, 4, 4, 4) + _FRAME_RESCALE = 5.0 / 3.0 + _SEQ_ALIGN = 64 + + def __init__(self): + super().__init__( + seperate_cfg=True, + input_params_posi={"prompt_embeds": "prompt_embeds", "text_token_tags": "text_token_tags"}, + input_params_nega={"prompt_embeds": "prompt_embeds", "text_token_tags": "text_token_tags"}, + input_params=("video_latents", "audio_latents", "keyframe_cond_anchor", "keyframe_indices", "ref_blocks"), + output_params=("packed",) + ) + + @staticmethod + def _to_device(packed: dict, device) -> dict: + return {k: v.to(device) if torch.is_tensor(v) else v for k, v in packed.items()} + + def _aligned_seq_len(self, used: int) -> int: + return ((used + self._SEQ_ALIGN - 1) // self._SEQ_ALIGN) * self._SEQ_ALIGN + + def _axis_from_sqrt_area(self, dim: int, patch: int, sqrt_area: float) -> torch.Tensor: + ratio = dim / sqrt_area + left = (1.0 - ratio) * 0.5 + right = left + ratio + grid = np.linspace(left, right, dim // patch, endpoint=False) * self._INTERP + return torch.from_numpy(grid).to(torch.float64) + + def _video_t_grid(self, n: int, origin: float) -> torch.Tensor: + spans = torch.tensor([self._FRAME_RESCALE * self._FRAME_PER_TOKEN[k % self._T_GROUP] for k in range(n)], dtype=torch.float64) + return origin + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)]) + + def _temporal_position_span(self, temporal_length: int) -> float: + spans = np.ones(int(temporal_length), dtype=np.float64) * self._FRAME_RESCALE + for token_index in range(self._T_GROUP): + spans[token_index::self._T_GROUP] *= self._FRAME_PER_TOKEN[token_index] + return float(spans.sum()) + + def _video_t_span(self, n: int) -> float: + return sum(self._FRAME_RESCALE * self._FRAME_PER_TOKEN[k % self._T_GROUP] for k in range(n)) + + def _frame_grid(self, latent_h: int, latent_w: int, sqrt_area): + h_grid = self._axis_from_sqrt_area(latent_h, 2, sqrt_area) + w_grid = self._axis_from_sqrt_area(latent_w, 2, sqrt_area) + hh, ww = torch.meshgrid(h_grid, w_grid, indexing="ij") + return torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1), w_grid + + def _video_grid(self, latent_t: int, frame: torch.Tensor, origin: float) -> torch.Tensor: + video_g = torch.empty(latent_t, frame.shape[0], 3, dtype=torch.float64) + video_g[:, :, 0] = self._video_t_grid(latent_t, origin)[:, None] + video_g[:, :, 1:] = frame[None] + return video_g.reshape(-1, 3) + + def _audio_w_axis(self, w_grid: torch.Tensor, audio_t: int, audio_rows: int) -> torch.Tensor: + return torch.cat([torch.full((audio_t,), float(w_grid[0]), dtype=torch.float64), + torch.full((audio_rows - audio_t,), float(w_grid[-1]), dtype=torch.float64)]) + + def _build_packed_fl2va(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframe_indices, audio_channel=2): + """fl2va layout: [text | cond | audio | video | pad].""" + frame_rows = (latent_h // 2) * (latent_w // 2) + video_rows = latent_t * frame_rows + audio_rows = audio_t * audio_channel + num_keyframes = len(keyframe_indices) + cond_rows = num_keyframes * frame_rows + used = text_len + cond_rows + audio_rows + video_rows + seq_len = self._aligned_seq_len(used) + + text_sl = slice(0, text_len) + cond_sl = slice(text_len, text_len + cond_rows) + audio_sl = slice(cond_sl.stop, cond_sl.stop + audio_rows) + video_sl = slice(audio_sl.stop, audio_sl.stop + video_rows) + + # img_pos covers both cond AND video rows, conditions first + img_pos = torch.cat([torch.arange(cond_sl.start, cond_sl.stop), torch.arange(video_sl.start, video_sl.stop)]) + audio_pos = torch.arange(audio_sl.start, audio_sl.stop) + + g = torch.zeros(seq_len, 3, dtype=torch.float64) + g[text_sl, 0] = torch.arange(text_len, dtype=torch.float64) + + sqrt_area = np.sqrt(latent_h * latent_w) + frame, w_grid = self._frame_grid(latent_h, latent_w, sqrt_area) + + # Condition rows: temporal position depends on frame_index + temporal_span = self._temporal_position_span(latent_t) + for i, idx in enumerate(keyframe_indices): + sl = slice(i * frame_rows, (i + 1) * frame_rows) + if idx == 0: + cond_t = float(text_len) + else: # idx == -1 + cond_t = float(text_len) + temporal_span - self._FRAME_RESCALE + cond_g = torch.empty(frame_rows, 3, dtype=torch.float64) + cond_g[:, 0] = cond_t + cond_g[:, 1:] = frame + g[cond_sl.start + sl.start: cond_sl.start + sl.stop] = cond_g + + g[video_sl] = self._video_grid(latent_t, frame, float(text_len)) + g[audio_sl, 0] = (float(text_len) + torch.arange(audio_t, dtype=torch.float64)).repeat(audio_channel) + g[audio_sl, 2] = self._audio_w_axis(w_grid, audio_t, audio_rows) + + token_tags = torch.full((seq_len,), -1, dtype=torch.long) + token_tags[text_sl] = 1 + token_tags[audio_sl] = 2 + token_tags[img_pos] = 0 # both cond and video rows are tagged as video (0) + + return { + "img_pos": img_pos, "audio_pos": audio_pos, "text_pos": torch.arange(0, text_len), + "img_position_ids": g[None], "token_tags": token_tags, + "cu_seqlens": torch.tensor([0, used, seq_len], dtype=torch.int32), "seq_len": seq_len, + } + + def _build_packed_ref2va(self, text_len, latent_t, latent_h, latent_w, audio_t, ref_blocks, audio_channel=2): + """ref2va layout: [text | ref_0 | ref_1 | ... | target_audio | target_video | pad]""" + ph, pw = latent_h // 2, latent_w // 2 + target_frame_rows = ph * pw + target_video_rows = latent_t * target_frame_rows + target_audio_rows = audio_t * audio_channel + + block_dims, total_ref_visual_rows, total_ref_audio_rows = [], 0, 0 + for b in ref_blocks: + kind = b["kind"] + info = {"kind": kind, "visual_rows": 0, "audio_rows": 0, "ref_audio_t": int(b.get("ref_audio_t", 0))} + if kind in ("image", "video", "video_audio"): + lt_r, lh_r, lw_r = int(b["latent_t"]), int(b["latent_h"]), int(b["latent_w"]) + info.update(visual_rows=lt_r * (lh_r // 2) * (lw_r // 2), latent_t=lt_r, latent_h=lh_r, latent_w=lw_r) + total_ref_visual_rows += info["visual_rows"] + info["audio_rows"] = info["ref_audio_t"] * audio_channel + total_ref_audio_rows += info["audio_rows"] + elif kind == "audio": + info["audio_rows"] = info["ref_audio_t"] * audio_channel + total_ref_audio_rows += info["audio_rows"] + else: + raise ValueError(f"unknown ref kind: {kind}") + block_dims.append(info) + + used = text_len + total_ref_visual_rows + total_ref_audio_rows + target_audio_rows + target_video_rows + seq_len = self._aligned_seq_len(used) + + g = torch.zeros(seq_len, 3, dtype=torch.float64) + g[0:text_len, 0] = torch.arange(text_len, dtype=torch.float64) + token_tags = torch.full((seq_len,), -1, dtype=torch.long) + token_tags[0:text_len] = 1 + + # Iterate through ref blocks, placing them contiguously. + cursor, t_cursor = text_len, float(text_len) + ref_visual_pos_parts, ref_audio_pos_parts = [], [] + # Target w_grid used for audio W-axis (channel separation) + target_sqrt_area = float(np.sqrt(latent_h * latent_w)) + target_w_grid = self._axis_from_sqrt_area(latent_w, 2, target_sqrt_area) + + for info in block_dims: + kind = info["kind"] + if kind == "image": + v_rows, lh_r, lw_r = info["visual_rows"], info["latent_h"], info["latent_w"] + # Own spatial grid + sqrt_area = float(np.sqrt(lh_r * lw_r)) + frame, w_grid = self._frame_grid(lh_r, lw_r, sqrt_area) + sl = slice(cursor, cursor + v_rows) + g[sl, 0] = t_cursor + g[sl, 1:] = frame + token_tags[sl] = 0 + ref_visual_pos_parts.append(torch.arange(sl.start, sl.stop)) + cursor += v_rows + t_cursor += 1.0 + + elif kind in ("video", "video_audio"): + a_rows, v_rows, ref_at = info["audio_rows"], info["visual_rows"], info["ref_audio_t"] + lt_r, lh_r, lw_r = info["latent_t"], info["latent_h"], info["latent_w"] + sqrt_area = float(np.sqrt(lh_r * lw_r)) + frame, rv_w_grid = self._frame_grid(lh_r, lw_r, sqrt_area) + + audio_sl = slice(cursor, cursor + a_rows) + visual_sl = slice(audio_sl.stop, audio_sl.stop + v_rows) + + a_t_grid = t_cursor + torch.arange(ref_at, dtype=torch.float64) + g[audio_sl, 0] = a_t_grid.repeat(audio_channel) + if ref_at: + # W-axis uses THIS reference video's own grid, not the target's. + g[audio_sl, 2] = self._audio_w_axis(rv_w_grid, ref_at, a_rows) + token_tags[audio_sl] = 2 + ref_audio_pos_parts.append(torch.arange(audio_sl.start, audio_sl.stop)) + + g[visual_sl] = self._video_grid(lt_r, frame, t_cursor) + token_tags[visual_sl] = 0 + ref_visual_pos_parts.append(torch.arange(visual_sl.start, visual_sl.stop)) + + cursor = visual_sl.stop + t_cursor += max(float(ref_at), self._video_t_span(lt_r)) + + elif kind == "audio": + a_rows, ref_at = info["audio_rows"], info["ref_audio_t"] + sl = slice(cursor, cursor + a_rows) + a_t_grid = t_cursor + torch.arange(ref_at, dtype=torch.float64) + g[sl, 0] = a_t_grid.repeat(audio_channel) + if ref_at: + g[sl, 2] = self._audio_w_axis(target_w_grid, ref_at, a_rows) + token_tags[sl] = 2 + ref_audio_pos_parts.append(torch.arange(sl.start, sl.stop)) + cursor += a_rows + t_cursor += float(ref_at) + + # Target audio + target video after ref blocks + target_audio_sl = slice(cursor, cursor + target_audio_rows) + target_video_sl = slice(target_audio_sl.stop, target_audio_sl.stop + target_video_rows) + + # Target spatial grid (own) + frame_t, w_grid_t = self._frame_grid(latent_h, latent_w, target_sqrt_area) + g[target_video_sl] = self._video_grid(latent_t, frame_t, t_cursor) + + target_audio_t_grid = t_cursor + torch.arange(audio_t, dtype=torch.float64) + g[target_audio_sl, 0] = target_audio_t_grid.repeat(audio_channel) + g[target_audio_sl, 2] = self._audio_w_axis(w_grid_t, audio_t, target_audio_rows) + + # Build combined img_pos / audio_pos + target_video_pos = torch.arange(target_video_sl.start, target_video_sl.stop) + target_audio_pos = torch.arange(target_audio_sl.start, target_audio_sl.stop) + + if ref_visual_pos_parts: + img_pos = torch.cat(ref_visual_pos_parts + [target_video_pos]) + else: + img_pos = target_video_pos + if ref_audio_pos_parts: + audio_pos = torch.cat(ref_audio_pos_parts + [target_audio_pos]) + else: + audio_pos = target_audio_pos + + token_tags[img_pos] = 0 + token_tags[target_audio_sl] = 2 + + text_pos = torch.arange(0, text_len) + cu = torch.tensor([0, used, seq_len], dtype=torch.int32) + + return { + "img_pos": img_pos, "audio_pos": audio_pos, "text_pos": text_pos, + "img_position_ids": g[None], "token_tags": token_tags, "cu_seqlens": cu, + "seq_len": seq_len, + } + + def process(self, pipe: MiniMaxH3Pipeline, prompt_embeds, video_latents, audio_latents, text_token_tags=None, keyframe_cond_anchor=None, keyframe_indices=None, ref_blocks=None): + text_len = prompt_embeds.shape[0] + video_latent_t, latent_h, latent_w = video_latents.shape[2:] + audio_latent_t = audio_latents.shape[-1] + if ref_blocks is not None: + packed = self._build_packed_ref2va(text_len, video_latent_t, latent_h, latent_w, audio_latent_t, ref_blocks) + else: + packed = self._build_packed_fl2va(text_len, video_latent_t, latent_h, latent_w, audio_latent_t, keyframe_indices if keyframe_cond_anchor is not None else []) + + packed["token_tags"][packed["text_pos"]] = text_token_tags.cpu() + if pipe.device == "mps": + packed["img_position_ids"] = packed["img_position_ids"].to(torch.float32) + return {"packed": self._to_device(packed, pipe.device)} + + +def model_fn_minimax_h3( + dit, + video_latents, + audio_latents, + packed, + prompt_embeds, + t_video, + t_audio, + keyframe_cond_anchor=None, + ref_visual_anchor=None, + ref_audio_anchor=None, + imgvid_cond_noise_aug=0.999, + audio_cond_noise_aug=1.0, + use_gradient_checkpointing=False, + use_gradient_checkpointing_offload=False, + **kwargs, +): + dtype, device = video_latents.dtype, video_latents.device + f, h, w = video_latents.shape[2:] + audio_channel, audio_t = audio_latents.shape[0], audio_latents.shape[-1] + video_rows = patchify_video(video_latents) + audio_rows = pack_audio(audio_latents) + + img_pos = packed["img_pos"] + audio_pos = packed["audio_pos"] + text_pos = packed["text_pos"] + cu = packed["cu_seqlens"] + seq_len = packed["seq_len"] + text_len = text_pos.shape[0] + # Video Sequence + cond_anchor = ref_visual_anchor if ref_visual_anchor is not None else keyframe_cond_anchor + cond_rows_count = 0 if cond_anchor is None else cond_anchor.shape[0] + x = torch.zeros(1, seq_len, 96, dtype=dtype, device=device) + x[0].index_copy_(0, img_pos[cond_rows_count:], video_rows) + if cond_anchor is not None: + x[0].index_copy_(0, img_pos[:cond_rows_count], cond_anchor) + # Audio Sequence + ref_audio_rows_count = 0 if ref_audio_anchor is None else ref_audio_anchor.shape[0] + audio_x = torch.zeros(1, seq_len, 32, dtype=dtype, device=device) + audio_x[0].index_copy_(0, audio_pos[ref_audio_rows_count:], audio_rows) + if ref_audio_anchor is not None: + audio_x[0].index_copy_(0, audio_pos[:ref_audio_rows_count], ref_audio_anchor) + + timesteps = torch.full((seq_len,), float(t_video), dtype=torch.float32, device=device) + timesteps[audio_pos] = float(t_audio) + timesteps[img_pos[:cond_rows_count]] = max(float(t_video), imgvid_cond_noise_aug) + timesteps[audio_pos[:ref_audio_rows_count]] = max(float(t_audio), audio_cond_noise_aug) + unique_timesteps, inverse_indices = torch.unique(timesteps, sorted=True, return_inverse=True) + + refiner_cu = torch.tensor([0, text_len, text_len], dtype=torch.int32, device=device) + v_video_rows, v_audio_rows = dit( + x=x, + audio_x=audio_x, + img_position_ids=packed["img_position_ids"], + unique_timesteps=unique_timesteps, + inverse_indices=inverse_indices, + token_tags=packed["token_tags"], + prompt_embeds=prompt_embeds, + img_pos_info={"position_ids": img_pos}, + audio_pos_info={"position_ids": audio_pos}, + text_pos_info={"position_ids": text_pos}, + img_pos_for_infer_output_info={"position_ids": img_pos}, + packed_seq_params={"cu_seqlens_q": cu, "max_seqlen_q": int(cu[1])}, + refiner_packed_seq_params={"cu_seqlens_q": refiner_cu, "max_seqlen_q": text_len}, + update_mask=None, + skip_mask_out_condition=True, + use_gradient_checkpointing=use_gradient_checkpointing, + use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, + ) + + v_video_rows = v_video_rows[cond_rows_count:] + v_audio_rows = v_audio_rows[ref_audio_rows_count:] + + v_video = unpatchify_video(v_video_rows, f, h, w) + v_audio = unpack_audio(v_audio_rows, audio_channel, audio_t) + return -v_video, -v_audio diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_dit.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_dit.py new file mode 100644 index 0000000000..2a4684e7a2 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_dit.py @@ -0,0 +1,398 @@ +from __future__ import annotations + +import math + +import torch +import torch.nn as nn + +from ..core.attention import attention_forward +from ..core.gradient import gradient_checkpoint_forward + +MINIMAX_H3_ADALN_MODALITY_NUM = 3 +_PATCH_T, _PATCH_H, _PATCH_W = 1, 2, 2 + + +def patchify_video(latent: torch.Tensor) -> torch.Tensor: + # [1,24,T,H,W] -> [T*(H/2)*(W/2), 96] + b, c, ft, fh, fw = (int(x) for x in latent.shape) + t, h, w = ft // _PATCH_T, fh // _PATCH_H, fw // _PATCH_W + packed = latent.reshape(b, c, t, _PATCH_T, h, _PATCH_H, w, _PATCH_W) + packed = torch.einsum("nctrhpwq->nthwcrpq", packed) + return packed.reshape(b * t * h * w, c * _PATCH_T * _PATCH_H * _PATCH_W).contiguous() + + +def unpatchify_video(rows: torch.Tensor, ft: int, fh: int, fw: int, channel: int = 24) -> torch.Tensor: + # [T*(H/2)*(W/2), 96] -> [1,24,T,H,W] (inverse of patchify_video; ft/fh/fw match its input) + t, h, w = ft // _PATCH_T, fh // _PATCH_H, fw // _PATCH_W + packed = rows.reshape(-1, t, h, w, channel, _PATCH_T, _PATCH_H, _PATCH_W) + latent = torch.einsum("nthwcrpq->nctrhpwq", packed) + return latent.reshape(-1, channel, ft, fh, fw).contiguous() + + +def pack_audio(latent: torch.Tensor) -> torch.Tensor: + # [audio_channel, 32, T] -> [audio_channel*T, 32] (channel-major) + ac, ld, steps = (int(x) for x in latent.shape) + return latent.permute(0, 2, 1).reshape(ac * steps, ld).contiguous() + + +def unpack_audio(rows: torch.Tensor, audio_channel: int, steps: int, latent_dim: int = 32) -> torch.Tensor: + # [audio_channel*T, 32] -> [audio_channel, 32, T] + return rows.reshape(audio_channel, steps, latent_dim).permute(0, 2, 1).contiguous() + + +def _norm(size: int, *, eps: float) -> nn.RMSNorm: + return nn.RMSNorm(size, eps=eps) + + +def _rotate_half(x: torch.Tensor) -> torch.Tensor: + x1, x2 = torch.chunk(x, 2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + + +def _apply_rope(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: + rot_dim = freqs.shape[-1] + x_rot, x_pass = x[..., :rot_dim], x[..., rot_dim:] + cos = torch.cos(freqs).to(x.dtype).unsqueeze(1) + sin = torch.sin(freqs).to(x.dtype).unsqueeze(1) + x_rot = (x_rot * cos) + (_rotate_half(x_rot) * sin) + return torch.cat((x_rot, x_pass), dim=-1) + + +def _modulate_scale_shift(x, shift, scale, indices): + # Cast back to x: index_select on the AdaLN params can promote the expression. + return (x * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)).to(x.dtype) + + +def _modulate_gate(x, gate, other, indices): + return (x + gate.index_select(0, indices) * other).to(x.dtype) + + +def _sdpa_varlen_attention(q, k, v, cu_seqlens, softmax_scale): + out = torch.empty_like(q) + bounds = cu_seqlens.tolist() + for start, stop in zip(bounds[:-1], bounds[1:]): + if stop == start: + continue + seg_q = q[start:stop].transpose(0, 1).unsqueeze(0) + seg_k = k[start:stop].transpose(0, 1).unsqueeze(0) + seg_v = v[start:stop].transpose(0, 1).unsqueeze(0) + seg_out = attention_forward(seg_q, seg_k, seg_v, scale=softmax_scale) + out[start:stop] = seg_out.squeeze(0).transpose(0, 1) + return out + + +class MiniMaxH3Rope(nn.Module): + def __init__(self, inv_freq_len: int) -> None: + super().__init__() + self.inv_freq_len = inv_freq_len + self.inv_freq = nn.Parameter(self._build_inv_freq()) + + def _build_inv_freq(self, device=None) -> torch.Tensor: + steps = torch.arange(0, self.inv_freq_len, dtype=torch.float32, device=device) + return 1.0 / (10000.0 ** (steps / self.inv_freq_len)) + + def forward(self, img_position_ids: torch.Tensor) -> torch.Tensor: + if img_position_ids.dim() != 3 or img_position_ids.shape[0] != 1: + raise ValueError(f"img_position_ids must be [1, S, 3], got {list(img_position_ids.shape)}") + pos = img_position_ids[0].to(torch.float32) + inv_freq = self._build_inv_freq(pos.device) + per_axis = pos.unsqueeze(-1) * inv_freq.view(1, 1, -1) + t_f, h_f, w_f = per_axis.unbind(dim=1) + half = torch.cat((t_f, h_f, w_f), dim=-1) + return torch.cat((half, half), dim=-1) + + +class MiniMaxH3TimeEmbedder(nn.Module): + def __init__(self, timestep_input_dim, time_embed_hidden_size, time_embed_dim): + super().__init__() + self.frequency_embedding_size = timestep_input_dim + self.proj_in = nn.Linear(timestep_input_dim, time_embed_hidden_size, bias=True) + self.proj_out = nn.Linear(time_embed_hidden_size, time_embed_dim, bias=True) + + def forward(self, t: torch.Tensor, *, dtype: torch.dtype) -> torch.Tensor: + half = self.frequency_embedding_size // 2 + freqs = torch.exp( + -math.log(10000.0) * torch.arange(half, dtype=torch.float32, device=t.device) / half + ) + args = t.to(torch.float32)[:, None] * freqs[None] + t_freq = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + hidden = self.proj_in(t_freq.to(dtype)) + hidden = nn.functional.silu(hidden) + return self.proj_out(hidden) + + +class MiniMaxH3Attention(nn.Module): + def __init__(self, hidden_size, num_attention_heads, attention_head_dim, qk_norm_eps): + super().__init__() + self.num_heads = num_attention_heads + self.head_dim = attention_head_dim + inner_dim = self.num_heads * self.head_dim + self.softmax_scale = self.head_dim**-0.5 + self.qkv_proj = nn.Linear(hidden_size, inner_dim * 3, bias=False) + self.q_norm = _norm(attention_head_dim, eps=qk_norm_eps) + self.k_norm = _norm(attention_head_dim, eps=qk_norm_eps) + self.out_proj = nn.Linear(inner_dim, hidden_size, bias=False) + + def forward(self, x, *, rope_freqs, cu_seqlens, max_seqlen=None): + total = x.shape[0] + qkv = self.qkv_proj(x) + qkv = qkv.view(total, self.num_heads, 3, self.head_dim) + q = qkv[:, :, 0, :] + k = qkv[:, :, 1, :] + v = qkv[:, :, 2, :] + q = self.q_norm(q) + k = self.k_norm(k) + if rope_freqs is not None: + q = _apply_rope(q, rope_freqs) + k = _apply_rope(k, rope_freqs) + out = _sdpa_varlen_attention(q, k, v, cu_seqlens=cu_seqlens, softmax_scale=self.softmax_scale) + out = out.reshape(total, self.num_heads * self.head_dim) + return self.out_proj(out) + + +class MiniMaxH3MLP(nn.Module): + def __init__(self, hidden_size, ffn_hidden_size): + super().__init__() + self.fc1 = nn.Linear(hidden_size, ffn_hidden_size * 2, bias=False) + self.fc2 = nn.Linear(ffn_hidden_size, hidden_size, bias=False) + + def forward(self, x): + hidden = self.fc1(x) + gate, up = hidden.chunk(2, dim=-1) + hidden = nn.functional.silu(gate) * up + return self.fc2(hidden) + + +class MiniMaxH3AdalnProj(nn.Module): + def __init__(self, hidden_size, time_embed_dim, out_features, *, expand_ratio, modality_num): + super().__init__() + if out_features != expand_ratio * hidden_size * modality_num: + raise ValueError( + f"adaln out_features mismatch: {out_features} != {expand_ratio}*{hidden_size}*{modality_num}" + ) + self.expand_ratio = expand_ratio + self.modality_num = modality_num + self.hidden_size = hidden_size + self.linear = nn.Linear(time_embed_dim, out_features, bias=True) + + def forward(self, t_emb): + x = nn.functional.silu(t_emb) + x = self.linear(x) + m = x.shape[0] + x = x.view(m * self.modality_num, self.expand_ratio * self.hidden_size) + return tuple(x.chunk(self.expand_ratio, dim=-1)) + + +class MiniMaxH3TokenRefinerBlock(nn.Module): + def __init__(self, hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps): + super().__init__() + self.norm1 = _norm(hidden_size, eps=norm_eps) + self.norm2 = _norm(hidden_size, eps=norm_eps) + self.attn = MiniMaxH3Attention(hidden_size, num_attention_heads, attention_head_dim, qk_norm_eps) + self.mlp = MiniMaxH3MLP(hidden_size, ffn_hidden_size) + + def forward(self, x, *, cu_seqlens, max_seqlen): + x = x + self.attn(self.norm1(x), rope_freqs=None, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) + x = x + self.mlp(self.norm2(x)) + return x + + +class MiniMaxH3TokenRefiner(nn.Module): + def __init__(self, num_layers, hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps, final_norm_eps): + super().__init__() + self.blocks = nn.ModuleList( + [MiniMaxH3TokenRefinerBlock(hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps) for _ in range(num_layers)] + ) + self.final_norm = _norm(hidden_size, eps=final_norm_eps) + + def forward(self, x, *, cu_seqlens, max_seqlen): + for block in self.blocks: + x = block(x, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) + return self.final_norm(x) + + +class MiniMaxH3DiTBlock(nn.Module): + def __init__(self, hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, time_embed_dim, adaln_out_features, norm_eps, qk_norm_eps): + super().__init__() + self.norm1 = _norm(hidden_size, eps=norm_eps) + self.norm2 = _norm(hidden_size, eps=norm_eps) + self.attn = MiniMaxH3Attention(hidden_size, num_attention_heads, attention_head_dim, qk_norm_eps) + self.mlp = MiniMaxH3MLP(hidden_size, ffn_hidden_size) + self.adaln_proj = MiniMaxH3AdalnProj(hidden_size, time_embed_dim, adaln_out_features, expand_ratio=6, modality_num=MINIMAX_H3_ADALN_MODALITY_NUM) + + def forward(self, x, *, t_emb, combined_indices, rope_freqs, cu_seqlens, max_seqlen): + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(t_emb) + residual = x + h = self.norm1(x) + h = _modulate_scale_shift(h, shift_msa, scale_msa, combined_indices) + h = self.attn(h, rope_freqs=rope_freqs, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) + x = _modulate_gate(residual, gate_msa, h, combined_indices) + residual = x + h = self.norm2(x) + h = _modulate_scale_shift(h, shift_mlp, scale_mlp, combined_indices) + h = self.mlp(h) + return _modulate_gate(residual, gate_mlp, h, combined_indices) + + +class MiniMaxH3FinalLayer(nn.Module): + def __init__(self, hidden_size, time_embed_dim, final_adaln_out_features, latents_dim, audio_latents_dim, patch_size, final_norm_eps): + super().__init__() + video_patch_dim = latents_dim * patch_size[0] * patch_size[1] * patch_size[2] + self.norm = _norm(hidden_size, eps=final_norm_eps) + self.adaln_proj = MiniMaxH3AdalnProj(hidden_size, time_embed_dim, final_adaln_out_features, expand_ratio=2, modality_num=1) + self.video_out = nn.Linear(hidden_size, video_patch_dim, bias=True) + self.audio_out = nn.Linear(hidden_size, audio_latents_dim, bias=True) + + def forward(self, x, *, t_emb, inverse_indices): + shift, scale = self.adaln_proj(t_emb) + h = self.norm(x) + h = _modulate_scale_shift(h, shift, scale, inverse_indices) + video = self.video_out(h) + audio = self.audio_out(h) + return video, audio + + +class MiniMaxH3DiT(nn.Module): + _repeated_blocks = ["MiniMaxH3DiTBlock"] + + def __init__( + self, + num_layers: int = 50, + token_refiner_num_layers: int = 2, + hidden_size: int = 5376, + num_attention_heads: int = 56, + attention_head_dim: int = 128, + ffn_hidden_size: int = 14336, + latents_dim: int = 24, + audio_latents_dim: int = 32, + patch_size: tuple = (1, 2, 2), + text_dim: int = 5120, + timestep_input_dim: int = 256, + time_embed_hidden_size: int = 5376, + time_embed_dim: int = 2688, + adaln_out_features: int = 96768, + final_adaln_out_features: int = 10752, + rope_inv_freq_len: int = 16, + norm_eps: float = 1e-5, + qk_norm_eps: float = 1e-5, + final_norm_eps: float = 1e-5, + **kwargs, + ): + super().__init__() + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.num_channels_latents = latents_dim + video_patch_dim = latents_dim * patch_size[0] * patch_size[1] * patch_size[2] + + self.video_patch_proj = nn.Linear(video_patch_dim, hidden_size, bias=True) + self.audio_patch_proj = nn.Linear(audio_latents_dim, hidden_size, bias=True) + self.condition_proj = nn.Linear(text_dim, hidden_size, bias=True) + self.time_embedder = MiniMaxH3TimeEmbedder(timestep_input_dim, time_embed_hidden_size, time_embed_dim) + self.rope = MiniMaxH3Rope(rope_inv_freq_len) + self.token_refiner = MiniMaxH3TokenRefiner( + token_refiner_num_layers, hidden_size, num_attention_heads, attention_head_dim, + ffn_hidden_size, norm_eps, qk_norm_eps, final_norm_eps, + ) + self.blocks = nn.ModuleList( + [MiniMaxH3DiTBlock(hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, time_embed_dim, adaln_out_features, norm_eps, qk_norm_eps) for _ in range(num_layers)] + ) + self.final_layer = MiniMaxH3FinalLayer(hidden_size, time_embed_dim, final_adaln_out_features, latents_dim, audio_latents_dim, patch_size, final_norm_eps) + + def _embed(self, *, x, audio_x, text_embeddings_selected, unique_timesteps, img_pos, audio_pos, text_pos, refiner_cu_seqlens, refiner_max_seqlen, seq_len, device): + dtype = text_embeddings_selected.dtype + x_rows = x.view(-1, x.shape[-1]).index_select(0, img_pos).to(dtype) + video_embed = self.video_patch_proj(x_rows) + audio_rows = audio_x.view(-1, audio_x.shape[-1]).index_select(0, audio_pos).to(dtype) + audio_embed = self.audio_patch_proj(audio_rows) + text_rows = text_embeddings_selected.to(device=device) + text_embed = self.condition_proj(text_rows) + text_embed = self.token_refiner(text_embed, cu_seqlens=refiner_cu_seqlens, max_seqlen=refiner_max_seqlen) + + embeddings = torch.zeros((seq_len, self.hidden_size), device=device, dtype=dtype) + embeddings.index_add_(0, text_pos, text_embed.to(dtype)[: text_pos.shape[0]]) + embeddings.index_add_(0, img_pos, video_embed.to(dtype)[: img_pos.shape[0]]) + embeddings.index_add_(0, audio_pos, audio_embed.to(dtype)[: audio_pos.shape[0]]) + + t_emb = self.time_embedder(unique_timesteps, dtype=dtype) + return embeddings, t_emb + + def forward( + self, + x, + audio_x, + img_position_ids, + unique_timesteps, + inverse_indices, + update_mask, + token_tags, + prompt_embeds, + img_pos_info, + audio_pos_info, + text_pos_info, + img_pos_for_infer_output_info, + packed_seq_params, + refiner_packed_seq_params, + use_gradient_checkpointing=False, + use_gradient_checkpointing_offload=False, + update_audio_mask=None, + skip_mask_out_condition=False, + ) -> tuple[torch.Tensor, torch.Tensor]: + inverse_indices = inverse_indices.view(-1).to(torch.long) + token_tags = token_tags.view(-1).to(torch.long) + text_selected = prompt_embeds + + img_pos = img_pos_info["position_ids"].view(-1).to(torch.long) + audio_pos = audio_pos_info["position_ids"].view(-1).to(torch.long) + text_pos = text_pos_info["position_ids"].view(-1).to(torch.long) + infer_out_pos = img_pos_for_infer_output_info["position_ids"].view(-1).to(torch.long) + + cu_seqlens = packed_seq_params["cu_seqlens_q"].to(torch.int32) + max_seqlen = int(packed_seq_params["max_seqlen_q"]) + refiner_cu = refiner_packed_seq_params["cu_seqlens_q"].to(torch.int32) + refiner_max = int(refiner_packed_seq_params["max_seqlen_q"]) + + if x.dim() != 3 or x.shape[0] != 1: + raise ValueError(f"x must be [1, S, C], got {list(x.shape)}") + seq_len = int(x.shape[1]) + device = x.device + + rope_freqs = self.rope(img_position_ids).to(device) + + decoder_input, t_emb = self._embed( + x=x, audio_x=audio_x, text_embeddings_selected=text_selected, + unique_timesteps=unique_timesteps.view(-1).to(device), + img_pos=img_pos.to(device), audio_pos=audio_pos.to(device), text_pos=text_pos.to(device), + refiner_cu_seqlens=refiner_cu.to(device), refiner_max_seqlen=refiner_max, + seq_len=seq_len, device=device, + ) + + combined_indices = (inverse_indices * MINIMAX_H3_ADALN_MODALITY_NUM + token_tags.clamp(min=0)).to(device) + inverse_indices = inverse_indices.to(device) + + hidden = decoder_input + cu_seqlens = cu_seqlens.to(device) + for block in self.blocks: + hidden = gradient_checkpoint_forward( + block, + use_gradient_checkpointing, + use_gradient_checkpointing_offload, + hidden, + t_emb=t_emb, + combined_indices=combined_indices, + rope_freqs=rope_freqs, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + ) + + video_logits, audio_logits = self.final_layer(hidden, t_emb=t_emb, inverse_indices=inverse_indices) + + video_logits = video_logits.index_select(0, infer_out_pos.to(device)) + audio_logits = audio_logits.index_select(0, audio_pos.to(device)) + if not skip_mask_out_condition: + update_mask = update_mask.view(-1).to(device) + if update_mask.shape[0] != video_logits.shape[0]: + raise ValueError(f"update_mask length mismatch: {update_mask.shape[0]} != {video_logits.shape[0]}") + video_logits = video_logits * update_mask.unsqueeze(-1) + if update_audio_mask is not None: + audio_logits = audio_logits * update_audio_mask.view(-1).unsqueeze(-1) + return video_logits, audio_logits diff --git a/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_text_encoder.py b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_text_encoder.py new file mode 100644 index 0000000000..2b3a894524 --- /dev/null +++ b/veomni/models/diffusers/minimax_h3/minimax_h3_core/minimax_h3_text_encoder.py @@ -0,0 +1,231 @@ +import torch +from transformers import Qwen3VLConfig, Qwen3VLModel + +VISION_START = "<|vision_start|>" +VISION_END = "<|vision_end|>" +IMAGE_PAD = "<|image_pad|>" +VIDEO_PAD = "<|video_pad|>" + +PRESENTATION_TEXT_TAG = 1 +PRESENTATION_VIDEO_TAG = 0 + +QWEN_VIDEO_SAMPLE_FPS = 2.0 +QWEN_TEMPORAL_PATCH = 2 +MINIMAX_SUPPORTED_FPS = 24 + +class MiniMaxH3TextEncoder(torch.nn.Module): + def __init__(self, num_retained_layers: int = 50): + super().__init__() + self.num_retained_layers = num_retained_layers + config = Qwen3VLConfig(**{ + "architectures": ["Qwen3VLForConditionalGeneration"], + "image_token_id": 151655, + "model_type": "qwen3_vl", + "text_config": { + "attention_bias": False, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 25600, + "max_position_embeddings": 262144, + "model_type": "qwen3_vl_text", + "num_attention_heads": 64, + "num_hidden_layers": num_retained_layers, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_interleaved": True, + "mrope_section": [24, 20, 20], + "rope_type": "default", + }, + "rope_theta": 5000000, + "use_cache": True, + "vocab_size": 151936, + }, + "tie_word_embeddings": False, + "transformers_version": "4.57.0.dev0", + "video_token_id": 151656, + "vision_config": { + "deepstack_visual_indexes": [8, 16, 24], + "depth": 27, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_vl", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 5120, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2, + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652, + }) + self.model = Qwen3VLModel(config) + self.model.language_model.norm = torch.nn.Identity() + self.config = config + self.image_token_id = config.image_token_id + self.video_token_id = config.video_token_id + + def forward( + self, + input_ids=None, + attention_mask=None, + pixel_values=None, + image_grid_thw=None, + pixel_values_videos=None, + video_grid_thw=None, + **kwargs, + ): + mm_token_type_ids = None + if pixel_values is not None or pixel_values_videos is not None: + mm_token_type_ids = torch.zeros_like(input_ids, dtype=torch.int32) + mm_token_type_ids[input_ids == self.image_token_id] = 1 + mm_token_type_ids[input_ids == self.video_token_id] = 2 + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + mm_token_type_ids=mm_token_type_ids, + output_hidden_states=False, + use_cache=False, + return_dict=True, + **kwargs, + ) + return outputs.last_hidden_state[0] + + +def image_token_counts(processor, images): + vision = processor.image_processor(images=images, return_tensors="pt") + merge = int(processor.image_processor.merge_size) ** 2 + counts = [int(vision["image_grid_thw"][i].prod().item()) // merge for i in range(len(images))] + return vision["pixel_values"], vision["image_grid_thw"], counts + + +def video_token_counts(processor, videos, timestamps_per_video): + vout = processor.video_processor(videos=videos, do_sample_frames=False, return_tensors="pt") + grid = vout["video_grid_thw"] + merge = int(processor.image_processor.merge_size) ** 2 + block_counts, block_timestamps = [], [] + for index, timestamps in enumerate(timestamps_per_video): + n_blocks = int(grid[index, 0]) + per_block = int(grid[index, 1]) * int(grid[index, 2]) // merge + block_counts.append([per_block] * n_blocks) + block_timestamps.append([float(t) for t in timestamps]) + return vout["pixel_values_videos"], grid, block_counts, block_timestamps + + +def _text_ids(tokenizer, text: str) -> list[int]: + return list(tokenizer(text, add_special_tokens=False)["input_ids"]) + + +def _vision_block_ids(tokenizer, pad_token: str, count: int) -> list[int]: + return ( + [tokenizer.convert_tokens_to_ids(VISION_START)] + + [tokenizer.convert_tokens_to_ids(pad_token)] * int(count) + + [tokenizer.convert_tokens_to_ids(VISION_END)] + ) + +class _Presentation: + def __init__(self): + self.ids: list[int] = [] + self.tags: list[int] = [] + + def text(self, token_ids: list[int]): + self.ids += token_ids + self.tags += [PRESENTATION_TEXT_TAG] * len(token_ids) + + def vision(self, token_ids: list[int]): + self.ids += token_ids + self.tags += [PRESENTATION_VIDEO_TAG] * len(token_ids) + + def build(self): + return (torch.tensor(self.ids, dtype=torch.long), torch.tensor(self.tags, dtype=torch.long)) + + +def presentation_t2va(tokenizer, prompt: str): + presentation = _Presentation() + presentation.text(_text_ids(tokenizer, prompt)) + return presentation.build() + + +def presentation_fl2va(tokenizer, prompt: str, image_token_counts): + presentation = _Presentation() + for index, count in enumerate(image_token_counts, start=1): + presentation.text(_text_ids(tokenizer, f": ")) + presentation.vision(_vision_block_ids(tokenizer, IMAGE_PAD, count)) + presentation.text(_text_ids(tokenizer, prompt)) + return presentation.build() + + +def presentation_ref2va(tokenizer, prompt: str, condition_labels, image_token_counts, video_block_token_counts, video_block_timestamps): + if not prompt: + raise ValueError("prompt must be non-empty") + presentation = _Presentation() + image_seen = 0 + video_seen = 0 + for cond_type, ordinal in condition_labels: + if cond_type == "image": + image_seen += 1 + if image_seen > len(image_token_counts): + raise ValueError("image_token_count required for an image reference") + count = int(image_token_counts[image_seen - 1]) + if count <= 0: + raise ValueError("image_token_count required for an image reference") + presentation.text(_text_ids(tokenizer, f": ")) + presentation.vision(_vision_block_ids(tokenizer, IMAGE_PAD, count)) + elif cond_type == "audio": + presentation.text(_text_ids(tokenizer, f"