[Model] Add MiniMax H3 diffusion support (#5691)

This commit is contained in:
Isotr0py authored and GitHub committed 2026-08-03 14:10:40 +08:00
1 parent 727f90958e
commit 24741961eb
45 files changed
+7356 -31

No files matched your search

@@ -0,0 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax H3 audio-video diffusion support."""
from .pipeline_minimax_h3 import (
MiniMaxH3Pipeline,
get_minimax_h3_post_process_func,
)
__all__ = [
"MiniMaxH3Pipeline",
"get_minimax_h3_post_process_func",
]
@@ -0,0 +1,167 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax H3 visual/audio condition-noise augmentation.
The request's condition timestep is applied to both the tensor value and the
DiT timestep. Tokenizer artifacts remain clean
and reusable; this module materializes the fixed noised anchors immediately
before the denoise loop.
"""
from __future__ import annotations
from collections.abc import Sequence
import torch
from .packed_tokens import minimax_h3_patchify_video_latent
# Channel-major packed audio rows always carry a stereo layout.
MINIMAX_H3_AUDIO_COND_CHANNELS = 2
def minimax_h3_imgvid_cond_noise_aug_rows(
clean_rows: torch.Tensor,
*,
condition_shapes: Sequence[Sequence[int]],
target_latent_t: int,
imgvid_cond_num_frames: int,
seed: int,
noise_aug: float,
) -> torch.Tensor:
"""Apply the imgvid-condition RF noise recipe to packed clean rows.
``condition_shapes`` contains ``(latent_t, latent_h, latent_w)`` in packed
visual-condition order. A new CPU generator with the same row seed is
created for every condition. Under the dependent-noise policy, each draw
uses the target temporal length plus the template's imgvid-condition frame
count, then slices the prefix matching the current condition.
"""
noise_aug = float(noise_aug)
if not 0.0 <= noise_aug <= 1.0:
raise ValueError(f"noise_aug must be in [0, 1], got {noise_aug}")
if noise_aug == 1.0:
return clean_rows
if clean_rows.ndim != 2 or int(clean_rows.shape[1]) != 96:
raise ValueError(f"clean imgvid condition rows must have shape [n, 96], got {list(clean_rows.shape)}")
target_latent_t = int(target_latent_t)
imgvid_cond_num_frames = int(imgvid_cond_num_frames)
if target_latent_t <= 0:
raise ValueError(f"target_latent_t must be positive, got {target_latent_t}")
if imgvid_cond_num_frames <= 0:
raise ValueError(
f"imgvid_cond_num_frames must be positive when condition rows exist, got {imgvid_cond_num_frames}"
)
parsed_shapes: list[tuple[int, int, int]] = []
expected_rows = 0
for raw_shape in condition_shapes:
if len(raw_shape) != 3:
raise ValueError(
f"each imgvid condition shape must be (latent_t, latent_h, latent_w), got {list(raw_shape)}"
)
latent_t, latent_h, latent_w = (int(value) for value in raw_shape)
if latent_t <= 0 or latent_h <= 0 or latent_w <= 0:
raise ValueError(f"imgvid condition shape must be positive, got {list(raw_shape)}")
if latent_h % 2 or latent_w % 2:
raise ValueError(
f"imgvid condition spatial dimensions must be divisible by 2, got {(latent_t, latent_h, latent_w)}"
)
parsed_shapes.append((latent_t, latent_h, latent_w))
expected_rows += latent_t * (latent_h // 2) * (latent_w // 2)
if not parsed_shapes:
raise ValueError("condition_shapes must not be empty")
if int(clean_rows.shape[0]) != expected_rows:
raise ValueError(
f"clean imgvid condition rows {int(clean_rows.shape[0])} != shape-derived rows {expected_rows}"
)
output_device = clean_rows.device
clean_rows = clean_rows.detach().to(device="cpu", dtype=torch.float32)
out: list[torch.Tensor] = []
row_offset = 0
timestep = torch.tensor(noise_aug, dtype=torch.float32, device="cpu")
for latent_t, latent_h, latent_w in parsed_shapes:
full_t = target_latent_t + imgvid_cond_num_frames
if full_t < latent_t:
raise ValueError(f"condition latent_t {latent_t} exceeds the noise draw length {full_t}")
generator = torch.Generator(device="cpu").manual_seed(int(seed))
noise = torch.randn(
1,
24,
full_t,
latent_h,
latent_w,
generator=generator,
dtype=torch.float32,
device="cpu",
)[:, :, :latent_t]
noise_rows = minimax_h3_patchify_video_latent(noise, patch_size=[1, 2, 2]).to(dtype=torch.float32)
row_count = int(noise_rows.shape[0])
clean_part = clean_rows[row_offset : row_offset + row_count].to(torch.float32)
out.append(timestep * clean_part + (1.0 - timestep) * noise_rows)
row_offset += row_count
return torch.cat(out, dim=0).to(device=output_device, dtype=torch.float32).contiguous()
def minimax_h3_audio_cond_noise_aug_rows(
clean_rows: torch.Tensor,
*,
condition_audio_t: Sequence[int],
seed: int,
noise_aug: float,
) -> torch.Tensor:
"""Apply the audio-condition RF noise recipe to packed clean rows.
``condition_audio_t`` contains the latent T of each audio-bearing
condition in canonical request order. Noise is drawn per condition
element, with a fresh CPU generator seeded with ``seed + 1`` for every
element. Consequently each condition restarts the
same RNG stream; concatenating the rows and drawing once would be
numerically different for ordered multi-reference requests.
The mix is intentionally evaluated on CPU in fp32 before the packed rows
are transferred to the DiT device.
"""
noise_aug = float(noise_aug)
if not 0.0 <= noise_aug <= 1.0:
raise ValueError(f"noise_aug must be in [0, 1], got {noise_aug}")
if noise_aug == 1.0:
return clean_rows
if clean_rows.ndim != 2 or int(clean_rows.shape[1]) != 32:
raise ValueError(f"clean audio condition rows must have shape [n, 32], got {list(clean_rows.shape)}")
audio_channels = MINIMAX_H3_AUDIO_COND_CHANNELS
parsed_audio_t = [int(value) for value in condition_audio_t]
if not parsed_audio_t:
raise ValueError("condition_audio_t must not be empty")
if any(value <= 0 for value in parsed_audio_t):
raise ValueError(f"condition audio latent lengths must be positive, got {parsed_audio_t}")
expected_rows = audio_channels * sum(parsed_audio_t)
if int(clean_rows.shape[0]) != expected_rows:
raise ValueError(f"clean audio condition rows {int(clean_rows.shape[0])} != shape-derived rows {expected_rows}")
out: list[torch.Tensor] = []
row_offset = 0
timestep = torch.tensor(noise_aug, dtype=torch.float32, device="cpu")
for audio_t in parsed_audio_t:
row_count = audio_channels * audio_t
clean_part = clean_rows[row_offset : row_offset + row_count].detach().to(device="cpu", dtype=torch.float32)
generator = torch.Generator(device="cpu").manual_seed(int(seed) + 1)
noise = torch.randn(
clean_part.shape,
generator=generator,
dtype=torch.float32,
device="cpu",
)
out.append(timestep * clean_part + (1.0 - timestep) * noise)
row_offset += row_count
return torch.cat(out, dim=0).to(device=clean_rows.device, dtype=torch.float32).contiguous()
__all__ = [
"minimax_h3_audio_cond_noise_aug_rows",
"minimax_h3_imgvid_cond_noise_aug_rows",
]
@@ -0,0 +1,249 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax H3 cfg-distilled full denoise loop.
Per step, the positive presentation is forwarded exactly once. Video and audio
target rows chain through the Euler-eta0 update while visual and audio condition
rows stay pinned to their noised step-0 anchors.
"""
from __future__ import annotations
from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from typing import Any
import torch
from .scheduling_minimax_h3_euler_ancestral import (
minimax_h3_euler_eta0_step,
minimax_h3_rf_v_to_x0,
)
MINIMAX_H3_IMGVID_COND_TIMESTEP = 0.999
# ref2va audio reference anchor timestep
MINIMAX_H3_AUDIO_REF_COND_TIMESTEP = 1.0
# Packed row widths: video rows are [1,2,2]-patchified 24-channel latents
# (24 * 1 * 2 * 2 = 96); audio rows carry the 32-dim audio latent.
MINIMAX_H3_VIDEO_ROW_WIDTH = 96
MINIMAX_H3_AUDIO_ROW_WIDTH = 32
class MiniMaxH3DenoiseBranch:
"""Static per-branch state: packed layout + fixed forward kwargs.
`packed` is a minimax_h3_packed_sequence(...) result (or equivalent layout
dict); `text_embeddings` is the branch's [text_len, 5120] hidden states;
`token_tags` must already carry any fl2va vision-span overrides.
"""
def __init__(
self,
*,
packed: dict[str, torch.Tensor],
text_embeddings: torch.Tensor,
token_tags: torch.Tensor,
device: torch.device,
) -> None:
seq_len = int(packed["seq_len"])
self.seq_len = seq_len
self.img_pos = packed["img_pos"].view(-1).to(torch.long)
self.audio_pos = packed["audio_pos"].view(-1).to(torch.long)
self.update_mask = packed["update_mask"].view(-1).to(torch.bool)
# ref2va: audio_pos may include reference-audio anchor rows
# (audio_update_mask False); absent means all rows are targets.
if "audio_update_mask" in packed:
self.audio_update_mask = packed["audio_update_mask"].view(-1).to(torch.bool)
else:
self.audio_update_mask = torch.ones(self.audio_pos.shape[0], dtype=torch.bool)
text_len = int(packed["text_pos"].view(-1).shape[0])
if list(text_embeddings.shape)[0] != text_len:
raise ValueError(f"text_embeddings rows {list(text_embeddings.shape)} != packed text_len {text_len}")
if int(token_tags.view(-1).shape[0]) != seq_len:
raise ValueError(f"token_tags length {int(token_tags.view(-1).shape[0])} != seq_len {seq_len}")
cu = packed["cu_seqlens"].to(torch.int32)
self.img_pos_dev = self.img_pos.to(device)
self.audio_pos_dev = self.audio_pos.to(device)
self.update_mask_dev = self.update_mask.to(device)
self.audio_update_mask_dev = self.audio_update_mask.to(device)
self.x_base = torch.zeros(1, seq_len, MINIMAX_H3_VIDEO_ROW_WIDTH, dtype=torch.float32, device=device)
self.audio_x_base = torch.zeros(1, seq_len, MINIMAX_H3_AUDIO_ROW_WIDTH, dtype=torch.float32, device=device)
text_pos_dev = packed["text_pos"].view(-1).to(torch.long).to(device)
self.static_kwargs: dict[str, Any] = {
"img_position_ids": packed["img_position_ids"][None].to(device),
"update_mask": self.update_mask_dev,
"token_tags": token_tags.view(-1).to(torch.long).to(device),
"skip_mask_out_condition": False,
"prompt_embeds": text_embeddings.to(device),
"img_pos_info": {"position_ids": self.img_pos_dev},
"audio_pos_info": {"position_ids": self.audio_pos_dev},
"text_pos_info": {"position_ids": text_pos_dev},
"img_pos_for_infer_output_info": {"position_ids": self.img_pos_dev},
"packed_seq_params": {
"cu_seqlens_q": cu.to(device),
"max_seqlen_q": int(cu[1]),
},
"refiner_packed_seq_params": {
"cu_seqlens_q": torch.tensor([0, text_len, text_len], dtype=torch.int32, device=device),
"max_seqlen_q": text_len,
},
}
def forward_kwargs(
self,
*,
video_rows: torch.Tensor,
audio_rows: torch.Tensor,
t_video: float,
t_audio: float,
imgvid_cond_timestep: float,
audio_ref_cond_timestep: float,
) -> dict[str, Any]:
x = self.x_base.clone()
x[0].index_copy_(0, self.img_pos_dev, video_rows)
audio_x = self.audio_x_base.clone()
audio_x[0].index_copy_(0, self.audio_pos_dev, audio_rows)
# Packed-sequence timestep semantics: non-media rows (text and
# padding) inherit the current video timestep. Later steps must reuse
# the previous step's updated rows; re-initializing from zeros is only
# valid at step 0.
timesteps = torch.full(
(self.seq_len,),
float(t_video),
dtype=torch.float32,
device=x.device,
)
timesteps[self.img_pos_dev[self.update_mask_dev]] = t_video
timesteps[self.img_pos_dev[~self.update_mask_dev]] = imgvid_cond_timestep
timesteps[self.audio_pos_dev[self.audio_update_mask_dev]] = t_audio
timesteps[self.audio_pos_dev[~self.audio_update_mask_dev]] = audio_ref_cond_timestep
unique_timesteps, inverse_indices = torch.unique(timesteps, sorted=True, return_inverse=True)
return {
**self.static_kwargs,
"x": x,
"audio_x": audio_x,
"unique_timesteps": unique_timesteps,
"inverse_indices": inverse_indices,
}
def minimax_h3_denoise_loop(
*,
model: Any,
positive: MiniMaxH3DenoiseBranch,
initial_video_rows: torch.Tensor,
initial_audio_rows: torch.Tensor,
keyframe_cond_rows: torch.Tensor | None,
audio_ref_rows: torch.Tensor | None = None,
sigmas_video: list[float],
sigmas_audio: list[float],
device: torch.device,
imgvid_cond_noise_aug_for_inference: float = MINIMAX_H3_IMGVID_COND_TIMESTEP,
audio_cond_noise_aug_for_inference: float = MINIMAX_H3_AUDIO_REF_COND_TIMESTEP,
on_step: Callable[[int, torch.Tensor, torch.Tensor], None] | None = None,
step_profiler: Callable[[int], AbstractContextManager] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the full denoise loop; returns final (video_rows, audio_rows).
``initial_video_rows`` covers all image rows of the positive layout. For a
conditional task, pass ``keyframe_cond_rows`` and/or ``audio_ref_rows`` to
pin those rows across every step. The model's raw positive velocity is the
update signal; MiniMax H3 only supports cfg-distilled checkpoints.
"""
if len(sigmas_video) != len(sigmas_audio):
raise ValueError("video/audio sigma schedules must have equal length")
if len(sigmas_video) < 2:
raise ValueError("sigma schedules need at least 2 entries")
n_cond = int((~positive.update_mask).sum())
if keyframe_cond_rows is None:
if n_cond != 0:
raise ValueError(f"layout has {n_cond} cond rows but keyframe_cond_rows is None")
else:
if int(keyframe_cond_rows.shape[0]) != n_cond:
raise ValueError(f"keyframe_cond_rows {int(keyframe_cond_rows.shape[0])} != layout cond rows {n_cond}")
video_rows = initial_video_rows.to(device=device, dtype=torch.float32).clone()
audio_rows = initial_audio_rows.to(device=device, dtype=torch.float32).clone()
update = positive.update_mask_dev
audio_update = positive.audio_update_mask_dev
if int(video_rows.shape[0]) != int(positive.img_pos.shape[0]):
raise ValueError(
f"initial video rows {int(video_rows.shape[0])} != positive layout rows {int(positive.img_pos.shape[0])}"
)
if int(audio_rows.shape[0]) != int(positive.audio_pos.shape[0]):
raise ValueError(
f"initial audio rows {int(audio_rows.shape[0])} != positive layout rows {int(positive.audio_pos.shape[0])}"
)
n_audio_ref = int((~positive.audio_update_mask).sum())
if audio_ref_rows is None:
if n_audio_ref != 0:
raise ValueError(f"layout has {n_audio_ref} audio ref rows but audio_ref_rows is None")
audio_anchor = None
else:
if int(audio_ref_rows.shape[0]) != n_audio_ref:
raise ValueError(f"audio_ref_rows {int(audio_ref_rows.shape[0])} != layout audio ref rows {n_audio_ref}")
audio_anchor = audio_ref_rows.to(device=device, dtype=torch.float32)
cond_anchor = keyframe_cond_rows.to(device=device, dtype=torch.float32) if keyframe_cond_rows is not None else None
if cond_anchor is not None:
video_rows[~update] = cond_anchor
if audio_anchor is not None:
audio_rows[~audio_update] = audio_anchor
num_steps = len(sigmas_video) - 1
for step in range(num_steps):
step_cm = step_profiler(step) if step_profiler is not None else nullcontext()
with step_cm:
s_v, s_v_next = sigmas_video[step], sigmas_video[step + 1]
s_a, s_a_next = sigmas_audio[step], sigmas_audio[step + 1]
t_v, t_a = 1.0 - s_v, 1.0 - s_a
imgvid_cond_t = max(t_v, float(imgvid_cond_noise_aug_for_inference))
audio_ref_cond_t = max(t_a, float(audio_cond_noise_aug_for_inference))
fk = positive.forward_kwargs(
video_rows=video_rows,
audio_rows=audio_rows,
t_video=t_v,
t_audio=t_a,
imgvid_cond_timestep=imgvid_cond_t,
audio_ref_cond_timestep=audio_ref_cond_t,
)
with torch.inference_mode():
v_video, v_audio = model(**fk)
mv_video_t = v_video.float()[update]
mv_audio_t = v_audio.float()[audio_update]
x0_video = minimax_h3_rf_v_to_x0(
video_rows[update],
mv_video_t,
torch.tensor(t_v, dtype=torch.float32, device=device),
)
new_target = minimax_h3_euler_eta0_step(video_rows[update], x0_video, sigma_curr=s_v, sigma_next=s_v_next)
video_rows = video_rows.clone()
video_rows[update] = new_target
if cond_anchor is not None:
video_rows[~update] = cond_anchor # per-step imgvid cond reset
x0_audio = minimax_h3_rf_v_to_x0(
audio_rows[audio_update],
mv_audio_t,
torch.tensor(t_a, dtype=torch.float32, device=device),
)
new_audio = minimax_h3_euler_eta0_step(
audio_rows[audio_update], x0_audio, sigma_curr=s_a, sigma_next=s_a_next
)
audio_rows = audio_rows.clone()
audio_rows[audio_update] = new_audio
if audio_anchor is not None:
audio_rows[~audio_update] = audio_anchor # per-step audio ref reset
if on_step is not None:
on_step(step, video_rows, audio_rows)
return video_rows, audio_rows
__all__ = [
"MINIMAX_H3_AUDIO_REF_COND_TIMESTEP",
"MINIMAX_H3_AUDIO_ROW_WIDTH",
"MINIMAX_H3_IMGVID_COND_TIMESTEP",
"MINIMAX_H3_VIDEO_ROW_WIDTH",
"MiniMaxH3DenoiseBranch",
"minimax_h3_denoise_loop",
]
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,572 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax H3 packed-sequence materialization for FL2VA and Ref2VA tasks.
Layout: [text L | imgvid_cond C | audio A(=t*2ch) | video_target V | pad P].
Builder rules:
- placeholder ids; block-derived pos infos/masks/cu_seqlens/document_id
- img_position_ids fp64 grid: text rows (row_idx,0,0); video/cond t counter
continues text_len with temporal interp spans (frame_rescale 5/3 x
frame_per_token (1,4,4,4,4)); each spatial sqrt_area axis uses evenly spaced
coordinates excluding the right endpoint, then scales them by INTERP;
audio channel-major blocks pinned to the w-grid extremes.
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
import numpy as np
import torch
MINIMAX_H3_FL2VA_KEYFRAME_SIGNATURES: tuple[tuple[int, ...], ...] = (
(0,),
(-1,),
(0, -1),
)
MINIMAX_H3_TEXT_ID = -5
MINIMAX_H3_IMGVID_COND_ID = -11
MINIMAX_H3_AUDIO_REF_COND_ID = -17
MINIMAX_H3_AUDIO_FIRST_ID = -15
MINIMAX_H3_AUDIO_ID = -14
MINIMAX_H3_VIDEO_FIRST_ID = -3
MINIMAX_H3_VIDEO_ID = -2
MINIMAX_H3_VIDEO_LAST_ID = -4
MINIMAX_H3_PAD_ID = -1
_INTERP = 32
_T_GROUP = 5
_FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
_FRAME_RESCALE = 5.0 / 3.0
_SEQ_ALIGN = 64
# MiniMax H3 packs video latents with a fixed [1, 2, 2] (t, h, w) patch.
_PATCH_T = 1
_PATCH_H = 2
_PATCH_W = 2
def _keyframe_cond_frame_indices(
*,
include_keyframe_cond: bool,
keyframe_frame_indices: list[int] | tuple[int, ...] | None,
) -> list[int]:
if not include_keyframe_cond:
if keyframe_frame_indices is not None:
raise ValueError("keyframe_frame_indices must be omitted when keyframe cond is not included")
return []
if keyframe_frame_indices is None:
raise ValueError("strict fl2va packed layout requires keyframe_frame_indices")
if any(isinstance(value, bool) or not isinstance(value, int) for value in keyframe_frame_indices):
raise ValueError("strict fl2va packed layout requires integer keyframe_frame_indices")
out = list(keyframe_frame_indices)
if tuple(out) not in MINIMAX_H3_FL2VA_KEYFRAME_SIGNATURES:
raise ValueError(
"strict fl2va packed layout requires keyframe_frame_indices in "
f"{MINIMAX_H3_FL2VA_KEYFRAME_SIGNATURES!r}, got {out!r}"
)
return out
def _resolve_keyframe_frame_indices(
frame_indices: Sequence[int],
*,
frame_count: int | None,
) -> list[int]:
if frame_indices and frame_count is None:
raise ValueError("frame_count is required when keyframe_frame_indices are provided")
if frame_count is None:
return []
if frame_count <= 0:
raise ValueError("frame_count must be positive")
seen: dict[int, int] = {}
resolved: list[int] = []
for block_index, semantic_index in enumerate(frame_indices):
if semantic_index == -1:
resolved_index = frame_count - 1
elif 0 <= semantic_index < frame_count:
resolved_index = semantic_index
else:
raise ValueError(f"keyframe frame index {semantic_index} must be -1 or in [0, {frame_count})")
previous = seen.get(resolved_index)
if previous is not None:
raise ValueError(
f"keyframe frame index at block {block_index} resolves to "
f"{resolved_index}, already bound by block {previous}"
)
seen[resolved_index] = block_index
resolved.append(resolved_index)
return resolved
def _temporal_position_span(temporal_length: int) -> float:
"""Temporal position span for patch_t=1, in fp64.
NOTE: intentionally NOT merged with ``_video_t_span``. This variant sums
via numpy (pairwise summation), matching the fl2va anchor computation,
while ``_video_t_span`` sums sequentially, matching the ref2va
t-origin accumulation. The two orders diverge in the last ulp
from n=16 onward, so each path must keep its own summation order.
"""
spans = np.ones(int(temporal_length), dtype=np.float64) * _FRAME_RESCALE
for token_index in range(_T_GROUP):
spans[token_index::_T_GROUP] *= _FRAME_PER_TOKEN[token_index]
return float(spans.sum())
def minimax_h3_packed_sequence(
*,
text_len: int,
latent_t: int,
latent_h: int,
latent_w: int,
audio_t: int,
audio_channel: int = 2,
include_keyframe_cond: bool,
keyframe_frame_indices: list[int] | tuple[int, ...] | None = None,
frame_count: int | None = None,
) -> dict[str, torch.Tensor]:
"""Build the packed-sequence structural fields for one CFG branch.
The used length is padded up to a multiple of 64.
"""
ph, pw = latent_h // _PATCH_H, latent_w // _PATCH_W
frame_rows = ph * pw
cond_frame_indices = _keyframe_cond_frame_indices(
include_keyframe_cond=include_keyframe_cond,
keyframe_frame_indices=keyframe_frame_indices,
)
resolved_cond_frame_indices = _resolve_keyframe_frame_indices(
cond_frame_indices,
frame_count=frame_count,
)
cond_rows = len(cond_frame_indices) * frame_rows
video_rows = latent_t * frame_rows
audio_rows = audio_t * audio_channel
used = text_len + cond_rows + audio_rows + video_rows
seq_len = ((used + _SEQ_ALIGN - 1) // _SEQ_ALIGN) * _SEQ_ALIGN
text_sl = slice(0, text_len)
cond_sl = slice(text_len, text_len + cond_rows)
audio_sl = slice(cond_sl.stop, cond_sl.stop + audio_rows)
video_sl = slice(audio_sl.stop, audio_sl.stop + video_rows)
pad_sl = slice(video_sl.stop, seq_len)
input_ids = torch.full((seq_len,), MINIMAX_H3_PAD_ID, dtype=torch.int64)
input_ids[text_sl] = MINIMAX_H3_TEXT_ID
if cond_rows:
input_ids[cond_sl] = MINIMAX_H3_IMGVID_COND_ID
input_ids[audio_sl] = MINIMAX_H3_AUDIO_ID
input_ids[audio_sl.start] = MINIMAX_H3_AUDIO_FIRST_ID
input_ids[video_sl] = MINIMAX_H3_VIDEO_ID
input_ids[video_sl.start] = MINIMAX_H3_VIDEO_FIRST_ID
input_ids[video_sl.stop - 1] = MINIMAX_H3_VIDEO_LAST_ID
image_mask = torch.zeros(seq_len, dtype=torch.bool)
image_mask[cond_sl] = True
image_mask[video_sl] = True
audio_mask = torch.zeros(seq_len, dtype=torch.bool)
audio_mask[audio_sl] = True
img_pos = torch.cat(
[
torch.arange(cond_sl.start, cond_sl.stop),
torch.arange(video_sl.start, video_sl.stop),
]
)
update_mask = torch.zeros(img_pos.shape[0], dtype=torch.bool)
update_mask[cond_rows:] = True
audio_pos = torch.arange(audio_sl.start, audio_sl.stop)
text_pos = torch.arange(0, text_len)
g = torch.zeros(seq_len, 3, dtype=torch.float64)
g[text_sl, 0] = torch.arange(text_len, dtype=torch.float64)
t_grid = _video_t_grid(latent_t, float(text_len))
sqrt_area = np.sqrt(latent_h * latent_w)
h_grid = _axis_from_sqrt_area(latent_h, _PATCH_H, sqrt_area)
w_grid = _axis_from_sqrt_area(latent_w, _PATCH_W, sqrt_area)
hh, ww = torch.meshgrid(h_grid, w_grid, indexing="ij")
frame = torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1)
video_g = torch.empty(latent_t, frame_rows, 3, dtype=torch.float64)
video_g[:, :, 0] = t_grid[:, None]
video_g[:, :, 1:] = frame[None]
for block_index, pixel_index in enumerate(resolved_cond_frame_indices):
sl = slice(
cond_sl.start + block_index * frame_rows,
cond_sl.start + (block_index + 1) * frame_rows,
)
if pixel_index == 0:
cond_t = float(text_len)
elif frame_count is not None and pixel_index == frame_count - 1:
cond_t = float(text_len) + _temporal_position_span(latent_t) - _FRAME_RESCALE
else:
raise ValueError(
f"fl2va packed layout only supports first/last keyframe anchors, got resolved frame index {pixel_index}"
)
g[sl, 0] = cond_t
g[sl, 1:] = frame
g[video_sl] = video_g.reshape(-1, 3)
audio_t_grid = float(text_len) + torch.arange(audio_t, dtype=torch.float64)
g[audio_sl, 0] = audio_t_grid.repeat(audio_channel)
g[audio_sl, 2] = torch.cat(
[
torch.full((audio_t,), float(w_grid[0]), dtype=torch.float64),
torch.full((audio_rows - audio_t,), float(w_grid[-1]), dtype=torch.float64),
]
)
token_tags = torch.full((seq_len,), -1, dtype=torch.long) # PADDING
token_tags[text_sl] = 1 # TEXT (fl2va image-segment override happens upstream)
token_tags[audio_sl] = 2 # AUDIO
token_tags[img_pos] = 0 # VIDEO
cu = torch.tensor([0, used, seq_len], dtype=torch.int32)
doc = torch.zeros(seq_len, dtype=torch.int32)
doc[pad_sl] = 1
return {
"seq_len": torch.tensor(seq_len),
"input_ids": input_ids,
"image_mask": image_mask,
"audio_mask": audio_mask,
"img_pos": img_pos,
"audio_pos": audio_pos,
"text_pos": text_pos,
"update_mask": update_mask,
"img_position_ids": g,
"token_tags": token_tags,
"cu_seqlens": cu,
"document_id": doc,
}
def _positive_int(
block: Mapping[str, object],
key: str,
path: str,
*,
allow_zero: bool = False,
) -> int:
value = block.get(key)
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError(f"{path}.{key} must be an integer")
if value < 0 or (value == 0 and not allow_zero):
predicate = "non-negative" if allow_zero else "positive"
raise ValueError(f"{path}.{key} must be {predicate}")
return int(value)
def _axis_from_sqrt_area(dim: int, patch: int, sqrt_area: float) -> torch.Tensor:
ratio = dim / sqrt_area
left = (1.0 - ratio) * 1.0 / 2.0
right = left + ratio * 1.0
grid = np.linspace(left, right, dim // patch, endpoint=False) * _INTERP
return torch.from_numpy(grid).to(torch.float64)
def _video_t_grid(n: int, origin: float) -> torch.Tensor:
spans = torch.tensor(
[_FRAME_RESCALE * _FRAME_PER_TOKEN[k % _T_GROUP] for k in range(n)],
dtype=torch.float64,
)
return origin + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)])
def _video_t_span(n: int) -> float:
# Sequential fp64 summation on purpose — see _temporal_position_span for
# why the two span implementations must not be unified.
return sum(_FRAME_RESCALE * _FRAME_PER_TOKEN[k % _T_GROUP] for k in range(n))
def _range_for_slice(sl: slice) -> torch.Tensor:
return torch.arange(sl.start, sl.stop, dtype=torch.long)
def _cat_ranges(parts: list[torch.Tensor]) -> torch.Tensor:
if parts:
return torch.cat(parts)
return torch.empty(0, dtype=torch.long)
def minimax_h3_packed_sequence_ref2va_blocks(
*,
text_len: int,
latent_t: int,
latent_h: int,
latent_w: int,
audio_t: int,
ref_blocks: Sequence[Mapping[str, object]],
audio_channel: int = 2,
seq_len: int | None = None,
) -> dict[str, torch.Tensor]:
"""General ref2va-family packed layout.
``ref_blocks`` are consumed in request/plan order:
- ``{"kind": "image", "latent_h": H, "latent_w": W}``
- ``{"kind": "audio", "ref_audio_t": T}``
- ``{"kind": "video"|"video_audio", "ref_audio_t": T,
"latent_t": RT, "latent_h": RH, "latent_w": RW}``
Video-bearing blocks pack their audio rows immediately before their video
rows; both share the same temporal origin and advance by the longer of the
audio and video spans. Standalone audio advances the target origin by its
own T, and image blocks advance it by one integer slot.
"""
if not isinstance(ref_blocks, Sequence) or isinstance(ref_blocks, (str, bytes)):
raise ValueError("ref_blocks must be a sequence")
parsed: list[dict[str, object]] = []
ref_visual_rows = 0
ref_audio_rows = 0
for index, raw in enumerate(ref_blocks):
path = f"ref_blocks[{index}]"
if not isinstance(raw, Mapping):
raise ValueError(f"{path} must be an object")
kind = raw.get("kind", raw.get("type"))
if not isinstance(kind, str) or not kind:
raise ValueError(f"{path}.kind must be a non-empty string")
if kind == "image":
rh = _positive_int(raw, "latent_h", path)
rw = _positive_int(raw, "latent_w", path)
rows = (rh // _PATCH_H) * (rw // _PATCH_W)
item = {"kind": kind, "latent_h": rh, "latent_w": rw, "rows": rows}
ref_visual_rows += rows
elif kind == "audio":
rt = _positive_int(raw, "ref_audio_t", path, allow_zero=True)
rows = rt * audio_channel
item = {"kind": kind, "ref_audio_t": rt, "audio_rows": rows}
ref_audio_rows += rows
elif kind in ("video", "video_audio"):
rt = _positive_int(raw, "ref_audio_t", path, allow_zero=True)
vt = _positive_int(raw, "latent_t", path)
vh = _positive_int(raw, "latent_h", path)
vw = _positive_int(raw, "latent_w", path)
frame_rows = (vh // _PATCH_H) * (vw // _PATCH_W)
audio_rows = rt * audio_channel
video_rows = vt * frame_rows
item = {
"kind": kind,
"ref_audio_t": rt,
"latent_t": vt,
"latent_h": vh,
"latent_w": vw,
"frame_rows": frame_rows,
"audio_rows": audio_rows,
"video_rows": video_rows,
}
ref_audio_rows += audio_rows
ref_visual_rows += video_rows
else:
raise ValueError(f"{path}.kind unsupported for ref2va: {kind!r}")
parsed.append(item)
ph, pw = latent_h // _PATCH_H, latent_w // _PATCH_W
frame_rows = ph * pw
video_rows = latent_t * frame_rows
audio_rows = audio_t * audio_channel
ref_rows = ref_visual_rows + ref_audio_rows
used = text_len + ref_rows + audio_rows + video_rows
if seq_len is None:
seq_len = ((used + _SEQ_ALIGN - 1) // _SEQ_ALIGN) * _SEQ_ALIGN
if seq_len < used:
raise ValueError(f"seq_len {seq_len} < used rows {used}")
text_sl = slice(0, text_len)
cursor = text_len
block_slices: list[dict[str, object]] = []
for item in parsed:
kind = str(item["kind"])
if kind == "image":
rows = int(item["rows"])
visual_sl = slice(cursor, cursor + rows)
cursor = visual_sl.stop
block_slices.append({**item, "visual_sl": visual_sl})
elif kind == "audio":
rows = int(item["audio_rows"])
audio_sl = slice(cursor, cursor + rows)
cursor = audio_sl.stop
block_slices.append({**item, "audio_sl": audio_sl})
else:
a_rows = int(item["audio_rows"])
v_rows = int(item["video_rows"])
audio_sl = slice(cursor, cursor + a_rows)
visual_sl = slice(audio_sl.stop, audio_sl.stop + v_rows)
cursor = visual_sl.stop
block_slices.append({**item, "audio_sl": audio_sl, "visual_sl": visual_sl})
audio_sl = slice(cursor, cursor + audio_rows)
video_sl = slice(audio_sl.stop, audio_sl.stop + video_rows)
pad_sl = slice(video_sl.stop, seq_len)
input_ids = torch.full((seq_len,), MINIMAX_H3_PAD_ID, dtype=torch.int64)
input_ids[text_sl] = MINIMAX_H3_TEXT_ID
image_mask = torch.zeros(seq_len, dtype=torch.bool)
audio_mask = torch.zeros(seq_len, dtype=torch.bool)
ref_img_pos_parts: list[torch.Tensor] = []
ref_audio_pos_parts: list[torch.Tensor] = []
g = torch.zeros(seq_len, 3, dtype=torch.float64)
g[text_sl, 0] = torch.arange(text_len, dtype=torch.float64)
target_area = np.sqrt(latent_h * latent_w)
h_grid = _axis_from_sqrt_area(latent_h, _PATCH_H, target_area)
w_grid = _axis_from_sqrt_area(latent_w, _PATCH_W, target_area)
hh, ww = torch.meshgrid(h_grid, w_grid, indexing="ij")
target_frame = torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1)
t_cursor = float(text_len)
for item in block_slices:
kind = str(item["kind"])
if kind == "image":
visual_sl = item["visual_sl"]
assert isinstance(visual_sl, slice)
input_ids[visual_sl] = MINIMAX_H3_IMGVID_COND_ID
image_mask[visual_sl] = True
ref_img_pos_parts.append(_range_for_slice(visual_sl))
rh = int(item["latent_h"])
rw = int(item["latent_w"])
area = np.sqrt(rh * rw)
ref_hh, ref_ww = torch.meshgrid(
_axis_from_sqrt_area(rh, _PATCH_H, area),
_axis_from_sqrt_area(rw, _PATCH_W, area),
indexing="ij",
)
g[visual_sl, 0] = t_cursor
g[visual_sl, 1] = ref_hh.reshape(-1)
g[visual_sl, 2] = ref_ww.reshape(-1)
t_cursor += 1.0
elif kind == "audio":
audio_ref_sl = item["audio_sl"]
assert isinstance(audio_ref_sl, slice)
ref_t = int(item["ref_audio_t"])
input_ids[audio_ref_sl] = MINIMAX_H3_AUDIO_REF_COND_ID
audio_mask[audio_ref_sl] = True
ref_audio_pos_parts.append(_range_for_slice(audio_ref_sl))
ref_t_grid = t_cursor + torch.arange(ref_t, dtype=torch.float64)
g[audio_ref_sl, 0] = ref_t_grid.repeat(audio_channel)
if ref_t:
g[audio_ref_sl, 2] = torch.cat(
[
torch.full((ref_t,), float(w_grid[0]), dtype=torch.float64),
torch.full(
(int(item["audio_rows"]) - ref_t,),
float(w_grid[-1]),
dtype=torch.float64,
),
]
)
t_cursor += float(ref_t)
else:
audio_ref_sl = item["audio_sl"]
visual_sl = item["visual_sl"]
assert isinstance(audio_ref_sl, slice)
assert isinstance(visual_sl, slice)
ref_t = int(item["ref_audio_t"])
vt = int(item["latent_t"])
vh = int(item["latent_h"])
vw = int(item["latent_w"])
input_ids[audio_ref_sl] = MINIMAX_H3_AUDIO_REF_COND_ID
input_ids[visual_sl] = MINIMAX_H3_IMGVID_COND_ID
audio_mask[audio_ref_sl] = True
image_mask[visual_sl] = True
ref_audio_pos_parts.append(_range_for_slice(audio_ref_sl))
ref_img_pos_parts.append(_range_for_slice(visual_sl))
ref_area = np.sqrt(vh * vw)
rv_h_grid = _axis_from_sqrt_area(vh, _PATCH_H, ref_area)
rv_w_grid = _axis_from_sqrt_area(vw, _PATCH_W, ref_area)
rv_hh, rv_ww = torch.meshgrid(rv_h_grid, rv_w_grid, indexing="ij")
ref_t_grid = t_cursor + torch.arange(ref_t, dtype=torch.float64)
g[audio_ref_sl, 0] = ref_t_grid.repeat(audio_channel)
if ref_t:
g[audio_ref_sl, 2] = torch.cat(
[
torch.full((ref_t,), float(rv_w_grid[0]), dtype=torch.float64),
torch.full(
(int(item["audio_rows"]) - ref_t,),
float(rv_w_grid[-1]),
dtype=torch.float64,
),
]
)
rv_frame = torch.stack([rv_hh.reshape(-1), rv_ww.reshape(-1)], dim=-1)
rv_g = torch.empty(vt, int(item["frame_rows"]), 3, dtype=torch.float64)
rv_g[:, :, 0] = _video_t_grid(vt, t_cursor)[:, None]
rv_g[:, :, 1:] = rv_frame[None]
g[visual_sl] = rv_g.reshape(-1, 3)
t_cursor += max(float(ref_t), _video_t_span(vt))
input_ids[audio_sl] = MINIMAX_H3_AUDIO_ID
input_ids[audio_sl.start] = MINIMAX_H3_AUDIO_FIRST_ID
input_ids[video_sl] = MINIMAX_H3_VIDEO_ID
input_ids[video_sl.start] = MINIMAX_H3_VIDEO_FIRST_ID
input_ids[video_sl.stop - 1] = MINIMAX_H3_VIDEO_LAST_ID
audio_mask[audio_sl] = True
image_mask[video_sl] = True
audio_t_grid = t_cursor + torch.arange(audio_t, dtype=torch.float64)
g[audio_sl, 0] = audio_t_grid.repeat(audio_channel)
g[audio_sl, 2] = torch.cat(
[
torch.full((audio_t,), float(w_grid[0]), dtype=torch.float64),
torch.full((audio_rows - audio_t,), float(w_grid[-1]), dtype=torch.float64),
]
)
video_g = torch.empty(latent_t, frame_rows, 3, dtype=torch.float64)
video_g[:, :, 0] = _video_t_grid(latent_t, t_cursor)[:, None]
video_g[:, :, 1:] = target_frame[None]
g[video_sl] = video_g.reshape(-1, 3)
target_img_pos = _range_for_slice(video_sl)
target_audio_pos = _range_for_slice(audio_sl)
img_pos = _cat_ranges(ref_img_pos_parts + [target_img_pos])
audio_pos = _cat_ranges(ref_audio_pos_parts + [target_audio_pos])
update_mask = torch.zeros(img_pos.shape[0], dtype=torch.bool)
update_mask[ref_visual_rows:] = True
audio_update_mask = torch.zeros(audio_pos.shape[0], dtype=torch.bool)
audio_update_mask[ref_audio_rows:] = True
text_pos = torch.arange(0, text_len)
token_tags = torch.full((seq_len,), -1, dtype=torch.long) # PADDING
token_tags[text_sl] = 1 # TEXT
if ref_audio_rows:
token_tags[_cat_ranges(ref_audio_pos_parts)] = 2 # AUDIO (reference)
token_tags[audio_sl] = 2 # AUDIO (target)
token_tags[img_pos] = 0 # VIDEO (refs + target)
cu = torch.tensor([0, used, seq_len], dtype=torch.int32)
doc = torch.zeros(seq_len, dtype=torch.int32)
doc[pad_sl] = 1
return {
"seq_len": torch.tensor(seq_len),
"input_ids": input_ids,
"image_mask": image_mask,
"audio_mask": audio_mask,
"img_pos": img_pos,
"audio_pos": audio_pos,
"text_pos": text_pos,
"update_mask": update_mask,
"audio_update_mask": audio_update_mask,
"img_position_ids": g,
"token_tags": token_tags,
"cu_seqlens": cu,
"document_id": doc,
}
__all__ = [
"MINIMAX_H3_AUDIO_FIRST_ID",
"MINIMAX_H3_AUDIO_ID",
"MINIMAX_H3_AUDIO_REF_COND_ID",
"MINIMAX_H3_IMGVID_COND_ID",
"MINIMAX_H3_PAD_ID",
"MINIMAX_H3_TEXT_ID",
"MINIMAX_H3_VIDEO_FIRST_ID",
"MINIMAX_H3_VIDEO_ID",
"MINIMAX_H3_VIDEO_LAST_ID",
"minimax_h3_packed_sequence",
"minimax_h3_packed_sequence_ref2va_blocks",
]
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from collections.abc import Sequence
import torch
def _int_tuple(value: Sequence[int], name: str, length: int) -> tuple[int, ...]:
if len(value) != length:
raise ValueError(f"{name} must have length {length}, got {list(value)!r}")
out = tuple(int(item) for item in value)
if any(item <= 0 for item in out):
raise ValueError(f"{name} values must be positive, got {list(value)!r}")
return out
def _rank(tensor: torch.Tensor, name: str, rank: int) -> None:
if tensor.ndim != rank:
raise ValueError(f"{name} must be rank {rank}, got shape={list(tensor.shape)}")
def minimax_h3_patchify_video_latent(
latent: torch.Tensor,
*,
patch_size: Sequence[int],
) -> torch.Tensor:
"""Pack video latent [B,C,T,H,W] into DiT token rows."""
_rank(latent, "video latent", 5)
pt, ph, pw = _int_tuple(patch_size, "patch_size", 3)
batch, channel, full_t, full_h, full_w = (int(dim) for dim in latent.shape)
if full_t % pt or full_h % ph or full_w % pw:
raise ValueError(
"video latent spatial/time dims must be divisible by patch_size: "
f"shape={list(latent.shape)}, patch_size={[pt, ph, pw]}"
)
t, h, w = full_t // pt, full_h // ph, full_w // pw
packed = latent.reshape(batch, channel, t, pt, h, ph, w, pw)
packed = torch.einsum("nctrhpwq->nthwcrpq", packed)
return packed.reshape(batch * t * h * w, channel * pt * ph * pw).contiguous()
def minimax_h3_unpatchify_video_tokens(
rows: torch.Tensor,
*,
latent_shape: Sequence[int],
patch_size: Sequence[int],
) -> torch.Tensor:
"""Unpack DiT video token rows into latent [B,C,T,H,W]."""
_rank(rows, "video token rows", 2)
t, h, w, channel = _int_tuple(latent_shape, "latent_shape", 4)
pt, ph, pw = _int_tuple(patch_size, "patch_size", 3)
expected_dim = pt * ph * pw * channel
if int(rows.shape[-1]) != expected_dim:
raise ValueError(
f"video token dim {int(rows.shape[-1])} != patch volume * channel "
f"{expected_dim} for latent_shape={list(latent_shape)}, "
f"patch_size={[pt, ph, pw]}"
)
rows_per_sample = t * h * w
if int(rows.shape[0]) % rows_per_sample:
raise ValueError(
f"video rows {int(rows.shape[0])} must be divisible by t*h*w "
f"{rows_per_sample} for latent_shape={list(latent_shape)}"
)
packed = rows.reshape(-1, t, h, w, channel, pt, ph, pw)
latent = torch.einsum("nthwcrpq->nctrhpwq", packed)
return latent.reshape(-1, channel, t * pt, h * ph, w * pw).contiguous()
def minimax_h3_pack_audio_latent(latent: torch.Tensor) -> torch.Tensor:
"""Pack audio VAE latent [audio_channel, latent_dim, T] into rows."""
_rank(latent, "audio latent", 3)
audio_channel, latent_dim, steps = (int(dim) for dim in latent.shape)
return (
latent.permute(0, 2, 1)
.reshape(
audio_channel * steps,
latent_dim,
)
.contiguous()
)
def minimax_h3_unpack_audio_tokens(
rows: torch.Tensor,
*,
audio_t: int,
audio_channel: int,
) -> torch.Tensor:
"""Unpack DiT audio token rows into audio VAE latent [C,latent_dim,T]."""
_rank(rows, "audio token rows", 2)
audio_t = int(audio_t)
audio_channel = int(audio_channel)
if audio_t <= 0 or audio_channel <= 0:
raise ValueError(f"audio_t and audio_channel must be positive, got {audio_t=} {audio_channel=}")
if int(rows.shape[0]) != audio_t:
raise ValueError(f"audio rows {int(rows.shape[0])} != audio_t {audio_t}")
if audio_t % audio_channel:
raise ValueError(f"audio_t must be divisible by audio_channel, got {audio_t=} {audio_channel=}")
native = rows.reshape(audio_channel, audio_t // audio_channel, int(rows.shape[-1]))
return native.permute(0, 2, 1).contiguous()
__all__ = [
"minimax_h3_pack_audio_latent",
"minimax_h3_patchify_video_latent",
"minimax_h3_unpack_audio_tokens",
"minimax_h3_unpatchify_video_tokens",
]
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax H3 Qwen presentation building.
Builds the positive presentation token stream:
- fl2va: '<Picture 1>: ' label + vision block (<|vision_start|> +
N*<|image_pad|> + <|vision_end|>) + prompt text.
- t2va: prompt text only (no vision block).
Prompt text passes through verbatim (no stripping or rewriting).
All presentation variants are emitted through the shared ``_Presentation``
accumulator so ids and AdaLN token tags cannot drift apart.
"""
from __future__ import annotations
from collections.abc import Sequence
from typing import Any
import torch
VISION_START = "<|vision_start|>"
VISION_END = "<|vision_end|>"
IMAGE_PAD = "<|image_pad|>"
VIDEO_PAD = "<|video_pad|>"
_TEXT_TAG = 1
_VIDEO_TAG = 0
def _text_ids(tokenizer: Any, text: str) -> list[int]:
return list(tokenizer(text, add_special_tokens=False)["input_ids"])
def _vision_block_ids(tokenizer: Any, pad_token: str, count: int) -> list[int]:
return (
[tokenizer.convert_tokens_to_ids(VISION_START)]
+ [tokenizer.convert_tokens_to_ids(pad_token)] * int(count)
+ [tokenizer.convert_tokens_to_ids(VISION_END)]
)
class _Presentation:
"""Accumulates aligned (ids, token_tags) presentation segments."""
def __init__(self) -> None:
self.ids: list[int] = []
self.tags: list[int] = []
def text(self, token_ids: list[int]) -> None:
self.ids += token_ids
self.tags += [_TEXT_TAG] * len(token_ids)
def vision(self, token_ids: list[int]) -> None:
self.ids += token_ids
self.tags += [_VIDEO_TAG] * len(token_ids)
def build(self) -> tuple[torch.Tensor, torch.Tensor]:
return (
torch.tensor(self.ids, dtype=torch.long),
torch.tensor(self.tags, dtype=torch.long),
)
def _timestamped_video_blocks(
presentation: _Presentation,
tokenizer: Any,
*,
counts: Sequence[int],
timestamps: Sequence[float],
context: str,
) -> None:
"""Emit per-temporal-block ``<{t:.1f} seconds>`` text + VIDEO vision."""
counts = [int(value) for value in counts]
timestamps = [float(value) for value in timestamps]
if not counts or len(counts) != len(timestamps):
raise ValueError(f"{context}video block token counts and timestamps must align")
for count, timestamp in zip(counts, timestamps):
if count <= 0:
raise ValueError(f"{context}video block token count must be positive")
presentation.text(_text_ids(tokenizer, f"<{timestamp:.1f} seconds>"))
presentation.vision(_vision_block_ids(tokenizer, VIDEO_PAD, count))
def minimax_h3_text_only_ids(tokenizer: Any, prompt: str) -> torch.Tensor:
"""t2va presentation: verbatim prompt, no special tokens."""
if not prompt:
raise ValueError("prompt must be non-empty")
return torch.tensor(_text_ids(tokenizer, prompt), dtype=torch.long)
def _multi_image_presentation(
tokenizer: Any,
*,
prompt: str,
image_token_counts: list[int],
) -> tuple[torch.Tensor, torch.Tensor]:
if not image_token_counts:
raise ValueError("image_token_counts must be non-empty")
presentation = _Presentation()
for index, count in enumerate(image_token_counts, start=1):
if int(count) <= 0:
raise ValueError("image_token_count must be positive")
presentation.text(_text_ids(tokenizer, f"<Picture {index}>: "))
presentation.vision(_vision_block_ids(tokenizer, IMAGE_PAD, count))
presentation.text(_text_ids(tokenizer, prompt))
return presentation.build()
def minimax_h3_multi_image_presentation_ids(
tokenizer: Any,
*,
prompt: str,
image_token_counts: list[int],
) -> torch.Tensor:
"""Positive fl2va presentation with one or more labeled image slots."""
ids, _ = _multi_image_presentation(tokenizer, prompt=prompt, image_token_counts=image_token_counts)
return ids
def minimax_h3_multi_image_presentation_token_tags(
tokenizer: Any,
*,
prompt: str,
image_token_counts: list[int],
) -> torch.Tensor:
"""AdaLN token tags aligned with minimax_h3_multi_image_presentation_ids."""
_, tags = _multi_image_presentation(tokenizer, prompt=prompt, image_token_counts=image_token_counts)
return tags
def minimax_h3_ref2va_presentation(
tokenizer: Any,
*,
prompt: str,
condition_labels: list[tuple[str, int]],
image_token_count: int | list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""ref2va positive presentation:
per condition in request order — image i: ``<Picture i>: `` label followed
by the vision block; audio j: ``<Audio j>: `` label only (audio content
never enters Qwen) — then the verbatim prompt. Returns (ids, token_tags)
with the vision block tagged VIDEO(0) and everything else TEXT(1).
condition_labels: [("image", 1), ("audio", 1), ...] with 1-based ordinals
per type.
"""
return minimax_h3_ref2va_video_presentation(
tokenizer,
prompt=prompt,
condition_labels=condition_labels,
image_token_count=image_token_count,
video_block_token_counts=None,
video_block_timestamps=None,
)
def _as_int_list(value: int | Sequence[int] | None, *, name: str) -> list[int]:
if value is None:
return []
if isinstance(value, int):
return [int(value)]
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
raise ValueError(f"{name} must be an int or a sequence of ints")
return [int(item) for item in value]
def _as_nested_int_list(
value: Sequence[int] | Sequence[Sequence[int]] | None,
*,
name: str,
) -> list[list[int]]:
if value is None:
return []
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
raise ValueError(f"{name} must be a sequence")
if len(value) == 0:
return []
first = value[0]
if isinstance(first, Sequence) and not isinstance(first, (str, bytes)):
out: list[list[int]] = []
for group in value:
if not isinstance(group, Sequence) or isinstance(group, (str, bytes)):
raise ValueError(f"{name} must not mix nested and flat entries")
out.append([int(item) for item in group])
return out
return [[int(item) for item in value]]
def _as_nested_float_list(
value: Sequence[float] | Sequence[Sequence[float]] | None,
*,
name: str,
) -> list[list[float]]:
if value is None:
return []
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
raise ValueError(f"{name} must be a sequence")
if len(value) == 0:
return []
first = value[0]
if isinstance(first, Sequence) and not isinstance(first, (str, bytes)):
out: list[list[float]] = []
for group in value:
if not isinstance(group, Sequence) or isinstance(group, (str, bytes)):
raise ValueError(f"{name} must not mix nested and flat entries")
out.append([float(item) for item in group])
return out
return [[float(item) for item in value]]
def minimax_h3_ref2va_video_presentation(
tokenizer: Any,
*,
prompt: str,
condition_labels: list[tuple[str, int]],
image_token_count: int | list[int] | None,
video_block_token_counts: list[int] | list[list[int]] | None,
video_block_timestamps: list[float] | list[list[float]] | None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""ref2va (optionally with video refs) positive presentation:
per condition in request order —
- image i: ``<Picture i>: `` label + one image vision block;
- audio j: ``<Audio j>: `` label only (audio content never enters Qwen);
- video k: ``<Video k>: `` label, then per temporal block a timestamp
text ``<{t:.1f} seconds>`` followed by a VIDEO vision block
(<|vision_start|> + <|video_pad|> x n + <|vision_end|>). Timestamps are
the mean of each merged frame pair (Qwen3VL temporal merge 2; odd frame
counts repeat the last frame), emitting the
``<0.2 seconds>`` ..
``<4.0 seconds>`` sequence — note Python bankers-rounding at .1f.
then the verbatim prompt. Vision blocks are tagged VIDEO(0), everything
else TEXT(1).
"""
if not prompt:
raise ValueError("prompt must be non-empty")
presentation = _Presentation()
image_token_counts = _as_int_list(image_token_count, name="image_token_count")
video_counts_by_ref = _as_nested_int_list(
video_block_token_counts,
name="video_block_token_counts",
)
video_timestamps_by_ref = _as_nested_float_list(
video_block_timestamps,
name="video_block_timestamps",
)
if len(video_counts_by_ref) != len(video_timestamps_by_ref):
raise ValueError("video block token counts and timestamps must align")
image_seen = 0
video_seen = 0
for cond_type, ordinal in condition_labels:
if cond_type == "image":
image_seen += 1
if image_seen > len(image_token_counts):
raise ValueError("image_token_count required for an image reference")
count = int(image_token_counts[image_seen - 1])
if count <= 0:
raise ValueError("image_token_count required for an image reference")
presentation.text(_text_ids(tokenizer, f"<Picture {ordinal}>: "))
presentation.vision(_vision_block_ids(tokenizer, IMAGE_PAD, count))
elif cond_type == "audio":
presentation.text(_text_ids(tokenizer, f"<Audio {ordinal}>: "))
elif cond_type == "video":
video_seen += 1
if video_seen > len(video_counts_by_ref):
raise ValueError("video reference requires block token counts and timestamps")
counts = video_counts_by_ref[video_seen - 1]
timestamps = video_timestamps_by_ref[video_seen - 1]
if not counts or not timestamps:
raise ValueError("video reference requires block token counts and timestamps")
presentation.text(_text_ids(tokenizer, f"<Video {ordinal}>: "))
_timestamped_video_blocks(
presentation,
tokenizer,
counts=counts,
timestamps=timestamps,
context="",
)
else:
raise ValueError(f"unsupported ref2va condition type {cond_type!r}")
if image_seen != len(image_token_counts):
raise ValueError("unused image_token_count entries")
if video_seen != len(video_counts_by_ref):
raise ValueError("unused video block token count entries")
presentation.text(_text_ids(tokenizer, prompt))
return presentation.build()
__all__ = [
"minimax_h3_multi_image_presentation_ids",
"minimax_h3_multi_image_presentation_token_tags",
"minimax_h3_ref2va_presentation",
"minimax_h3_ref2va_video_presentation",
"minimax_h3_text_only_ids",
]
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax H3 Ref2VA reference-video preparation."""
from __future__ import annotations
import json
import math
import os
import subprocess
from pathlib import Path
from typing import Any
import numpy as np
import torch
MINIMAX_H3_FPS = 24.0
MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS = 2.0
MINIMAX_H3_QWEN_TEMPORAL_PATCH = 2
MINIMAX_H3_BASE_SHORT_EDGE = 768
MINIMAX_H3_MAX_PIXELS = 768 * 1344
MINIMAX_H3_CANVAS_MULTIPLE = 32
def _nearest_multiple(value: float, multiple: int) -> int:
return max(multiple, int(round(float(value) / multiple)) * multiple)
def _probe_video(path: str) -> dict[str, Any]:
result = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-select_streams",
"v:0",
"-count_frames",
"-show_entries",
("stream=width,height,r_frame_rate,nb_read_frames,nb_frames,sample_aspect_ratio:stream_tags=rotate"),
"-of",
"json",
path,
],
check=True,
capture_output=True,
text=True,
)
streams = json.loads(result.stdout).get("streams") or []
if not streams:
raise ValueError(f"media has no video stream: {path}")
stream = streams[0]
numerator, denominator = str(stream["r_frame_rate"]).split("/")
raw_count = stream.get("nb_read_frames") or stream.get("nb_frames")
if raw_count in (None, "", "N/A"):
raise ValueError(f"cannot determine video frame count: {path}")
return {
"width": int(stream["width"]),
"height": int(stream["height"]),
"fps": float(numerator) / float(denominator),
"frame_count": int(raw_count),
}
def _has_audio(path: str) -> bool:
result = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-select_streams",
"a:0",
"-show_entries",
"stream=index",
"-of",
"json",
path,
],
check=True,
capture_output=True,
text=True,
)
return bool(json.loads(result.stdout).get("streams") or [])
def _reference_video_shape(width: int, height: int) -> tuple[int, int]:
ratio = float(width) / float(height)
if not 0.25 <= ratio <= 4.0:
raise ValueError(f"reference video aspect ratio must be in [1:4, 4:1], got {width}x{height}")
if ratio >= 1.0:
target_width = MINIMAX_H3_BASE_SHORT_EDGE * ratio
target_height = float(MINIMAX_H3_BASE_SHORT_EDGE)
else:
target_width = float(MINIMAX_H3_BASE_SHORT_EDGE)
target_height = MINIMAX_H3_BASE_SHORT_EDGE / ratio
area = target_width * target_height
if area > MINIMAX_H3_MAX_PIXELS:
scale = math.sqrt(MINIMAX_H3_MAX_PIXELS / area)
target_width *= scale
target_height *= scale
return (
_nearest_multiple(target_width, MINIMAX_H3_CANVAS_MULTIPLE),
_nearest_multiple(target_height, MINIMAX_H3_CANVAS_MULTIPLE),
)
def _transcode_reference_video(
source: str,
*,
target_width: int,
target_height: int,
target_frame_count: int,
workdir: str,
) -> str:
output = str(Path(workdir) / "prepared.mp4")
subprocess.run(
[
"ffmpeg",
"-y",
"-loglevel",
"error",
"-i",
source,
"-map",
"0:v:0",
"-an",
"-vf",
(f"fps={MINIMAX_H3_FPS:g},scale={target_width}:{target_height}:flags=lanczos,setsar=1"),
"-frames:v",
str(target_frame_count),
"-metadata:s:v:0",
"rotate=0",
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
output,
],
check=True,
)
return output
def prepare_reference_videos(
values: Any,
*,
target_frame_count: int,
workdir: str,
) -> list[dict[str, Any]]:
if isinstance(values, (str, os.PathLike)):
values = [values]
if not isinstance(values, (list, tuple)) or not values:
raise ValueError("MiniMax H3 Ref2VA video input must be a path or a non-empty list of paths")
prepared: list[dict[str, Any]] = []
for index, value in enumerate(values):
if not isinstance(value, (str, os.PathLike)):
raise TypeError(
f"MiniMax H3 multi-video Ref2VA currently requires file paths, got item {index}: {type(value)!r}"
)
source = str(value)
meta = _probe_video(source)
width, height = _reference_video_shape(meta["width"], meta["height"])
item_workdir = Path(workdir) / f"video_{index}"
item_workdir.mkdir(parents=True)
prepared_path = _transcode_reference_video(
source,
target_width=width,
target_height=height,
target_frame_count=target_frame_count,
workdir=str(item_workdir),
)
prepared.append(
{
"original_path": source,
"prepared_path": prepared_path,
"input_has_audio": _has_audio(source),
"width": width,
"height": height,
}
)
return prepared
def load_video_frames(path: str) -> np.ndarray:
try:
import decord
except ImportError:
import av
with av.open(path) as container:
frames = [frame.to_ndarray(format="rgb24") for frame in container.decode(video=0)]
if not frames:
raise ValueError(f"video has no frames: {path}")
return np.stack(frames)
reader = decord.VideoReader(path)
if len(reader) <= 0:
raise ValueError(f"video has no frames: {path}")
frames = reader.get_batch(list(range(len(reader))))
return frames.asnumpy() if hasattr(frames, "asnumpy") else np.asarray(frames)
def sample_reference_video_frames(
prepared_path: str,
*,
workdir: str,
) -> dict[str, Any]:
from PIL import Image
meta = _probe_video(prepared_path)
ratio = MINIMAX_H3_FPS / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS
indices: list[int] = []
cursor = 0.0
while True:
frame_index = int(round(cursor))
if frame_index >= meta["frame_count"]:
break
if not indices or frame_index > indices[-1]:
indices.append(frame_index)
cursor += ratio
if not indices:
raise ValueError(f"no frames sampled from {prepared_path}")
frame_dir = Path(workdir)
frame_dir.mkdir(parents=True, exist_ok=True)
frames = []
for output_index, source_index in enumerate(indices, start=1):
output = frame_dir / f"frame_{output_index:06d}.png"
subprocess.run(
[
"ffmpeg",
"-y",
"-loglevel",
"error",
"-i",
prepared_path,
"-vf",
f"select=eq(n\\,{source_index})",
"-vsync",
"vfr",
"-frames:v",
"1",
str(output),
],
check=True,
)
frames.append(np.asarray(Image.open(output).convert("RGB")))
timestamps = [index / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS for index in range(len(indices))]
timestamps += [timestamps[-1]] * ((-len(timestamps)) % MINIMAX_H3_QWEN_TEMPORAL_PATCH)
block_timestamps = [
(timestamps[index] + timestamps[index + MINIMAX_H3_QWEN_TEMPORAL_PATCH - 1]) / 2
for index in range(
0,
len(timestamps),
MINIMAX_H3_QWEN_TEMPORAL_PATCH,
)
]
return {
"frames": frames,
"block_timestamps": block_timestamps,
}
def load_video_audio(path: str) -> tuple[torch.Tensor, int]:
import tempfile
import torchaudio
with tempfile.TemporaryDirectory(prefix="minimax_h3_video_audio_") as tmpdir:
output = str(Path(tmpdir) / "audio.wav")
subprocess.run(
[
"ffmpeg",
"-y",
"-loglevel",
"error",
"-i",
path,
"-vn",
"-ac",
"2",
"-ar",
"44100",
"-f",
"wav",
output,
],
check=True,
)
return torchaudio.load(output)
__all__ = [
"load_video_audio",
"load_video_frames",
"prepare_reference_videos",
"sample_reference_video_frames",
]
@@ -0,0 +1,179 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import math
from typing import Any
import torch
def _require_finite_tensor(tensor: torch.Tensor, name: str) -> None:
if not bool(torch.isfinite(tensor).all().item()):
raise ValueError(f"{name} must be finite")
def _validate_unit_timestep(timestep: torch.Tensor, name: str) -> None:
if not isinstance(timestep, torch.Tensor):
raise ValueError(f"{name} must be a torch.Tensor")
if not torch.is_floating_point(timestep):
raise ValueError(f"{name} must be a floating point tensor")
_require_finite_tensor(timestep, name)
out_of_range = (timestep < 0) | (timestep > 1)
if bool(out_of_range.any().item()):
raise ValueError(f"{name} must be in [0, 1]")
def _validate_sigma(value: float, name: str) -> float:
sigma = float(value)
if not math.isfinite(sigma):
raise ValueError(f"{name} must be finite")
if sigma < 0.0:
raise ValueError(f"{name} must be non-negative")
return sigma
def _validate_timestep_sigma_pair(
timestep: torch.Tensor,
sigma_curr: float,
name: str,
) -> float:
_validate_unit_timestep(timestep, f"{name}_timestep")
sigma = _validate_sigma(sigma_curr, f"{name}_sigma_curr")
expected = 1.0 - timestep.detach().to(dtype=torch.float32)
actual = torch.full_like(expected, sigma)
if not torch.allclose(actual, expected, rtol=1e-5, atol=1e-5):
raise ValueError(f"{name}_sigma_curr must equal 1 - {name}_timestep")
return sigma
def minimax_h3_rf_v_to_x0(
xt: torch.Tensor,
v: torch.Tensor,
timestep: torch.Tensor,
) -> torch.Tensor:
if xt.shape != v.shape:
raise ValueError(f"xt and v shapes must match, got {xt.shape} vs {v.shape}")
if not torch.is_floating_point(xt):
raise ValueError("xt must be a floating point tensor")
if not torch.is_floating_point(v):
raise ValueError("v must be a floating point tensor")
_require_finite_tensor(xt, "xt")
_require_finite_tensor(v, "v")
_validate_unit_timestep(timestep, "timestep")
cond_t = timestep.to(device=xt.device, dtype=xt.dtype)
while cond_t.ndim < xt.ndim:
cond_t = cond_t.unsqueeze(-1)
sigma_t = 1 - cond_t
x0 = xt + sigma_t * v
_require_finite_tensor(x0, "x0")
return x0
def minimax_h3_euler_eta0_step(
state: torch.Tensor,
denoised: torch.Tensor,
*,
sigma_curr: float,
sigma_next: float,
) -> torch.Tensor:
if state.shape != denoised.shape:
raise ValueError(f"state and denoised shapes must match, got {state.shape} vs {denoised.shape}")
if not torch.is_floating_point(state):
raise ValueError("state must be a floating point tensor")
if not torch.is_floating_point(denoised):
raise ValueError("denoised must be a floating point tensor")
_require_finite_tensor(state, "state")
_require_finite_tensor(denoised, "denoised")
sigma_curr = _validate_sigma(sigma_curr, "sigma_curr")
sigma_next = _validate_sigma(sigma_next, "sigma_next")
if sigma_curr == 0.0:
if sigma_next != 0.0:
raise ValueError("sigma_next must be 0 when sigma_curr is 0")
return state
compute_dtype = torch.float32
if state.dtype not in (torch.float16, torch.bfloat16):
compute_dtype = state.dtype
sigma_curr_t = state.new_tensor(sigma_curr, dtype=compute_dtype)
sigma_next_t = state.new_tensor(sigma_next, dtype=compute_dtype)
sigma_ratio = sigma_next_t / sigma_curr_t
out = sigma_ratio * state.to(dtype=compute_dtype) + (1.0 - sigma_ratio) * denoised.to(dtype=compute_dtype)
out = out.to(dtype=state.dtype)
_require_finite_tensor(out, "euler_eta0_step output")
return out
class MiniMaxH3EulerAncestralEta0SchedulerAdapter:
def __init__(self, **config: Any) -> None:
if config:
raise ValueError(f"{type(self).__name__} does not accept config fields: {sorted(config)}")
def set_shift(self, _flow_shift: float) -> None:
"""Ignore flow shift, matching the previous loader-specific path."""
def step_denoising(
self,
*,
input_visual_latent: torch.Tensor,
input_audio_latent: torch.Tensor,
timestep: torch.Tensor,
noise_pred_visual: torch.Tensor,
noise_pred_audio: torch.Tensor,
sigma_curr: float,
sigma_next: float,
video_timestep: torch.Tensor | None = None,
audio_timestep: torch.Tensor | None = None,
video_sigma_curr: float | None = None,
video_sigma_next: float | None = None,
audio_sigma_curr: float | None = None,
audio_sigma_next: float | None = None,
) -> dict[str, torch.Tensor]:
visual_timestep = timestep if video_timestep is None else video_timestep
audio_timestep = timestep if audio_timestep is None else audio_timestep
visual_sigma_curr = sigma_curr if video_sigma_curr is None else video_sigma_curr
visual_sigma_next = sigma_next if video_sigma_next is None else video_sigma_next
audio_sigma_curr = sigma_curr if audio_sigma_curr is None else audio_sigma_curr
audio_sigma_next = sigma_next if audio_sigma_next is None else audio_sigma_next
visual_sigma_curr = _validate_timestep_sigma_pair(
visual_timestep,
visual_sigma_curr,
"video",
)
audio_sigma_curr = _validate_timestep_sigma_pair(
audio_timestep,
audio_sigma_curr,
"audio",
)
denoised_visual = minimax_h3_rf_v_to_x0(
input_visual_latent,
noise_pred_visual,
visual_timestep,
)
denoised_audio = minimax_h3_rf_v_to_x0(
input_audio_latent,
noise_pred_audio,
audio_timestep,
)
return {
"output_visual_latent": minimax_h3_euler_eta0_step(
input_visual_latent,
denoised_visual,
sigma_curr=visual_sigma_curr,
sigma_next=visual_sigma_next,
),
"output_audio_latent": minimax_h3_euler_eta0_step(
input_audio_latent,
denoised_audio,
sigma_curr=audio_sigma_curr,
sigma_next=audio_sigma_next,
),
}
EntryClass = MiniMaxH3EulerAncestralEta0SchedulerAdapter
__all__ = [
"MiniMaxH3EulerAncestralEta0SchedulerAdapter",
"minimax_h3_euler_eta0_step",
"minimax_h3_rf_v_to_x0",
]
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
def _align_frame_count(frame_count: int) -> int:
"""Snap ``frame_count`` up to the MiniMax H3 17n+5 frame boundary."""
if frame_count <= 0:
return 1
current = int(frame_count)
while current % 17 != 5:
current += 1
return current
def _video_latent_t(frame_count: int) -> int:
if frame_count <= 5:
return 2
return ((int(frame_count) - 5) // 17) * 5 + 2
def _frame_count_from_video_latent_t(out_t: int) -> int:
if out_t == 1:
return 1
if out_t < 2 or (out_t - 2) % 5 != 0:
raise ValueError("MiniMax H3 video latent T must be 1 or match 5n+2")
return 17 * ((int(out_t) - 2) // 5) + 5
def _audio_latent_t(duration_seconds: float) -> int:
# Rounding happens at the 40 Hz audio latent boundary.
return int(round(float(duration_seconds) * 40.0))
def _time_shift_sigmas(
*,
num_steps: int = 50,
shift_scale: float = 6.0,
) -> list[float]:
if shift_scale <= 0:
raise ValueError("MiniMax H3 shift_scale must be > 0")
if num_steps <= 0:
raise ValueError("MiniMax H3 num_steps must be > 0")
import torch
# The rectified-flow sigma range is fixed at [1.0, 0.0].
base = torch.linspace(
1.0,
0.0,
int(num_steps),
device="cpu",
dtype=torch.float32,
)
shifted = float(shift_scale) * base / (1 + (float(shift_scale) - 1) * base)
shifted, _ = torch.unique_consecutive(shifted, return_counts=True)
# A one-point request is still exactly one point. Normal serving uses
# multiple points, but preserving the requested cardinality keeps
# ``num_inference_steps`` the sole schedule-size control.
if num_steps > 1 and shifted[-1].item() > 0.0:
shifted = torch.cat([shifted, torch.tensor([0.0], dtype=shifted.dtype)])
return [float(value) for value in shifted.tolist()]
class MiniMaxH3ShapePlanner:
"""Compute MiniMax H3 frame, latent-shape, and time-shift values."""
def align_frame_count(self, frame_count: int) -> int:
return _align_frame_count(frame_count)
def video_latent_t(self, frame_count: int) -> int:
return _video_latent_t(frame_count)
def frame_count_from_video_latent_t(self, out_t: int) -> int:
return _frame_count_from_video_latent_t(out_t)
def audio_latent_t(self, duration_seconds: float) -> int:
return _audio_latent_t(duration_seconds)
def time_shift_sigmas(
self,
*,
num_steps: int = 50,
shift_scale: float = 6.0,
) -> list[float]:
return _time_shift_sigmas(
num_steps=num_steps,
shift_scale=shift_scale,
)
MINIMAX_H3_SHAPE_PLANNER = MiniMaxH3ShapePlanner()
def minimax_h3_align_frame_count(frame_count: int) -> int:
return MINIMAX_H3_SHAPE_PLANNER.align_frame_count(frame_count)
def minimax_h3_frame_count_from_video_latent_t(out_t: int) -> int:
return MINIMAX_H3_SHAPE_PLANNER.frame_count_from_video_latent_t(out_t)
def minimax_h3_time_shift_sigmas(
*,
num_steps: int = 50,
shift_scale: float = 6.0,
) -> list[float]:
return MINIMAX_H3_SHAPE_PLANNER.time_shift_sigmas(
num_steps=num_steps,
shift_scale=shift_scale,
)
@@ -0,0 +1,360 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax H3 remote-code VAE adapters and exact latent contracts."""
from __future__ import annotations
import importlib
import json
from contextlib import AbstractContextManager
from pathlib import Path
from typing import Any
import torch
import torch.distributed as dist
import torch.nn as nn
from PIL import Image
from transformers.dynamic_module_utils import get_class_from_dynamic_module
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
DistributedVaeMixin,
)
from vllm_omni.diffusion.distributed.parallel_state import get_dit_group
from .packed_tokens import minimax_h3_patchify_video_latent
MINIMAX_H3_KEYFRAME_ENCODE_SEED = 42
MINIMAX_H3_AUDIO_SAMPLE_RATE = 32000
MINIMAX_H3_AUDIO_CHANNELS = 2
def _load_component_config(component_path: str) -> dict[str, Any]:
config_path = Path(component_path) / "config.json"
config = json.loads(config_path.read_text(encoding="utf-8"))
channels = int(config["latent_channels"])
for key in ("latents_mean", "latents_std"):
values = config.get(key)
if not isinstance(values, list) or len(values) != channels:
raise ValueError(f"{config_path}: {key} must contain {channels} values")
return config
def _load_remote_component(
component_path: str,
config: dict[str, Any],
) -> nn.Module:
auto_map = config.get("auto_map") or {}
class_reference = auto_map.get("AutoModel")
if not isinstance(class_reference, str):
raise ValueError(f"{component_path}/config.json must define auto_map.AutoModel")
component_cls = get_class_from_dynamic_module(
class_reference,
component_path,
)
return component_cls.from_pretrained(component_path)
class _AudioVAEDeterminismContext(AbstractContextManager):
def __enter__(self):
backends = torch.backends
self._saved = (
backends.cuda.matmul.allow_tf32,
backends.cudnn.allow_tf32,
backends.cudnn.benchmark,
backends.cudnn.deterministic,
backends.cudnn.enabled,
backends.cuda.flash_sdp_enabled(),
backends.cuda.mem_efficient_sdp_enabled(),
backends.cuda.math_sdp_enabled(),
)
backends.cuda.matmul.allow_tf32 = False
backends.cudnn.allow_tf32 = False
backends.cudnn.benchmark = False
backends.cudnn.deterministic = True
backends.cudnn.enabled = False
backends.cuda.enable_flash_sdp(False)
backends.cuda.enable_mem_efficient_sdp(False)
backends.cuda.enable_math_sdp(True)
return self
def __exit__(self, exc_type, exc, traceback):
backends = torch.backends
(
backends.cuda.matmul.allow_tf32,
backends.cudnn.allow_tf32,
backends.cudnn.benchmark,
backends.cudnn.deterministic,
backends.cudnn.enabled,
flash,
memory_efficient,
math_sdp,
) = self._saved
backends.cuda.enable_flash_sdp(flash)
backends.cuda.enable_mem_efficient_sdp(memory_efficient)
backends.cuda.enable_math_sdp(math_sdp)
return False
class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
"""Adapter around the checkpoint's native parallel-tiled video VAE."""
def __init__(
self,
component_path: str,
*,
device: torch.device,
) -> None:
super().__init__()
self.config_dict = _load_component_config(component_path)
self.remote = _load_remote_component(
component_path,
self.config_dict,
)
# Match the reference loader contract: video VAE weights stay FP32.
# Keyframe encoding is numerically sensitive to first casting the
# checkpoint through FP16; decode still runs under FP16 autocast.
self.remote.eval().to(device=device, dtype=torch.float32)
self.model = self.remote.model
self.use_tiling = True
self.use_slicing = False
self.parallel_size = 1
def set_parallel_size(
self,
parallel_size: int,
mode: str = "tile",
) -> None:
if mode != "tile":
raise ValueError(f"MiniMax H3 VAE supports its native tile parallel mode only, got {mode!r}")
group = get_dit_group()
world_size = dist.get_world_size(group)
rank = dist.get_rank(group)
parallel_size = int(parallel_size)
if parallel_size not in (1, world_size):
raise ValueError(
"MiniMax H3 native VAE patch parallelism currently requires "
"vae_patch_parallel_size=1 or the full DiT group size "
f"({world_size}), got {parallel_size}"
)
self.parallel_size = parallel_size
enabled = parallel_size > 1
package = self.remote.__class__.__module__.rsplit(".", 1)[0]
parallel_module = importlib.import_module(f"{package}.parallel")
state = parallel_module.get_parallel_state()
state.clear()
state.update(
group_size=parallel_size,
group_rank=rank if enabled else 0,
local_process_group=group if enabled else None,
sp_size=parallel_size,
sp_rank=rank if enabled else 0,
sp_enabled=enabled,
sp_process_group=group if enabled else None,
tp_size=1,
tp_rank=0,
)
self.model.parallel_tiling = enabled
def is_distributed_enabled(self) -> bool:
return self.parallel_size > 1 and dist.is_initialized()
@torch.inference_mode()
def encode_image(self, image: Image.Image) -> torch.Tensor:
previous_parallel = self.model.parallel_tiling
self.model.parallel_tiling = False
parameter = next(self.parameters())
previous_dtype = parameter.dtype
if previous_dtype != torch.float32:
self.to(torch.float32)
devices = [parameter.device] if parameter.device.type == "cuda" else []
try:
with torch.random.fork_rng(devices=devices):
torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
for device in devices:
with torch.cuda.device(device):
torch.cuda.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
latent = self.model.encode_images(
image,
use_fp16_latent=True,
)[0]
finally:
self.model.parallel_tiling = previous_parallel
if previous_dtype != torch.float32:
self.to(previous_dtype)
# Match the reference contract exactly: normalization and patchify
# happen on CPU in FP32 after the sampled encode. The condition noise
# path is sensitive enough that doing these elementwise operations on
# CUDA can noticeably change the final conditioned video.
latent = latent.float().cpu()
if latent.ndim == 4:
latent = latent[None]
channels = int(self.config_dict["latent_channels"])
mean = torch.tensor(
self.config_dict["latents_mean"],
).view(1, channels, 1, 1, 1)
std = torch.tensor(
self.config_dict["latents_std"],
).view(1, channels, 1, 1, 1)
return minimax_h3_patchify_video_latent(
(latent - mean) / std,
patch_size=(1, 2, 2),
).float()
@torch.inference_mode()
def encode_video(
self,
frames: Any,
) -> tuple[torch.Tensor, tuple[int, int, int]]:
parameter = next(self.parameters())
previous_dtype = parameter.dtype
if previous_dtype != torch.float32:
self.to(torch.float32)
devices = [parameter.device] if parameter.device.type == "cuda" else []
try:
with torch.random.fork_rng(devices=devices):
torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
for device in devices:
with torch.cuda.device(device):
torch.cuda.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
latent = self.model.encode_videos(
frames,
use_fp16_latent=True,
)[0]
finally:
if previous_dtype != torch.float32:
self.to(previous_dtype)
latent = latent.float().cpu()
if latent.ndim == 4:
latent = latent[None]
channels = int(self.config_dict["latent_channels"])
if latent.ndim != 5 or int(latent.shape[1]) != channels:
raise ValueError(f"unexpected reference video latent shape {tuple(latent.shape)}")
shape = (
int(latent.shape[2]),
int(latent.shape[3]),
int(latent.shape[4]),
)
mean = torch.tensor(
self.config_dict["latents_mean"],
).view(1, channels, 1, 1, 1)
std = torch.tensor(
self.config_dict["latents_std"],
).view(1, channels, 1, 1, 1)
rows = minimax_h3_patchify_video_latent(
(latent - mean) / std,
patch_size=(1, 2, 2),
).float()
return rows, shape
@torch.inference_mode()
def decode_latent(self, latent: torch.Tensor) -> torch.Tensor:
channels = int(self.config_dict["latent_channels"])
mean = torch.tensor(
self.config_dict["latents_mean"],
device=latent.device,
dtype=latent.dtype,
).view(1, channels, 1, 1, 1)
std = torch.tensor(
self.config_dict["latents_std"],
device=latent.device,
dtype=latent.dtype,
).view(1, channels, 1, 1, 1)
decoded = self.model.decode_base(latent * std + mean)
frames = self.model.processor.revert_tensor(decoded)
if frames.ndim == 4:
frames = frames.unsqueeze(0).transpose(1, 2)
if frames.ndim != 5:
raise ValueError(f"unexpected decoded video shape {tuple(frames.shape)}")
return frames.float()
class MiniMaxH3AudioVAE(nn.Module):
def __init__(
self,
component_path: str,
*,
device: torch.device,
) -> None:
super().__init__()
self.config_dict = _load_component_config(component_path)
self.remote = _load_remote_component(
component_path,
self.config_dict,
)
# The checkpoint's audio VAE contract is FP32 for both reference
# encoding and waveform decoding.
self.remote.eval().to(device=device, dtype=torch.float32)
self.model = self.remote.model
self.sample_rate = int(self.config_dict["sample_rate"])
@torch.inference_mode()
def encode_waveform(
self,
waveform: torch.Tensor,
sample_rate: int,
) -> tuple[torch.Tensor, int]:
import torchaudio
waveform = waveform.float()
if waveform.ndim == 1:
waveform = waveform[None]
if int(sample_rate) != MINIMAX_H3_AUDIO_SAMPLE_RATE:
waveform = torchaudio.transforms.Resample(
int(sample_rate),
MINIMAX_H3_AUDIO_SAMPLE_RATE,
)(waveform)
if waveform.shape[0] < MINIMAX_H3_AUDIO_CHANNELS:
waveform = waveform.repeat(
MINIMAX_H3_AUDIO_CHANNELS,
1,
)
waveform = waveform[:MINIMAX_H3_AUDIO_CHANNELS]
device = next(self.model.parameters()).device
waveform = waveform.to(device)
with _AudioVAEDeterminismContext():
audio = self.model.preprocess(
waveform.unsqueeze(1),
MINIMAX_H3_AUDIO_SAMPLE_RATE,
)
latent = self.model.encoder(audio)
if bool(getattr(self.model, "attn_proj", False)):
latent = self.model.pre_block(latent.transpose(1, 2)).transpose(1, 2)
latent = self.model.mean_proj(latent).float().cpu()
channels = int(self.config_dict["latent_channels"])
if latent.shape[-1] != channels:
if latent.shape[1] != channels:
raise ValueError(f"cannot canonicalize audio latent {tuple(latent.shape)}")
latent = latent.transpose(1, 2).contiguous()
mean = torch.tensor(
self.config_dict["latents_mean"],
).view(1, 1, channels)
std = torch.tensor(
self.config_dict["latents_std"],
).view(1, 1, channels)
rows = ((latent - mean) / std).reshape(-1, channels)
return rows.float(), int(latent.shape[1])
@torch.inference_mode()
def decode_latent(self, latent: torch.Tensor) -> torch.Tensor:
channels = int(self.config_dict["latent_channels"])
mean = torch.tensor(
self.config_dict["latents_mean"],
device=latent.device,
dtype=latent.dtype,
).view(1, channels, 1)
std = torch.tensor(
self.config_dict["latents_std"],
device=latent.device,
dtype=latent.dtype,
).view(1, channels, 1)
waveform = self.remote.decode(latent * std + mean)
if waveform.ndim != 3 or waveform.shape[1] != 1:
raise ValueError(f"unexpected decoded audio shape {tuple(waveform.shape)}")
return waveform.permute(1, 0, 2).contiguous().float()
__all__ = ["MiniMaxH3AudioVAE", "MiniMaxH3VideoVAE"]