[Model] Add MiniMax H3 diffusion support (#5691)
This commit is contained in:
1 parent
727f90958e
commit
24741961eb
45 files changed
+7356
-31
No files matched your search
@@ -0,0 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax H3 audio-video diffusion support."""
|
||||
|
||||
from .pipeline_minimax_h3 import (
|
||||
MiniMaxH3Pipeline,
|
||||
get_minimax_h3_post_process_func,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MiniMaxH3Pipeline",
|
||||
"get_minimax_h3_post_process_func",
|
||||
]
|
||||
@@ -0,0 +1,167 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax H3 visual/audio condition-noise augmentation.
|
||||
|
||||
The request's condition timestep is applied to both the tensor value and the
|
||||
DiT timestep. Tokenizer artifacts remain clean
|
||||
and reusable; this module materializes the fixed noised anchors immediately
|
||||
before the denoise loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from .packed_tokens import minimax_h3_patchify_video_latent
|
||||
|
||||
# Channel-major packed audio rows always carry a stereo layout.
|
||||
MINIMAX_H3_AUDIO_COND_CHANNELS = 2
|
||||
|
||||
|
||||
def minimax_h3_imgvid_cond_noise_aug_rows(
|
||||
clean_rows: torch.Tensor,
|
||||
*,
|
||||
condition_shapes: Sequence[Sequence[int]],
|
||||
target_latent_t: int,
|
||||
imgvid_cond_num_frames: int,
|
||||
seed: int,
|
||||
noise_aug: float,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the imgvid-condition RF noise recipe to packed clean rows.
|
||||
|
||||
``condition_shapes`` contains ``(latent_t, latent_h, latent_w)`` in packed
|
||||
visual-condition order. A new CPU generator with the same row seed is
|
||||
created for every condition. Under the dependent-noise policy, each draw
|
||||
uses the target temporal length plus the template's imgvid-condition frame
|
||||
count, then slices the prefix matching the current condition.
|
||||
"""
|
||||
|
||||
noise_aug = float(noise_aug)
|
||||
if not 0.0 <= noise_aug <= 1.0:
|
||||
raise ValueError(f"noise_aug must be in [0, 1], got {noise_aug}")
|
||||
if noise_aug == 1.0:
|
||||
return clean_rows
|
||||
if clean_rows.ndim != 2 or int(clean_rows.shape[1]) != 96:
|
||||
raise ValueError(f"clean imgvid condition rows must have shape [n, 96], got {list(clean_rows.shape)}")
|
||||
|
||||
target_latent_t = int(target_latent_t)
|
||||
imgvid_cond_num_frames = int(imgvid_cond_num_frames)
|
||||
if target_latent_t <= 0:
|
||||
raise ValueError(f"target_latent_t must be positive, got {target_latent_t}")
|
||||
if imgvid_cond_num_frames <= 0:
|
||||
raise ValueError(
|
||||
f"imgvid_cond_num_frames must be positive when condition rows exist, got {imgvid_cond_num_frames}"
|
||||
)
|
||||
|
||||
parsed_shapes: list[tuple[int, int, int]] = []
|
||||
expected_rows = 0
|
||||
for raw_shape in condition_shapes:
|
||||
if len(raw_shape) != 3:
|
||||
raise ValueError(
|
||||
f"each imgvid condition shape must be (latent_t, latent_h, latent_w), got {list(raw_shape)}"
|
||||
)
|
||||
latent_t, latent_h, latent_w = (int(value) for value in raw_shape)
|
||||
if latent_t <= 0 or latent_h <= 0 or latent_w <= 0:
|
||||
raise ValueError(f"imgvid condition shape must be positive, got {list(raw_shape)}")
|
||||
if latent_h % 2 or latent_w % 2:
|
||||
raise ValueError(
|
||||
f"imgvid condition spatial dimensions must be divisible by 2, got {(latent_t, latent_h, latent_w)}"
|
||||
)
|
||||
parsed_shapes.append((latent_t, latent_h, latent_w))
|
||||
expected_rows += latent_t * (latent_h // 2) * (latent_w // 2)
|
||||
if not parsed_shapes:
|
||||
raise ValueError("condition_shapes must not be empty")
|
||||
if int(clean_rows.shape[0]) != expected_rows:
|
||||
raise ValueError(
|
||||
f"clean imgvid condition rows {int(clean_rows.shape[0])} != shape-derived rows {expected_rows}"
|
||||
)
|
||||
|
||||
output_device = clean_rows.device
|
||||
clean_rows = clean_rows.detach().to(device="cpu", dtype=torch.float32)
|
||||
out: list[torch.Tensor] = []
|
||||
row_offset = 0
|
||||
timestep = torch.tensor(noise_aug, dtype=torch.float32, device="cpu")
|
||||
for latent_t, latent_h, latent_w in parsed_shapes:
|
||||
full_t = target_latent_t + imgvid_cond_num_frames
|
||||
if full_t < latent_t:
|
||||
raise ValueError(f"condition latent_t {latent_t} exceeds the noise draw length {full_t}")
|
||||
generator = torch.Generator(device="cpu").manual_seed(int(seed))
|
||||
noise = torch.randn(
|
||||
1,
|
||||
24,
|
||||
full_t,
|
||||
latent_h,
|
||||
latent_w,
|
||||
generator=generator,
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)[:, :, :latent_t]
|
||||
noise_rows = minimax_h3_patchify_video_latent(noise, patch_size=[1, 2, 2]).to(dtype=torch.float32)
|
||||
row_count = int(noise_rows.shape[0])
|
||||
clean_part = clean_rows[row_offset : row_offset + row_count].to(torch.float32)
|
||||
out.append(timestep * clean_part + (1.0 - timestep) * noise_rows)
|
||||
row_offset += row_count
|
||||
return torch.cat(out, dim=0).to(device=output_device, dtype=torch.float32).contiguous()
|
||||
|
||||
|
||||
def minimax_h3_audio_cond_noise_aug_rows(
|
||||
clean_rows: torch.Tensor,
|
||||
*,
|
||||
condition_audio_t: Sequence[int],
|
||||
seed: int,
|
||||
noise_aug: float,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the audio-condition RF noise recipe to packed clean rows.
|
||||
|
||||
``condition_audio_t`` contains the latent T of each audio-bearing
|
||||
condition in canonical request order. Noise is drawn per condition
|
||||
element, with a fresh CPU generator seeded with ``seed + 1`` for every
|
||||
element. Consequently each condition restarts the
|
||||
same RNG stream; concatenating the rows and drawing once would be
|
||||
numerically different for ordered multi-reference requests.
|
||||
|
||||
The mix is intentionally evaluated on CPU in fp32 before the packed rows
|
||||
are transferred to the DiT device.
|
||||
"""
|
||||
|
||||
noise_aug = float(noise_aug)
|
||||
if not 0.0 <= noise_aug <= 1.0:
|
||||
raise ValueError(f"noise_aug must be in [0, 1], got {noise_aug}")
|
||||
if noise_aug == 1.0:
|
||||
return clean_rows
|
||||
if clean_rows.ndim != 2 or int(clean_rows.shape[1]) != 32:
|
||||
raise ValueError(f"clean audio condition rows must have shape [n, 32], got {list(clean_rows.shape)}")
|
||||
|
||||
audio_channels = MINIMAX_H3_AUDIO_COND_CHANNELS
|
||||
parsed_audio_t = [int(value) for value in condition_audio_t]
|
||||
if not parsed_audio_t:
|
||||
raise ValueError("condition_audio_t must not be empty")
|
||||
if any(value <= 0 for value in parsed_audio_t):
|
||||
raise ValueError(f"condition audio latent lengths must be positive, got {parsed_audio_t}")
|
||||
expected_rows = audio_channels * sum(parsed_audio_t)
|
||||
if int(clean_rows.shape[0]) != expected_rows:
|
||||
raise ValueError(f"clean audio condition rows {int(clean_rows.shape[0])} != shape-derived rows {expected_rows}")
|
||||
|
||||
out: list[torch.Tensor] = []
|
||||
row_offset = 0
|
||||
timestep = torch.tensor(noise_aug, dtype=torch.float32, device="cpu")
|
||||
for audio_t in parsed_audio_t:
|
||||
row_count = audio_channels * audio_t
|
||||
clean_part = clean_rows[row_offset : row_offset + row_count].detach().to(device="cpu", dtype=torch.float32)
|
||||
generator = torch.Generator(device="cpu").manual_seed(int(seed) + 1)
|
||||
noise = torch.randn(
|
||||
clean_part.shape,
|
||||
generator=generator,
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
out.append(timestep * clean_part + (1.0 - timestep) * noise)
|
||||
row_offset += row_count
|
||||
return torch.cat(out, dim=0).to(device=clean_rows.device, dtype=torch.float32).contiguous()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"minimax_h3_audio_cond_noise_aug_rows",
|
||||
"minimax_h3_imgvid_cond_noise_aug_rows",
|
||||
]
|
||||
@@ -0,0 +1,249 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax H3 cfg-distilled full denoise loop.
|
||||
|
||||
Per step, the positive presentation is forwarded exactly once. Video and audio
|
||||
target rows chain through the Euler-eta0 update while visual and audio condition
|
||||
rows stay pinned to their noised step-0 anchors.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from .scheduling_minimax_h3_euler_ancestral import (
|
||||
minimax_h3_euler_eta0_step,
|
||||
minimax_h3_rf_v_to_x0,
|
||||
)
|
||||
|
||||
MINIMAX_H3_IMGVID_COND_TIMESTEP = 0.999
|
||||
# ref2va audio reference anchor timestep
|
||||
MINIMAX_H3_AUDIO_REF_COND_TIMESTEP = 1.0
|
||||
# Packed row widths: video rows are [1,2,2]-patchified 24-channel latents
|
||||
# (24 * 1 * 2 * 2 = 96); audio rows carry the 32-dim audio latent.
|
||||
MINIMAX_H3_VIDEO_ROW_WIDTH = 96
|
||||
MINIMAX_H3_AUDIO_ROW_WIDTH = 32
|
||||
|
||||
|
||||
class MiniMaxH3DenoiseBranch:
|
||||
"""Static per-branch state: packed layout + fixed forward kwargs.
|
||||
|
||||
`packed` is a minimax_h3_packed_sequence(...) result (or equivalent layout
|
||||
dict); `text_embeddings` is the branch's [text_len, 5120] hidden states;
|
||||
`token_tags` must already carry any fl2va vision-span overrides.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
packed: dict[str, torch.Tensor],
|
||||
text_embeddings: torch.Tensor,
|
||||
token_tags: torch.Tensor,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
seq_len = int(packed["seq_len"])
|
||||
self.seq_len = seq_len
|
||||
self.img_pos = packed["img_pos"].view(-1).to(torch.long)
|
||||
self.audio_pos = packed["audio_pos"].view(-1).to(torch.long)
|
||||
self.update_mask = packed["update_mask"].view(-1).to(torch.bool)
|
||||
# ref2va: audio_pos may include reference-audio anchor rows
|
||||
# (audio_update_mask False); absent means all rows are targets.
|
||||
if "audio_update_mask" in packed:
|
||||
self.audio_update_mask = packed["audio_update_mask"].view(-1).to(torch.bool)
|
||||
else:
|
||||
self.audio_update_mask = torch.ones(self.audio_pos.shape[0], dtype=torch.bool)
|
||||
text_len = int(packed["text_pos"].view(-1).shape[0])
|
||||
if list(text_embeddings.shape)[0] != text_len:
|
||||
raise ValueError(f"text_embeddings rows {list(text_embeddings.shape)} != packed text_len {text_len}")
|
||||
if int(token_tags.view(-1).shape[0]) != seq_len:
|
||||
raise ValueError(f"token_tags length {int(token_tags.view(-1).shape[0])} != seq_len {seq_len}")
|
||||
cu = packed["cu_seqlens"].to(torch.int32)
|
||||
self.img_pos_dev = self.img_pos.to(device)
|
||||
self.audio_pos_dev = self.audio_pos.to(device)
|
||||
self.update_mask_dev = self.update_mask.to(device)
|
||||
self.audio_update_mask_dev = self.audio_update_mask.to(device)
|
||||
self.x_base = torch.zeros(1, seq_len, MINIMAX_H3_VIDEO_ROW_WIDTH, dtype=torch.float32, device=device)
|
||||
self.audio_x_base = torch.zeros(1, seq_len, MINIMAX_H3_AUDIO_ROW_WIDTH, dtype=torch.float32, device=device)
|
||||
text_pos_dev = packed["text_pos"].view(-1).to(torch.long).to(device)
|
||||
self.static_kwargs: dict[str, Any] = {
|
||||
"img_position_ids": packed["img_position_ids"][None].to(device),
|
||||
"update_mask": self.update_mask_dev,
|
||||
"token_tags": token_tags.view(-1).to(torch.long).to(device),
|
||||
"skip_mask_out_condition": False,
|
||||
"prompt_embeds": text_embeddings.to(device),
|
||||
"img_pos_info": {"position_ids": self.img_pos_dev},
|
||||
"audio_pos_info": {"position_ids": self.audio_pos_dev},
|
||||
"text_pos_info": {"position_ids": text_pos_dev},
|
||||
"img_pos_for_infer_output_info": {"position_ids": self.img_pos_dev},
|
||||
"packed_seq_params": {
|
||||
"cu_seqlens_q": cu.to(device),
|
||||
"max_seqlen_q": int(cu[1]),
|
||||
},
|
||||
"refiner_packed_seq_params": {
|
||||
"cu_seqlens_q": torch.tensor([0, text_len, text_len], dtype=torch.int32, device=device),
|
||||
"max_seqlen_q": text_len,
|
||||
},
|
||||
}
|
||||
|
||||
def forward_kwargs(
|
||||
self,
|
||||
*,
|
||||
video_rows: torch.Tensor,
|
||||
audio_rows: torch.Tensor,
|
||||
t_video: float,
|
||||
t_audio: float,
|
||||
imgvid_cond_timestep: float,
|
||||
audio_ref_cond_timestep: float,
|
||||
) -> dict[str, Any]:
|
||||
x = self.x_base.clone()
|
||||
x[0].index_copy_(0, self.img_pos_dev, video_rows)
|
||||
audio_x = self.audio_x_base.clone()
|
||||
audio_x[0].index_copy_(0, self.audio_pos_dev, audio_rows)
|
||||
# Packed-sequence timestep semantics: non-media rows (text and
|
||||
# padding) inherit the current video timestep. Later steps must reuse
|
||||
# the previous step's updated rows; re-initializing from zeros is only
|
||||
# valid at step 0.
|
||||
timesteps = torch.full(
|
||||
(self.seq_len,),
|
||||
float(t_video),
|
||||
dtype=torch.float32,
|
||||
device=x.device,
|
||||
)
|
||||
timesteps[self.img_pos_dev[self.update_mask_dev]] = t_video
|
||||
timesteps[self.img_pos_dev[~self.update_mask_dev]] = imgvid_cond_timestep
|
||||
timesteps[self.audio_pos_dev[self.audio_update_mask_dev]] = t_audio
|
||||
timesteps[self.audio_pos_dev[~self.audio_update_mask_dev]] = audio_ref_cond_timestep
|
||||
unique_timesteps, inverse_indices = torch.unique(timesteps, sorted=True, return_inverse=True)
|
||||
return {
|
||||
**self.static_kwargs,
|
||||
"x": x,
|
||||
"audio_x": audio_x,
|
||||
"unique_timesteps": unique_timesteps,
|
||||
"inverse_indices": inverse_indices,
|
||||
}
|
||||
|
||||
|
||||
def minimax_h3_denoise_loop(
|
||||
*,
|
||||
model: Any,
|
||||
positive: MiniMaxH3DenoiseBranch,
|
||||
initial_video_rows: torch.Tensor,
|
||||
initial_audio_rows: torch.Tensor,
|
||||
keyframe_cond_rows: torch.Tensor | None,
|
||||
audio_ref_rows: torch.Tensor | None = None,
|
||||
sigmas_video: list[float],
|
||||
sigmas_audio: list[float],
|
||||
device: torch.device,
|
||||
imgvid_cond_noise_aug_for_inference: float = MINIMAX_H3_IMGVID_COND_TIMESTEP,
|
||||
audio_cond_noise_aug_for_inference: float = MINIMAX_H3_AUDIO_REF_COND_TIMESTEP,
|
||||
on_step: Callable[[int, torch.Tensor, torch.Tensor], None] | None = None,
|
||||
step_profiler: Callable[[int], AbstractContextManager] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the full denoise loop; returns final (video_rows, audio_rows).
|
||||
|
||||
``initial_video_rows`` covers all image rows of the positive layout. For a
|
||||
conditional task, pass ``keyframe_cond_rows`` and/or ``audio_ref_rows`` to
|
||||
pin those rows across every step. The model's raw positive velocity is the
|
||||
update signal; MiniMax H3 only supports cfg-distilled checkpoints.
|
||||
"""
|
||||
if len(sigmas_video) != len(sigmas_audio):
|
||||
raise ValueError("video/audio sigma schedules must have equal length")
|
||||
if len(sigmas_video) < 2:
|
||||
raise ValueError("sigma schedules need at least 2 entries")
|
||||
n_cond = int((~positive.update_mask).sum())
|
||||
if keyframe_cond_rows is None:
|
||||
if n_cond != 0:
|
||||
raise ValueError(f"layout has {n_cond} cond rows but keyframe_cond_rows is None")
|
||||
else:
|
||||
if int(keyframe_cond_rows.shape[0]) != n_cond:
|
||||
raise ValueError(f"keyframe_cond_rows {int(keyframe_cond_rows.shape[0])} != layout cond rows {n_cond}")
|
||||
video_rows = initial_video_rows.to(device=device, dtype=torch.float32).clone()
|
||||
audio_rows = initial_audio_rows.to(device=device, dtype=torch.float32).clone()
|
||||
update = positive.update_mask_dev
|
||||
audio_update = positive.audio_update_mask_dev
|
||||
if int(video_rows.shape[0]) != int(positive.img_pos.shape[0]):
|
||||
raise ValueError(
|
||||
f"initial video rows {int(video_rows.shape[0])} != positive layout rows {int(positive.img_pos.shape[0])}"
|
||||
)
|
||||
if int(audio_rows.shape[0]) != int(positive.audio_pos.shape[0]):
|
||||
raise ValueError(
|
||||
f"initial audio rows {int(audio_rows.shape[0])} != positive layout rows {int(positive.audio_pos.shape[0])}"
|
||||
)
|
||||
n_audio_ref = int((~positive.audio_update_mask).sum())
|
||||
if audio_ref_rows is None:
|
||||
if n_audio_ref != 0:
|
||||
raise ValueError(f"layout has {n_audio_ref} audio ref rows but audio_ref_rows is None")
|
||||
audio_anchor = None
|
||||
else:
|
||||
if int(audio_ref_rows.shape[0]) != n_audio_ref:
|
||||
raise ValueError(f"audio_ref_rows {int(audio_ref_rows.shape[0])} != layout audio ref rows {n_audio_ref}")
|
||||
audio_anchor = audio_ref_rows.to(device=device, dtype=torch.float32)
|
||||
cond_anchor = keyframe_cond_rows.to(device=device, dtype=torch.float32) if keyframe_cond_rows is not None else None
|
||||
if cond_anchor is not None:
|
||||
video_rows[~update] = cond_anchor
|
||||
if audio_anchor is not None:
|
||||
audio_rows[~audio_update] = audio_anchor
|
||||
|
||||
num_steps = len(sigmas_video) - 1
|
||||
for step in range(num_steps):
|
||||
step_cm = step_profiler(step) if step_profiler is not None else nullcontext()
|
||||
with step_cm:
|
||||
s_v, s_v_next = sigmas_video[step], sigmas_video[step + 1]
|
||||
s_a, s_a_next = sigmas_audio[step], sigmas_audio[step + 1]
|
||||
t_v, t_a = 1.0 - s_v, 1.0 - s_a
|
||||
imgvid_cond_t = max(t_v, float(imgvid_cond_noise_aug_for_inference))
|
||||
audio_ref_cond_t = max(t_a, float(audio_cond_noise_aug_for_inference))
|
||||
|
||||
fk = positive.forward_kwargs(
|
||||
video_rows=video_rows,
|
||||
audio_rows=audio_rows,
|
||||
t_video=t_v,
|
||||
t_audio=t_a,
|
||||
imgvid_cond_timestep=imgvid_cond_t,
|
||||
audio_ref_cond_timestep=audio_ref_cond_t,
|
||||
)
|
||||
with torch.inference_mode():
|
||||
v_video, v_audio = model(**fk)
|
||||
mv_video_t = v_video.float()[update]
|
||||
mv_audio_t = v_audio.float()[audio_update]
|
||||
|
||||
x0_video = minimax_h3_rf_v_to_x0(
|
||||
video_rows[update],
|
||||
mv_video_t,
|
||||
torch.tensor(t_v, dtype=torch.float32, device=device),
|
||||
)
|
||||
new_target = minimax_h3_euler_eta0_step(video_rows[update], x0_video, sigma_curr=s_v, sigma_next=s_v_next)
|
||||
video_rows = video_rows.clone()
|
||||
video_rows[update] = new_target
|
||||
if cond_anchor is not None:
|
||||
video_rows[~update] = cond_anchor # per-step imgvid cond reset
|
||||
|
||||
x0_audio = minimax_h3_rf_v_to_x0(
|
||||
audio_rows[audio_update],
|
||||
mv_audio_t,
|
||||
torch.tensor(t_a, dtype=torch.float32, device=device),
|
||||
)
|
||||
new_audio = minimax_h3_euler_eta0_step(
|
||||
audio_rows[audio_update], x0_audio, sigma_curr=s_a, sigma_next=s_a_next
|
||||
)
|
||||
audio_rows = audio_rows.clone()
|
||||
audio_rows[audio_update] = new_audio
|
||||
if audio_anchor is not None:
|
||||
audio_rows[~audio_update] = audio_anchor # per-step audio ref reset
|
||||
if on_step is not None:
|
||||
on_step(step, video_rows, audio_rows)
|
||||
|
||||
return video_rows, audio_rows
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MINIMAX_H3_AUDIO_REF_COND_TIMESTEP",
|
||||
"MINIMAX_H3_AUDIO_ROW_WIDTH",
|
||||
"MINIMAX_H3_IMGVID_COND_TIMESTEP",
|
||||
"MINIMAX_H3_VIDEO_ROW_WIDTH",
|
||||
"MiniMaxH3DenoiseBranch",
|
||||
"minimax_h3_denoise_loop",
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,572 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax H3 packed-sequence materialization for FL2VA and Ref2VA tasks.
|
||||
|
||||
Layout: [text L | imgvid_cond C | audio A(=t*2ch) | video_target V | pad P].
|
||||
Builder rules:
|
||||
- placeholder ids; block-derived pos infos/masks/cu_seqlens/document_id
|
||||
- img_position_ids fp64 grid: text rows (row_idx,0,0); video/cond t counter
|
||||
continues text_len with temporal interp spans (frame_rescale 5/3 x
|
||||
frame_per_token (1,4,4,4,4)); each spatial sqrt_area axis uses evenly spaced
|
||||
coordinates excluding the right endpoint, then scales them by INTERP;
|
||||
audio channel-major blocks pinned to the w-grid extremes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
MINIMAX_H3_FL2VA_KEYFRAME_SIGNATURES: tuple[tuple[int, ...], ...] = (
|
||||
(0,),
|
||||
(-1,),
|
||||
(0, -1),
|
||||
)
|
||||
|
||||
MINIMAX_H3_TEXT_ID = -5
|
||||
MINIMAX_H3_IMGVID_COND_ID = -11
|
||||
MINIMAX_H3_AUDIO_REF_COND_ID = -17
|
||||
MINIMAX_H3_AUDIO_FIRST_ID = -15
|
||||
MINIMAX_H3_AUDIO_ID = -14
|
||||
MINIMAX_H3_VIDEO_FIRST_ID = -3
|
||||
MINIMAX_H3_VIDEO_ID = -2
|
||||
MINIMAX_H3_VIDEO_LAST_ID = -4
|
||||
MINIMAX_H3_PAD_ID = -1
|
||||
|
||||
_INTERP = 32
|
||||
_T_GROUP = 5
|
||||
_FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
||||
_FRAME_RESCALE = 5.0 / 3.0
|
||||
_SEQ_ALIGN = 64
|
||||
# MiniMax H3 packs video latents with a fixed [1, 2, 2] (t, h, w) patch.
|
||||
_PATCH_T = 1
|
||||
_PATCH_H = 2
|
||||
_PATCH_W = 2
|
||||
|
||||
|
||||
def _keyframe_cond_frame_indices(
|
||||
*,
|
||||
include_keyframe_cond: bool,
|
||||
keyframe_frame_indices: list[int] | tuple[int, ...] | None,
|
||||
) -> list[int]:
|
||||
if not include_keyframe_cond:
|
||||
if keyframe_frame_indices is not None:
|
||||
raise ValueError("keyframe_frame_indices must be omitted when keyframe cond is not included")
|
||||
return []
|
||||
if keyframe_frame_indices is None:
|
||||
raise ValueError("strict fl2va packed layout requires keyframe_frame_indices")
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) for value in keyframe_frame_indices):
|
||||
raise ValueError("strict fl2va packed layout requires integer keyframe_frame_indices")
|
||||
out = list(keyframe_frame_indices)
|
||||
if tuple(out) not in MINIMAX_H3_FL2VA_KEYFRAME_SIGNATURES:
|
||||
raise ValueError(
|
||||
"strict fl2va packed layout requires keyframe_frame_indices in "
|
||||
f"{MINIMAX_H3_FL2VA_KEYFRAME_SIGNATURES!r}, got {out!r}"
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _resolve_keyframe_frame_indices(
|
||||
frame_indices: Sequence[int],
|
||||
*,
|
||||
frame_count: int | None,
|
||||
) -> list[int]:
|
||||
if frame_indices and frame_count is None:
|
||||
raise ValueError("frame_count is required when keyframe_frame_indices are provided")
|
||||
if frame_count is None:
|
||||
return []
|
||||
if frame_count <= 0:
|
||||
raise ValueError("frame_count must be positive")
|
||||
seen: dict[int, int] = {}
|
||||
resolved: list[int] = []
|
||||
for block_index, semantic_index in enumerate(frame_indices):
|
||||
if semantic_index == -1:
|
||||
resolved_index = frame_count - 1
|
||||
elif 0 <= semantic_index < frame_count:
|
||||
resolved_index = semantic_index
|
||||
else:
|
||||
raise ValueError(f"keyframe frame index {semantic_index} must be -1 or in [0, {frame_count})")
|
||||
previous = seen.get(resolved_index)
|
||||
if previous is not None:
|
||||
raise ValueError(
|
||||
f"keyframe frame index at block {block_index} resolves to "
|
||||
f"{resolved_index}, already bound by block {previous}"
|
||||
)
|
||||
seen[resolved_index] = block_index
|
||||
resolved.append(resolved_index)
|
||||
return resolved
|
||||
|
||||
|
||||
def _temporal_position_span(temporal_length: int) -> float:
|
||||
"""Temporal position span for patch_t=1, in fp64.
|
||||
|
||||
NOTE: intentionally NOT merged with ``_video_t_span``. This variant sums
|
||||
via numpy (pairwise summation), matching the fl2va anchor computation,
|
||||
while ``_video_t_span`` sums sequentially, matching the ref2va
|
||||
t-origin accumulation. The two orders diverge in the last ulp
|
||||
from n=16 onward, so each path must keep its own summation order.
|
||||
"""
|
||||
spans = np.ones(int(temporal_length), dtype=np.float64) * _FRAME_RESCALE
|
||||
for token_index in range(_T_GROUP):
|
||||
spans[token_index::_T_GROUP] *= _FRAME_PER_TOKEN[token_index]
|
||||
return float(spans.sum())
|
||||
|
||||
|
||||
def minimax_h3_packed_sequence(
|
||||
*,
|
||||
text_len: int,
|
||||
latent_t: int,
|
||||
latent_h: int,
|
||||
latent_w: int,
|
||||
audio_t: int,
|
||||
audio_channel: int = 2,
|
||||
include_keyframe_cond: bool,
|
||||
keyframe_frame_indices: list[int] | tuple[int, ...] | None = None,
|
||||
frame_count: int | None = None,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Build the packed-sequence structural fields for one CFG branch.
|
||||
|
||||
The used length is padded up to a multiple of 64.
|
||||
"""
|
||||
ph, pw = latent_h // _PATCH_H, latent_w // _PATCH_W
|
||||
frame_rows = ph * pw
|
||||
cond_frame_indices = _keyframe_cond_frame_indices(
|
||||
include_keyframe_cond=include_keyframe_cond,
|
||||
keyframe_frame_indices=keyframe_frame_indices,
|
||||
)
|
||||
resolved_cond_frame_indices = _resolve_keyframe_frame_indices(
|
||||
cond_frame_indices,
|
||||
frame_count=frame_count,
|
||||
)
|
||||
cond_rows = len(cond_frame_indices) * frame_rows
|
||||
video_rows = latent_t * frame_rows
|
||||
audio_rows = audio_t * audio_channel
|
||||
used = text_len + cond_rows + audio_rows + video_rows
|
||||
seq_len = ((used + _SEQ_ALIGN - 1) // _SEQ_ALIGN) * _SEQ_ALIGN
|
||||
|
||||
text_sl = slice(0, text_len)
|
||||
cond_sl = slice(text_len, text_len + cond_rows)
|
||||
audio_sl = slice(cond_sl.stop, cond_sl.stop + audio_rows)
|
||||
video_sl = slice(audio_sl.stop, audio_sl.stop + video_rows)
|
||||
pad_sl = slice(video_sl.stop, seq_len)
|
||||
|
||||
input_ids = torch.full((seq_len,), MINIMAX_H3_PAD_ID, dtype=torch.int64)
|
||||
input_ids[text_sl] = MINIMAX_H3_TEXT_ID
|
||||
if cond_rows:
|
||||
input_ids[cond_sl] = MINIMAX_H3_IMGVID_COND_ID
|
||||
input_ids[audio_sl] = MINIMAX_H3_AUDIO_ID
|
||||
input_ids[audio_sl.start] = MINIMAX_H3_AUDIO_FIRST_ID
|
||||
input_ids[video_sl] = MINIMAX_H3_VIDEO_ID
|
||||
input_ids[video_sl.start] = MINIMAX_H3_VIDEO_FIRST_ID
|
||||
input_ids[video_sl.stop - 1] = MINIMAX_H3_VIDEO_LAST_ID
|
||||
|
||||
image_mask = torch.zeros(seq_len, dtype=torch.bool)
|
||||
image_mask[cond_sl] = True
|
||||
image_mask[video_sl] = True
|
||||
audio_mask = torch.zeros(seq_len, dtype=torch.bool)
|
||||
audio_mask[audio_sl] = True
|
||||
|
||||
img_pos = torch.cat(
|
||||
[
|
||||
torch.arange(cond_sl.start, cond_sl.stop),
|
||||
torch.arange(video_sl.start, video_sl.stop),
|
||||
]
|
||||
)
|
||||
update_mask = torch.zeros(img_pos.shape[0], dtype=torch.bool)
|
||||
update_mask[cond_rows:] = True
|
||||
audio_pos = torch.arange(audio_sl.start, audio_sl.stop)
|
||||
text_pos = torch.arange(0, text_len)
|
||||
|
||||
g = torch.zeros(seq_len, 3, dtype=torch.float64)
|
||||
g[text_sl, 0] = torch.arange(text_len, dtype=torch.float64)
|
||||
|
||||
t_grid = _video_t_grid(latent_t, float(text_len))
|
||||
sqrt_area = np.sqrt(latent_h * latent_w)
|
||||
h_grid = _axis_from_sqrt_area(latent_h, _PATCH_H, sqrt_area)
|
||||
w_grid = _axis_from_sqrt_area(latent_w, _PATCH_W, sqrt_area)
|
||||
hh, ww = torch.meshgrid(h_grid, w_grid, indexing="ij")
|
||||
frame = torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1)
|
||||
video_g = torch.empty(latent_t, frame_rows, 3, dtype=torch.float64)
|
||||
video_g[:, :, 0] = t_grid[:, None]
|
||||
video_g[:, :, 1:] = frame[None]
|
||||
for block_index, pixel_index in enumerate(resolved_cond_frame_indices):
|
||||
sl = slice(
|
||||
cond_sl.start + block_index * frame_rows,
|
||||
cond_sl.start + (block_index + 1) * frame_rows,
|
||||
)
|
||||
if pixel_index == 0:
|
||||
cond_t = float(text_len)
|
||||
elif frame_count is not None and pixel_index == frame_count - 1:
|
||||
cond_t = float(text_len) + _temporal_position_span(latent_t) - _FRAME_RESCALE
|
||||
else:
|
||||
raise ValueError(
|
||||
f"fl2va packed layout only supports first/last keyframe anchors, got resolved frame index {pixel_index}"
|
||||
)
|
||||
g[sl, 0] = cond_t
|
||||
g[sl, 1:] = frame
|
||||
g[video_sl] = video_g.reshape(-1, 3)
|
||||
audio_t_grid = float(text_len) + torch.arange(audio_t, dtype=torch.float64)
|
||||
g[audio_sl, 0] = audio_t_grid.repeat(audio_channel)
|
||||
g[audio_sl, 2] = torch.cat(
|
||||
[
|
||||
torch.full((audio_t,), float(w_grid[0]), dtype=torch.float64),
|
||||
torch.full((audio_rows - audio_t,), float(w_grid[-1]), dtype=torch.float64),
|
||||
]
|
||||
)
|
||||
|
||||
token_tags = torch.full((seq_len,), -1, dtype=torch.long) # PADDING
|
||||
token_tags[text_sl] = 1 # TEXT (fl2va image-segment override happens upstream)
|
||||
token_tags[audio_sl] = 2 # AUDIO
|
||||
token_tags[img_pos] = 0 # VIDEO
|
||||
|
||||
cu = torch.tensor([0, used, seq_len], dtype=torch.int32)
|
||||
doc = torch.zeros(seq_len, dtype=torch.int32)
|
||||
doc[pad_sl] = 1
|
||||
return {
|
||||
"seq_len": torch.tensor(seq_len),
|
||||
"input_ids": input_ids,
|
||||
"image_mask": image_mask,
|
||||
"audio_mask": audio_mask,
|
||||
"img_pos": img_pos,
|
||||
"audio_pos": audio_pos,
|
||||
"text_pos": text_pos,
|
||||
"update_mask": update_mask,
|
||||
"img_position_ids": g,
|
||||
"token_tags": token_tags,
|
||||
"cu_seqlens": cu,
|
||||
"document_id": doc,
|
||||
}
|
||||
|
||||
|
||||
def _positive_int(
|
||||
block: Mapping[str, object],
|
||||
key: str,
|
||||
path: str,
|
||||
*,
|
||||
allow_zero: bool = False,
|
||||
) -> int:
|
||||
value = block.get(key)
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ValueError(f"{path}.{key} must be an integer")
|
||||
if value < 0 or (value == 0 and not allow_zero):
|
||||
predicate = "non-negative" if allow_zero else "positive"
|
||||
raise ValueError(f"{path}.{key} must be {predicate}")
|
||||
return int(value)
|
||||
|
||||
|
||||
def _axis_from_sqrt_area(dim: int, patch: int, sqrt_area: float) -> torch.Tensor:
|
||||
ratio = dim / sqrt_area
|
||||
left = (1.0 - ratio) * 1.0 / 2.0
|
||||
right = left + ratio * 1.0
|
||||
grid = np.linspace(left, right, dim // patch, endpoint=False) * _INTERP
|
||||
return torch.from_numpy(grid).to(torch.float64)
|
||||
|
||||
|
||||
def _video_t_grid(n: int, origin: float) -> torch.Tensor:
|
||||
spans = torch.tensor(
|
||||
[_FRAME_RESCALE * _FRAME_PER_TOKEN[k % _T_GROUP] for k in range(n)],
|
||||
dtype=torch.float64,
|
||||
)
|
||||
return origin + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)])
|
||||
|
||||
|
||||
def _video_t_span(n: int) -> float:
|
||||
# Sequential fp64 summation on purpose — see _temporal_position_span for
|
||||
# why the two span implementations must not be unified.
|
||||
return sum(_FRAME_RESCALE * _FRAME_PER_TOKEN[k % _T_GROUP] for k in range(n))
|
||||
|
||||
|
||||
def _range_for_slice(sl: slice) -> torch.Tensor:
|
||||
return torch.arange(sl.start, sl.stop, dtype=torch.long)
|
||||
|
||||
|
||||
def _cat_ranges(parts: list[torch.Tensor]) -> torch.Tensor:
|
||||
if parts:
|
||||
return torch.cat(parts)
|
||||
return torch.empty(0, dtype=torch.long)
|
||||
|
||||
|
||||
def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
*,
|
||||
text_len: int,
|
||||
latent_t: int,
|
||||
latent_h: int,
|
||||
latent_w: int,
|
||||
audio_t: int,
|
||||
ref_blocks: Sequence[Mapping[str, object]],
|
||||
audio_channel: int = 2,
|
||||
seq_len: int | None = None,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""General ref2va-family packed layout.
|
||||
|
||||
``ref_blocks`` are consumed in request/plan order:
|
||||
- ``{"kind": "image", "latent_h": H, "latent_w": W}``
|
||||
- ``{"kind": "audio", "ref_audio_t": T}``
|
||||
- ``{"kind": "video"|"video_audio", "ref_audio_t": T,
|
||||
"latent_t": RT, "latent_h": RH, "latent_w": RW}``
|
||||
|
||||
Video-bearing blocks pack their audio rows immediately before their video
|
||||
rows; both share the same temporal origin and advance by the longer of the
|
||||
audio and video spans. Standalone audio advances the target origin by its
|
||||
own T, and image blocks advance it by one integer slot.
|
||||
"""
|
||||
if not isinstance(ref_blocks, Sequence) or isinstance(ref_blocks, (str, bytes)):
|
||||
raise ValueError("ref_blocks must be a sequence")
|
||||
|
||||
parsed: list[dict[str, object]] = []
|
||||
ref_visual_rows = 0
|
||||
ref_audio_rows = 0
|
||||
for index, raw in enumerate(ref_blocks):
|
||||
path = f"ref_blocks[{index}]"
|
||||
if not isinstance(raw, Mapping):
|
||||
raise ValueError(f"{path} must be an object")
|
||||
kind = raw.get("kind", raw.get("type"))
|
||||
if not isinstance(kind, str) or not kind:
|
||||
raise ValueError(f"{path}.kind must be a non-empty string")
|
||||
if kind == "image":
|
||||
rh = _positive_int(raw, "latent_h", path)
|
||||
rw = _positive_int(raw, "latent_w", path)
|
||||
rows = (rh // _PATCH_H) * (rw // _PATCH_W)
|
||||
item = {"kind": kind, "latent_h": rh, "latent_w": rw, "rows": rows}
|
||||
ref_visual_rows += rows
|
||||
elif kind == "audio":
|
||||
rt = _positive_int(raw, "ref_audio_t", path, allow_zero=True)
|
||||
rows = rt * audio_channel
|
||||
item = {"kind": kind, "ref_audio_t": rt, "audio_rows": rows}
|
||||
ref_audio_rows += rows
|
||||
elif kind in ("video", "video_audio"):
|
||||
rt = _positive_int(raw, "ref_audio_t", path, allow_zero=True)
|
||||
vt = _positive_int(raw, "latent_t", path)
|
||||
vh = _positive_int(raw, "latent_h", path)
|
||||
vw = _positive_int(raw, "latent_w", path)
|
||||
frame_rows = (vh // _PATCH_H) * (vw // _PATCH_W)
|
||||
audio_rows = rt * audio_channel
|
||||
video_rows = vt * frame_rows
|
||||
item = {
|
||||
"kind": kind,
|
||||
"ref_audio_t": rt,
|
||||
"latent_t": vt,
|
||||
"latent_h": vh,
|
||||
"latent_w": vw,
|
||||
"frame_rows": frame_rows,
|
||||
"audio_rows": audio_rows,
|
||||
"video_rows": video_rows,
|
||||
}
|
||||
ref_audio_rows += audio_rows
|
||||
ref_visual_rows += video_rows
|
||||
else:
|
||||
raise ValueError(f"{path}.kind unsupported for ref2va: {kind!r}")
|
||||
parsed.append(item)
|
||||
|
||||
ph, pw = latent_h // _PATCH_H, latent_w // _PATCH_W
|
||||
frame_rows = ph * pw
|
||||
video_rows = latent_t * frame_rows
|
||||
audio_rows = audio_t * audio_channel
|
||||
ref_rows = ref_visual_rows + ref_audio_rows
|
||||
used = text_len + ref_rows + audio_rows + video_rows
|
||||
if seq_len is None:
|
||||
seq_len = ((used + _SEQ_ALIGN - 1) // _SEQ_ALIGN) * _SEQ_ALIGN
|
||||
if seq_len < used:
|
||||
raise ValueError(f"seq_len {seq_len} < used rows {used}")
|
||||
|
||||
text_sl = slice(0, text_len)
|
||||
cursor = text_len
|
||||
block_slices: list[dict[str, object]] = []
|
||||
for item in parsed:
|
||||
kind = str(item["kind"])
|
||||
if kind == "image":
|
||||
rows = int(item["rows"])
|
||||
visual_sl = slice(cursor, cursor + rows)
|
||||
cursor = visual_sl.stop
|
||||
block_slices.append({**item, "visual_sl": visual_sl})
|
||||
elif kind == "audio":
|
||||
rows = int(item["audio_rows"])
|
||||
audio_sl = slice(cursor, cursor + rows)
|
||||
cursor = audio_sl.stop
|
||||
block_slices.append({**item, "audio_sl": audio_sl})
|
||||
else:
|
||||
a_rows = int(item["audio_rows"])
|
||||
v_rows = int(item["video_rows"])
|
||||
audio_sl = slice(cursor, cursor + a_rows)
|
||||
visual_sl = slice(audio_sl.stop, audio_sl.stop + v_rows)
|
||||
cursor = visual_sl.stop
|
||||
block_slices.append({**item, "audio_sl": audio_sl, "visual_sl": visual_sl})
|
||||
|
||||
audio_sl = slice(cursor, cursor + audio_rows)
|
||||
video_sl = slice(audio_sl.stop, audio_sl.stop + video_rows)
|
||||
pad_sl = slice(video_sl.stop, seq_len)
|
||||
|
||||
input_ids = torch.full((seq_len,), MINIMAX_H3_PAD_ID, dtype=torch.int64)
|
||||
input_ids[text_sl] = MINIMAX_H3_TEXT_ID
|
||||
image_mask = torch.zeros(seq_len, dtype=torch.bool)
|
||||
audio_mask = torch.zeros(seq_len, dtype=torch.bool)
|
||||
|
||||
ref_img_pos_parts: list[torch.Tensor] = []
|
||||
ref_audio_pos_parts: list[torch.Tensor] = []
|
||||
g = torch.zeros(seq_len, 3, dtype=torch.float64)
|
||||
g[text_sl, 0] = torch.arange(text_len, dtype=torch.float64)
|
||||
|
||||
target_area = np.sqrt(latent_h * latent_w)
|
||||
h_grid = _axis_from_sqrt_area(latent_h, _PATCH_H, target_area)
|
||||
w_grid = _axis_from_sqrt_area(latent_w, _PATCH_W, target_area)
|
||||
hh, ww = torch.meshgrid(h_grid, w_grid, indexing="ij")
|
||||
target_frame = torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1)
|
||||
|
||||
t_cursor = float(text_len)
|
||||
for item in block_slices:
|
||||
kind = str(item["kind"])
|
||||
if kind == "image":
|
||||
visual_sl = item["visual_sl"]
|
||||
assert isinstance(visual_sl, slice)
|
||||
input_ids[visual_sl] = MINIMAX_H3_IMGVID_COND_ID
|
||||
image_mask[visual_sl] = True
|
||||
ref_img_pos_parts.append(_range_for_slice(visual_sl))
|
||||
rh = int(item["latent_h"])
|
||||
rw = int(item["latent_w"])
|
||||
area = np.sqrt(rh * rw)
|
||||
ref_hh, ref_ww = torch.meshgrid(
|
||||
_axis_from_sqrt_area(rh, _PATCH_H, area),
|
||||
_axis_from_sqrt_area(rw, _PATCH_W, area),
|
||||
indexing="ij",
|
||||
)
|
||||
g[visual_sl, 0] = t_cursor
|
||||
g[visual_sl, 1] = ref_hh.reshape(-1)
|
||||
g[visual_sl, 2] = ref_ww.reshape(-1)
|
||||
t_cursor += 1.0
|
||||
elif kind == "audio":
|
||||
audio_ref_sl = item["audio_sl"]
|
||||
assert isinstance(audio_ref_sl, slice)
|
||||
ref_t = int(item["ref_audio_t"])
|
||||
input_ids[audio_ref_sl] = MINIMAX_H3_AUDIO_REF_COND_ID
|
||||
audio_mask[audio_ref_sl] = True
|
||||
ref_audio_pos_parts.append(_range_for_slice(audio_ref_sl))
|
||||
ref_t_grid = t_cursor + torch.arange(ref_t, dtype=torch.float64)
|
||||
g[audio_ref_sl, 0] = ref_t_grid.repeat(audio_channel)
|
||||
if ref_t:
|
||||
g[audio_ref_sl, 2] = torch.cat(
|
||||
[
|
||||
torch.full((ref_t,), float(w_grid[0]), dtype=torch.float64),
|
||||
torch.full(
|
||||
(int(item["audio_rows"]) - ref_t,),
|
||||
float(w_grid[-1]),
|
||||
dtype=torch.float64,
|
||||
),
|
||||
]
|
||||
)
|
||||
t_cursor += float(ref_t)
|
||||
else:
|
||||
audio_ref_sl = item["audio_sl"]
|
||||
visual_sl = item["visual_sl"]
|
||||
assert isinstance(audio_ref_sl, slice)
|
||||
assert isinstance(visual_sl, slice)
|
||||
ref_t = int(item["ref_audio_t"])
|
||||
vt = int(item["latent_t"])
|
||||
vh = int(item["latent_h"])
|
||||
vw = int(item["latent_w"])
|
||||
input_ids[audio_ref_sl] = MINIMAX_H3_AUDIO_REF_COND_ID
|
||||
input_ids[visual_sl] = MINIMAX_H3_IMGVID_COND_ID
|
||||
audio_mask[audio_ref_sl] = True
|
||||
image_mask[visual_sl] = True
|
||||
ref_audio_pos_parts.append(_range_for_slice(audio_ref_sl))
|
||||
ref_img_pos_parts.append(_range_for_slice(visual_sl))
|
||||
|
||||
ref_area = np.sqrt(vh * vw)
|
||||
rv_h_grid = _axis_from_sqrt_area(vh, _PATCH_H, ref_area)
|
||||
rv_w_grid = _axis_from_sqrt_area(vw, _PATCH_W, ref_area)
|
||||
rv_hh, rv_ww = torch.meshgrid(rv_h_grid, rv_w_grid, indexing="ij")
|
||||
|
||||
ref_t_grid = t_cursor + torch.arange(ref_t, dtype=torch.float64)
|
||||
g[audio_ref_sl, 0] = ref_t_grid.repeat(audio_channel)
|
||||
if ref_t:
|
||||
g[audio_ref_sl, 2] = torch.cat(
|
||||
[
|
||||
torch.full((ref_t,), float(rv_w_grid[0]), dtype=torch.float64),
|
||||
torch.full(
|
||||
(int(item["audio_rows"]) - ref_t,),
|
||||
float(rv_w_grid[-1]),
|
||||
dtype=torch.float64,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
rv_frame = torch.stack([rv_hh.reshape(-1), rv_ww.reshape(-1)], dim=-1)
|
||||
rv_g = torch.empty(vt, int(item["frame_rows"]), 3, dtype=torch.float64)
|
||||
rv_g[:, :, 0] = _video_t_grid(vt, t_cursor)[:, None]
|
||||
rv_g[:, :, 1:] = rv_frame[None]
|
||||
g[visual_sl] = rv_g.reshape(-1, 3)
|
||||
t_cursor += max(float(ref_t), _video_t_span(vt))
|
||||
|
||||
input_ids[audio_sl] = MINIMAX_H3_AUDIO_ID
|
||||
input_ids[audio_sl.start] = MINIMAX_H3_AUDIO_FIRST_ID
|
||||
input_ids[video_sl] = MINIMAX_H3_VIDEO_ID
|
||||
input_ids[video_sl.start] = MINIMAX_H3_VIDEO_FIRST_ID
|
||||
input_ids[video_sl.stop - 1] = MINIMAX_H3_VIDEO_LAST_ID
|
||||
audio_mask[audio_sl] = True
|
||||
image_mask[video_sl] = True
|
||||
|
||||
audio_t_grid = t_cursor + torch.arange(audio_t, dtype=torch.float64)
|
||||
g[audio_sl, 0] = audio_t_grid.repeat(audio_channel)
|
||||
g[audio_sl, 2] = torch.cat(
|
||||
[
|
||||
torch.full((audio_t,), float(w_grid[0]), dtype=torch.float64),
|
||||
torch.full((audio_rows - audio_t,), float(w_grid[-1]), dtype=torch.float64),
|
||||
]
|
||||
)
|
||||
|
||||
video_g = torch.empty(latent_t, frame_rows, 3, dtype=torch.float64)
|
||||
video_g[:, :, 0] = _video_t_grid(latent_t, t_cursor)[:, None]
|
||||
video_g[:, :, 1:] = target_frame[None]
|
||||
g[video_sl] = video_g.reshape(-1, 3)
|
||||
|
||||
target_img_pos = _range_for_slice(video_sl)
|
||||
target_audio_pos = _range_for_slice(audio_sl)
|
||||
img_pos = _cat_ranges(ref_img_pos_parts + [target_img_pos])
|
||||
audio_pos = _cat_ranges(ref_audio_pos_parts + [target_audio_pos])
|
||||
|
||||
update_mask = torch.zeros(img_pos.shape[0], dtype=torch.bool)
|
||||
update_mask[ref_visual_rows:] = True
|
||||
audio_update_mask = torch.zeros(audio_pos.shape[0], dtype=torch.bool)
|
||||
audio_update_mask[ref_audio_rows:] = True
|
||||
text_pos = torch.arange(0, text_len)
|
||||
|
||||
token_tags = torch.full((seq_len,), -1, dtype=torch.long) # PADDING
|
||||
token_tags[text_sl] = 1 # TEXT
|
||||
if ref_audio_rows:
|
||||
token_tags[_cat_ranges(ref_audio_pos_parts)] = 2 # AUDIO (reference)
|
||||
token_tags[audio_sl] = 2 # AUDIO (target)
|
||||
token_tags[img_pos] = 0 # VIDEO (refs + target)
|
||||
|
||||
cu = torch.tensor([0, used, seq_len], dtype=torch.int32)
|
||||
doc = torch.zeros(seq_len, dtype=torch.int32)
|
||||
doc[pad_sl] = 1
|
||||
return {
|
||||
"seq_len": torch.tensor(seq_len),
|
||||
"input_ids": input_ids,
|
||||
"image_mask": image_mask,
|
||||
"audio_mask": audio_mask,
|
||||
"img_pos": img_pos,
|
||||
"audio_pos": audio_pos,
|
||||
"text_pos": text_pos,
|
||||
"update_mask": update_mask,
|
||||
"audio_update_mask": audio_update_mask,
|
||||
"img_position_ids": g,
|
||||
"token_tags": token_tags,
|
||||
"cu_seqlens": cu,
|
||||
"document_id": doc,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MINIMAX_H3_AUDIO_FIRST_ID",
|
||||
"MINIMAX_H3_AUDIO_ID",
|
||||
"MINIMAX_H3_AUDIO_REF_COND_ID",
|
||||
"MINIMAX_H3_IMGVID_COND_ID",
|
||||
"MINIMAX_H3_PAD_ID",
|
||||
"MINIMAX_H3_TEXT_ID",
|
||||
"MINIMAX_H3_VIDEO_FIRST_ID",
|
||||
"MINIMAX_H3_VIDEO_ID",
|
||||
"MINIMAX_H3_VIDEO_LAST_ID",
|
||||
"minimax_h3_packed_sequence",
|
||||
"minimax_h3_packed_sequence_ref2va_blocks",
|
||||
]
|
||||
@@ -0,0 +1,114 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def _int_tuple(value: Sequence[int], name: str, length: int) -> tuple[int, ...]:
|
||||
if len(value) != length:
|
||||
raise ValueError(f"{name} must have length {length}, got {list(value)!r}")
|
||||
out = tuple(int(item) for item in value)
|
||||
if any(item <= 0 for item in out):
|
||||
raise ValueError(f"{name} values must be positive, got {list(value)!r}")
|
||||
return out
|
||||
|
||||
|
||||
def _rank(tensor: torch.Tensor, name: str, rank: int) -> None:
|
||||
if tensor.ndim != rank:
|
||||
raise ValueError(f"{name} must be rank {rank}, got shape={list(tensor.shape)}")
|
||||
|
||||
|
||||
def minimax_h3_patchify_video_latent(
|
||||
latent: torch.Tensor,
|
||||
*,
|
||||
patch_size: Sequence[int],
|
||||
) -> torch.Tensor:
|
||||
"""Pack video latent [B,C,T,H,W] into DiT token rows."""
|
||||
|
||||
_rank(latent, "video latent", 5)
|
||||
pt, ph, pw = _int_tuple(patch_size, "patch_size", 3)
|
||||
batch, channel, full_t, full_h, full_w = (int(dim) for dim in latent.shape)
|
||||
if full_t % pt or full_h % ph or full_w % pw:
|
||||
raise ValueError(
|
||||
"video latent spatial/time dims must be divisible by patch_size: "
|
||||
f"shape={list(latent.shape)}, patch_size={[pt, ph, pw]}"
|
||||
)
|
||||
t, h, w = full_t // pt, full_h // ph, full_w // pw
|
||||
packed = latent.reshape(batch, channel, t, pt, h, ph, w, pw)
|
||||
packed = torch.einsum("nctrhpwq->nthwcrpq", packed)
|
||||
return packed.reshape(batch * t * h * w, channel * pt * ph * pw).contiguous()
|
||||
|
||||
|
||||
def minimax_h3_unpatchify_video_tokens(
|
||||
rows: torch.Tensor,
|
||||
*,
|
||||
latent_shape: Sequence[int],
|
||||
patch_size: Sequence[int],
|
||||
) -> torch.Tensor:
|
||||
"""Unpack DiT video token rows into latent [B,C,T,H,W]."""
|
||||
|
||||
_rank(rows, "video token rows", 2)
|
||||
t, h, w, channel = _int_tuple(latent_shape, "latent_shape", 4)
|
||||
pt, ph, pw = _int_tuple(patch_size, "patch_size", 3)
|
||||
expected_dim = pt * ph * pw * channel
|
||||
if int(rows.shape[-1]) != expected_dim:
|
||||
raise ValueError(
|
||||
f"video token dim {int(rows.shape[-1])} != patch volume * channel "
|
||||
f"{expected_dim} for latent_shape={list(latent_shape)}, "
|
||||
f"patch_size={[pt, ph, pw]}"
|
||||
)
|
||||
rows_per_sample = t * h * w
|
||||
if int(rows.shape[0]) % rows_per_sample:
|
||||
raise ValueError(
|
||||
f"video rows {int(rows.shape[0])} must be divisible by t*h*w "
|
||||
f"{rows_per_sample} for latent_shape={list(latent_shape)}"
|
||||
)
|
||||
packed = rows.reshape(-1, t, h, w, channel, pt, ph, pw)
|
||||
latent = torch.einsum("nthwcrpq->nctrhpwq", packed)
|
||||
return latent.reshape(-1, channel, t * pt, h * ph, w * pw).contiguous()
|
||||
|
||||
|
||||
def minimax_h3_pack_audio_latent(latent: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack audio VAE latent [audio_channel, latent_dim, T] into rows."""
|
||||
|
||||
_rank(latent, "audio latent", 3)
|
||||
audio_channel, latent_dim, steps = (int(dim) for dim in latent.shape)
|
||||
return (
|
||||
latent.permute(0, 2, 1)
|
||||
.reshape(
|
||||
audio_channel * steps,
|
||||
latent_dim,
|
||||
)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
def minimax_h3_unpack_audio_tokens(
|
||||
rows: torch.Tensor,
|
||||
*,
|
||||
audio_t: int,
|
||||
audio_channel: int,
|
||||
) -> torch.Tensor:
|
||||
"""Unpack DiT audio token rows into audio VAE latent [C,latent_dim,T]."""
|
||||
|
||||
_rank(rows, "audio token rows", 2)
|
||||
audio_t = int(audio_t)
|
||||
audio_channel = int(audio_channel)
|
||||
if audio_t <= 0 or audio_channel <= 0:
|
||||
raise ValueError(f"audio_t and audio_channel must be positive, got {audio_t=} {audio_channel=}")
|
||||
if int(rows.shape[0]) != audio_t:
|
||||
raise ValueError(f"audio rows {int(rows.shape[0])} != audio_t {audio_t}")
|
||||
if audio_t % audio_channel:
|
||||
raise ValueError(f"audio_t must be divisible by audio_channel, got {audio_t=} {audio_channel=}")
|
||||
native = rows.reshape(audio_channel, audio_t // audio_channel, int(rows.shape[-1]))
|
||||
return native.permute(0, 2, 1).contiguous()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"minimax_h3_pack_audio_latent",
|
||||
"minimax_h3_patchify_video_latent",
|
||||
"minimax_h3_unpack_audio_tokens",
|
||||
"minimax_h3_unpatchify_video_tokens",
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax H3 Qwen presentation building.
|
||||
|
||||
Builds the positive presentation token stream:
|
||||
- fl2va: '<Picture 1>: ' label + vision block (<|vision_start|> +
|
||||
N*<|image_pad|> + <|vision_end|>) + prompt text.
|
||||
- t2va: prompt text only (no vision block).
|
||||
Prompt text passes through verbatim (no stripping or rewriting).
|
||||
|
||||
All presentation variants are emitted through the shared ``_Presentation``
|
||||
accumulator so ids and AdaLN token tags cannot drift apart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
VISION_START = "<|vision_start|>"
|
||||
VISION_END = "<|vision_end|>"
|
||||
IMAGE_PAD = "<|image_pad|>"
|
||||
VIDEO_PAD = "<|video_pad|>"
|
||||
|
||||
_TEXT_TAG = 1
|
||||
_VIDEO_TAG = 0
|
||||
|
||||
|
||||
def _text_ids(tokenizer: Any, text: str) -> list[int]:
|
||||
return list(tokenizer(text, add_special_tokens=False)["input_ids"])
|
||||
|
||||
|
||||
def _vision_block_ids(tokenizer: Any, pad_token: str, count: int) -> list[int]:
|
||||
return (
|
||||
[tokenizer.convert_tokens_to_ids(VISION_START)]
|
||||
+ [tokenizer.convert_tokens_to_ids(pad_token)] * int(count)
|
||||
+ [tokenizer.convert_tokens_to_ids(VISION_END)]
|
||||
)
|
||||
|
||||
|
||||
class _Presentation:
|
||||
"""Accumulates aligned (ids, token_tags) presentation segments."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.ids: list[int] = []
|
||||
self.tags: list[int] = []
|
||||
|
||||
def text(self, token_ids: list[int]) -> None:
|
||||
self.ids += token_ids
|
||||
self.tags += [_TEXT_TAG] * len(token_ids)
|
||||
|
||||
def vision(self, token_ids: list[int]) -> None:
|
||||
self.ids += token_ids
|
||||
self.tags += [_VIDEO_TAG] * len(token_ids)
|
||||
|
||||
def build(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.tensor(self.ids, dtype=torch.long),
|
||||
torch.tensor(self.tags, dtype=torch.long),
|
||||
)
|
||||
|
||||
|
||||
def _timestamped_video_blocks(
|
||||
presentation: _Presentation,
|
||||
tokenizer: Any,
|
||||
*,
|
||||
counts: Sequence[int],
|
||||
timestamps: Sequence[float],
|
||||
context: str,
|
||||
) -> None:
|
||||
"""Emit per-temporal-block ``<{t:.1f} seconds>`` text + VIDEO vision."""
|
||||
|
||||
counts = [int(value) for value in counts]
|
||||
timestamps = [float(value) for value in timestamps]
|
||||
if not counts or len(counts) != len(timestamps):
|
||||
raise ValueError(f"{context}video block token counts and timestamps must align")
|
||||
for count, timestamp in zip(counts, timestamps):
|
||||
if count <= 0:
|
||||
raise ValueError(f"{context}video block token count must be positive")
|
||||
presentation.text(_text_ids(tokenizer, f"<{timestamp:.1f} seconds>"))
|
||||
presentation.vision(_vision_block_ids(tokenizer, VIDEO_PAD, count))
|
||||
|
||||
|
||||
def minimax_h3_text_only_ids(tokenizer: Any, prompt: str) -> torch.Tensor:
|
||||
"""t2va presentation: verbatim prompt, no special tokens."""
|
||||
if not prompt:
|
||||
raise ValueError("prompt must be non-empty")
|
||||
return torch.tensor(_text_ids(tokenizer, prompt), dtype=torch.long)
|
||||
|
||||
|
||||
def _multi_image_presentation(
|
||||
tokenizer: Any,
|
||||
*,
|
||||
prompt: str,
|
||||
image_token_counts: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not image_token_counts:
|
||||
raise ValueError("image_token_counts must be non-empty")
|
||||
presentation = _Presentation()
|
||||
for index, count in enumerate(image_token_counts, start=1):
|
||||
if int(count) <= 0:
|
||||
raise ValueError("image_token_count must be positive")
|
||||
presentation.text(_text_ids(tokenizer, f"<Picture {index}>: "))
|
||||
presentation.vision(_vision_block_ids(tokenizer, IMAGE_PAD, count))
|
||||
presentation.text(_text_ids(tokenizer, prompt))
|
||||
return presentation.build()
|
||||
|
||||
|
||||
def minimax_h3_multi_image_presentation_ids(
|
||||
tokenizer: Any,
|
||||
*,
|
||||
prompt: str,
|
||||
image_token_counts: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Positive fl2va presentation with one or more labeled image slots."""
|
||||
ids, _ = _multi_image_presentation(tokenizer, prompt=prompt, image_token_counts=image_token_counts)
|
||||
return ids
|
||||
|
||||
|
||||
def minimax_h3_multi_image_presentation_token_tags(
|
||||
tokenizer: Any,
|
||||
*,
|
||||
prompt: str,
|
||||
image_token_counts: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""AdaLN token tags aligned with minimax_h3_multi_image_presentation_ids."""
|
||||
_, tags = _multi_image_presentation(tokenizer, prompt=prompt, image_token_counts=image_token_counts)
|
||||
return tags
|
||||
|
||||
|
||||
def minimax_h3_ref2va_presentation(
|
||||
tokenizer: Any,
|
||||
*,
|
||||
prompt: str,
|
||||
condition_labels: list[tuple[str, int]],
|
||||
image_token_count: int | list[int] | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""ref2va positive presentation:
|
||||
|
||||
per condition in request order — image i: ``<Picture i>: `` label followed
|
||||
by the vision block; audio j: ``<Audio j>: `` label only (audio content
|
||||
never enters Qwen) — then the verbatim prompt. Returns (ids, token_tags)
|
||||
with the vision block tagged VIDEO(0) and everything else TEXT(1).
|
||||
|
||||
condition_labels: [("image", 1), ("audio", 1), ...] with 1-based ordinals
|
||||
per type.
|
||||
"""
|
||||
return minimax_h3_ref2va_video_presentation(
|
||||
tokenizer,
|
||||
prompt=prompt,
|
||||
condition_labels=condition_labels,
|
||||
image_token_count=image_token_count,
|
||||
video_block_token_counts=None,
|
||||
video_block_timestamps=None,
|
||||
)
|
||||
|
||||
|
||||
def _as_int_list(value: int | Sequence[int] | None, *, name: str) -> list[int]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, int):
|
||||
return [int(value)]
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
||||
raise ValueError(f"{name} must be an int or a sequence of ints")
|
||||
return [int(item) for item in value]
|
||||
|
||||
|
||||
def _as_nested_int_list(
|
||||
value: Sequence[int] | Sequence[Sequence[int]] | None,
|
||||
*,
|
||||
name: str,
|
||||
) -> list[list[int]]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
||||
raise ValueError(f"{name} must be a sequence")
|
||||
if len(value) == 0:
|
||||
return []
|
||||
first = value[0]
|
||||
if isinstance(first, Sequence) and not isinstance(first, (str, bytes)):
|
||||
out: list[list[int]] = []
|
||||
for group in value:
|
||||
if not isinstance(group, Sequence) or isinstance(group, (str, bytes)):
|
||||
raise ValueError(f"{name} must not mix nested and flat entries")
|
||||
out.append([int(item) for item in group])
|
||||
return out
|
||||
return [[int(item) for item in value]]
|
||||
|
||||
|
||||
def _as_nested_float_list(
|
||||
value: Sequence[float] | Sequence[Sequence[float]] | None,
|
||||
*,
|
||||
name: str,
|
||||
) -> list[list[float]]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
||||
raise ValueError(f"{name} must be a sequence")
|
||||
if len(value) == 0:
|
||||
return []
|
||||
first = value[0]
|
||||
if isinstance(first, Sequence) and not isinstance(first, (str, bytes)):
|
||||
out: list[list[float]] = []
|
||||
for group in value:
|
||||
if not isinstance(group, Sequence) or isinstance(group, (str, bytes)):
|
||||
raise ValueError(f"{name} must not mix nested and flat entries")
|
||||
out.append([float(item) for item in group])
|
||||
return out
|
||||
return [[float(item) for item in value]]
|
||||
|
||||
|
||||
def minimax_h3_ref2va_video_presentation(
|
||||
tokenizer: Any,
|
||||
*,
|
||||
prompt: str,
|
||||
condition_labels: list[tuple[str, int]],
|
||||
image_token_count: int | list[int] | None,
|
||||
video_block_token_counts: list[int] | list[list[int]] | None,
|
||||
video_block_timestamps: list[float] | list[list[float]] | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""ref2va (optionally with video refs) positive presentation:
|
||||
|
||||
per condition in request order —
|
||||
- image i: ``<Picture i>: `` label + one image vision block;
|
||||
- audio j: ``<Audio j>: `` label only (audio content never enters Qwen);
|
||||
- video k: ``<Video k>: `` label, then per temporal block a timestamp
|
||||
text ``<{t:.1f} seconds>`` followed by a VIDEO vision block
|
||||
(<|vision_start|> + <|video_pad|> x n + <|vision_end|>). Timestamps are
|
||||
the mean of each merged frame pair (Qwen3VL temporal merge 2; odd frame
|
||||
counts repeat the last frame), emitting the
|
||||
``<0.2 seconds>`` ..
|
||||
``<4.0 seconds>`` sequence — note Python bankers-rounding at .1f.
|
||||
then the verbatim prompt. Vision blocks are tagged VIDEO(0), everything
|
||||
else TEXT(1).
|
||||
"""
|
||||
if not prompt:
|
||||
raise ValueError("prompt must be non-empty")
|
||||
presentation = _Presentation()
|
||||
image_token_counts = _as_int_list(image_token_count, name="image_token_count")
|
||||
video_counts_by_ref = _as_nested_int_list(
|
||||
video_block_token_counts,
|
||||
name="video_block_token_counts",
|
||||
)
|
||||
video_timestamps_by_ref = _as_nested_float_list(
|
||||
video_block_timestamps,
|
||||
name="video_block_timestamps",
|
||||
)
|
||||
if len(video_counts_by_ref) != len(video_timestamps_by_ref):
|
||||
raise ValueError("video block token counts and timestamps must align")
|
||||
image_seen = 0
|
||||
video_seen = 0
|
||||
for cond_type, ordinal in condition_labels:
|
||||
if cond_type == "image":
|
||||
image_seen += 1
|
||||
if image_seen > len(image_token_counts):
|
||||
raise ValueError("image_token_count required for an image reference")
|
||||
count = int(image_token_counts[image_seen - 1])
|
||||
if count <= 0:
|
||||
raise ValueError("image_token_count required for an image reference")
|
||||
presentation.text(_text_ids(tokenizer, f"<Picture {ordinal}>: "))
|
||||
presentation.vision(_vision_block_ids(tokenizer, IMAGE_PAD, count))
|
||||
elif cond_type == "audio":
|
||||
presentation.text(_text_ids(tokenizer, f"<Audio {ordinal}>: "))
|
||||
elif cond_type == "video":
|
||||
video_seen += 1
|
||||
if video_seen > len(video_counts_by_ref):
|
||||
raise ValueError("video reference requires block token counts and timestamps")
|
||||
counts = video_counts_by_ref[video_seen - 1]
|
||||
timestamps = video_timestamps_by_ref[video_seen - 1]
|
||||
if not counts or not timestamps:
|
||||
raise ValueError("video reference requires block token counts and timestamps")
|
||||
presentation.text(_text_ids(tokenizer, f"<Video {ordinal}>: "))
|
||||
_timestamped_video_blocks(
|
||||
presentation,
|
||||
tokenizer,
|
||||
counts=counts,
|
||||
timestamps=timestamps,
|
||||
context="",
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"unsupported ref2va condition type {cond_type!r}")
|
||||
if image_seen != len(image_token_counts):
|
||||
raise ValueError("unused image_token_count entries")
|
||||
if video_seen != len(video_counts_by_ref):
|
||||
raise ValueError("unused video block token count entries")
|
||||
presentation.text(_text_ids(tokenizer, prompt))
|
||||
return presentation.build()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"minimax_h3_multi_image_presentation_ids",
|
||||
"minimax_h3_multi_image_presentation_token_tags",
|
||||
"minimax_h3_ref2va_presentation",
|
||||
"minimax_h3_ref2va_video_presentation",
|
||||
"minimax_h3_text_only_ids",
|
||||
]
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax H3 Ref2VA reference-video preparation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
MINIMAX_H3_FPS = 24.0
|
||||
MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS = 2.0
|
||||
MINIMAX_H3_QWEN_TEMPORAL_PATCH = 2
|
||||
MINIMAX_H3_BASE_SHORT_EDGE = 768
|
||||
MINIMAX_H3_MAX_PIXELS = 768 * 1344
|
||||
MINIMAX_H3_CANVAS_MULTIPLE = 32
|
||||
|
||||
|
||||
def _nearest_multiple(value: float, multiple: int) -> int:
|
||||
return max(multiple, int(round(float(value) / multiple)) * multiple)
|
||||
|
||||
|
||||
def _probe_video(path: str) -> dict[str, Any]:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"v:0",
|
||||
"-count_frames",
|
||||
"-show_entries",
|
||||
("stream=width,height,r_frame_rate,nb_read_frames,nb_frames,sample_aspect_ratio:stream_tags=rotate"),
|
||||
"-of",
|
||||
"json",
|
||||
path,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
streams = json.loads(result.stdout).get("streams") or []
|
||||
if not streams:
|
||||
raise ValueError(f"media has no video stream: {path}")
|
||||
stream = streams[0]
|
||||
numerator, denominator = str(stream["r_frame_rate"]).split("/")
|
||||
raw_count = stream.get("nb_read_frames") or stream.get("nb_frames")
|
||||
if raw_count in (None, "", "N/A"):
|
||||
raise ValueError(f"cannot determine video frame count: {path}")
|
||||
return {
|
||||
"width": int(stream["width"]),
|
||||
"height": int(stream["height"]),
|
||||
"fps": float(numerator) / float(denominator),
|
||||
"frame_count": int(raw_count),
|
||||
}
|
||||
|
||||
|
||||
def _has_audio(path: str) -> bool:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"a:0",
|
||||
"-show_entries",
|
||||
"stream=index",
|
||||
"-of",
|
||||
"json",
|
||||
path,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return bool(json.loads(result.stdout).get("streams") or [])
|
||||
|
||||
|
||||
def _reference_video_shape(width: int, height: int) -> tuple[int, int]:
|
||||
ratio = float(width) / float(height)
|
||||
if not 0.25 <= ratio <= 4.0:
|
||||
raise ValueError(f"reference video aspect ratio must be in [1:4, 4:1], got {width}x{height}")
|
||||
if ratio >= 1.0:
|
||||
target_width = MINIMAX_H3_BASE_SHORT_EDGE * ratio
|
||||
target_height = float(MINIMAX_H3_BASE_SHORT_EDGE)
|
||||
else:
|
||||
target_width = float(MINIMAX_H3_BASE_SHORT_EDGE)
|
||||
target_height = MINIMAX_H3_BASE_SHORT_EDGE / ratio
|
||||
area = target_width * target_height
|
||||
if area > MINIMAX_H3_MAX_PIXELS:
|
||||
scale = math.sqrt(MINIMAX_H3_MAX_PIXELS / area)
|
||||
target_width *= scale
|
||||
target_height *= scale
|
||||
return (
|
||||
_nearest_multiple(target_width, MINIMAX_H3_CANVAS_MULTIPLE),
|
||||
_nearest_multiple(target_height, MINIMAX_H3_CANVAS_MULTIPLE),
|
||||
)
|
||||
|
||||
|
||||
def _transcode_reference_video(
|
||||
source: str,
|
||||
*,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
target_frame_count: int,
|
||||
workdir: str,
|
||||
) -> str:
|
||||
output = str(Path(workdir) / "prepared.mp4")
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
source,
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-an",
|
||||
"-vf",
|
||||
(f"fps={MINIMAX_H3_FPS:g},scale={target_width}:{target_height}:flags=lanczos,setsar=1"),
|
||||
"-frames:v",
|
||||
str(target_frame_count),
|
||||
"-metadata:s:v:0",
|
||||
"rotate=0",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
output,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def prepare_reference_videos(
|
||||
values: Any,
|
||||
*,
|
||||
target_frame_count: int,
|
||||
workdir: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
if isinstance(values, (str, os.PathLike)):
|
||||
values = [values]
|
||||
if not isinstance(values, (list, tuple)) or not values:
|
||||
raise ValueError("MiniMax H3 Ref2VA video input must be a path or a non-empty list of paths")
|
||||
prepared: list[dict[str, Any]] = []
|
||||
for index, value in enumerate(values):
|
||||
if not isinstance(value, (str, os.PathLike)):
|
||||
raise TypeError(
|
||||
f"MiniMax H3 multi-video Ref2VA currently requires file paths, got item {index}: {type(value)!r}"
|
||||
)
|
||||
source = str(value)
|
||||
meta = _probe_video(source)
|
||||
width, height = _reference_video_shape(meta["width"], meta["height"])
|
||||
item_workdir = Path(workdir) / f"video_{index}"
|
||||
item_workdir.mkdir(parents=True)
|
||||
prepared_path = _transcode_reference_video(
|
||||
source,
|
||||
target_width=width,
|
||||
target_height=height,
|
||||
target_frame_count=target_frame_count,
|
||||
workdir=str(item_workdir),
|
||||
)
|
||||
prepared.append(
|
||||
{
|
||||
"original_path": source,
|
||||
"prepared_path": prepared_path,
|
||||
"input_has_audio": _has_audio(source),
|
||||
"width": width,
|
||||
"height": height,
|
||||
}
|
||||
)
|
||||
return prepared
|
||||
|
||||
|
||||
def load_video_frames(path: str) -> np.ndarray:
|
||||
try:
|
||||
import decord
|
||||
except ImportError:
|
||||
import av
|
||||
|
||||
with av.open(path) as container:
|
||||
frames = [frame.to_ndarray(format="rgb24") for frame in container.decode(video=0)]
|
||||
if not frames:
|
||||
raise ValueError(f"video has no frames: {path}")
|
||||
return np.stack(frames)
|
||||
|
||||
reader = decord.VideoReader(path)
|
||||
if len(reader) <= 0:
|
||||
raise ValueError(f"video has no frames: {path}")
|
||||
frames = reader.get_batch(list(range(len(reader))))
|
||||
return frames.asnumpy() if hasattr(frames, "asnumpy") else np.asarray(frames)
|
||||
|
||||
|
||||
def sample_reference_video_frames(
|
||||
prepared_path: str,
|
||||
*,
|
||||
workdir: str,
|
||||
) -> dict[str, Any]:
|
||||
from PIL import Image
|
||||
|
||||
meta = _probe_video(prepared_path)
|
||||
ratio = MINIMAX_H3_FPS / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS
|
||||
indices: list[int] = []
|
||||
cursor = 0.0
|
||||
while True:
|
||||
frame_index = int(round(cursor))
|
||||
if frame_index >= meta["frame_count"]:
|
||||
break
|
||||
if not indices or frame_index > indices[-1]:
|
||||
indices.append(frame_index)
|
||||
cursor += ratio
|
||||
if not indices:
|
||||
raise ValueError(f"no frames sampled from {prepared_path}")
|
||||
|
||||
frame_dir = Path(workdir)
|
||||
frame_dir.mkdir(parents=True, exist_ok=True)
|
||||
frames = []
|
||||
for output_index, source_index in enumerate(indices, start=1):
|
||||
output = frame_dir / f"frame_{output_index:06d}.png"
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
prepared_path,
|
||||
"-vf",
|
||||
f"select=eq(n\\,{source_index})",
|
||||
"-vsync",
|
||||
"vfr",
|
||||
"-frames:v",
|
||||
"1",
|
||||
str(output),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
frames.append(np.asarray(Image.open(output).convert("RGB")))
|
||||
|
||||
timestamps = [index / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS for index in range(len(indices))]
|
||||
timestamps += [timestamps[-1]] * ((-len(timestamps)) % MINIMAX_H3_QWEN_TEMPORAL_PATCH)
|
||||
block_timestamps = [
|
||||
(timestamps[index] + timestamps[index + MINIMAX_H3_QWEN_TEMPORAL_PATCH - 1]) / 2
|
||||
for index in range(
|
||||
0,
|
||||
len(timestamps),
|
||||
MINIMAX_H3_QWEN_TEMPORAL_PATCH,
|
||||
)
|
||||
]
|
||||
return {
|
||||
"frames": frames,
|
||||
"block_timestamps": block_timestamps,
|
||||
}
|
||||
|
||||
|
||||
def load_video_audio(path: str) -> tuple[torch.Tensor, int]:
|
||||
import tempfile
|
||||
|
||||
import torchaudio
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="minimax_h3_video_audio_") as tmpdir:
|
||||
output = str(Path(tmpdir) / "audio.wav")
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
path,
|
||||
"-vn",
|
||||
"-ac",
|
||||
"2",
|
||||
"-ar",
|
||||
"44100",
|
||||
"-f",
|
||||
"wav",
|
||||
output,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
return torchaudio.load(output)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"load_video_audio",
|
||||
"load_video_frames",
|
||||
"prepare_reference_videos",
|
||||
"sample_reference_video_frames",
|
||||
]
|
||||
@@ -0,0 +1,179 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def _require_finite_tensor(tensor: torch.Tensor, name: str) -> None:
|
||||
if not bool(torch.isfinite(tensor).all().item()):
|
||||
raise ValueError(f"{name} must be finite")
|
||||
|
||||
|
||||
def _validate_unit_timestep(timestep: torch.Tensor, name: str) -> None:
|
||||
if not isinstance(timestep, torch.Tensor):
|
||||
raise ValueError(f"{name} must be a torch.Tensor")
|
||||
if not torch.is_floating_point(timestep):
|
||||
raise ValueError(f"{name} must be a floating point tensor")
|
||||
_require_finite_tensor(timestep, name)
|
||||
out_of_range = (timestep < 0) | (timestep > 1)
|
||||
if bool(out_of_range.any().item()):
|
||||
raise ValueError(f"{name} must be in [0, 1]")
|
||||
|
||||
|
||||
def _validate_sigma(value: float, name: str) -> float:
|
||||
sigma = float(value)
|
||||
if not math.isfinite(sigma):
|
||||
raise ValueError(f"{name} must be finite")
|
||||
if sigma < 0.0:
|
||||
raise ValueError(f"{name} must be non-negative")
|
||||
return sigma
|
||||
|
||||
|
||||
def _validate_timestep_sigma_pair(
|
||||
timestep: torch.Tensor,
|
||||
sigma_curr: float,
|
||||
name: str,
|
||||
) -> float:
|
||||
_validate_unit_timestep(timestep, f"{name}_timestep")
|
||||
sigma = _validate_sigma(sigma_curr, f"{name}_sigma_curr")
|
||||
expected = 1.0 - timestep.detach().to(dtype=torch.float32)
|
||||
actual = torch.full_like(expected, sigma)
|
||||
if not torch.allclose(actual, expected, rtol=1e-5, atol=1e-5):
|
||||
raise ValueError(f"{name}_sigma_curr must equal 1 - {name}_timestep")
|
||||
return sigma
|
||||
|
||||
|
||||
def minimax_h3_rf_v_to_x0(
|
||||
xt: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if xt.shape != v.shape:
|
||||
raise ValueError(f"xt and v shapes must match, got {xt.shape} vs {v.shape}")
|
||||
if not torch.is_floating_point(xt):
|
||||
raise ValueError("xt must be a floating point tensor")
|
||||
if not torch.is_floating_point(v):
|
||||
raise ValueError("v must be a floating point tensor")
|
||||
_require_finite_tensor(xt, "xt")
|
||||
_require_finite_tensor(v, "v")
|
||||
_validate_unit_timestep(timestep, "timestep")
|
||||
cond_t = timestep.to(device=xt.device, dtype=xt.dtype)
|
||||
while cond_t.ndim < xt.ndim:
|
||||
cond_t = cond_t.unsqueeze(-1)
|
||||
sigma_t = 1 - cond_t
|
||||
x0 = xt + sigma_t * v
|
||||
_require_finite_tensor(x0, "x0")
|
||||
return x0
|
||||
|
||||
|
||||
def minimax_h3_euler_eta0_step(
|
||||
state: torch.Tensor,
|
||||
denoised: torch.Tensor,
|
||||
*,
|
||||
sigma_curr: float,
|
||||
sigma_next: float,
|
||||
) -> torch.Tensor:
|
||||
if state.shape != denoised.shape:
|
||||
raise ValueError(f"state and denoised shapes must match, got {state.shape} vs {denoised.shape}")
|
||||
if not torch.is_floating_point(state):
|
||||
raise ValueError("state must be a floating point tensor")
|
||||
if not torch.is_floating_point(denoised):
|
||||
raise ValueError("denoised must be a floating point tensor")
|
||||
_require_finite_tensor(state, "state")
|
||||
_require_finite_tensor(denoised, "denoised")
|
||||
sigma_curr = _validate_sigma(sigma_curr, "sigma_curr")
|
||||
sigma_next = _validate_sigma(sigma_next, "sigma_next")
|
||||
if sigma_curr == 0.0:
|
||||
if sigma_next != 0.0:
|
||||
raise ValueError("sigma_next must be 0 when sigma_curr is 0")
|
||||
return state
|
||||
compute_dtype = torch.float32
|
||||
if state.dtype not in (torch.float16, torch.bfloat16):
|
||||
compute_dtype = state.dtype
|
||||
sigma_curr_t = state.new_tensor(sigma_curr, dtype=compute_dtype)
|
||||
sigma_next_t = state.new_tensor(sigma_next, dtype=compute_dtype)
|
||||
sigma_ratio = sigma_next_t / sigma_curr_t
|
||||
out = sigma_ratio * state.to(dtype=compute_dtype) + (1.0 - sigma_ratio) * denoised.to(dtype=compute_dtype)
|
||||
out = out.to(dtype=state.dtype)
|
||||
_require_finite_tensor(out, "euler_eta0_step output")
|
||||
return out
|
||||
|
||||
|
||||
class MiniMaxH3EulerAncestralEta0SchedulerAdapter:
|
||||
def __init__(self, **config: Any) -> None:
|
||||
if config:
|
||||
raise ValueError(f"{type(self).__name__} does not accept config fields: {sorted(config)}")
|
||||
|
||||
def set_shift(self, _flow_shift: float) -> None:
|
||||
"""Ignore flow shift, matching the previous loader-specific path."""
|
||||
|
||||
def step_denoising(
|
||||
self,
|
||||
*,
|
||||
input_visual_latent: torch.Tensor,
|
||||
input_audio_latent: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
noise_pred_visual: torch.Tensor,
|
||||
noise_pred_audio: torch.Tensor,
|
||||
sigma_curr: float,
|
||||
sigma_next: float,
|
||||
video_timestep: torch.Tensor | None = None,
|
||||
audio_timestep: torch.Tensor | None = None,
|
||||
video_sigma_curr: float | None = None,
|
||||
video_sigma_next: float | None = None,
|
||||
audio_sigma_curr: float | None = None,
|
||||
audio_sigma_next: float | None = None,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
visual_timestep = timestep if video_timestep is None else video_timestep
|
||||
audio_timestep = timestep if audio_timestep is None else audio_timestep
|
||||
visual_sigma_curr = sigma_curr if video_sigma_curr is None else video_sigma_curr
|
||||
visual_sigma_next = sigma_next if video_sigma_next is None else video_sigma_next
|
||||
audio_sigma_curr = sigma_curr if audio_sigma_curr is None else audio_sigma_curr
|
||||
audio_sigma_next = sigma_next if audio_sigma_next is None else audio_sigma_next
|
||||
visual_sigma_curr = _validate_timestep_sigma_pair(
|
||||
visual_timestep,
|
||||
visual_sigma_curr,
|
||||
"video",
|
||||
)
|
||||
audio_sigma_curr = _validate_timestep_sigma_pair(
|
||||
audio_timestep,
|
||||
audio_sigma_curr,
|
||||
"audio",
|
||||
)
|
||||
|
||||
denoised_visual = minimax_h3_rf_v_to_x0(
|
||||
input_visual_latent,
|
||||
noise_pred_visual,
|
||||
visual_timestep,
|
||||
)
|
||||
denoised_audio = minimax_h3_rf_v_to_x0(
|
||||
input_audio_latent,
|
||||
noise_pred_audio,
|
||||
audio_timestep,
|
||||
)
|
||||
return {
|
||||
"output_visual_latent": minimax_h3_euler_eta0_step(
|
||||
input_visual_latent,
|
||||
denoised_visual,
|
||||
sigma_curr=visual_sigma_curr,
|
||||
sigma_next=visual_sigma_next,
|
||||
),
|
||||
"output_audio_latent": minimax_h3_euler_eta0_step(
|
||||
input_audio_latent,
|
||||
denoised_audio,
|
||||
sigma_curr=audio_sigma_curr,
|
||||
sigma_next=audio_sigma_next,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
EntryClass = MiniMaxH3EulerAncestralEta0SchedulerAdapter
|
||||
|
||||
__all__ = [
|
||||
"MiniMaxH3EulerAncestralEta0SchedulerAdapter",
|
||||
"minimax_h3_euler_eta0_step",
|
||||
"minimax_h3_rf_v_to_x0",
|
||||
]
|
||||
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def _align_frame_count(frame_count: int) -> int:
|
||||
"""Snap ``frame_count`` up to the MiniMax H3 17n+5 frame boundary."""
|
||||
if frame_count <= 0:
|
||||
return 1
|
||||
current = int(frame_count)
|
||||
while current % 17 != 5:
|
||||
current += 1
|
||||
return current
|
||||
|
||||
|
||||
def _video_latent_t(frame_count: int) -> int:
|
||||
if frame_count <= 5:
|
||||
return 2
|
||||
return ((int(frame_count) - 5) // 17) * 5 + 2
|
||||
|
||||
|
||||
def _frame_count_from_video_latent_t(out_t: int) -> int:
|
||||
if out_t == 1:
|
||||
return 1
|
||||
if out_t < 2 or (out_t - 2) % 5 != 0:
|
||||
raise ValueError("MiniMax H3 video latent T must be 1 or match 5n+2")
|
||||
return 17 * ((int(out_t) - 2) // 5) + 5
|
||||
|
||||
|
||||
def _audio_latent_t(duration_seconds: float) -> int:
|
||||
# Rounding happens at the 40 Hz audio latent boundary.
|
||||
return int(round(float(duration_seconds) * 40.0))
|
||||
|
||||
|
||||
def _time_shift_sigmas(
|
||||
*,
|
||||
num_steps: int = 50,
|
||||
shift_scale: float = 6.0,
|
||||
) -> list[float]:
|
||||
if shift_scale <= 0:
|
||||
raise ValueError("MiniMax H3 shift_scale must be > 0")
|
||||
if num_steps <= 0:
|
||||
raise ValueError("MiniMax H3 num_steps must be > 0")
|
||||
|
||||
import torch
|
||||
|
||||
# The rectified-flow sigma range is fixed at [1.0, 0.0].
|
||||
base = torch.linspace(
|
||||
1.0,
|
||||
0.0,
|
||||
int(num_steps),
|
||||
device="cpu",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
shifted = float(shift_scale) * base / (1 + (float(shift_scale) - 1) * base)
|
||||
shifted, _ = torch.unique_consecutive(shifted, return_counts=True)
|
||||
# A one-point request is still exactly one point. Normal serving uses
|
||||
# multiple points, but preserving the requested cardinality keeps
|
||||
# ``num_inference_steps`` the sole schedule-size control.
|
||||
if num_steps > 1 and shifted[-1].item() > 0.0:
|
||||
shifted = torch.cat([shifted, torch.tensor([0.0], dtype=shifted.dtype)])
|
||||
return [float(value) for value in shifted.tolist()]
|
||||
|
||||
|
||||
class MiniMaxH3ShapePlanner:
|
||||
"""Compute MiniMax H3 frame, latent-shape, and time-shift values."""
|
||||
|
||||
def align_frame_count(self, frame_count: int) -> int:
|
||||
return _align_frame_count(frame_count)
|
||||
|
||||
def video_latent_t(self, frame_count: int) -> int:
|
||||
return _video_latent_t(frame_count)
|
||||
|
||||
def frame_count_from_video_latent_t(self, out_t: int) -> int:
|
||||
return _frame_count_from_video_latent_t(out_t)
|
||||
|
||||
def audio_latent_t(self, duration_seconds: float) -> int:
|
||||
return _audio_latent_t(duration_seconds)
|
||||
|
||||
def time_shift_sigmas(
|
||||
self,
|
||||
*,
|
||||
num_steps: int = 50,
|
||||
shift_scale: float = 6.0,
|
||||
) -> list[float]:
|
||||
return _time_shift_sigmas(
|
||||
num_steps=num_steps,
|
||||
shift_scale=shift_scale,
|
||||
)
|
||||
|
||||
|
||||
MINIMAX_H3_SHAPE_PLANNER = MiniMaxH3ShapePlanner()
|
||||
|
||||
|
||||
def minimax_h3_align_frame_count(frame_count: int) -> int:
|
||||
return MINIMAX_H3_SHAPE_PLANNER.align_frame_count(frame_count)
|
||||
|
||||
|
||||
def minimax_h3_frame_count_from_video_latent_t(out_t: int) -> int:
|
||||
return MINIMAX_H3_SHAPE_PLANNER.frame_count_from_video_latent_t(out_t)
|
||||
|
||||
|
||||
def minimax_h3_time_shift_sigmas(
|
||||
*,
|
||||
num_steps: int = 50,
|
||||
shift_scale: float = 6.0,
|
||||
) -> list[float]:
|
||||
return MINIMAX_H3_SHAPE_PLANNER.time_shift_sigmas(
|
||||
num_steps=num_steps,
|
||||
shift_scale=shift_scale,
|
||||
)
|
||||
@@ -0,0 +1,360 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax H3 remote-code VAE adapters and exact latent contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
from contextlib import AbstractContextManager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
from PIL import Image
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
||||
DistributedVaeMixin,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_dit_group
|
||||
|
||||
from .packed_tokens import minimax_h3_patchify_video_latent
|
||||
|
||||
MINIMAX_H3_KEYFRAME_ENCODE_SEED = 42
|
||||
MINIMAX_H3_AUDIO_SAMPLE_RATE = 32000
|
||||
MINIMAX_H3_AUDIO_CHANNELS = 2
|
||||
|
||||
|
||||
def _load_component_config(component_path: str) -> dict[str, Any]:
|
||||
config_path = Path(component_path) / "config.json"
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
channels = int(config["latent_channels"])
|
||||
for key in ("latents_mean", "latents_std"):
|
||||
values = config.get(key)
|
||||
if not isinstance(values, list) or len(values) != channels:
|
||||
raise ValueError(f"{config_path}: {key} must contain {channels} values")
|
||||
return config
|
||||
|
||||
|
||||
def _load_remote_component(
|
||||
component_path: str,
|
||||
config: dict[str, Any],
|
||||
) -> nn.Module:
|
||||
auto_map = config.get("auto_map") or {}
|
||||
class_reference = auto_map.get("AutoModel")
|
||||
if not isinstance(class_reference, str):
|
||||
raise ValueError(f"{component_path}/config.json must define auto_map.AutoModel")
|
||||
component_cls = get_class_from_dynamic_module(
|
||||
class_reference,
|
||||
component_path,
|
||||
)
|
||||
return component_cls.from_pretrained(component_path)
|
||||
|
||||
|
||||
class _AudioVAEDeterminismContext(AbstractContextManager):
|
||||
def __enter__(self):
|
||||
backends = torch.backends
|
||||
self._saved = (
|
||||
backends.cuda.matmul.allow_tf32,
|
||||
backends.cudnn.allow_tf32,
|
||||
backends.cudnn.benchmark,
|
||||
backends.cudnn.deterministic,
|
||||
backends.cudnn.enabled,
|
||||
backends.cuda.flash_sdp_enabled(),
|
||||
backends.cuda.mem_efficient_sdp_enabled(),
|
||||
backends.cuda.math_sdp_enabled(),
|
||||
)
|
||||
backends.cuda.matmul.allow_tf32 = False
|
||||
backends.cudnn.allow_tf32 = False
|
||||
backends.cudnn.benchmark = False
|
||||
backends.cudnn.deterministic = True
|
||||
backends.cudnn.enabled = False
|
||||
backends.cuda.enable_flash_sdp(False)
|
||||
backends.cuda.enable_mem_efficient_sdp(False)
|
||||
backends.cuda.enable_math_sdp(True)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
backends = torch.backends
|
||||
(
|
||||
backends.cuda.matmul.allow_tf32,
|
||||
backends.cudnn.allow_tf32,
|
||||
backends.cudnn.benchmark,
|
||||
backends.cudnn.deterministic,
|
||||
backends.cudnn.enabled,
|
||||
flash,
|
||||
memory_efficient,
|
||||
math_sdp,
|
||||
) = self._saved
|
||||
backends.cuda.enable_flash_sdp(flash)
|
||||
backends.cuda.enable_mem_efficient_sdp(memory_efficient)
|
||||
backends.cuda.enable_math_sdp(math_sdp)
|
||||
return False
|
||||
|
||||
|
||||
class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
"""Adapter around the checkpoint's native parallel-tiled video VAE."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
component_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config_dict = _load_component_config(component_path)
|
||||
self.remote = _load_remote_component(
|
||||
component_path,
|
||||
self.config_dict,
|
||||
)
|
||||
# Match the reference loader contract: video VAE weights stay FP32.
|
||||
# Keyframe encoding is numerically sensitive to first casting the
|
||||
# checkpoint through FP16; decode still runs under FP16 autocast.
|
||||
self.remote.eval().to(device=device, dtype=torch.float32)
|
||||
self.model = self.remote.model
|
||||
self.use_tiling = True
|
||||
self.use_slicing = False
|
||||
self.parallel_size = 1
|
||||
|
||||
def set_parallel_size(
|
||||
self,
|
||||
parallel_size: int,
|
||||
mode: str = "tile",
|
||||
) -> None:
|
||||
if mode != "tile":
|
||||
raise ValueError(f"MiniMax H3 VAE supports its native tile parallel mode only, got {mode!r}")
|
||||
group = get_dit_group()
|
||||
world_size = dist.get_world_size(group)
|
||||
rank = dist.get_rank(group)
|
||||
parallel_size = int(parallel_size)
|
||||
if parallel_size not in (1, world_size):
|
||||
raise ValueError(
|
||||
"MiniMax H3 native VAE patch parallelism currently requires "
|
||||
"vae_patch_parallel_size=1 or the full DiT group size "
|
||||
f"({world_size}), got {parallel_size}"
|
||||
)
|
||||
self.parallel_size = parallel_size
|
||||
enabled = parallel_size > 1
|
||||
|
||||
package = self.remote.__class__.__module__.rsplit(".", 1)[0]
|
||||
parallel_module = importlib.import_module(f"{package}.parallel")
|
||||
state = parallel_module.get_parallel_state()
|
||||
state.clear()
|
||||
state.update(
|
||||
group_size=parallel_size,
|
||||
group_rank=rank if enabled else 0,
|
||||
local_process_group=group if enabled else None,
|
||||
sp_size=parallel_size,
|
||||
sp_rank=rank if enabled else 0,
|
||||
sp_enabled=enabled,
|
||||
sp_process_group=group if enabled else None,
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
)
|
||||
self.model.parallel_tiling = enabled
|
||||
|
||||
def is_distributed_enabled(self) -> bool:
|
||||
return self.parallel_size > 1 and dist.is_initialized()
|
||||
|
||||
@torch.inference_mode()
|
||||
def encode_image(self, image: Image.Image) -> torch.Tensor:
|
||||
previous_parallel = self.model.parallel_tiling
|
||||
self.model.parallel_tiling = False
|
||||
parameter = next(self.parameters())
|
||||
previous_dtype = parameter.dtype
|
||||
if previous_dtype != torch.float32:
|
||||
self.to(torch.float32)
|
||||
devices = [parameter.device] if parameter.device.type == "cuda" else []
|
||||
try:
|
||||
with torch.random.fork_rng(devices=devices):
|
||||
torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
|
||||
for device in devices:
|
||||
with torch.cuda.device(device):
|
||||
torch.cuda.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
|
||||
latent = self.model.encode_images(
|
||||
image,
|
||||
use_fp16_latent=True,
|
||||
)[0]
|
||||
finally:
|
||||
self.model.parallel_tiling = previous_parallel
|
||||
if previous_dtype != torch.float32:
|
||||
self.to(previous_dtype)
|
||||
|
||||
# Match the reference contract exactly: normalization and patchify
|
||||
# happen on CPU in FP32 after the sampled encode. The condition noise
|
||||
# path is sensitive enough that doing these elementwise operations on
|
||||
# CUDA can noticeably change the final conditioned video.
|
||||
latent = latent.float().cpu()
|
||||
if latent.ndim == 4:
|
||||
latent = latent[None]
|
||||
channels = int(self.config_dict["latent_channels"])
|
||||
mean = torch.tensor(
|
||||
self.config_dict["latents_mean"],
|
||||
).view(1, channels, 1, 1, 1)
|
||||
std = torch.tensor(
|
||||
self.config_dict["latents_std"],
|
||||
).view(1, channels, 1, 1, 1)
|
||||
return minimax_h3_patchify_video_latent(
|
||||
(latent - mean) / std,
|
||||
patch_size=(1, 2, 2),
|
||||
).float()
|
||||
|
||||
@torch.inference_mode()
|
||||
def encode_video(
|
||||
self,
|
||||
frames: Any,
|
||||
) -> tuple[torch.Tensor, tuple[int, int, int]]:
|
||||
parameter = next(self.parameters())
|
||||
previous_dtype = parameter.dtype
|
||||
if previous_dtype != torch.float32:
|
||||
self.to(torch.float32)
|
||||
devices = [parameter.device] if parameter.device.type == "cuda" else []
|
||||
try:
|
||||
with torch.random.fork_rng(devices=devices):
|
||||
torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
|
||||
for device in devices:
|
||||
with torch.cuda.device(device):
|
||||
torch.cuda.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
|
||||
latent = self.model.encode_videos(
|
||||
frames,
|
||||
use_fp16_latent=True,
|
||||
)[0]
|
||||
finally:
|
||||
if previous_dtype != torch.float32:
|
||||
self.to(previous_dtype)
|
||||
|
||||
latent = latent.float().cpu()
|
||||
if latent.ndim == 4:
|
||||
latent = latent[None]
|
||||
channels = int(self.config_dict["latent_channels"])
|
||||
if latent.ndim != 5 or int(latent.shape[1]) != channels:
|
||||
raise ValueError(f"unexpected reference video latent shape {tuple(latent.shape)}")
|
||||
shape = (
|
||||
int(latent.shape[2]),
|
||||
int(latent.shape[3]),
|
||||
int(latent.shape[4]),
|
||||
)
|
||||
mean = torch.tensor(
|
||||
self.config_dict["latents_mean"],
|
||||
).view(1, channels, 1, 1, 1)
|
||||
std = torch.tensor(
|
||||
self.config_dict["latents_std"],
|
||||
).view(1, channels, 1, 1, 1)
|
||||
rows = minimax_h3_patchify_video_latent(
|
||||
(latent - mean) / std,
|
||||
patch_size=(1, 2, 2),
|
||||
).float()
|
||||
return rows, shape
|
||||
|
||||
@torch.inference_mode()
|
||||
def decode_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
channels = int(self.config_dict["latent_channels"])
|
||||
mean = torch.tensor(
|
||||
self.config_dict["latents_mean"],
|
||||
device=latent.device,
|
||||
dtype=latent.dtype,
|
||||
).view(1, channels, 1, 1, 1)
|
||||
std = torch.tensor(
|
||||
self.config_dict["latents_std"],
|
||||
device=latent.device,
|
||||
dtype=latent.dtype,
|
||||
).view(1, channels, 1, 1, 1)
|
||||
decoded = self.model.decode_base(latent * std + mean)
|
||||
frames = self.model.processor.revert_tensor(decoded)
|
||||
if frames.ndim == 4:
|
||||
frames = frames.unsqueeze(0).transpose(1, 2)
|
||||
if frames.ndim != 5:
|
||||
raise ValueError(f"unexpected decoded video shape {tuple(frames.shape)}")
|
||||
return frames.float()
|
||||
|
||||
|
||||
class MiniMaxH3AudioVAE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
component_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config_dict = _load_component_config(component_path)
|
||||
self.remote = _load_remote_component(
|
||||
component_path,
|
||||
self.config_dict,
|
||||
)
|
||||
# The checkpoint's audio VAE contract is FP32 for both reference
|
||||
# encoding and waveform decoding.
|
||||
self.remote.eval().to(device=device, dtype=torch.float32)
|
||||
self.model = self.remote.model
|
||||
self.sample_rate = int(self.config_dict["sample_rate"])
|
||||
|
||||
@torch.inference_mode()
|
||||
def encode_waveform(
|
||||
self,
|
||||
waveform: torch.Tensor,
|
||||
sample_rate: int,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
import torchaudio
|
||||
|
||||
waveform = waveform.float()
|
||||
if waveform.ndim == 1:
|
||||
waveform = waveform[None]
|
||||
if int(sample_rate) != MINIMAX_H3_AUDIO_SAMPLE_RATE:
|
||||
waveform = torchaudio.transforms.Resample(
|
||||
int(sample_rate),
|
||||
MINIMAX_H3_AUDIO_SAMPLE_RATE,
|
||||
)(waveform)
|
||||
if waveform.shape[0] < MINIMAX_H3_AUDIO_CHANNELS:
|
||||
waveform = waveform.repeat(
|
||||
MINIMAX_H3_AUDIO_CHANNELS,
|
||||
1,
|
||||
)
|
||||
waveform = waveform[:MINIMAX_H3_AUDIO_CHANNELS]
|
||||
device = next(self.model.parameters()).device
|
||||
waveform = waveform.to(device)
|
||||
|
||||
with _AudioVAEDeterminismContext():
|
||||
audio = self.model.preprocess(
|
||||
waveform.unsqueeze(1),
|
||||
MINIMAX_H3_AUDIO_SAMPLE_RATE,
|
||||
)
|
||||
latent = self.model.encoder(audio)
|
||||
if bool(getattr(self.model, "attn_proj", False)):
|
||||
latent = self.model.pre_block(latent.transpose(1, 2)).transpose(1, 2)
|
||||
latent = self.model.mean_proj(latent).float().cpu()
|
||||
|
||||
channels = int(self.config_dict["latent_channels"])
|
||||
if latent.shape[-1] != channels:
|
||||
if latent.shape[1] != channels:
|
||||
raise ValueError(f"cannot canonicalize audio latent {tuple(latent.shape)}")
|
||||
latent = latent.transpose(1, 2).contiguous()
|
||||
mean = torch.tensor(
|
||||
self.config_dict["latents_mean"],
|
||||
).view(1, 1, channels)
|
||||
std = torch.tensor(
|
||||
self.config_dict["latents_std"],
|
||||
).view(1, 1, channels)
|
||||
rows = ((latent - mean) / std).reshape(-1, channels)
|
||||
return rows.float(), int(latent.shape[1])
|
||||
|
||||
@torch.inference_mode()
|
||||
def decode_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
channels = int(self.config_dict["latent_channels"])
|
||||
mean = torch.tensor(
|
||||
self.config_dict["latents_mean"],
|
||||
device=latent.device,
|
||||
dtype=latent.dtype,
|
||||
).view(1, channels, 1)
|
||||
std = torch.tensor(
|
||||
self.config_dict["latents_std"],
|
||||
device=latent.device,
|
||||
dtype=latent.dtype,
|
||||
).view(1, channels, 1)
|
||||
waveform = self.remote.decode(latent * std + mean)
|
||||
if waveform.ndim != 3 or waveform.shape[1] != 1:
|
||||
raise ValueError(f"unexpected decoded audio shape {tuple(waveform.shape)}")
|
||||
return waveform.permute(1, 0, 2).contiguous().float()
|
||||
|
||||
|
||||
__all__ = ["MiniMaxH3AudioVAE", "MiniMaxH3VideoVAE"]
|
||||
Reference in new issue
Block a user