[feat] Extend disaggregated MiniMax-H3 stage with VAE encoders (#6939)

Signed-off-by: asukaqaq-s <1311722138@qq.com>
Signed-off-by: Jader <yjader@foxmail.com>
Co-authored-by: Jader <yjader@foxmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-09-16 16:52:35 +08:00
1 parent e78a5d01f7
commit d554e6b67e
27 files changed
+3663 -2399

No files matched your search

@@ -1,16 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
"""MiniMax H3 checkpoint resolution for the AR text-encoder stage."""
from pathlib import Path
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
MINIMAX_H3_TEXT_ENCODER_DOWNLOAD_PATTERNS = {
"fl2va": ["FL2VA/text_encoder/**"],
"ref2va": ["Ref2VA/text_encoder/**"],
MINIMAX_H3_ENCODER_DOWNLOAD_PATTERNS = {
partition: [
f"{subdir}/text_encoder/**",
f"{subdir}/video_vae/**",
f"{subdir}/audio_vae/**",
]
for partition, subdir in {"fl2va": "FL2VA", "ref2va": "Ref2VA"}.items()
}
MINIMAX_H3_ENCODER_DOWNLOAD_PATTERNS["combined"] = MINIMAX_H3_ENCODER_DOWNLOAD_PATTERNS["fl2va"]
def resolve_minimax_h3_partition(
@@ -32,7 +35,7 @@ def resolve_minimax_h3_partition(
return "ref2va" if task == "ref2va" else "fl2va"
def resolve_minimax_h3_model_root(
def resolve_minimax_h3_encoder_model_root(
model: str,
revision: str | None,
task_type: str | None,
@@ -50,7 +53,7 @@ def resolve_minimax_h3_model_root(
snapshot = download_weights_from_hf_specific(
model_name_or_path=model,
cache_dir=None,
allow_patterns=MINIMAX_H3_TEXT_ENCODER_DOWNLOAD_PATTERNS[partition],
allow_patterns=MINIMAX_H3_ENCODER_DOWNLOAD_PATTERNS[partition],
revision=revision,
require_all=True,
)
@@ -59,4 +62,11 @@ def resolve_minimax_h3_model_root(
return str(Path(snapshot) / subdir / "text_encoder")
__all__ = ["resolve_minimax_h3_model_root", "resolve_minimax_h3_partition"]
resolve_minimax_h3_model_root = resolve_minimax_h3_encoder_model_root
__all__ = [
"resolve_minimax_h3_encoder_model_root",
"resolve_minimax_h3_model_root",
"resolve_minimax_h3_partition",
]
@@ -4,8 +4,8 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, fields
from typing import Any
import torch
@@ -119,3 +119,502 @@ class MiniMaxH3TextConditioning:
"hidden_states": self.hidden_states,
"token_tags": self.token_tags,
}
MINIMAX_H3_ENCODER_REQUEST_KEY = "minimax_h3_encoder_request"
MINIMAX_H3_ENCODER_LAYOUT_KEY = "minimax_h3_encoder_layout"
STAGE_SCHEMA_VERSION = 1
_ENCODER_WIRE_SCHEMA_ID = 1
_TASK_TO_CODE = {"t2va": 1, "fl2va": 2, "ref2va": 3}
_CODE_TO_TASK = {value: key for key, value in _TASK_TO_CODE.items()}
_REF_KIND_TO_CODE = {"image": 1, "video": 2, "video_audio": 3, "audio": 4}
_CODE_TO_REF_KIND = {value: key for key, value in _REF_KIND_TO_CODE.items()}
_INTEGER_DTYPES = frozenset(
{
torch.int8,
torch.int16,
torch.int32,
torch.int64,
torch.uint8,
}
)
_VIDEO_CONDITION_WIDTH = 96
_AUDIO_CONDITION_WIDTH = 32
_AUDIO_CONDITION_CHANNELS = 2
def _wire_layout(payload: Mapping[str, Any]) -> torch.Tensor:
private_metadata = payload.get("kv_metadata")
if not isinstance(private_metadata, Mapping):
raise ValueError("MiniMax H3 wire payload requires private layout metadata")
layout = private_metadata.get(MINIMAX_H3_ENCODER_LAYOUT_KEY)
if isinstance(layout, torch.Tensor):
return layout
raise ValueError("MiniMax H3 wire payload requires one packed encoder layout tensor")
def _validate_condition_tensors(
visual: torch.Tensor | None,
visual_shapes: Sequence[Sequence[int]],
audio: torch.Tensor | None,
audio_lengths: Sequence[int],
) -> None:
if (visual is None) != (not visual_shapes):
raise ValueError("MiniMax H3 visual condition and shapes must be present together")
if (audio is None) != (not audio_lengths):
raise ValueError("MiniMax H3 audio condition and lengths must be present together")
parsed_shapes = [tuple(int(item) for item in shape) for shape in visual_shapes]
if any(
len(shape) != 3 or any(item <= 0 for item in shape) or shape[1] % 2 or shape[2] % 2 for shape in parsed_shapes
):
raise ValueError("MiniMax H3 visual condition shapes must be positive [T, H, W] triplets with even H/W")
if visual is not None:
expected_rows = sum(t * (h // 2) * (w // 2) for t, h, w in parsed_shapes)
if (
visual.dtype != torch.float32
or visual.ndim != 2
or tuple(visual.shape)
!= (
expected_rows,
_VIDEO_CONDITION_WIDTH,
)
):
raise ValueError(
"MiniMax H3 visual condition must be FP32 with shape "
f"[{expected_rows}, {_VIDEO_CONDITION_WIDTH}], got {visual.dtype} {tuple(visual.shape)}"
)
parsed_lengths = [int(length) for length in audio_lengths]
if any(length <= 0 for length in parsed_lengths):
raise ValueError("MiniMax H3 audio condition lengths must be positive")
if audio is not None:
expected_rows = _AUDIO_CONDITION_CHANNELS * sum(parsed_lengths)
if (
audio.dtype != torch.float32
or audio.ndim != 2
or tuple(audio.shape)
!= (
expected_rows,
_AUDIO_CONDITION_WIDTH,
)
):
raise ValueError(
"MiniMax H3 audio condition must be FP32 with shape "
f"[{expected_rows}, {_AUDIO_CONDITION_WIDTH}], got {audio.dtype} {tuple(audio.shape)}"
)
def _ref_block_rows(ref_blocks: Sequence[Mapping[str, Any]]) -> list[list[int]]:
rows: list[list[int]] = []
for block in ref_blocks:
kind = str(block.get("kind") or "")
kind_code = _REF_KIND_TO_CODE.get(kind)
if kind_code is None:
raise ValueError(f"unsupported MiniMax H3 ref kind {kind!r}")
rows.append(
[
kind_code,
int(block.get("ref_audio_t", 0)),
int(block.get("latent_t", 0)),
int(block.get("latent_h", 0)),
int(block.get("latent_w", 0)),
]
)
return rows
def _decode_ref_blocks(value: torch.Tensor) -> tuple[dict[str, Any], ...]:
if value.ndim != 2 or value.shape[1] != 5:
raise ValueError("MiniMax H3 ref_blocks metadata must have shape [N, 5]")
blocks: list[dict[str, Any]] = []
for kind_code, ref_audio_t, ref_t, ref_h, ref_w in value.detach().cpu().tolist():
kind = _CODE_TO_REF_KIND.get(int(kind_code))
if kind is None:
raise ValueError(f"unsupported MiniMax H3 ref kind code {kind_code!r}")
block: dict[str, Any] = {"kind": kind}
for key, item in (
("ref_audio_t", ref_audio_t),
("latent_t", ref_t),
("latent_h", ref_h),
("latent_w", ref_w),
):
if item:
block[key] = int(item)
blocks.append(block)
return tuple(blocks)
@dataclass(frozen=True)
class MiniMaxH3EncoderMediaInput:
task: str
height: int
width: int
num_frames: int
latent_t: int
audio_t: int
images: tuple[torch.Tensor, ...] = ()
videos: tuple[torch.Tensor, ...] = ()
video_audios: tuple[tuple[torch.Tensor, int] | None, ...] = ()
audios: tuple[tuple[torch.Tensor, int], ...] = ()
keyframe_frame_indices: tuple[int, ...] = ()
@classmethod
def from_mm_tensors(
cls,
values: Sequence[torch.Tensor],
metadata: Mapping[str, Any],
) -> MiniMaxH3EncoderMediaInput:
if not isinstance(values, Sequence) or isinstance(values, (str, bytes)):
raise ValueError("MiniMax H3 encoder media must be a tensor sequence")
tensors = list(values)
if any(not isinstance(item, torch.Tensor) for item in tensors):
raise ValueError("MiniMax H3 encoder media input must contain only tensors")
task = str(metadata["task"])
if task not in _TASK_TO_CODE:
raise ValueError(f"unsupported MiniMax H3 task {task!r}")
height = int(metadata["height"])
width = int(metadata["width"])
num_frames = int(metadata["num_frames"])
latent_t = int(metadata["latent_t"])
audio_t = int(metadata["audio_t"])
image_count = int(metadata.get("image_count", 0))
video_count = int(metadata.get("video_count", 0))
audio_count = int(metadata.get("audio_count", 0))
if min(height, width, num_frames, latent_t, audio_t) <= 0:
raise ValueError("MiniMax H3 encoder media dimensions must be positive")
if min(image_count, video_count, audio_count) < 0:
raise ValueError("MiniMax H3 encoder media counts must be non-negative")
audio_flags = tuple(bool(value) for value in metadata.get("video_audio_flags", ()))
video_audio_sample_rates = tuple(int(value) for value in metadata.get("video_audio_sample_rates", ()))
audio_sample_rates = tuple(int(value) for value in metadata.get("audio_sample_rates", ()))
if len(audio_flags) != video_count or len(video_audio_sample_rates) != video_count:
raise ValueError("MiniMax H3 video audio flags must align with videos")
if len(audio_sample_rates) != audio_count:
raise ValueError("MiniMax H3 audio sample rates must align with audio inputs")
if any((rate > 0) != has_audio for rate, has_audio in zip(video_audio_sample_rates, audio_flags, strict=True)):
raise ValueError("MiniMax H3 embedded audio sample rates must align with audio flags")
if any(rate <= 0 for rate in audio_sample_rates):
raise ValueError("MiniMax H3 audio sample rates must be positive")
cursor = 0
images = tuple(tensors[cursor : cursor + image_count])
cursor += image_count
videos = tuple(tensors[cursor : cursor + video_count])
cursor += video_count
if len(images) != image_count or any(image.ndim != 3 or image.shape[-1] != 3 for image in images):
raise ValueError("MiniMax H3 encoder images must have shape [H, W, 3]")
if len(videos) != video_count or any(video.ndim != 4 or video.shape[-1] != 3 for video in videos):
raise ValueError("MiniMax H3 encoder videos must have shape [T, H, W, 3]")
video_audios: list[tuple[torch.Tensor, int] | None] = []
for has_audio, sample_rate in zip(audio_flags, video_audio_sample_rates, strict=True):
if not has_audio:
video_audios.append(None)
continue
if cursor >= len(tensors):
raise ValueError("MiniMax H3 encoder media input is truncated")
waveform = tensors[cursor]
cursor += 1
if waveform.ndim not in (1, 2):
raise ValueError("MiniMax H3 embedded audio must have shape [samples] or [channels, samples]")
video_audios.append((waveform, sample_rate))
audios: list[tuple[torch.Tensor, int]] = []
for sample_rate in audio_sample_rates:
if cursor >= len(tensors):
raise ValueError("MiniMax H3 encoder media input is truncated")
waveform = tensors[cursor]
cursor += 1
if waveform.ndim not in (1, 2):
raise ValueError("MiniMax H3 audio must have shape [samples] or [channels, samples]")
audios.append((waveform, sample_rate))
if cursor != len(tensors):
raise ValueError(f"MiniMax H3 encoder media input has {len(tensors) - cursor} trailing tensors")
return cls(
task=task,
height=height,
width=width,
num_frames=num_frames,
latent_t=latent_t,
audio_t=audio_t,
images=images,
videos=videos,
video_audios=tuple(video_audios),
audios=tuple(audios),
keyframe_frame_indices=tuple(int(value) for value in metadata.get("keyframe_frame_indices", ())),
)
def to_mm_tensors(self) -> list[torch.Tensor]:
task_code = _TASK_TO_CODE.get(self.task)
if task_code is None:
raise ValueError(f"unsupported MiniMax H3 task {self.task!r}")
if min(self.height, self.width, self.num_frames, self.latent_t, self.audio_t) <= 0:
raise ValueError("MiniMax H3 encoder media dimensions must be positive")
if len(self.video_audios) != len(self.videos):
raise ValueError("MiniMax H3 video audio slots must align with videos")
if any(image.ndim != 3 or image.shape[-1] != 3 for image in self.images):
raise ValueError("MiniMax H3 encoder images must have shape [H, W, 3]")
if any(video.ndim != 4 or video.shape[-1] != 3 for video in self.videos):
raise ValueError("MiniMax H3 encoder videos must have shape [T, H, W, 3]")
audio_items = [item for item in self.video_audios if item is not None] + list(self.audios)
if any(waveform.ndim not in (1, 2) or int(sample_rate) <= 0 for waveform, sample_rate in audio_items):
raise ValueError("MiniMax H3 encoder audio must have a waveform and positive sample rate")
tensors = [*self.images, *self.videos]
for item in self.video_audios:
if item is None:
continue
waveform, _sample_rate = item
tensors.append(waveform)
tensors.extend(waveform for waveform, _sample_rate in self.audios)
return tensors
def to_metadata(self) -> dict[str, Any]:
return {
"task": self.task,
"height": self.height,
"width": self.width,
"num_frames": self.num_frames,
"latent_t": self.latent_t,
"audio_t": self.audio_t,
"image_count": len(self.images),
"video_count": len(self.videos),
"audio_count": len(self.audios),
"video_audio_flags": [item is not None for item in self.video_audios],
"video_audio_sample_rates": [item[1] if item is not None else 0 for item in self.video_audios],
"audio_sample_rates": [sample_rate for _waveform, sample_rate in self.audios],
"keyframe_frame_indices": list(self.keyframe_frame_indices),
}
@dataclass(frozen=True)
class MiniMaxH3EncoderMediaConditioning:
task: str
height: int
width: int
num_frames: int
latent_t: int
audio_t: int
visual_condition: torch.Tensor | None = None
visual_condition_shapes: tuple[tuple[int, int, int], ...] = ()
audio_condition: torch.Tensor | None = None
audio_condition_lengths: tuple[int, ...] = ()
ref_blocks: tuple[dict[str, Any], ...] = ()
keyframe_frame_indices: tuple[int, ...] = ()
def to_omni_components(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
task_code = _TASK_TO_CODE.get(self.task)
if task_code is None:
raise ValueError(f"unsupported MiniMax H3 task {self.task!r}")
if min(self.height, self.width, self.num_frames, self.latent_t, self.audio_t) <= 0:
raise ValueError("MiniMax H3 encoder media dimensions must be positive")
_validate_condition_tensors(
self.visual_condition,
self.visual_condition_shapes,
self.audio_condition,
self.audio_condition_lengths,
)
empty = torch.empty((0,), dtype=torch.float32)
ref_rows = _ref_block_rows(self.ref_blocks)
layout = [
_ENCODER_WIRE_SCHEMA_ID,
STAGE_SCHEMA_VERSION,
task_code,
self.height,
self.width,
self.num_frames,
self.latent_t,
self.audio_t,
len(self.visual_condition_shapes),
len(self.audio_condition_lengths),
len(ref_rows),
len(self.keyframe_frame_indices),
]
layout.extend(item for shape in self.visual_condition_shapes for item in shape)
layout.extend(self.audio_condition_lengths)
layout.extend(item for row in ref_rows for item in row)
layout.extend(self.keyframe_frame_indices)
return (
self.visual_condition if self.visual_condition is not None else empty,
self.audio_condition if self.audio_condition is not None else empty,
torch.tensor(layout, dtype=torch.int64),
)
@dataclass(frozen=True)
class MiniMaxH3EncoderConditioning:
hidden_states: torch.Tensor
token_tags: torch.Tensor
task: str
height: int
width: int
num_frames: int
latent_t: int
audio_t: int
visual_condition: torch.Tensor | None = None
visual_condition_shapes: tuple[tuple[int, int, int], ...] = ()
audio_condition: torch.Tensor | None = None
audio_condition_lengths: tuple[int, ...] = ()
ref_blocks: tuple[dict[str, Any], ...] = ()
keyframe_frame_indices: tuple[int, ...] = ()
@classmethod
def from_components(
cls,
text: MiniMaxH3TextConditioning,
media: MiniMaxH3EncoderMediaConditioning,
) -> MiniMaxH3EncoderConditioning:
"""Combine local encoder results without a stage-wire round trip."""
text = MiniMaxH3TextConditioning.from_payload(text.to_payload())
_validate_condition_tensors(
media.visual_condition,
media.visual_condition_shapes,
media.audio_condition,
media.audio_condition_lengths,
)
return cls(
hidden_states=text.hidden_states,
token_tags=text.token_tags,
**{item.name: getattr(media, item.name) for item in fields(media)},
)
@classmethod
def from_omni_payload(cls, payload: Mapping[str, Any]) -> MiniMaxH3EncoderConditioning:
hidden_payload = payload.get("hidden_states")
embed_payload = payload.get("embed")
meta_payload = payload.get("meta")
hidden_states = hidden_payload.get("output") if isinstance(hidden_payload, Mapping) else None
token_tags = meta_payload.get("token_role_ids") if isinstance(meta_payload, Mapping) else None
visual = embed_payload.get("embedding") if isinstance(embed_payload, Mapping) else None
audio = embed_payload.get("speech_feat") if isinstance(embed_payload, Mapping) else None
if not all(isinstance(item, torch.Tensor) for item in (hidden_states, token_tags, visual, audio)):
raise ValueError("MiniMax H3 encoder wire requires text, visual and audio tensors")
assert isinstance(hidden_states, torch.Tensor)
assert isinstance(token_tags, torch.Tensor)
assert isinstance(visual, torch.Tensor)
assert isinstance(audio, torch.Tensor)
if token_tags.ndim == 2 and token_tags.shape[-1] == 1:
token_tags = token_tags.squeeze(-1)
text = MiniMaxH3TextConditioning.from_payload({"hidden_states": hidden_states, "token_tags": token_tags})
for name, value in (("visual", visual), ("audio", audio)):
if value.numel() == 0 and (value.dtype != torch.float32 or tuple(value.shape) != (0,)):
raise ValueError(f"MiniMax H3 empty {name} condition slot must be FP32 with shape [0]")
if not isinstance(meta_payload, Mapping):
raise ValueError("MiniMax H3 wire payload requires a meta mapping")
layout = _wire_layout(payload)
if layout.dtype not in _INTEGER_DTYPES or layout.ndim != 1:
raise ValueError("MiniMax H3 encoder layout must be a one-dimensional integer tensor")
values = [int(item) for item in layout.detach().cpu().tolist()]
if len(values) < 12:
raise ValueError("MiniMax H3 encoder layout header is truncated")
(
schema_id,
version,
task_code,
height,
width,
num_frames,
latent_t,
audio_t,
visual_count,
audio_count,
ref_count,
keyframe_count,
) = values[:12]
if schema_id != _ENCODER_WIRE_SCHEMA_ID or version != STAGE_SCHEMA_VERSION:
raise ValueError(f"unsupported MiniMax H3 encoder wire schema {schema_id}:{version}")
task = _CODE_TO_TASK.get(task_code)
if task is None:
raise ValueError(f"unsupported MiniMax H3 wire task code {task_code!r}")
if min(height, width, num_frames, latent_t, audio_t) <= 0:
raise ValueError("MiniMax H3 encoder wire dimensions must be positive")
if min(visual_count, audio_count, ref_count, keyframe_count) < 0:
raise ValueError("MiniMax H3 encoder layout counts must be non-negative")
expected_size = 12 + 3 * visual_count + audio_count + 5 * ref_count + keyframe_count
if len(values) != expected_size:
raise ValueError(f"MiniMax H3 encoder layout has {len(values)} integers, expected {expected_size}")
cursor = 12
visual_shapes = tuple(
tuple(values[cursor + 3 * index : cursor + 3 * (index + 1)]) for index in range(visual_count)
)
cursor += 3 * visual_count
audio_lengths = tuple(values[cursor : cursor + audio_count])
cursor += audio_count
ref_rows = values[cursor : cursor + 5 * ref_count]
cursor += 5 * ref_count
ref_blocks_tensor = torch.tensor(ref_rows, dtype=torch.int64).reshape(-1, 5)
keyframe_frame_indices = tuple(values[cursor : cursor + keyframe_count])
visual_condition = visual if visual.numel() else None
audio_condition = audio if audio.numel() else None
_validate_condition_tensors(
visual_condition,
visual_shapes,
audio_condition,
audio_lengths,
)
return cls(
hidden_states=text.hidden_states,
token_tags=text.token_tags,
task=task,
height=height,
width=width,
num_frames=num_frames,
latent_t=latent_t,
audio_t=audio_t,
visual_condition=visual_condition,
visual_condition_shapes=visual_shapes,
audio_condition=audio_condition,
audio_condition_lengths=audio_lengths,
ref_blocks=_decode_ref_blocks(ref_blocks_tensor),
keyframe_frame_indices=keyframe_frame_indices,
)
def to_omni_payload(self) -> dict[str, Any]:
text = MiniMaxH3TextConditioning.from_payload(
{
"hidden_states": self.hidden_states,
"token_tags": self.token_tags,
}
)
visual, audio, layout = MiniMaxH3EncoderMediaConditioning(
task=self.task,
height=self.height,
width=self.width,
num_frames=self.num_frames,
latent_t=self.latent_t,
audio_t=self.audio_t,
visual_condition=self.visual_condition,
visual_condition_shapes=self.visual_condition_shapes,
audio_condition=self.audio_condition,
audio_condition_lengths=self.audio_condition_lengths,
ref_blocks=self.ref_blocks,
keyframe_frame_indices=self.keyframe_frame_indices,
).to_omni_components()
return {
"hidden_states": {"output": text.hidden_states},
"embed": {
"embedding": visual,
"speech_feat": audio,
},
"meta": {"token_role_ids": text.token_tags},
"kv_metadata": {MINIMAX_H3_ENCODER_LAYOUT_KEY: layout},
}
__all__ = [
"MINIMAX_H3_CONDITION_LABELS_KEY",
"MINIMAX_H3_ENCODER_LAYOUT_KEY",
"MINIMAX_H3_ENCODER_REQUEST_KEY",
"MINIMAX_H3_PRESENTATION_TASK_KEY",
"MINIMAX_H3_TEXT_CONDITIONING_SCHEMA",
"MINIMAX_H3_TEXT_HIDDEN_SIZE",
"STAGE_SCHEMA_VERSION",
"MiniMaxH3EncoderConditioning",
"MiniMaxH3EncoderMediaConditioning",
"MiniMaxH3EncoderMediaInput",
"MiniMaxH3TextConditioning",
]
@@ -0,0 +1,252 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_tp_group
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.minimax_h3.vae import (
MiniMaxH3AudioVAE,
MiniMaxH3VideoVAE,
)
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
MINIMAX_H3_ENCODER_LAYOUT_KEY,
MINIMAX_H3_ENCODER_REQUEST_KEY,
MiniMaxH3EncoderMediaConditioning,
MiniMaxH3EncoderMediaInput,
)
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import encode_media
from vllm_omni.model_executor.models.minimax_h3.text_encoder import (
MiniMaxH3TextEncoderBackbone,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
_MEDIA_CONDITIONING_CACHE_KEY = "_minimax_h3_encoder_media_conditioning"
_MEDIA_CACHE_KEY = "_minimax_h3_encoder_media"
_COMPONENT_CONFIG_KEY = "minimax_h3_encoder_components"
_COMPONENT_ROLES = frozenset({"text_encoder", "video_vae", "audio_vae"})
def _partition_root(model_path: str) -> Path:
path = Path(model_path)
if path.name in {"text_encoder", "video_vae", "audio_vae"}:
return path.parent
return path
@dataclass(frozen=True)
class MiniMaxH3EncoderComponentConfig:
text_parallel_mode: str
video_parallel_mode: str
audio_parallel_mode: str
def __post_init__(self) -> None:
if self.text_parallel_mode != "tp":
raise ValueError("MiniMax H3 text encoder currently supports parallel_mode='tp' only")
if self.video_parallel_mode not in {"leader", "patch"}:
raise ValueError("MiniMax H3 video VAE parallel_mode must be 'leader' or 'patch'")
if self.audio_parallel_mode != "leader":
raise ValueError("MiniMax H3 audio VAE currently supports parallel_mode='leader' only")
@staticmethod
def _parallel_mode(raw: Mapping[str, Any], role: str) -> str:
value = raw[role]
if not isinstance(value, Mapping):
raise TypeError(f"MiniMax H3 encoder role {role!r} must be a mapping")
if set(value) != {"parallel_mode"}:
raise ValueError(f"MiniMax H3 encoder role {role!r} requires only parallel_mode")
return str(value["parallel_mode"])
@classmethod
def from_vllm_config(cls, vllm_config: VllmConfig) -> MiniMaxH3EncoderComponentConfig:
hf_config = vllm_config.model_config.hf_config
raw = getattr(hf_config, _COMPONENT_CONFIG_KEY, None)
if not isinstance(raw, Mapping):
raise ValueError(f"MiniMax H3 Encoder Stage requires explicit {_COMPONENT_CONFIG_KEY}")
roles = set(raw)
if roles != _COMPONENT_ROLES:
missing = sorted(_COMPONENT_ROLES - roles)
unknown = sorted(roles - _COMPONENT_ROLES)
raise ValueError(
f"{_COMPONENT_CONFIG_KEY} requires exactly {sorted(_COMPONENT_ROLES)}; "
f"missing={missing}, unknown={unknown}"
)
return cls(
text_parallel_mode=cls._parallel_mode(raw, "text_encoder"),
video_parallel_mode=cls._parallel_mode(raw, "video_vae"),
audio_parallel_mode=cls._parallel_mode(raw, "audio_vae"),
)
class MiniMaxH3Encoder(MiniMaxH3TextEncoderBackbone):
requires_request_sample_eligibility = True
enable_update_additional_information = True
omni_payload_at_request_end = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model") -> None:
super().__init__(vllm_config=vllm_config, prefix=prefix)
try:
tp_group = get_tp_group()
tp_rank = int(tp_group.rank_in_group)
except (AssertionError, RuntimeError):
tp_group = None
tp_rank = 0
self._component_leader = tp_rank == 0
self.component_config = MiniMaxH3EncoderComponentConfig.from_vllm_config(vllm_config)
root = _partition_root(vllm_config.model_config.model)
missing = [name for name in ("text_encoder", "video_vae", "audio_vae") if not (root / name).is_dir()]
if missing:
raise RuntimeError(f"MiniMax H3 Encoder Stage requires all three encoders; missing {missing}")
device = get_local_device()
self.video_vae: MiniMaxH3VideoVAE | None = None
self.audio_vae: MiniMaxH3AudioVAE | None = None
if self._component_leader or self.component_config.video_parallel_mode == "patch":
self.video_vae = MiniMaxH3VideoVAE(
str(root / "video_vae"),
device=device,
encode_only=True,
trust_remote_code=vllm_config.model_config.trust_remote_code,
)
if self.component_config.video_parallel_mode == "patch":
if tp_group is None:
raise RuntimeError("MiniMax H3 video VAE patch parallelism requires an initialized TP group")
self.video_vae.set_parallel_size(
tp_group.world_size,
process_group=tp_group.device_group,
)
if self._component_leader:
self.audio_vae = MiniMaxH3AudioVAE(
str(root / "audio_vae"),
device=device,
encode_only=True,
trust_remote_code=vllm_config.model_config.trust_remote_code,
)
@staticmethod
def _media_input(info: dict[str, Any]) -> MiniMaxH3EncoderMediaInput:
cached = info.get(_MEDIA_CACHE_KEY)
if isinstance(cached, MiniMaxH3EncoderMediaInput):
return cached
meta = info.get("meta")
metadata = meta.get(MINIMAX_H3_ENCODER_REQUEST_KEY) if isinstance(meta, Mapping) else None
if not isinstance(metadata, Mapping):
raise RuntimeError("MiniMax H3 unified encoder requires request metadata")
hidden_states = info.get("hidden_states")
raw_media = hidden_states.get("layers") if isinstance(hidden_states, Mapping) else None
if raw_media is None:
media_tensors: list[torch.Tensor] = []
elif isinstance(raw_media, Mapping):
try:
ordered_items = sorted(raw_media.items(), key=lambda item: int(item[0]))
except (TypeError, ValueError) as exc:
raise RuntimeError("MiniMax H3 encoder media keys must be integer indices") from exc
if [int(key) for key, _ in ordered_items] != list(range(len(ordered_items))):
raise RuntimeError("MiniMax H3 encoder media indices must be contiguous from zero")
media_tensors = [value for _, value in ordered_items]
else:
raise RuntimeError("MiniMax H3 encoder media must be a namespaced tensor mapping")
if any(not isinstance(value, torch.Tensor) for value in media_tensors):
raise RuntimeError("MiniMax H3 encoder media must contain only tensors")
media = MiniMaxH3EncoderMediaInput.from_mm_tensors(media_tensors, metadata)
if media.to_metadata() != dict(metadata):
raise RuntimeError("MiniMax H3 encoder media does not match request metadata")
info[_MEDIA_CACHE_KEY] = media
return media
def _encode_media(self, media: MiniMaxH3EncoderMediaInput) -> MiniMaxH3EncoderMediaConditioning | None:
if not self._component_leader and self.component_config.video_parallel_mode == "leader":
return None
return encode_media(
media,
video_vae=self.video_vae,
audio_vae=self.audio_vae,
emit_conditioning=self._component_leader,
)
@staticmethod
def _has_encoder_input(info: object) -> bool:
if not isinstance(info, Mapping):
return False
meta = info.get("meta")
return isinstance(meta, Mapping) and isinstance(meta.get(MINIMAX_H3_ENCODER_REQUEST_KEY), Mapping)
def make_omni_output(
self,
model_outputs: torch.Tensor | OmniOutput,
**kwargs: Any,
) -> OmniOutput:
output = super().make_omni_output(model_outputs, **kwargs)
infos = kwargs.get("model_intermediate_buffer") or []
if not isinstance(infos, Sequence) or not infos:
return output
sample_eligible = kwargs.get("request_sample_eligible")
if sample_eligible is None:
sample_eligible = [True] * len(infos)
if len(sample_eligible) != len(infos):
raise RuntimeError(
f"MiniMax H3 unified encoder received {len(sample_eligible)} sampling flags for {len(infos)} requests"
)
per_request: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None] = []
release_workspace = False
for eligible, info in zip(sample_eligible, infos, strict=True):
if not eligible:
per_request.append(None)
continue
if not isinstance(info, dict) or not self._has_encoder_input(info):
raise RuntimeError("MiniMax H3 unified encoder request metadata is missing from the runner")
release_workspace = True
if _MEDIA_CONDITIONING_CACHE_KEY not in info:
info[_MEDIA_CONDITIONING_CACHE_KEY] = self._encode_media(self._media_input(info))
if not self._component_leader:
continue
cached = info.get(_MEDIA_CONDITIONING_CACHE_KEY)
if not isinstance(cached, MiniMaxH3EncoderMediaConditioning):
raise RuntimeError("MiniMax H3 encoder components did not produce conditioning")
per_request.append(cached.to_omni_components())
if release_workspace:
torch.accelerator.empty_cache()
if not self._component_leader:
return output
if not any(item is not None for item in per_request):
return output
empty = torch.empty((0,), dtype=torch.float32)
multimodal_outputs = dict(output.multimodal_outputs or {})
dotted_key = f"kv_metadata.{MINIMAX_H3_ENCODER_LAYOUT_KEY}"
multimodal_outputs[dotted_key] = [
item[2] if item is not None else torch.empty((0,), dtype=torch.int64) for item in per_request
]
embed = dict(multimodal_outputs.get("embed") or {})
embed["embedding"] = [item[0] if item is not None else empty for item in per_request]
embed["speech_feat"] = [item[1] if item is not None else empty for item in per_request]
multimodal_outputs["embed"] = embed
return OmniOutput(
text_hidden_states=output.text_hidden_states,
multimodal_outputs=multimodal_outputs,
intermediate_tensors=output.intermediate_tensors,
next_token_id=output.next_token_id,
)
def load_weights(
self,
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loaded = super().load_weights(weights)
for component_name in ("video_vae", "audio_vae"):
component = getattr(self, component_name)
if component is not None:
loaded.update(f"{component_name}.{name}" for name, _ in component.named_parameters())
return loaded
__all__ = ["MiniMaxH3Encoder"]
@@ -0,0 +1,569 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared request preparation and condition codec execution for MiniMax H3."""
from __future__ import annotations
import math
import os
import tempfile
from collections.abc import Callable, Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from typing import Any
import numpy as np
import torch
from PIL import Image
from vllm_omni.errors import OmniClientError
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
MiniMaxH3EncoderMediaConditioning,
MiniMaxH3EncoderMediaInput,
)
from vllm_omni.model_executor.models.minimax_h3.preprocessing import (
MINIMAX_H3_OUTPUT_SHORT_EDGE,
load_minimax_h3_images,
resolve_minimax_h3_aspect_ratio,
resolve_minimax_h3_output_canvas,
resolve_minimax_h3_reference_image_shape,
)
from vllm_omni.model_executor.models.minimax_h3.reference_video import (
MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS,
load_audio_file,
load_video_audio,
load_video_frames,
prepare_reference_videos,
sample_reference_video_frames,
validate_reference_audio_files,
validate_reference_audio_waveforms,
)
MINIMAX_H3_FPS = 24
MINIMAX_H3_MIN_OUTPUT_SECONDS = 4.0
MINIMAX_H3_MAX_OUTPUT_SECONDS = 15.0
def _items(value: Any) -> list[Any]:
if value is None:
return []
if isinstance(value, list):
return value
if isinstance(value, tuple) and not (len(value) == 2 and isinstance(value[1], Mapping)):
return list(value)
return [value]
def _audio_items(value: Any) -> list[Any]:
if value is None:
return []
if isinstance(value, (list, tuple)) and len(value) == 2 and isinstance(value[1], (int, np.integer)):
return [value]
return list(value) if isinstance(value, (list, tuple)) else [value]
def _load_audio(value: Any) -> tuple[torch.Tensor, int]:
if isinstance(value, (list, tuple)) and not (len(value) == 2 and isinstance(value[1], (int, np.integer))):
audios = _load_audios(value)
if len(audios) != 1:
raise OmniClientError(f"MiniMax H3 expected one audio, got {len(audios)}")
return audios[0]
if isinstance(value, (str, os.PathLike)):
return load_audio_file(str(value))
if isinstance(value, (list, tuple)) and len(value) == 2:
waveform, sample_rate = value
return torch.as_tensor(waveform).float(), int(sample_rate)
if isinstance(value, dict):
waveform = value.get("waveform", value.get("array"))
sample_rate = value.get("sample_rate", value.get("sampling_rate"))
if waveform is not None and sample_rate is not None:
return torch.as_tensor(waveform).float(), int(sample_rate)
raise OmniClientError("MiniMax H3 audio input must be a path, (waveform, sample_rate), or a waveform mapping")
def _load_audios(value: Any) -> list[tuple[torch.Tensor, int]]:
if isinstance(value, (list, tuple)) and not (len(value) == 2 and isinstance(value[1], (int, np.integer))):
if not value:
raise OmniClientError("MiniMax H3 audio input must not be empty")
return [_load_audio(item) for item in value]
return [_load_audio(value)]
def _as_int_list(value: Any, *, name: str) -> list[int]:
if isinstance(value, bool):
raise OmniClientError(f"{name} must be an integer or a list of integers")
if isinstance(value, (int, np.integer)):
return [int(value)]
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
result = list(value)
if not result:
raise OmniClientError(f"{name} must not be empty")
if any(isinstance(item, bool) or not isinstance(item, (int, np.integer)) for item in result):
raise OmniClientError(f"{name} must contain only integers")
return [int(item) for item in result]
raise OmniClientError(f"{name} must be an integer or a list of integers")
def _resolve_fl2va_keyframe_indices(extra: Mapping[str, Any], image_count: int) -> list[int]:
target = extra.get("target")
target = target if isinstance(target, Mapping) else {}
raw = extra.get("frame_indices", extra.get("frame_index"))
if raw is None:
raw = target.get("frame_indices", target.get("frame_index"))
raw_indices = ([0] if image_count == 1 else [0, -1]) if raw is None else _as_int_list(raw, name="frame_indices")
if len(raw_indices) != image_count:
raise OmniClientError(
f"MiniMax H3 FL2VA requires one frame index per image: got {raw_indices!r} for {image_count} image(s)"
)
if tuple(raw_indices) not in ((0,), (-1,), (0, -1)):
raise OmniClientError("MiniMax H3 FL2VA frame_indices must be [0], [-1], or [0, -1]")
return raw_indices
def _validate_ref2va_reference_counts(image_count: int, video_count: int, audio_count: int) -> None:
if min(image_count, video_count, audio_count) < 0:
raise OmniClientError("MiniMax H3 reference counts must be non-negative")
if image_count + video_count == 0:
raise OmniClientError("ref2va requires at least one image or video reference")
if image_count > 9:
raise OmniClientError("ref2va accepts at most 9 image references")
if video_count > 3:
raise OmniClientError("ref2va accepts at most 3 video references")
if audio_count > 3:
raise OmniClientError("ref2va accepts at most 3 standalone audio references")
if image_count + video_count + audio_count > 12:
raise OmniClientError("ref2va accepts at most 12 total references")
def _validate_reference_image(image: Image.Image) -> None:
resolve_minimax_h3_reference_image_shape(image)
def resolve_minimax_h3_shape(
task: str,
sampling: Any,
image: Image.Image | None,
) -> tuple[int, int, int, int, int]:
# The diffusion package exports a pipeline that imports this shared module.
from vllm_omni.diffusion.models.minimax_h3.time_request import (
MINIMAX_H3_SHAPE_PLANNER,
minimax_h3_align_frame_count,
)
fps = int(getattr(sampling, "fps", None) or MINIMAX_H3_FPS)
if fps != MINIMAX_H3_FPS:
raise OmniClientError(f"MiniMax H3 output fps is fixed at {MINIMAX_H3_FPS}")
extra = sampling.extra_args or {}
target = extra.get("target")
if target is not None and not isinstance(target, Mapping):
raise OmniClientError("MiniMax H3 extra_args['target'] must be an object")
target = target if isinstance(target, Mapping) else {}
duration = target.get("duration_seconds", extra.get("duration_seconds", extra.get("duration")))
if duration is not None:
if isinstance(duration, bool):
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration!r}")
try:
duration = float(duration)
except (TypeError, ValueError) as exc:
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration!r}") from exc
if (
not math.isfinite(duration)
or not MINIMAX_H3_MIN_OUTPUT_SECONDS <= duration <= MINIMAX_H3_MAX_OUTPUT_SECONDS
):
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration}")
requested_frames = int(round(duration * fps))
elif int(getattr(sampling, "num_frames", None) or 1) > 1:
requested_frames = int(sampling.num_frames)
else:
requested_frames = 124 if task == "ref2va" else 209
if not MINIMAX_H3_MIN_OUTPUT_SECONDS <= requested_frames / fps <= MINIMAX_H3_MAX_OUTPUT_SECONDS:
raise OmniClientError(
f"MiniMax H3 output duration must be in [4, 15] seconds, got {requested_frames / fps:.3f}"
)
num_frames = minimax_h3_align_frame_count(requested_frames)
height = getattr(sampling, "height", None)
width = getattr(sampling, "width", None)
aspect_ratio = resolve_minimax_h3_aspect_ratio(
task,
target.get("aspect_ratio", extra.get("aspect_ratio")),
image,
)
raw_short_edge = target.get("short_edge", extra.get("short_edge", MINIMAX_H3_OUTPUT_SHORT_EDGE))
if isinstance(raw_short_edge, bool) or not isinstance(raw_short_edge, (int, np.integer)):
raise OmniClientError(
f"MiniMax H3 target.short_edge must be {MINIMAX_H3_OUTPUT_SHORT_EDGE}, got {raw_short_edge!r}"
)
if height is None or width is None:
height, width = resolve_minimax_h3_output_canvas(aspect_ratio, int(raw_short_edge))
height = int(height) // 32 * 32
width = int(width) // 32 * 32
if min(height, width) <= 0:
raise OmniClientError(f"invalid MiniMax H3 canvas {width}x{height}")
if width > 4 * height or height > 4 * width:
raise OmniClientError("MiniMax H3 canvas aspect ratio must be in [1:4, 4:1]")
return (
height,
width,
num_frames,
MINIMAX_H3_SHAPE_PLANNER.video_latent_t(num_frames),
MINIMAX_H3_SHAPE_PLANNER.audio_latent_t(num_frames / fps),
)
def _resolve_task(
extra_args: Mapping[str, Any],
multi_modal_data: Mapping[str, Any],
) -> str:
requested = extra_args.get("task")
if requested is not None:
return str(requested).lower()
if multi_modal_data.get("video") is not None or multi_modal_data.get("audio") is not None:
return "ref2va"
if multi_modal_data.get("image") is not None:
return "fl2va"
return "t2va"
def _prepare_encoder_images(
task: str,
images: list[Image.Image],
*,
height: int,
width: int,
) -> list[Any]:
if not images:
return []
if task == "ref2va":
return [
image.resize(
resolve_minimax_h3_reference_image_shape(image),
Image.Resampling.LANCZOS,
)
for image in images
]
if task != "fl2va":
return images
return [image.resize((width, height), Image.Resampling.LANCZOS) for image in images]
def _image_to_tensor(image: Image.Image) -> torch.Tensor:
array = np.asarray(image.convert("RGB"), dtype=np.uint8)
return torch.from_numpy(np.array(array, copy=True)).contiguous()
def _frames_to_tensor(frames: Sequence[Any]) -> torch.Tensor:
if len(frames) == 0:
raise ValueError("MiniMax H3 reference video must contain frames")
return torch.stack(
[
_image_to_tensor(frame if isinstance(frame, Image.Image) else Image.fromarray(np.asarray(frame)))
for frame in frames
]
)
def _reuse_prepared_reference_videos(
prepared: list[dict[str, Any]] | None,
*,
expected_count: int,
) -> list[dict[str, Any]] | None:
if prepared is None:
return None
if len(prepared) != expected_count:
raise OmniClientError("MiniMax H3 prepared-reference-video count does not match the request")
for item in prepared:
if not os.path.isfile(item["prepared_path"]):
raise OmniClientError(f"MiniMax H3 prepared reference video is unavailable: {item['prepared_path']}")
return prepared
@dataclass(frozen=True)
class PreparedEncoderInputs:
"""CPU-owned media and Qwen presentation inputs, independent of a runner."""
prompt: str
media: MiniMaxH3EncoderMediaInput
images: list[Image.Image]
qwen_videos: list[tuple[np.ndarray, dict[str, Any]]]
video_timestamps: list[list[float]]
condition_labels: list[tuple[str, int]]
def _effective_audio_inputs(
video_audios: Sequence[tuple[torch.Tensor, int] | None],
standalone_audios: Sequence[tuple[torch.Tensor, int]],
*,
max_standalone_seconds: float,
) -> list[tuple[torch.Tensor, int]]:
"""Return the waveforms exactly as the audio WVAE will consume them."""
audio_inputs = [item for item in video_audios if item is not None]
audio_inputs.extend(
(waveform[..., : int(round(max_standalone_seconds * sample_rate))], sample_rate)
for waveform, sample_rate in standalone_audios
)
return audio_inputs
def prepare_encoder_inputs(
prompt: Any,
sampling: Any,
*,
task: str | None = None,
prepared_reference_videos: list[dict[str, Any]] | None = None,
) -> PreparedEncoderInputs:
"""Decode references once and prepare the shared text/codec presentation."""
if isinstance(prompt, str):
prompt = {"prompt": prompt}
if not isinstance(prompt, dict):
raise TypeError(f"MiniMax H3 expects a string or dict prompt, got {type(prompt)!r}")
text = str(prompt.get("prompt") or "")
if not text:
raise OmniClientError("MiniMax H3 requires a non-empty prompt")
multi_modal_data = prompt.get("multi_modal_data") or {}
if not isinstance(multi_modal_data, Mapping):
raise TypeError("multi_modal_data must be a mapping")
image_values = _items(multi_modal_data.get("image"))
videos = _items(multi_modal_data.get("video"))
raw_audio = multi_modal_data.get("audio")
audio_values = _audio_items(raw_audio)
diffusion_sampling_params = sampling
extra_args = getattr(diffusion_sampling_params, "extra_args", None) or {}
task = task if task is not None else _resolve_task(extra_args, multi_modal_data)
raw_images = load_minimax_h3_images(image_values) if image_values else []
if task == "t2va":
if raw_images or videos or audio_values:
raise OmniClientError("t2va does not accept image, video, or audio conditions")
elif task == "fl2va":
if not raw_images or videos or audio_values:
raise OmniClientError("fl2va requires image conditions only")
if len(raw_images) > 2:
raise OmniClientError("fl2va accepts at most first and last images")
for image in raw_images:
_validate_reference_image(image)
elif task == "ref2va":
_validate_ref2va_reference_counts(len(raw_images), len(videos), len(audio_values))
if not raw_images and not videos:
raise OmniClientError("ref2va requires an image or video condition")
else:
raise OmniClientError(f"unsupported MiniMax H3 task {task!r}")
height, width, num_frames, latent_t, audio_t = resolve_minimax_h3_shape(
task,
diffusion_sampling_params,
raw_images[0] if raw_images else None,
)
images = _prepare_encoder_images(
task,
raw_images,
height=height,
width=width,
)
keyframe_indices = _resolve_fl2va_keyframe_indices(extra_args, len(images)) if task == "fl2va" else []
video_timestamps: list[list[float]] = []
qwen_video_inputs: list[tuple[np.ndarray, dict[str, Any]]] = []
condition_labels: list[tuple[str, int]] = []
encoded_video_inputs: list[torch.Tensor] = []
video_audio_inputs: list[tuple[torch.Tensor, int] | None] = []
if task == "fl2va":
condition_labels.extend(("image", index) for index in range(1, len(images) + 1))
elif task == "ref2va":
condition_labels.extend(("image", index) for index in range(1, len(images) + 1))
prepared_videos: list[dict[str, Any]] = []
if videos:
with tempfile.TemporaryDirectory(prefix="minimax_h3_encoder_") as workdir:
prepared_videos = _reuse_prepared_reference_videos(
prepared_reference_videos, expected_count=len(videos)
)
if prepared_videos is None:
prepared_videos = prepare_reference_videos(
videos,
target_frame_count=num_frames,
workdir=workdir,
start_time_seconds=extra_args.get("start_time_seconds"),
)
for item in prepared_videos:
full_frames = load_video_frames(item["prepared_path"])
encoded_video_inputs.append(_frames_to_tensor(full_frames))
sampled = sample_reference_video_frames(
item["prepared_path"],
decoded_frames=full_frames,
)
video_timestamps.append(sampled["block_timestamps"])
frames = np.stack(sampled["frames"])
frame_count = int(frames.shape[0])
qwen_video_inputs.append(
(
frames,
{
"total_num_frames": frame_count,
"fps": MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS,
"duration": frame_count / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS,
"video_backend": "minimax_h3",
"frames_indices": list(range(frame_count)),
"do_sample_frames": False,
},
)
)
if item["input_has_audio"]:
waveform, sample_rate = load_video_audio(
item["original_path"],
start_time_seconds=float(item.get("start_time_seconds", 0.0)),
duration_seconds=item.get(
"audio_duration_seconds",
item.get("duration_seconds"),
),
)
video_audio_inputs.append((waveform.float().contiguous(), int(sample_rate)))
else:
video_audio_inputs.append(None)
audio_index = 0
for video_index, item in enumerate(prepared_videos, start=1):
if item["input_has_audio"]:
audio_index += 1
condition_labels.append(("audio", audio_index))
condition_labels.append(("video", video_index))
for _ in audio_values:
audio_index += 1
condition_labels.append(("audio", audio_index))
if raw_audio is not None:
validate_reference_audio_files(raw_audio)
standalone_audios = _load_audios(raw_audio) if raw_audio is not None else []
validate_reference_audio_waveforms(standalone_audios)
media_input = MiniMaxH3EncoderMediaInput(
task=task,
height=height,
width=width,
num_frames=num_frames,
latent_t=latent_t,
audio_t=audio_t,
images=tuple(_image_to_tensor(image) for image in images),
videos=tuple(encoded_video_inputs),
video_audios=tuple(video_audio_inputs),
audios=tuple((waveform.float().contiguous(), int(sample_rate)) for waveform, sample_rate in standalone_audios),
keyframe_frame_indices=tuple(keyframe_indices),
)
audio_inputs = _effective_audio_inputs(
media_input.video_audios,
media_input.audios,
max_standalone_seconds=float(media_input.num_frames) / MINIMAX_H3_FPS,
)
embedded_audio_count = sum(item is not None for item in media_input.video_audios)
# Video soundtracks and standalone references have separate 15-second budgets.
validate_reference_audio_waveforms(audio_inputs[:embedded_audio_count])
validate_reference_audio_waveforms(audio_inputs[embedded_audio_count:])
return PreparedEncoderInputs(
prompt=text,
media=media_input,
images=images,
qwen_videos=qwen_video_inputs,
video_timestamps=video_timestamps,
condition_labels=condition_labels,
)
def _image_from_tensor(value: torch.Tensor) -> Image.Image:
if value.ndim != 3 or value.shape[-1] != 3:
raise ValueError(f"MiniMax H3 image tensor must have shape [H, W, 3], got {tuple(value.shape)}")
array = value.detach().cpu().to(torch.uint8).contiguous().numpy()
return Image.fromarray(array, mode="RGB")
def encode_media(
media: MiniMaxH3EncoderMediaInput,
*,
video_vae: Any,
audio_vae: Any,
emit_conditioning: bool,
component_scope: Callable[[Any], AbstractContextManager[Any]] = nullcontext,
) -> MiniMaxH3EncoderMediaConditioning | None:
"""Run codecs on participating ranks; only the output rank runs audio.
The caller selects participating ranks and configures the VAE process group.
Every visual participant must call this function with the same references.
Component scopes allow the diffusion runner to stage one codec at a time.
"""
visual_rows: list[torch.Tensor] = []
visual_shapes: list[tuple[int, int, int]] = []
if media.images or media.videos:
if video_vae is None:
raise RuntimeError("MiniMax H3 video VAE is not resident on this rank")
with component_scope(video_vae):
for value in media.images:
image = _image_from_tensor(value)
visual_rows.append(video_vae.encode_image(image))
visual_shapes.append((1, image.height // 16, image.width // 16))
for value in media.videos:
frames = np.asarray(value.detach().cpu().to(torch.uint8).contiguous().numpy())
rows, shape = video_vae.encode_video(frames)
visual_rows.append(rows)
visual_shapes.append(tuple(int(item) for item in shape))
if not emit_conditioning:
return None
audio_rows: list[torch.Tensor] = []
audio_lengths: list[int] = []
embedded_audio_count = sum(item is not None for item in media.video_audios)
audio_inputs = _effective_audio_inputs(
media.video_audios,
media.audios,
max_standalone_seconds=float(media.num_frames) / MINIMAX_H3_FPS,
)
if audio_inputs:
if audio_vae is None:
raise RuntimeError("MiniMax H3 audio WVAE is not resident on the encoder leader")
with component_scope(audio_vae):
for waveform, sample_rate in audio_inputs:
rows, length = audio_vae.encode_waveform(waveform, sample_rate)
audio_rows.append(rows)
audio_lengths.append(int(length))
if audio_lengths:
if any(length < 80 or length > 600 for length in audio_lengths):
raise ValueError("MiniMax H3 audio references must each be between 2 and 15 seconds")
if sum(audio_lengths[:embedded_audio_count]) > 600 or sum(audio_lengths[embedded_audio_count:]) > 600:
raise ValueError("MiniMax H3 audio references must be at most 15 seconds in total")
ref_blocks: list[dict[str, Any]] = []
image_shapes = visual_shapes[: len(media.images)]
video_shapes = visual_shapes[len(media.images) :]
ref_blocks.extend({"kind": "image", "latent_h": shape[1], "latent_w": shape[2]} for shape in image_shapes)
embedded_lengths = iter(audio_lengths[:embedded_audio_count])
for shape, embedded in zip(video_shapes, media.video_audios, strict=True):
ref_audio_t = int(next(embedded_lengths)) if embedded is not None else 0
ref_blocks.append(
{
"kind": "video_audio" if ref_audio_t else "video",
"ref_audio_t": ref_audio_t,
"latent_t": shape[0],
"latent_h": shape[1],
"latent_w": shape[2],
}
)
ref_blocks.extend(
{
"kind": "audio",
"ref_audio_t": int(length),
}
for length in audio_lengths[embedded_audio_count:]
)
return MiniMaxH3EncoderMediaConditioning(
task=media.task,
height=media.height,
width=media.width,
num_frames=media.num_frames,
latent_t=media.latent_t,
audio_t=media.audio_t,
visual_condition=torch.cat(visual_rows) if visual_rows else None,
visual_condition_shapes=tuple(visual_shapes),
audio_condition=torch.cat(audio_rows) if audio_rows else None,
audio_condition_lengths=tuple(audio_lengths),
ref_blocks=tuple(ref_blocks),
keyframe_frame_indices=media.keyframe_frame_indices,
)
@@ -16,18 +16,18 @@ MINIMAX_H3_PIPELINE = PipelineConfig(
model_type="minimax_h3_disaggregated",
default_deploy_config_name="minimax_h3_disaggregated.yaml",
stage_cli_aliases={"text_encoder_tp_size": (0, "tensor_parallel_size")},
model_arch="MiniMaxH3TextEncoder",
model_arch="MiniMaxH3Encoder",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="text_encoder",
model_stage="encoder",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
requires_multimodal_data=True,
model_arch="MiniMaxH3TextEncoder",
model_arch="MiniMaxH3Encoder",
engine_output_type="latent",
prompt_transform_func=f"{_PROCESSOR}.prepare_text_encoder_prompt",
prompt_transform_func=f"{_PROCESSOR}.prepare_encoder_prompt",
sampling_constraints={
"max_tokens": 1,
"temperature": 0.0,
@@ -42,9 +42,9 @@ MINIMAX_H3_PIPELINE = PipelineConfig(
input_sources=(0,),
final_output=True,
final_output_type="video",
requires_multimodal_data=True,
requires_multimodal_data=False,
model_arch="MiniMaxH3Pipeline",
custom_process_input_func=f"{_PROCESSOR}.text_encoder2diffusion",
custom_process_input_func=f"{_PROCESSOR}.encoder2diffusion",
omni_kv_config={"need_recv_cache": False},
model_path_resolver=f"{_DIFFUSION_PIPELINE}.resolve_minimax_h3_diffusion_model_path",
inline_diffusion=True,
@@ -531,14 +531,27 @@ def _decode_video_frames_ffmpeg(
)
def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]:
meta = _probe_video(prepared_path)
def sample_reference_video_frames(
prepared_path: str,
*,
decoded_frames: np.ndarray | None = None,
) -> dict[str, Any]:
if decoded_frames is None:
meta = _probe_video(prepared_path)
frame_count = int(meta["frame_count"])
else:
decoded_frames = np.asarray(decoded_frames)
if decoded_frames.ndim != 4 or decoded_frames.shape[-1] != 3 or len(decoded_frames) <= 0:
raise OmniClientError(
f"decoded reference video frames must have shape [T, H, W, 3], got {decoded_frames.shape}"
)
frame_count = len(decoded_frames)
ratio = MINIMAX_H3_FPS / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS
indices: list[int] = []
cursor = 0.0
while True:
frame_index = int(round(cursor))
if frame_index >= meta["frame_count"]:
if frame_index >= frame_count:
break
if not indices or frame_index > indices[-1]:
indices.append(frame_index)
@@ -546,16 +559,16 @@ def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]:
if not indices:
raise OmniClientError(f"no frames sampled from {prepared_path}")
# The preparation step emits a lossless RGB stream. Decode only the sampled
# frames in one ffmpeg process so Qwen3VL sees the exact prepared pixels
# without decoding the entire high-bitrate stream into host memory.
decoded_frames = _decode_video_frames_ffmpeg(
prepared_path,
frame_count=int(meta["frame_count"]),
indices=indices,
width=meta["width"],
height=meta["height"],
)
if decoded_frames is None:
decoded_frames = _decode_video_frames_ffmpeg(
prepared_path,
frame_count=frame_count,
indices=indices,
width=meta["width"],
height=meta["height"],
)
else:
decoded_frames = decoded_frames[indices]
frames = [np.asarray(frame) for frame in decoded_frames]
timestamps = [index / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS for index in range(len(indices))]
@@ -176,7 +176,7 @@ class _ResidualMerge(nn.Module):
info=Qwen3VLProcessingInfo,
dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class MiniMaxH3TextEncoder(Qwen3VLForConditionalGeneration):
class MiniMaxH3TextEncoderBackbone(Qwen3VLForConditionalGeneration):
"""Qwen3-VL encoder used by MiniMax H3.
MiniMax H3 consumes the residual stream after decoder layer 50. The
@@ -305,3 +305,6 @@ class MiniMaxH3TextEncoder(Qwen3VLForConditionalGeneration):
encoder_weights(),
mapper=self.hf_to_vllm_mapper,
)
MiniMaxH3TextEncoder = MiniMaxH3TextEncoderBackbone
+3 -3
View File
@@ -476,10 +476,10 @@ _OMNI_MODELS = {
"qwen3_vl",
"AuraQwen3VLForConditionalGeneration",
),
"MiniMaxH3TextEncoder": (
"MiniMaxH3Encoder": (
"minimax_h3",
"text_encoder",
"MiniMaxH3TextEncoder",
"encoder",
"MiniMaxH3Encoder",
),
## MiniMax-Music3 (text-to-music; AR talker -> flow-matching acoustic decoder)
"MiniMaxMusic3TalkerForConditionalGeneration": (