[feat] Extend disaggregated MiniMax-H3 stage with VAE encoders (#6939)
Signed-off-by: asukaqaq-s <1311722138@qq.com> Signed-off-by: Jader <yjader@foxmail.com> Co-authored-by: Jader <yjader@foxmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
27 files changed
+3663
-2399
No files matched your search
@@ -1,16 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniMax H3 checkpoint resolution for the AR text-encoder stage."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
|
||||
MINIMAX_H3_TEXT_ENCODER_DOWNLOAD_PATTERNS = {
|
||||
"fl2va": ["FL2VA/text_encoder/**"],
|
||||
"ref2va": ["Ref2VA/text_encoder/**"],
|
||||
MINIMAX_H3_ENCODER_DOWNLOAD_PATTERNS = {
|
||||
partition: [
|
||||
f"{subdir}/text_encoder/**",
|
||||
f"{subdir}/video_vae/**",
|
||||
f"{subdir}/audio_vae/**",
|
||||
]
|
||||
for partition, subdir in {"fl2va": "FL2VA", "ref2va": "Ref2VA"}.items()
|
||||
}
|
||||
MINIMAX_H3_ENCODER_DOWNLOAD_PATTERNS["combined"] = MINIMAX_H3_ENCODER_DOWNLOAD_PATTERNS["fl2va"]
|
||||
|
||||
|
||||
def resolve_minimax_h3_partition(
|
||||
@@ -32,7 +35,7 @@ def resolve_minimax_h3_partition(
|
||||
return "ref2va" if task == "ref2va" else "fl2va"
|
||||
|
||||
|
||||
def resolve_minimax_h3_model_root(
|
||||
def resolve_minimax_h3_encoder_model_root(
|
||||
model: str,
|
||||
revision: str | None,
|
||||
task_type: str | None,
|
||||
@@ -50,7 +53,7 @@ def resolve_minimax_h3_model_root(
|
||||
snapshot = download_weights_from_hf_specific(
|
||||
model_name_or_path=model,
|
||||
cache_dir=None,
|
||||
allow_patterns=MINIMAX_H3_TEXT_ENCODER_DOWNLOAD_PATTERNS[partition],
|
||||
allow_patterns=MINIMAX_H3_ENCODER_DOWNLOAD_PATTERNS[partition],
|
||||
revision=revision,
|
||||
require_all=True,
|
||||
)
|
||||
@@ -59,4 +62,11 @@ def resolve_minimax_h3_model_root(
|
||||
return str(Path(snapshot) / subdir / "text_encoder")
|
||||
|
||||
|
||||
__all__ = ["resolve_minimax_h3_model_root", "resolve_minimax_h3_partition"]
|
||||
resolve_minimax_h3_model_root = resolve_minimax_h3_encoder_model_root
|
||||
|
||||
|
||||
__all__ = [
|
||||
"resolve_minimax_h3_encoder_model_root",
|
||||
"resolve_minimax_h3_model_root",
|
||||
"resolve_minimax_h3_partition",
|
||||
]
|
||||
@@ -4,8 +4,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, fields
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -119,3 +119,502 @@ class MiniMaxH3TextConditioning:
|
||||
"hidden_states": self.hidden_states,
|
||||
"token_tags": self.token_tags,
|
||||
}
|
||||
|
||||
|
||||
MINIMAX_H3_ENCODER_REQUEST_KEY = "minimax_h3_encoder_request"
|
||||
MINIMAX_H3_ENCODER_LAYOUT_KEY = "minimax_h3_encoder_layout"
|
||||
STAGE_SCHEMA_VERSION = 1
|
||||
_ENCODER_WIRE_SCHEMA_ID = 1
|
||||
|
||||
_TASK_TO_CODE = {"t2va": 1, "fl2va": 2, "ref2va": 3}
|
||||
_CODE_TO_TASK = {value: key for key, value in _TASK_TO_CODE.items()}
|
||||
_REF_KIND_TO_CODE = {"image": 1, "video": 2, "video_audio": 3, "audio": 4}
|
||||
_CODE_TO_REF_KIND = {value: key for key, value in _REF_KIND_TO_CODE.items()}
|
||||
_INTEGER_DTYPES = frozenset(
|
||||
{
|
||||
torch.int8,
|
||||
torch.int16,
|
||||
torch.int32,
|
||||
torch.int64,
|
||||
torch.uint8,
|
||||
}
|
||||
)
|
||||
_VIDEO_CONDITION_WIDTH = 96
|
||||
_AUDIO_CONDITION_WIDTH = 32
|
||||
_AUDIO_CONDITION_CHANNELS = 2
|
||||
|
||||
|
||||
def _wire_layout(payload: Mapping[str, Any]) -> torch.Tensor:
|
||||
private_metadata = payload.get("kv_metadata")
|
||||
if not isinstance(private_metadata, Mapping):
|
||||
raise ValueError("MiniMax H3 wire payload requires private layout metadata")
|
||||
layout = private_metadata.get(MINIMAX_H3_ENCODER_LAYOUT_KEY)
|
||||
if isinstance(layout, torch.Tensor):
|
||||
return layout
|
||||
raise ValueError("MiniMax H3 wire payload requires one packed encoder layout tensor")
|
||||
|
||||
|
||||
def _validate_condition_tensors(
|
||||
visual: torch.Tensor | None,
|
||||
visual_shapes: Sequence[Sequence[int]],
|
||||
audio: torch.Tensor | None,
|
||||
audio_lengths: Sequence[int],
|
||||
) -> None:
|
||||
if (visual is None) != (not visual_shapes):
|
||||
raise ValueError("MiniMax H3 visual condition and shapes must be present together")
|
||||
if (audio is None) != (not audio_lengths):
|
||||
raise ValueError("MiniMax H3 audio condition and lengths must be present together")
|
||||
|
||||
parsed_shapes = [tuple(int(item) for item in shape) for shape in visual_shapes]
|
||||
if any(
|
||||
len(shape) != 3 or any(item <= 0 for item in shape) or shape[1] % 2 or shape[2] % 2 for shape in parsed_shapes
|
||||
):
|
||||
raise ValueError("MiniMax H3 visual condition shapes must be positive [T, H, W] triplets with even H/W")
|
||||
if visual is not None:
|
||||
expected_rows = sum(t * (h // 2) * (w // 2) for t, h, w in parsed_shapes)
|
||||
if (
|
||||
visual.dtype != torch.float32
|
||||
or visual.ndim != 2
|
||||
or tuple(visual.shape)
|
||||
!= (
|
||||
expected_rows,
|
||||
_VIDEO_CONDITION_WIDTH,
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"MiniMax H3 visual condition must be FP32 with shape "
|
||||
f"[{expected_rows}, {_VIDEO_CONDITION_WIDTH}], got {visual.dtype} {tuple(visual.shape)}"
|
||||
)
|
||||
|
||||
parsed_lengths = [int(length) for length in audio_lengths]
|
||||
if any(length <= 0 for length in parsed_lengths):
|
||||
raise ValueError("MiniMax H3 audio condition lengths must be positive")
|
||||
if audio is not None:
|
||||
expected_rows = _AUDIO_CONDITION_CHANNELS * sum(parsed_lengths)
|
||||
if (
|
||||
audio.dtype != torch.float32
|
||||
or audio.ndim != 2
|
||||
or tuple(audio.shape)
|
||||
!= (
|
||||
expected_rows,
|
||||
_AUDIO_CONDITION_WIDTH,
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"MiniMax H3 audio condition must be FP32 with shape "
|
||||
f"[{expected_rows}, {_AUDIO_CONDITION_WIDTH}], got {audio.dtype} {tuple(audio.shape)}"
|
||||
)
|
||||
|
||||
|
||||
def _ref_block_rows(ref_blocks: Sequence[Mapping[str, Any]]) -> list[list[int]]:
|
||||
rows: list[list[int]] = []
|
||||
for block in ref_blocks:
|
||||
kind = str(block.get("kind") or "")
|
||||
kind_code = _REF_KIND_TO_CODE.get(kind)
|
||||
if kind_code is None:
|
||||
raise ValueError(f"unsupported MiniMax H3 ref kind {kind!r}")
|
||||
rows.append(
|
||||
[
|
||||
kind_code,
|
||||
int(block.get("ref_audio_t", 0)),
|
||||
int(block.get("latent_t", 0)),
|
||||
int(block.get("latent_h", 0)),
|
||||
int(block.get("latent_w", 0)),
|
||||
]
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _decode_ref_blocks(value: torch.Tensor) -> tuple[dict[str, Any], ...]:
|
||||
if value.ndim != 2 or value.shape[1] != 5:
|
||||
raise ValueError("MiniMax H3 ref_blocks metadata must have shape [N, 5]")
|
||||
blocks: list[dict[str, Any]] = []
|
||||
for kind_code, ref_audio_t, ref_t, ref_h, ref_w in value.detach().cpu().tolist():
|
||||
kind = _CODE_TO_REF_KIND.get(int(kind_code))
|
||||
if kind is None:
|
||||
raise ValueError(f"unsupported MiniMax H3 ref kind code {kind_code!r}")
|
||||
block: dict[str, Any] = {"kind": kind}
|
||||
for key, item in (
|
||||
("ref_audio_t", ref_audio_t),
|
||||
("latent_t", ref_t),
|
||||
("latent_h", ref_h),
|
||||
("latent_w", ref_w),
|
||||
):
|
||||
if item:
|
||||
block[key] = int(item)
|
||||
blocks.append(block)
|
||||
return tuple(blocks)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MiniMaxH3EncoderMediaInput:
|
||||
task: str
|
||||
height: int
|
||||
width: int
|
||||
num_frames: int
|
||||
latent_t: int
|
||||
audio_t: int
|
||||
images: tuple[torch.Tensor, ...] = ()
|
||||
videos: tuple[torch.Tensor, ...] = ()
|
||||
video_audios: tuple[tuple[torch.Tensor, int] | None, ...] = ()
|
||||
audios: tuple[tuple[torch.Tensor, int], ...] = ()
|
||||
keyframe_frame_indices: tuple[int, ...] = ()
|
||||
|
||||
@classmethod
|
||||
def from_mm_tensors(
|
||||
cls,
|
||||
values: Sequence[torch.Tensor],
|
||||
metadata: Mapping[str, Any],
|
||||
) -> MiniMaxH3EncoderMediaInput:
|
||||
if not isinstance(values, Sequence) or isinstance(values, (str, bytes)):
|
||||
raise ValueError("MiniMax H3 encoder media must be a tensor sequence")
|
||||
tensors = list(values)
|
||||
if any(not isinstance(item, torch.Tensor) for item in tensors):
|
||||
raise ValueError("MiniMax H3 encoder media input must contain only tensors")
|
||||
task = str(metadata["task"])
|
||||
if task not in _TASK_TO_CODE:
|
||||
raise ValueError(f"unsupported MiniMax H3 task {task!r}")
|
||||
height = int(metadata["height"])
|
||||
width = int(metadata["width"])
|
||||
num_frames = int(metadata["num_frames"])
|
||||
latent_t = int(metadata["latent_t"])
|
||||
audio_t = int(metadata["audio_t"])
|
||||
image_count = int(metadata.get("image_count", 0))
|
||||
video_count = int(metadata.get("video_count", 0))
|
||||
audio_count = int(metadata.get("audio_count", 0))
|
||||
if min(height, width, num_frames, latent_t, audio_t) <= 0:
|
||||
raise ValueError("MiniMax H3 encoder media dimensions must be positive")
|
||||
if min(image_count, video_count, audio_count) < 0:
|
||||
raise ValueError("MiniMax H3 encoder media counts must be non-negative")
|
||||
|
||||
audio_flags = tuple(bool(value) for value in metadata.get("video_audio_flags", ()))
|
||||
video_audio_sample_rates = tuple(int(value) for value in metadata.get("video_audio_sample_rates", ()))
|
||||
audio_sample_rates = tuple(int(value) for value in metadata.get("audio_sample_rates", ()))
|
||||
if len(audio_flags) != video_count or len(video_audio_sample_rates) != video_count:
|
||||
raise ValueError("MiniMax H3 video audio flags must align with videos")
|
||||
if len(audio_sample_rates) != audio_count:
|
||||
raise ValueError("MiniMax H3 audio sample rates must align with audio inputs")
|
||||
if any((rate > 0) != has_audio for rate, has_audio in zip(video_audio_sample_rates, audio_flags, strict=True)):
|
||||
raise ValueError("MiniMax H3 embedded audio sample rates must align with audio flags")
|
||||
if any(rate <= 0 for rate in audio_sample_rates):
|
||||
raise ValueError("MiniMax H3 audio sample rates must be positive")
|
||||
|
||||
cursor = 0
|
||||
images = tuple(tensors[cursor : cursor + image_count])
|
||||
cursor += image_count
|
||||
videos = tuple(tensors[cursor : cursor + video_count])
|
||||
cursor += video_count
|
||||
if len(images) != image_count or any(image.ndim != 3 or image.shape[-1] != 3 for image in images):
|
||||
raise ValueError("MiniMax H3 encoder images must have shape [H, W, 3]")
|
||||
if len(videos) != video_count or any(video.ndim != 4 or video.shape[-1] != 3 for video in videos):
|
||||
raise ValueError("MiniMax H3 encoder videos must have shape [T, H, W, 3]")
|
||||
|
||||
video_audios: list[tuple[torch.Tensor, int] | None] = []
|
||||
for has_audio, sample_rate in zip(audio_flags, video_audio_sample_rates, strict=True):
|
||||
if not has_audio:
|
||||
video_audios.append(None)
|
||||
continue
|
||||
if cursor >= len(tensors):
|
||||
raise ValueError("MiniMax H3 encoder media input is truncated")
|
||||
waveform = tensors[cursor]
|
||||
cursor += 1
|
||||
if waveform.ndim not in (1, 2):
|
||||
raise ValueError("MiniMax H3 embedded audio must have shape [samples] or [channels, samples]")
|
||||
video_audios.append((waveform, sample_rate))
|
||||
|
||||
audios: list[tuple[torch.Tensor, int]] = []
|
||||
for sample_rate in audio_sample_rates:
|
||||
if cursor >= len(tensors):
|
||||
raise ValueError("MiniMax H3 encoder media input is truncated")
|
||||
waveform = tensors[cursor]
|
||||
cursor += 1
|
||||
if waveform.ndim not in (1, 2):
|
||||
raise ValueError("MiniMax H3 audio must have shape [samples] or [channels, samples]")
|
||||
audios.append((waveform, sample_rate))
|
||||
if cursor != len(tensors):
|
||||
raise ValueError(f"MiniMax H3 encoder media input has {len(tensors) - cursor} trailing tensors")
|
||||
return cls(
|
||||
task=task,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
latent_t=latent_t,
|
||||
audio_t=audio_t,
|
||||
images=images,
|
||||
videos=videos,
|
||||
video_audios=tuple(video_audios),
|
||||
audios=tuple(audios),
|
||||
keyframe_frame_indices=tuple(int(value) for value in metadata.get("keyframe_frame_indices", ())),
|
||||
)
|
||||
|
||||
def to_mm_tensors(self) -> list[torch.Tensor]:
|
||||
task_code = _TASK_TO_CODE.get(self.task)
|
||||
if task_code is None:
|
||||
raise ValueError(f"unsupported MiniMax H3 task {self.task!r}")
|
||||
if min(self.height, self.width, self.num_frames, self.latent_t, self.audio_t) <= 0:
|
||||
raise ValueError("MiniMax H3 encoder media dimensions must be positive")
|
||||
if len(self.video_audios) != len(self.videos):
|
||||
raise ValueError("MiniMax H3 video audio slots must align with videos")
|
||||
if any(image.ndim != 3 or image.shape[-1] != 3 for image in self.images):
|
||||
raise ValueError("MiniMax H3 encoder images must have shape [H, W, 3]")
|
||||
if any(video.ndim != 4 or video.shape[-1] != 3 for video in self.videos):
|
||||
raise ValueError("MiniMax H3 encoder videos must have shape [T, H, W, 3]")
|
||||
audio_items = [item for item in self.video_audios if item is not None] + list(self.audios)
|
||||
if any(waveform.ndim not in (1, 2) or int(sample_rate) <= 0 for waveform, sample_rate in audio_items):
|
||||
raise ValueError("MiniMax H3 encoder audio must have a waveform and positive sample rate")
|
||||
tensors = [*self.images, *self.videos]
|
||||
for item in self.video_audios:
|
||||
if item is None:
|
||||
continue
|
||||
waveform, _sample_rate = item
|
||||
tensors.append(waveform)
|
||||
tensors.extend(waveform for waveform, _sample_rate in self.audios)
|
||||
return tensors
|
||||
|
||||
def to_metadata(self) -> dict[str, Any]:
|
||||
return {
|
||||
"task": self.task,
|
||||
"height": self.height,
|
||||
"width": self.width,
|
||||
"num_frames": self.num_frames,
|
||||
"latent_t": self.latent_t,
|
||||
"audio_t": self.audio_t,
|
||||
"image_count": len(self.images),
|
||||
"video_count": len(self.videos),
|
||||
"audio_count": len(self.audios),
|
||||
"video_audio_flags": [item is not None for item in self.video_audios],
|
||||
"video_audio_sample_rates": [item[1] if item is not None else 0 for item in self.video_audios],
|
||||
"audio_sample_rates": [sample_rate for _waveform, sample_rate in self.audios],
|
||||
"keyframe_frame_indices": list(self.keyframe_frame_indices),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MiniMaxH3EncoderMediaConditioning:
|
||||
task: str
|
||||
height: int
|
||||
width: int
|
||||
num_frames: int
|
||||
latent_t: int
|
||||
audio_t: int
|
||||
visual_condition: torch.Tensor | None = None
|
||||
visual_condition_shapes: tuple[tuple[int, int, int], ...] = ()
|
||||
audio_condition: torch.Tensor | None = None
|
||||
audio_condition_lengths: tuple[int, ...] = ()
|
||||
ref_blocks: tuple[dict[str, Any], ...] = ()
|
||||
keyframe_frame_indices: tuple[int, ...] = ()
|
||||
|
||||
def to_omni_components(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
task_code = _TASK_TO_CODE.get(self.task)
|
||||
if task_code is None:
|
||||
raise ValueError(f"unsupported MiniMax H3 task {self.task!r}")
|
||||
if min(self.height, self.width, self.num_frames, self.latent_t, self.audio_t) <= 0:
|
||||
raise ValueError("MiniMax H3 encoder media dimensions must be positive")
|
||||
_validate_condition_tensors(
|
||||
self.visual_condition,
|
||||
self.visual_condition_shapes,
|
||||
self.audio_condition,
|
||||
self.audio_condition_lengths,
|
||||
)
|
||||
empty = torch.empty((0,), dtype=torch.float32)
|
||||
ref_rows = _ref_block_rows(self.ref_blocks)
|
||||
layout = [
|
||||
_ENCODER_WIRE_SCHEMA_ID,
|
||||
STAGE_SCHEMA_VERSION,
|
||||
task_code,
|
||||
self.height,
|
||||
self.width,
|
||||
self.num_frames,
|
||||
self.latent_t,
|
||||
self.audio_t,
|
||||
len(self.visual_condition_shapes),
|
||||
len(self.audio_condition_lengths),
|
||||
len(ref_rows),
|
||||
len(self.keyframe_frame_indices),
|
||||
]
|
||||
layout.extend(item for shape in self.visual_condition_shapes for item in shape)
|
||||
layout.extend(self.audio_condition_lengths)
|
||||
layout.extend(item for row in ref_rows for item in row)
|
||||
layout.extend(self.keyframe_frame_indices)
|
||||
return (
|
||||
self.visual_condition if self.visual_condition is not None else empty,
|
||||
self.audio_condition if self.audio_condition is not None else empty,
|
||||
torch.tensor(layout, dtype=torch.int64),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MiniMaxH3EncoderConditioning:
|
||||
hidden_states: torch.Tensor
|
||||
token_tags: torch.Tensor
|
||||
task: str
|
||||
height: int
|
||||
width: int
|
||||
num_frames: int
|
||||
latent_t: int
|
||||
audio_t: int
|
||||
visual_condition: torch.Tensor | None = None
|
||||
visual_condition_shapes: tuple[tuple[int, int, int], ...] = ()
|
||||
audio_condition: torch.Tensor | None = None
|
||||
audio_condition_lengths: tuple[int, ...] = ()
|
||||
ref_blocks: tuple[dict[str, Any], ...] = ()
|
||||
keyframe_frame_indices: tuple[int, ...] = ()
|
||||
|
||||
@classmethod
|
||||
def from_components(
|
||||
cls,
|
||||
text: MiniMaxH3TextConditioning,
|
||||
media: MiniMaxH3EncoderMediaConditioning,
|
||||
) -> MiniMaxH3EncoderConditioning:
|
||||
"""Combine local encoder results without a stage-wire round trip."""
|
||||
text = MiniMaxH3TextConditioning.from_payload(text.to_payload())
|
||||
_validate_condition_tensors(
|
||||
media.visual_condition,
|
||||
media.visual_condition_shapes,
|
||||
media.audio_condition,
|
||||
media.audio_condition_lengths,
|
||||
)
|
||||
return cls(
|
||||
hidden_states=text.hidden_states,
|
||||
token_tags=text.token_tags,
|
||||
**{item.name: getattr(media, item.name) for item in fields(media)},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_omni_payload(cls, payload: Mapping[str, Any]) -> MiniMaxH3EncoderConditioning:
|
||||
hidden_payload = payload.get("hidden_states")
|
||||
embed_payload = payload.get("embed")
|
||||
meta_payload = payload.get("meta")
|
||||
hidden_states = hidden_payload.get("output") if isinstance(hidden_payload, Mapping) else None
|
||||
token_tags = meta_payload.get("token_role_ids") if isinstance(meta_payload, Mapping) else None
|
||||
visual = embed_payload.get("embedding") if isinstance(embed_payload, Mapping) else None
|
||||
audio = embed_payload.get("speech_feat") if isinstance(embed_payload, Mapping) else None
|
||||
if not all(isinstance(item, torch.Tensor) for item in (hidden_states, token_tags, visual, audio)):
|
||||
raise ValueError("MiniMax H3 encoder wire requires text, visual and audio tensors")
|
||||
assert isinstance(hidden_states, torch.Tensor)
|
||||
assert isinstance(token_tags, torch.Tensor)
|
||||
assert isinstance(visual, torch.Tensor)
|
||||
assert isinstance(audio, torch.Tensor)
|
||||
if token_tags.ndim == 2 and token_tags.shape[-1] == 1:
|
||||
token_tags = token_tags.squeeze(-1)
|
||||
text = MiniMaxH3TextConditioning.from_payload({"hidden_states": hidden_states, "token_tags": token_tags})
|
||||
for name, value in (("visual", visual), ("audio", audio)):
|
||||
if value.numel() == 0 and (value.dtype != torch.float32 or tuple(value.shape) != (0,)):
|
||||
raise ValueError(f"MiniMax H3 empty {name} condition slot must be FP32 with shape [0]")
|
||||
if not isinstance(meta_payload, Mapping):
|
||||
raise ValueError("MiniMax H3 wire payload requires a meta mapping")
|
||||
layout = _wire_layout(payload)
|
||||
if layout.dtype not in _INTEGER_DTYPES or layout.ndim != 1:
|
||||
raise ValueError("MiniMax H3 encoder layout must be a one-dimensional integer tensor")
|
||||
values = [int(item) for item in layout.detach().cpu().tolist()]
|
||||
if len(values) < 12:
|
||||
raise ValueError("MiniMax H3 encoder layout header is truncated")
|
||||
(
|
||||
schema_id,
|
||||
version,
|
||||
task_code,
|
||||
height,
|
||||
width,
|
||||
num_frames,
|
||||
latent_t,
|
||||
audio_t,
|
||||
visual_count,
|
||||
audio_count,
|
||||
ref_count,
|
||||
keyframe_count,
|
||||
) = values[:12]
|
||||
if schema_id != _ENCODER_WIRE_SCHEMA_ID or version != STAGE_SCHEMA_VERSION:
|
||||
raise ValueError(f"unsupported MiniMax H3 encoder wire schema {schema_id}:{version}")
|
||||
task = _CODE_TO_TASK.get(task_code)
|
||||
if task is None:
|
||||
raise ValueError(f"unsupported MiniMax H3 wire task code {task_code!r}")
|
||||
if min(height, width, num_frames, latent_t, audio_t) <= 0:
|
||||
raise ValueError("MiniMax H3 encoder wire dimensions must be positive")
|
||||
if min(visual_count, audio_count, ref_count, keyframe_count) < 0:
|
||||
raise ValueError("MiniMax H3 encoder layout counts must be non-negative")
|
||||
expected_size = 12 + 3 * visual_count + audio_count + 5 * ref_count + keyframe_count
|
||||
if len(values) != expected_size:
|
||||
raise ValueError(f"MiniMax H3 encoder layout has {len(values)} integers, expected {expected_size}")
|
||||
cursor = 12
|
||||
visual_shapes = tuple(
|
||||
tuple(values[cursor + 3 * index : cursor + 3 * (index + 1)]) for index in range(visual_count)
|
||||
)
|
||||
cursor += 3 * visual_count
|
||||
audio_lengths = tuple(values[cursor : cursor + audio_count])
|
||||
cursor += audio_count
|
||||
ref_rows = values[cursor : cursor + 5 * ref_count]
|
||||
cursor += 5 * ref_count
|
||||
ref_blocks_tensor = torch.tensor(ref_rows, dtype=torch.int64).reshape(-1, 5)
|
||||
keyframe_frame_indices = tuple(values[cursor : cursor + keyframe_count])
|
||||
visual_condition = visual if visual.numel() else None
|
||||
audio_condition = audio if audio.numel() else None
|
||||
_validate_condition_tensors(
|
||||
visual_condition,
|
||||
visual_shapes,
|
||||
audio_condition,
|
||||
audio_lengths,
|
||||
)
|
||||
return cls(
|
||||
hidden_states=text.hidden_states,
|
||||
token_tags=text.token_tags,
|
||||
task=task,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
latent_t=latent_t,
|
||||
audio_t=audio_t,
|
||||
visual_condition=visual_condition,
|
||||
visual_condition_shapes=visual_shapes,
|
||||
audio_condition=audio_condition,
|
||||
audio_condition_lengths=audio_lengths,
|
||||
ref_blocks=_decode_ref_blocks(ref_blocks_tensor),
|
||||
keyframe_frame_indices=keyframe_frame_indices,
|
||||
)
|
||||
|
||||
def to_omni_payload(self) -> dict[str, Any]:
|
||||
text = MiniMaxH3TextConditioning.from_payload(
|
||||
{
|
||||
"hidden_states": self.hidden_states,
|
||||
"token_tags": self.token_tags,
|
||||
}
|
||||
)
|
||||
visual, audio, layout = MiniMaxH3EncoderMediaConditioning(
|
||||
task=self.task,
|
||||
height=self.height,
|
||||
width=self.width,
|
||||
num_frames=self.num_frames,
|
||||
latent_t=self.latent_t,
|
||||
audio_t=self.audio_t,
|
||||
visual_condition=self.visual_condition,
|
||||
visual_condition_shapes=self.visual_condition_shapes,
|
||||
audio_condition=self.audio_condition,
|
||||
audio_condition_lengths=self.audio_condition_lengths,
|
||||
ref_blocks=self.ref_blocks,
|
||||
keyframe_frame_indices=self.keyframe_frame_indices,
|
||||
).to_omni_components()
|
||||
return {
|
||||
"hidden_states": {"output": text.hidden_states},
|
||||
"embed": {
|
||||
"embedding": visual,
|
||||
"speech_feat": audio,
|
||||
},
|
||||
"meta": {"token_role_ids": text.token_tags},
|
||||
"kv_metadata": {MINIMAX_H3_ENCODER_LAYOUT_KEY: layout},
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MINIMAX_H3_CONDITION_LABELS_KEY",
|
||||
"MINIMAX_H3_ENCODER_LAYOUT_KEY",
|
||||
"MINIMAX_H3_ENCODER_REQUEST_KEY",
|
||||
"MINIMAX_H3_PRESENTATION_TASK_KEY",
|
||||
"MINIMAX_H3_TEXT_CONDITIONING_SCHEMA",
|
||||
"MINIMAX_H3_TEXT_HIDDEN_SIZE",
|
||||
"STAGE_SCHEMA_VERSION",
|
||||
"MiniMaxH3EncoderConditioning",
|
||||
"MiniMaxH3EncoderMediaConditioning",
|
||||
"MiniMaxH3EncoderMediaInput",
|
||||
"MiniMaxH3TextConditioning",
|
||||
]
|
||||
@@ -0,0 +1,252 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.models.minimax_h3.vae import (
|
||||
MiniMaxH3AudioVAE,
|
||||
MiniMaxH3VideoVAE,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
|
||||
MINIMAX_H3_ENCODER_LAYOUT_KEY,
|
||||
MINIMAX_H3_ENCODER_REQUEST_KEY,
|
||||
MiniMaxH3EncoderMediaConditioning,
|
||||
MiniMaxH3EncoderMediaInput,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import encode_media
|
||||
from vllm_omni.model_executor.models.minimax_h3.text_encoder import (
|
||||
MiniMaxH3TextEncoderBackbone,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
_MEDIA_CONDITIONING_CACHE_KEY = "_minimax_h3_encoder_media_conditioning"
|
||||
_MEDIA_CACHE_KEY = "_minimax_h3_encoder_media"
|
||||
_COMPONENT_CONFIG_KEY = "minimax_h3_encoder_components"
|
||||
_COMPONENT_ROLES = frozenset({"text_encoder", "video_vae", "audio_vae"})
|
||||
|
||||
|
||||
def _partition_root(model_path: str) -> Path:
|
||||
path = Path(model_path)
|
||||
if path.name in {"text_encoder", "video_vae", "audio_vae"}:
|
||||
return path.parent
|
||||
return path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MiniMaxH3EncoderComponentConfig:
|
||||
text_parallel_mode: str
|
||||
video_parallel_mode: str
|
||||
audio_parallel_mode: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.text_parallel_mode != "tp":
|
||||
raise ValueError("MiniMax H3 text encoder currently supports parallel_mode='tp' only")
|
||||
if self.video_parallel_mode not in {"leader", "patch"}:
|
||||
raise ValueError("MiniMax H3 video VAE parallel_mode must be 'leader' or 'patch'")
|
||||
if self.audio_parallel_mode != "leader":
|
||||
raise ValueError("MiniMax H3 audio VAE currently supports parallel_mode='leader' only")
|
||||
|
||||
@staticmethod
|
||||
def _parallel_mode(raw: Mapping[str, Any], role: str) -> str:
|
||||
value = raw[role]
|
||||
if not isinstance(value, Mapping):
|
||||
raise TypeError(f"MiniMax H3 encoder role {role!r} must be a mapping")
|
||||
if set(value) != {"parallel_mode"}:
|
||||
raise ValueError(f"MiniMax H3 encoder role {role!r} requires only parallel_mode")
|
||||
return str(value["parallel_mode"])
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(cls, vllm_config: VllmConfig) -> MiniMaxH3EncoderComponentConfig:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
raw = getattr(hf_config, _COMPONENT_CONFIG_KEY, None)
|
||||
if not isinstance(raw, Mapping):
|
||||
raise ValueError(f"MiniMax H3 Encoder Stage requires explicit {_COMPONENT_CONFIG_KEY}")
|
||||
roles = set(raw)
|
||||
if roles != _COMPONENT_ROLES:
|
||||
missing = sorted(_COMPONENT_ROLES - roles)
|
||||
unknown = sorted(roles - _COMPONENT_ROLES)
|
||||
raise ValueError(
|
||||
f"{_COMPONENT_CONFIG_KEY} requires exactly {sorted(_COMPONENT_ROLES)}; "
|
||||
f"missing={missing}, unknown={unknown}"
|
||||
)
|
||||
return cls(
|
||||
text_parallel_mode=cls._parallel_mode(raw, "text_encoder"),
|
||||
video_parallel_mode=cls._parallel_mode(raw, "video_vae"),
|
||||
audio_parallel_mode=cls._parallel_mode(raw, "audio_vae"),
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxH3Encoder(MiniMaxH3TextEncoderBackbone):
|
||||
requires_request_sample_eligibility = True
|
||||
enable_update_additional_information = True
|
||||
omni_payload_at_request_end = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model") -> None:
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
try:
|
||||
tp_group = get_tp_group()
|
||||
tp_rank = int(tp_group.rank_in_group)
|
||||
except (AssertionError, RuntimeError):
|
||||
tp_group = None
|
||||
tp_rank = 0
|
||||
self._component_leader = tp_rank == 0
|
||||
self.component_config = MiniMaxH3EncoderComponentConfig.from_vllm_config(vllm_config)
|
||||
root = _partition_root(vllm_config.model_config.model)
|
||||
missing = [name for name in ("text_encoder", "video_vae", "audio_vae") if not (root / name).is_dir()]
|
||||
if missing:
|
||||
raise RuntimeError(f"MiniMax H3 Encoder Stage requires all three encoders; missing {missing}")
|
||||
device = get_local_device()
|
||||
self.video_vae: MiniMaxH3VideoVAE | None = None
|
||||
self.audio_vae: MiniMaxH3AudioVAE | None = None
|
||||
if self._component_leader or self.component_config.video_parallel_mode == "patch":
|
||||
self.video_vae = MiniMaxH3VideoVAE(
|
||||
str(root / "video_vae"),
|
||||
device=device,
|
||||
encode_only=True,
|
||||
trust_remote_code=vllm_config.model_config.trust_remote_code,
|
||||
)
|
||||
if self.component_config.video_parallel_mode == "patch":
|
||||
if tp_group is None:
|
||||
raise RuntimeError("MiniMax H3 video VAE patch parallelism requires an initialized TP group")
|
||||
self.video_vae.set_parallel_size(
|
||||
tp_group.world_size,
|
||||
process_group=tp_group.device_group,
|
||||
)
|
||||
if self._component_leader:
|
||||
self.audio_vae = MiniMaxH3AudioVAE(
|
||||
str(root / "audio_vae"),
|
||||
device=device,
|
||||
encode_only=True,
|
||||
trust_remote_code=vllm_config.model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _media_input(info: dict[str, Any]) -> MiniMaxH3EncoderMediaInput:
|
||||
cached = info.get(_MEDIA_CACHE_KEY)
|
||||
if isinstance(cached, MiniMaxH3EncoderMediaInput):
|
||||
return cached
|
||||
meta = info.get("meta")
|
||||
metadata = meta.get(MINIMAX_H3_ENCODER_REQUEST_KEY) if isinstance(meta, Mapping) else None
|
||||
if not isinstance(metadata, Mapping):
|
||||
raise RuntimeError("MiniMax H3 unified encoder requires request metadata")
|
||||
|
||||
hidden_states = info.get("hidden_states")
|
||||
raw_media = hidden_states.get("layers") if isinstance(hidden_states, Mapping) else None
|
||||
if raw_media is None:
|
||||
media_tensors: list[torch.Tensor] = []
|
||||
elif isinstance(raw_media, Mapping):
|
||||
try:
|
||||
ordered_items = sorted(raw_media.items(), key=lambda item: int(item[0]))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RuntimeError("MiniMax H3 encoder media keys must be integer indices") from exc
|
||||
if [int(key) for key, _ in ordered_items] != list(range(len(ordered_items))):
|
||||
raise RuntimeError("MiniMax H3 encoder media indices must be contiguous from zero")
|
||||
media_tensors = [value for _, value in ordered_items]
|
||||
else:
|
||||
raise RuntimeError("MiniMax H3 encoder media must be a namespaced tensor mapping")
|
||||
if any(not isinstance(value, torch.Tensor) for value in media_tensors):
|
||||
raise RuntimeError("MiniMax H3 encoder media must contain only tensors")
|
||||
media = MiniMaxH3EncoderMediaInput.from_mm_tensors(media_tensors, metadata)
|
||||
if media.to_metadata() != dict(metadata):
|
||||
raise RuntimeError("MiniMax H3 encoder media does not match request metadata")
|
||||
info[_MEDIA_CACHE_KEY] = media
|
||||
return media
|
||||
|
||||
def _encode_media(self, media: MiniMaxH3EncoderMediaInput) -> MiniMaxH3EncoderMediaConditioning | None:
|
||||
if not self._component_leader and self.component_config.video_parallel_mode == "leader":
|
||||
return None
|
||||
return encode_media(
|
||||
media,
|
||||
video_vae=self.video_vae,
|
||||
audio_vae=self.audio_vae,
|
||||
emit_conditioning=self._component_leader,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _has_encoder_input(info: object) -> bool:
|
||||
if not isinstance(info, Mapping):
|
||||
return False
|
||||
meta = info.get("meta")
|
||||
return isinstance(meta, Mapping) and isinstance(meta.get(MINIMAX_H3_ENCODER_REQUEST_KEY), Mapping)
|
||||
|
||||
def make_omni_output(
|
||||
self,
|
||||
model_outputs: torch.Tensor | OmniOutput,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
output = super().make_omni_output(model_outputs, **kwargs)
|
||||
infos = kwargs.get("model_intermediate_buffer") or []
|
||||
if not isinstance(infos, Sequence) or not infos:
|
||||
return output
|
||||
sample_eligible = kwargs.get("request_sample_eligible")
|
||||
if sample_eligible is None:
|
||||
sample_eligible = [True] * len(infos)
|
||||
if len(sample_eligible) != len(infos):
|
||||
raise RuntimeError(
|
||||
f"MiniMax H3 unified encoder received {len(sample_eligible)} sampling flags for {len(infos)} requests"
|
||||
)
|
||||
per_request: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None] = []
|
||||
release_workspace = False
|
||||
for eligible, info in zip(sample_eligible, infos, strict=True):
|
||||
if not eligible:
|
||||
per_request.append(None)
|
||||
continue
|
||||
if not isinstance(info, dict) or not self._has_encoder_input(info):
|
||||
raise RuntimeError("MiniMax H3 unified encoder request metadata is missing from the runner")
|
||||
release_workspace = True
|
||||
if _MEDIA_CONDITIONING_CACHE_KEY not in info:
|
||||
info[_MEDIA_CONDITIONING_CACHE_KEY] = self._encode_media(self._media_input(info))
|
||||
if not self._component_leader:
|
||||
continue
|
||||
cached = info.get(_MEDIA_CONDITIONING_CACHE_KEY)
|
||||
if not isinstance(cached, MiniMaxH3EncoderMediaConditioning):
|
||||
raise RuntimeError("MiniMax H3 encoder components did not produce conditioning")
|
||||
per_request.append(cached.to_omni_components())
|
||||
|
||||
if release_workspace:
|
||||
torch.accelerator.empty_cache()
|
||||
if not self._component_leader:
|
||||
return output
|
||||
|
||||
if not any(item is not None for item in per_request):
|
||||
return output
|
||||
empty = torch.empty((0,), dtype=torch.float32)
|
||||
multimodal_outputs = dict(output.multimodal_outputs or {})
|
||||
dotted_key = f"kv_metadata.{MINIMAX_H3_ENCODER_LAYOUT_KEY}"
|
||||
multimodal_outputs[dotted_key] = [
|
||||
item[2] if item is not None else torch.empty((0,), dtype=torch.int64) for item in per_request
|
||||
]
|
||||
|
||||
embed = dict(multimodal_outputs.get("embed") or {})
|
||||
embed["embedding"] = [item[0] if item is not None else empty for item in per_request]
|
||||
embed["speech_feat"] = [item[1] if item is not None else empty for item in per_request]
|
||||
multimodal_outputs["embed"] = embed
|
||||
return OmniOutput(
|
||||
text_hidden_states=output.text_hidden_states,
|
||||
multimodal_outputs=multimodal_outputs,
|
||||
intermediate_tensors=output.intermediate_tensors,
|
||||
next_token_id=output.next_token_id,
|
||||
)
|
||||
|
||||
def load_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loaded = super().load_weights(weights)
|
||||
for component_name in ("video_vae", "audio_vae"):
|
||||
component = getattr(self, component_name)
|
||||
if component is not None:
|
||||
loaded.update(f"{component_name}.{name}" for name, _ in component.named_parameters())
|
||||
return loaded
|
||||
|
||||
|
||||
__all__ = ["MiniMaxH3Encoder"]
|
||||
@@ -0,0 +1,569 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared request preparation and condition codec execution for MiniMax H3."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
|
||||
MiniMaxH3EncoderMediaConditioning,
|
||||
MiniMaxH3EncoderMediaInput,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.preprocessing import (
|
||||
MINIMAX_H3_OUTPUT_SHORT_EDGE,
|
||||
load_minimax_h3_images,
|
||||
resolve_minimax_h3_aspect_ratio,
|
||||
resolve_minimax_h3_output_canvas,
|
||||
resolve_minimax_h3_reference_image_shape,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.reference_video import (
|
||||
MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS,
|
||||
load_audio_file,
|
||||
load_video_audio,
|
||||
load_video_frames,
|
||||
prepare_reference_videos,
|
||||
sample_reference_video_frames,
|
||||
validate_reference_audio_files,
|
||||
validate_reference_audio_waveforms,
|
||||
)
|
||||
|
||||
MINIMAX_H3_FPS = 24
|
||||
MINIMAX_H3_MIN_OUTPUT_SECONDS = 4.0
|
||||
MINIMAX_H3_MAX_OUTPUT_SECONDS = 15.0
|
||||
|
||||
|
||||
def _items(value: Any) -> list[Any]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if isinstance(value, tuple) and not (len(value) == 2 and isinstance(value[1], Mapping)):
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
def _audio_items(value: Any) -> list[Any]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple)) and len(value) == 2 and isinstance(value[1], (int, np.integer)):
|
||||
return [value]
|
||||
return list(value) if isinstance(value, (list, tuple)) else [value]
|
||||
|
||||
|
||||
def _load_audio(value: Any) -> tuple[torch.Tensor, int]:
|
||||
if isinstance(value, (list, tuple)) and not (len(value) == 2 and isinstance(value[1], (int, np.integer))):
|
||||
audios = _load_audios(value)
|
||||
if len(audios) != 1:
|
||||
raise OmniClientError(f"MiniMax H3 expected one audio, got {len(audios)}")
|
||||
return audios[0]
|
||||
if isinstance(value, (str, os.PathLike)):
|
||||
return load_audio_file(str(value))
|
||||
if isinstance(value, (list, tuple)) and len(value) == 2:
|
||||
waveform, sample_rate = value
|
||||
return torch.as_tensor(waveform).float(), int(sample_rate)
|
||||
if isinstance(value, dict):
|
||||
waveform = value.get("waveform", value.get("array"))
|
||||
sample_rate = value.get("sample_rate", value.get("sampling_rate"))
|
||||
if waveform is not None and sample_rate is not None:
|
||||
return torch.as_tensor(waveform).float(), int(sample_rate)
|
||||
raise OmniClientError("MiniMax H3 audio input must be a path, (waveform, sample_rate), or a waveform mapping")
|
||||
|
||||
|
||||
def _load_audios(value: Any) -> list[tuple[torch.Tensor, int]]:
|
||||
if isinstance(value, (list, tuple)) and not (len(value) == 2 and isinstance(value[1], (int, np.integer))):
|
||||
if not value:
|
||||
raise OmniClientError("MiniMax H3 audio input must not be empty")
|
||||
return [_load_audio(item) for item in value]
|
||||
return [_load_audio(value)]
|
||||
|
||||
|
||||
def _as_int_list(value: Any, *, name: str) -> list[int]:
|
||||
if isinstance(value, bool):
|
||||
raise OmniClientError(f"{name} must be an integer or a list of integers")
|
||||
if isinstance(value, (int, np.integer)):
|
||||
return [int(value)]
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
|
||||
result = list(value)
|
||||
if not result:
|
||||
raise OmniClientError(f"{name} must not be empty")
|
||||
if any(isinstance(item, bool) or not isinstance(item, (int, np.integer)) for item in result):
|
||||
raise OmniClientError(f"{name} must contain only integers")
|
||||
return [int(item) for item in result]
|
||||
raise OmniClientError(f"{name} must be an integer or a list of integers")
|
||||
|
||||
|
||||
def _resolve_fl2va_keyframe_indices(extra: Mapping[str, Any], image_count: int) -> list[int]:
|
||||
target = extra.get("target")
|
||||
target = target if isinstance(target, Mapping) else {}
|
||||
raw = extra.get("frame_indices", extra.get("frame_index"))
|
||||
if raw is None:
|
||||
raw = target.get("frame_indices", target.get("frame_index"))
|
||||
raw_indices = ([0] if image_count == 1 else [0, -1]) if raw is None else _as_int_list(raw, name="frame_indices")
|
||||
if len(raw_indices) != image_count:
|
||||
raise OmniClientError(
|
||||
f"MiniMax H3 FL2VA requires one frame index per image: got {raw_indices!r} for {image_count} image(s)"
|
||||
)
|
||||
if tuple(raw_indices) not in ((0,), (-1,), (0, -1)):
|
||||
raise OmniClientError("MiniMax H3 FL2VA frame_indices must be [0], [-1], or [0, -1]")
|
||||
return raw_indices
|
||||
|
||||
|
||||
def _validate_ref2va_reference_counts(image_count: int, video_count: int, audio_count: int) -> None:
|
||||
if min(image_count, video_count, audio_count) < 0:
|
||||
raise OmniClientError("MiniMax H3 reference counts must be non-negative")
|
||||
if image_count + video_count == 0:
|
||||
raise OmniClientError("ref2va requires at least one image or video reference")
|
||||
if image_count > 9:
|
||||
raise OmniClientError("ref2va accepts at most 9 image references")
|
||||
if video_count > 3:
|
||||
raise OmniClientError("ref2va accepts at most 3 video references")
|
||||
if audio_count > 3:
|
||||
raise OmniClientError("ref2va accepts at most 3 standalone audio references")
|
||||
if image_count + video_count + audio_count > 12:
|
||||
raise OmniClientError("ref2va accepts at most 12 total references")
|
||||
|
||||
|
||||
def _validate_reference_image(image: Image.Image) -> None:
|
||||
resolve_minimax_h3_reference_image_shape(image)
|
||||
|
||||
|
||||
def resolve_minimax_h3_shape(
|
||||
task: str,
|
||||
sampling: Any,
|
||||
image: Image.Image | None,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
# The diffusion package exports a pipeline that imports this shared module.
|
||||
from vllm_omni.diffusion.models.minimax_h3.time_request import (
|
||||
MINIMAX_H3_SHAPE_PLANNER,
|
||||
minimax_h3_align_frame_count,
|
||||
)
|
||||
|
||||
fps = int(getattr(sampling, "fps", None) or MINIMAX_H3_FPS)
|
||||
if fps != MINIMAX_H3_FPS:
|
||||
raise OmniClientError(f"MiniMax H3 output fps is fixed at {MINIMAX_H3_FPS}")
|
||||
extra = sampling.extra_args or {}
|
||||
target = extra.get("target")
|
||||
if target is not None and not isinstance(target, Mapping):
|
||||
raise OmniClientError("MiniMax H3 extra_args['target'] must be an object")
|
||||
target = target if isinstance(target, Mapping) else {}
|
||||
duration = target.get("duration_seconds", extra.get("duration_seconds", extra.get("duration")))
|
||||
if duration is not None:
|
||||
if isinstance(duration, bool):
|
||||
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration!r}")
|
||||
try:
|
||||
duration = float(duration)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration!r}") from exc
|
||||
if (
|
||||
not math.isfinite(duration)
|
||||
or not MINIMAX_H3_MIN_OUTPUT_SECONDS <= duration <= MINIMAX_H3_MAX_OUTPUT_SECONDS
|
||||
):
|
||||
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration}")
|
||||
requested_frames = int(round(duration * fps))
|
||||
elif int(getattr(sampling, "num_frames", None) or 1) > 1:
|
||||
requested_frames = int(sampling.num_frames)
|
||||
else:
|
||||
requested_frames = 124 if task == "ref2va" else 209
|
||||
if not MINIMAX_H3_MIN_OUTPUT_SECONDS <= requested_frames / fps <= MINIMAX_H3_MAX_OUTPUT_SECONDS:
|
||||
raise OmniClientError(
|
||||
f"MiniMax H3 output duration must be in [4, 15] seconds, got {requested_frames / fps:.3f}"
|
||||
)
|
||||
num_frames = minimax_h3_align_frame_count(requested_frames)
|
||||
|
||||
height = getattr(sampling, "height", None)
|
||||
width = getattr(sampling, "width", None)
|
||||
aspect_ratio = resolve_minimax_h3_aspect_ratio(
|
||||
task,
|
||||
target.get("aspect_ratio", extra.get("aspect_ratio")),
|
||||
image,
|
||||
)
|
||||
raw_short_edge = target.get("short_edge", extra.get("short_edge", MINIMAX_H3_OUTPUT_SHORT_EDGE))
|
||||
if isinstance(raw_short_edge, bool) or not isinstance(raw_short_edge, (int, np.integer)):
|
||||
raise OmniClientError(
|
||||
f"MiniMax H3 target.short_edge must be {MINIMAX_H3_OUTPUT_SHORT_EDGE}, got {raw_short_edge!r}"
|
||||
)
|
||||
if height is None or width is None:
|
||||
height, width = resolve_minimax_h3_output_canvas(aspect_ratio, int(raw_short_edge))
|
||||
height = int(height) // 32 * 32
|
||||
width = int(width) // 32 * 32
|
||||
if min(height, width) <= 0:
|
||||
raise OmniClientError(f"invalid MiniMax H3 canvas {width}x{height}")
|
||||
if width > 4 * height or height > 4 * width:
|
||||
raise OmniClientError("MiniMax H3 canvas aspect ratio must be in [1:4, 4:1]")
|
||||
return (
|
||||
height,
|
||||
width,
|
||||
num_frames,
|
||||
MINIMAX_H3_SHAPE_PLANNER.video_latent_t(num_frames),
|
||||
MINIMAX_H3_SHAPE_PLANNER.audio_latent_t(num_frames / fps),
|
||||
)
|
||||
|
||||
|
||||
def _resolve_task(
|
||||
extra_args: Mapping[str, Any],
|
||||
multi_modal_data: Mapping[str, Any],
|
||||
) -> str:
|
||||
requested = extra_args.get("task")
|
||||
if requested is not None:
|
||||
return str(requested).lower()
|
||||
if multi_modal_data.get("video") is not None or multi_modal_data.get("audio") is not None:
|
||||
return "ref2va"
|
||||
if multi_modal_data.get("image") is not None:
|
||||
return "fl2va"
|
||||
return "t2va"
|
||||
|
||||
|
||||
def _prepare_encoder_images(
|
||||
task: str,
|
||||
images: list[Image.Image],
|
||||
*,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> list[Any]:
|
||||
if not images:
|
||||
return []
|
||||
if task == "ref2va":
|
||||
return [
|
||||
image.resize(
|
||||
resolve_minimax_h3_reference_image_shape(image),
|
||||
Image.Resampling.LANCZOS,
|
||||
)
|
||||
for image in images
|
||||
]
|
||||
if task != "fl2va":
|
||||
return images
|
||||
return [image.resize((width, height), Image.Resampling.LANCZOS) for image in images]
|
||||
|
||||
|
||||
def _image_to_tensor(image: Image.Image) -> torch.Tensor:
|
||||
array = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
return torch.from_numpy(np.array(array, copy=True)).contiguous()
|
||||
|
||||
|
||||
def _frames_to_tensor(frames: Sequence[Any]) -> torch.Tensor:
|
||||
if len(frames) == 0:
|
||||
raise ValueError("MiniMax H3 reference video must contain frames")
|
||||
return torch.stack(
|
||||
[
|
||||
_image_to_tensor(frame if isinstance(frame, Image.Image) else Image.fromarray(np.asarray(frame)))
|
||||
for frame in frames
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _reuse_prepared_reference_videos(
|
||||
prepared: list[dict[str, Any]] | None,
|
||||
*,
|
||||
expected_count: int,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
if prepared is None:
|
||||
return None
|
||||
if len(prepared) != expected_count:
|
||||
raise OmniClientError("MiniMax H3 prepared-reference-video count does not match the request")
|
||||
for item in prepared:
|
||||
if not os.path.isfile(item["prepared_path"]):
|
||||
raise OmniClientError(f"MiniMax H3 prepared reference video is unavailable: {item['prepared_path']}")
|
||||
return prepared
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedEncoderInputs:
|
||||
"""CPU-owned media and Qwen presentation inputs, independent of a runner."""
|
||||
|
||||
prompt: str
|
||||
media: MiniMaxH3EncoderMediaInput
|
||||
images: list[Image.Image]
|
||||
qwen_videos: list[tuple[np.ndarray, dict[str, Any]]]
|
||||
video_timestamps: list[list[float]]
|
||||
condition_labels: list[tuple[str, int]]
|
||||
|
||||
|
||||
def _effective_audio_inputs(
|
||||
video_audios: Sequence[tuple[torch.Tensor, int] | None],
|
||||
standalone_audios: Sequence[tuple[torch.Tensor, int]],
|
||||
*,
|
||||
max_standalone_seconds: float,
|
||||
) -> list[tuple[torch.Tensor, int]]:
|
||||
"""Return the waveforms exactly as the audio WVAE will consume them."""
|
||||
audio_inputs = [item for item in video_audios if item is not None]
|
||||
audio_inputs.extend(
|
||||
(waveform[..., : int(round(max_standalone_seconds * sample_rate))], sample_rate)
|
||||
for waveform, sample_rate in standalone_audios
|
||||
)
|
||||
return audio_inputs
|
||||
|
||||
|
||||
def prepare_encoder_inputs(
|
||||
prompt: Any,
|
||||
sampling: Any,
|
||||
*,
|
||||
task: str | None = None,
|
||||
prepared_reference_videos: list[dict[str, Any]] | None = None,
|
||||
) -> PreparedEncoderInputs:
|
||||
"""Decode references once and prepare the shared text/codec presentation."""
|
||||
if isinstance(prompt, str):
|
||||
prompt = {"prompt": prompt}
|
||||
if not isinstance(prompt, dict):
|
||||
raise TypeError(f"MiniMax H3 expects a string or dict prompt, got {type(prompt)!r}")
|
||||
|
||||
text = str(prompt.get("prompt") or "")
|
||||
if not text:
|
||||
raise OmniClientError("MiniMax H3 requires a non-empty prompt")
|
||||
multi_modal_data = prompt.get("multi_modal_data") or {}
|
||||
if not isinstance(multi_modal_data, Mapping):
|
||||
raise TypeError("multi_modal_data must be a mapping")
|
||||
|
||||
image_values = _items(multi_modal_data.get("image"))
|
||||
videos = _items(multi_modal_data.get("video"))
|
||||
raw_audio = multi_modal_data.get("audio")
|
||||
audio_values = _audio_items(raw_audio)
|
||||
diffusion_sampling_params = sampling
|
||||
extra_args = getattr(diffusion_sampling_params, "extra_args", None) or {}
|
||||
task = task if task is not None else _resolve_task(extra_args, multi_modal_data)
|
||||
raw_images = load_minimax_h3_images(image_values) if image_values else []
|
||||
|
||||
if task == "t2va":
|
||||
if raw_images or videos or audio_values:
|
||||
raise OmniClientError("t2va does not accept image, video, or audio conditions")
|
||||
elif task == "fl2va":
|
||||
if not raw_images or videos or audio_values:
|
||||
raise OmniClientError("fl2va requires image conditions only")
|
||||
if len(raw_images) > 2:
|
||||
raise OmniClientError("fl2va accepts at most first and last images")
|
||||
for image in raw_images:
|
||||
_validate_reference_image(image)
|
||||
elif task == "ref2va":
|
||||
_validate_ref2va_reference_counts(len(raw_images), len(videos), len(audio_values))
|
||||
if not raw_images and not videos:
|
||||
raise OmniClientError("ref2va requires an image or video condition")
|
||||
else:
|
||||
raise OmniClientError(f"unsupported MiniMax H3 task {task!r}")
|
||||
|
||||
height, width, num_frames, latent_t, audio_t = resolve_minimax_h3_shape(
|
||||
task,
|
||||
diffusion_sampling_params,
|
||||
raw_images[0] if raw_images else None,
|
||||
)
|
||||
images = _prepare_encoder_images(
|
||||
task,
|
||||
raw_images,
|
||||
height=height,
|
||||
width=width,
|
||||
)
|
||||
keyframe_indices = _resolve_fl2va_keyframe_indices(extra_args, len(images)) if task == "fl2va" else []
|
||||
video_timestamps: list[list[float]] = []
|
||||
qwen_video_inputs: list[tuple[np.ndarray, dict[str, Any]]] = []
|
||||
condition_labels: list[tuple[str, int]] = []
|
||||
encoded_video_inputs: list[torch.Tensor] = []
|
||||
video_audio_inputs: list[tuple[torch.Tensor, int] | None] = []
|
||||
|
||||
if task == "fl2va":
|
||||
condition_labels.extend(("image", index) for index in range(1, len(images) + 1))
|
||||
elif task == "ref2va":
|
||||
condition_labels.extend(("image", index) for index in range(1, len(images) + 1))
|
||||
prepared_videos: list[dict[str, Any]] = []
|
||||
if videos:
|
||||
with tempfile.TemporaryDirectory(prefix="minimax_h3_encoder_") as workdir:
|
||||
prepared_videos = _reuse_prepared_reference_videos(
|
||||
prepared_reference_videos, expected_count=len(videos)
|
||||
)
|
||||
if prepared_videos is None:
|
||||
prepared_videos = prepare_reference_videos(
|
||||
videos,
|
||||
target_frame_count=num_frames,
|
||||
workdir=workdir,
|
||||
start_time_seconds=extra_args.get("start_time_seconds"),
|
||||
)
|
||||
for item in prepared_videos:
|
||||
full_frames = load_video_frames(item["prepared_path"])
|
||||
encoded_video_inputs.append(_frames_to_tensor(full_frames))
|
||||
sampled = sample_reference_video_frames(
|
||||
item["prepared_path"],
|
||||
decoded_frames=full_frames,
|
||||
)
|
||||
video_timestamps.append(sampled["block_timestamps"])
|
||||
frames = np.stack(sampled["frames"])
|
||||
frame_count = int(frames.shape[0])
|
||||
qwen_video_inputs.append(
|
||||
(
|
||||
frames,
|
||||
{
|
||||
"total_num_frames": frame_count,
|
||||
"fps": MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS,
|
||||
"duration": frame_count / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS,
|
||||
"video_backend": "minimax_h3",
|
||||
"frames_indices": list(range(frame_count)),
|
||||
"do_sample_frames": False,
|
||||
},
|
||||
)
|
||||
)
|
||||
if item["input_has_audio"]:
|
||||
waveform, sample_rate = load_video_audio(
|
||||
item["original_path"],
|
||||
start_time_seconds=float(item.get("start_time_seconds", 0.0)),
|
||||
duration_seconds=item.get(
|
||||
"audio_duration_seconds",
|
||||
item.get("duration_seconds"),
|
||||
),
|
||||
)
|
||||
video_audio_inputs.append((waveform.float().contiguous(), int(sample_rate)))
|
||||
else:
|
||||
video_audio_inputs.append(None)
|
||||
audio_index = 0
|
||||
for video_index, item in enumerate(prepared_videos, start=1):
|
||||
if item["input_has_audio"]:
|
||||
audio_index += 1
|
||||
condition_labels.append(("audio", audio_index))
|
||||
condition_labels.append(("video", video_index))
|
||||
for _ in audio_values:
|
||||
audio_index += 1
|
||||
condition_labels.append(("audio", audio_index))
|
||||
|
||||
if raw_audio is not None:
|
||||
validate_reference_audio_files(raw_audio)
|
||||
standalone_audios = _load_audios(raw_audio) if raw_audio is not None else []
|
||||
validate_reference_audio_waveforms(standalone_audios)
|
||||
media_input = MiniMaxH3EncoderMediaInput(
|
||||
task=task,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
latent_t=latent_t,
|
||||
audio_t=audio_t,
|
||||
images=tuple(_image_to_tensor(image) for image in images),
|
||||
videos=tuple(encoded_video_inputs),
|
||||
video_audios=tuple(video_audio_inputs),
|
||||
audios=tuple((waveform.float().contiguous(), int(sample_rate)) for waveform, sample_rate in standalone_audios),
|
||||
keyframe_frame_indices=tuple(keyframe_indices),
|
||||
)
|
||||
audio_inputs = _effective_audio_inputs(
|
||||
media_input.video_audios,
|
||||
media_input.audios,
|
||||
max_standalone_seconds=float(media_input.num_frames) / MINIMAX_H3_FPS,
|
||||
)
|
||||
embedded_audio_count = sum(item is not None for item in media_input.video_audios)
|
||||
# Video soundtracks and standalone references have separate 15-second budgets.
|
||||
validate_reference_audio_waveforms(audio_inputs[:embedded_audio_count])
|
||||
validate_reference_audio_waveforms(audio_inputs[embedded_audio_count:])
|
||||
|
||||
return PreparedEncoderInputs(
|
||||
prompt=text,
|
||||
media=media_input,
|
||||
images=images,
|
||||
qwen_videos=qwen_video_inputs,
|
||||
video_timestamps=video_timestamps,
|
||||
condition_labels=condition_labels,
|
||||
)
|
||||
|
||||
|
||||
def _image_from_tensor(value: torch.Tensor) -> Image.Image:
|
||||
if value.ndim != 3 or value.shape[-1] != 3:
|
||||
raise ValueError(f"MiniMax H3 image tensor must have shape [H, W, 3], got {tuple(value.shape)}")
|
||||
array = value.detach().cpu().to(torch.uint8).contiguous().numpy()
|
||||
return Image.fromarray(array, mode="RGB")
|
||||
|
||||
|
||||
def encode_media(
|
||||
media: MiniMaxH3EncoderMediaInput,
|
||||
*,
|
||||
video_vae: Any,
|
||||
audio_vae: Any,
|
||||
emit_conditioning: bool,
|
||||
component_scope: Callable[[Any], AbstractContextManager[Any]] = nullcontext,
|
||||
) -> MiniMaxH3EncoderMediaConditioning | None:
|
||||
"""Run codecs on participating ranks; only the output rank runs audio.
|
||||
|
||||
The caller selects participating ranks and configures the VAE process group.
|
||||
Every visual participant must call this function with the same references.
|
||||
Component scopes allow the diffusion runner to stage one codec at a time.
|
||||
"""
|
||||
visual_rows: list[torch.Tensor] = []
|
||||
visual_shapes: list[tuple[int, int, int]] = []
|
||||
if media.images or media.videos:
|
||||
if video_vae is None:
|
||||
raise RuntimeError("MiniMax H3 video VAE is not resident on this rank")
|
||||
with component_scope(video_vae):
|
||||
for value in media.images:
|
||||
image = _image_from_tensor(value)
|
||||
visual_rows.append(video_vae.encode_image(image))
|
||||
visual_shapes.append((1, image.height // 16, image.width // 16))
|
||||
for value in media.videos:
|
||||
frames = np.asarray(value.detach().cpu().to(torch.uint8).contiguous().numpy())
|
||||
rows, shape = video_vae.encode_video(frames)
|
||||
visual_rows.append(rows)
|
||||
visual_shapes.append(tuple(int(item) for item in shape))
|
||||
|
||||
if not emit_conditioning:
|
||||
return None
|
||||
|
||||
audio_rows: list[torch.Tensor] = []
|
||||
audio_lengths: list[int] = []
|
||||
embedded_audio_count = sum(item is not None for item in media.video_audios)
|
||||
audio_inputs = _effective_audio_inputs(
|
||||
media.video_audios,
|
||||
media.audios,
|
||||
max_standalone_seconds=float(media.num_frames) / MINIMAX_H3_FPS,
|
||||
)
|
||||
if audio_inputs:
|
||||
if audio_vae is None:
|
||||
raise RuntimeError("MiniMax H3 audio WVAE is not resident on the encoder leader")
|
||||
with component_scope(audio_vae):
|
||||
for waveform, sample_rate in audio_inputs:
|
||||
rows, length = audio_vae.encode_waveform(waveform, sample_rate)
|
||||
audio_rows.append(rows)
|
||||
audio_lengths.append(int(length))
|
||||
if audio_lengths:
|
||||
if any(length < 80 or length > 600 for length in audio_lengths):
|
||||
raise ValueError("MiniMax H3 audio references must each be between 2 and 15 seconds")
|
||||
if sum(audio_lengths[:embedded_audio_count]) > 600 or sum(audio_lengths[embedded_audio_count:]) > 600:
|
||||
raise ValueError("MiniMax H3 audio references must be at most 15 seconds in total")
|
||||
|
||||
ref_blocks: list[dict[str, Any]] = []
|
||||
image_shapes = visual_shapes[: len(media.images)]
|
||||
video_shapes = visual_shapes[len(media.images) :]
|
||||
ref_blocks.extend({"kind": "image", "latent_h": shape[1], "latent_w": shape[2]} for shape in image_shapes)
|
||||
embedded_lengths = iter(audio_lengths[:embedded_audio_count])
|
||||
for shape, embedded in zip(video_shapes, media.video_audios, strict=True):
|
||||
ref_audio_t = int(next(embedded_lengths)) if embedded is not None else 0
|
||||
ref_blocks.append(
|
||||
{
|
||||
"kind": "video_audio" if ref_audio_t else "video",
|
||||
"ref_audio_t": ref_audio_t,
|
||||
"latent_t": shape[0],
|
||||
"latent_h": shape[1],
|
||||
"latent_w": shape[2],
|
||||
}
|
||||
)
|
||||
ref_blocks.extend(
|
||||
{
|
||||
"kind": "audio",
|
||||
"ref_audio_t": int(length),
|
||||
}
|
||||
for length in audio_lengths[embedded_audio_count:]
|
||||
)
|
||||
return MiniMaxH3EncoderMediaConditioning(
|
||||
task=media.task,
|
||||
height=media.height,
|
||||
width=media.width,
|
||||
num_frames=media.num_frames,
|
||||
latent_t=media.latent_t,
|
||||
audio_t=media.audio_t,
|
||||
visual_condition=torch.cat(visual_rows) if visual_rows else None,
|
||||
visual_condition_shapes=tuple(visual_shapes),
|
||||
audio_condition=torch.cat(audio_rows) if audio_rows else None,
|
||||
audio_condition_lengths=tuple(audio_lengths),
|
||||
ref_blocks=tuple(ref_blocks),
|
||||
keyframe_frame_indices=media.keyframe_frame_indices,
|
||||
)
|
||||
@@ -16,18 +16,18 @@ MINIMAX_H3_PIPELINE = PipelineConfig(
|
||||
model_type="minimax_h3_disaggregated",
|
||||
default_deploy_config_name="minimax_h3_disaggregated.yaml",
|
||||
stage_cli_aliases={"text_encoder_tp_size": (0, "tensor_parallel_size")},
|
||||
model_arch="MiniMaxH3TextEncoder",
|
||||
model_arch="MiniMaxH3Encoder",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="text_encoder",
|
||||
model_stage="encoder",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
model_arch="MiniMaxH3TextEncoder",
|
||||
model_arch="MiniMaxH3Encoder",
|
||||
engine_output_type="latent",
|
||||
prompt_transform_func=f"{_PROCESSOR}.prepare_text_encoder_prompt",
|
||||
prompt_transform_func=f"{_PROCESSOR}.prepare_encoder_prompt",
|
||||
sampling_constraints={
|
||||
"max_tokens": 1,
|
||||
"temperature": 0.0,
|
||||
@@ -42,9 +42,9 @@ MINIMAX_H3_PIPELINE = PipelineConfig(
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="video",
|
||||
requires_multimodal_data=True,
|
||||
requires_multimodal_data=False,
|
||||
model_arch="MiniMaxH3Pipeline",
|
||||
custom_process_input_func=f"{_PROCESSOR}.text_encoder2diffusion",
|
||||
custom_process_input_func=f"{_PROCESSOR}.encoder2diffusion",
|
||||
omni_kv_config={"need_recv_cache": False},
|
||||
model_path_resolver=f"{_DIFFUSION_PIPELINE}.resolve_minimax_h3_diffusion_model_path",
|
||||
inline_diffusion=True,
|
||||
|
||||
@@ -531,14 +531,27 @@ def _decode_video_frames_ffmpeg(
|
||||
)
|
||||
|
||||
|
||||
def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]:
|
||||
meta = _probe_video(prepared_path)
|
||||
def sample_reference_video_frames(
|
||||
prepared_path: str,
|
||||
*,
|
||||
decoded_frames: np.ndarray | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if decoded_frames is None:
|
||||
meta = _probe_video(prepared_path)
|
||||
frame_count = int(meta["frame_count"])
|
||||
else:
|
||||
decoded_frames = np.asarray(decoded_frames)
|
||||
if decoded_frames.ndim != 4 or decoded_frames.shape[-1] != 3 or len(decoded_frames) <= 0:
|
||||
raise OmniClientError(
|
||||
f"decoded reference video frames must have shape [T, H, W, 3], got {decoded_frames.shape}"
|
||||
)
|
||||
frame_count = len(decoded_frames)
|
||||
ratio = MINIMAX_H3_FPS / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS
|
||||
indices: list[int] = []
|
||||
cursor = 0.0
|
||||
while True:
|
||||
frame_index = int(round(cursor))
|
||||
if frame_index >= meta["frame_count"]:
|
||||
if frame_index >= frame_count:
|
||||
break
|
||||
if not indices or frame_index > indices[-1]:
|
||||
indices.append(frame_index)
|
||||
@@ -546,16 +559,16 @@ def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]:
|
||||
if not indices:
|
||||
raise OmniClientError(f"no frames sampled from {prepared_path}")
|
||||
|
||||
# The preparation step emits a lossless RGB stream. Decode only the sampled
|
||||
# frames in one ffmpeg process so Qwen3VL sees the exact prepared pixels
|
||||
# without decoding the entire high-bitrate stream into host memory.
|
||||
decoded_frames = _decode_video_frames_ffmpeg(
|
||||
prepared_path,
|
||||
frame_count=int(meta["frame_count"]),
|
||||
indices=indices,
|
||||
width=meta["width"],
|
||||
height=meta["height"],
|
||||
)
|
||||
if decoded_frames is None:
|
||||
decoded_frames = _decode_video_frames_ffmpeg(
|
||||
prepared_path,
|
||||
frame_count=frame_count,
|
||||
indices=indices,
|
||||
width=meta["width"],
|
||||
height=meta["height"],
|
||||
)
|
||||
else:
|
||||
decoded_frames = decoded_frames[indices]
|
||||
frames = [np.asarray(frame) for frame in decoded_frames]
|
||||
|
||||
timestamps = [index / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS for index in range(len(indices))]
|
||||
|
||||
@@ -176,7 +176,7 @@ class _ResidualMerge(nn.Module):
|
||||
info=Qwen3VLProcessingInfo,
|
||||
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
||||
)
|
||||
class MiniMaxH3TextEncoder(Qwen3VLForConditionalGeneration):
|
||||
class MiniMaxH3TextEncoderBackbone(Qwen3VLForConditionalGeneration):
|
||||
"""Qwen3-VL encoder used by MiniMax H3.
|
||||
|
||||
MiniMax H3 consumes the residual stream after decoder layer 50. The
|
||||
@@ -305,3 +305,6 @@ class MiniMaxH3TextEncoder(Qwen3VLForConditionalGeneration):
|
||||
encoder_weights(),
|
||||
mapper=self.hf_to_vllm_mapper,
|
||||
)
|
||||
|
||||
|
||||
MiniMaxH3TextEncoder = MiniMaxH3TextEncoderBackbone
|
||||
@@ -476,10 +476,10 @@ _OMNI_MODELS = {
|
||||
"qwen3_vl",
|
||||
"AuraQwen3VLForConditionalGeneration",
|
||||
),
|
||||
"MiniMaxH3TextEncoder": (
|
||||
"MiniMaxH3Encoder": (
|
||||
"minimax_h3",
|
||||
"text_encoder",
|
||||
"MiniMaxH3TextEncoder",
|
||||
"encoder",
|
||||
"MiniMaxH3Encoder",
|
||||
),
|
||||
## MiniMax-Music3 (text-to-music; AR talker -> flow-matching acoustic decoder)
|
||||
"MiniMaxMusic3TalkerForConditionalGeneration": (
|
||||
|
||||
Reference in new issue
Block a user