[Model] Add MiniMax-H3 long-video latent continuation with driving audio (#7838)

Signed-off-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
汪志鹏 authored and GitHub committed 2026-09-20 23:44:33 +08:00
1 parent 16734bbdf3
commit 139a47a578
17 files changed
+1089 -50

No files matched your search

@@ -358,6 +358,7 @@ class MiniMaxH3EncoderMediaInput:
video_audios: tuple[tuple[torch.Tensor, int] | None, ...] = ()
audios: tuple[tuple[torch.Tensor, int], ...] = ()
keyframe_frame_indices: tuple[int, ...] = ()
audio_mode: str = "native"
video_edit: torch.Tensor | None = None
video_edit_mask: torch.Tensor | None = None
audio_edit: tuple[torch.Tensor, int] | None = None
@@ -468,6 +469,7 @@ class MiniMaxH3EncoderMediaInput:
if cursor != len(tensors):
raise ValueError(f"MiniMax H3 encoder media input has {len(tensors) - cursor} trailing tensors")
return cls(
audio_mode=str(metadata.get("audio_mode", "native")),
task=task,
height=height,
width=width,
@@ -541,6 +543,7 @@ class MiniMaxH3EncoderMediaInput:
def to_metadata(self) -> dict[str, Any]:
metadata = {
"audio_mode": self.audio_mode,
"task": self.task,
"height": self.height,
"width": self.width,
@@ -22,6 +22,7 @@ from vllm_omni.model_executor.models.minimax_h3.conditioning import (
MiniMaxH3EncoderMediaConditioning,
MiniMaxH3EncoderMediaInput,
)
from vllm_omni.model_executor.models.minimax_h3.long_video import max_output_seconds, resolve_long_video_mode
from vllm_omni.model_executor.models.minimax_h3.preprocessing import (
MINIMAX_H3_OUTPUT_SHORT_EDGE,
load_minimax_h3_images,
@@ -160,28 +161,25 @@ def resolve_minimax_h3_shape(
if target is not None and not isinstance(target, Mapping):
raise OmniClientError("MiniMax H3 extra_args['target'] must be an object")
target = target if isinstance(target, Mapping) else {}
max_seconds = max_output_seconds(extra, task)
duration_range = f"in [4, {max_seconds:g}] seconds"
duration = target.get("duration_seconds", extra.get("duration_seconds", extra.get("duration")))
if duration is not None:
if isinstance(duration, bool):
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration!r}")
raise OmniClientError(f"MiniMax H3 output duration must be {duration_range}, got {duration!r}")
try:
duration = float(duration)
except (TypeError, ValueError) as exc:
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration!r}") from exc
if (
not math.isfinite(duration)
or not MINIMAX_H3_MIN_OUTPUT_SECONDS <= duration <= MINIMAX_H3_MAX_OUTPUT_SECONDS
):
raise OmniClientError(f"MiniMax H3 output duration must be in [4, 15] seconds, got {duration}")
raise OmniClientError(f"MiniMax H3 output duration must be {duration_range}, got {duration!r}") from exc
if not math.isfinite(duration) or not MINIMAX_H3_MIN_OUTPUT_SECONDS <= duration <= max_seconds:
raise OmniClientError(f"MiniMax H3 output duration must be {duration_range}, got {duration}")
requested_frames = int(round(duration * fps))
elif int(getattr(sampling, "num_frames", None) or 1) > 1:
requested_frames = int(sampling.num_frames)
else:
requested_frames = 124 if task == "ref2va" else 209
if not MINIMAX_H3_MIN_OUTPUT_SECONDS <= requested_frames / fps <= MINIMAX_H3_MAX_OUTPUT_SECONDS:
raise OmniClientError(
f"MiniMax H3 output duration must be in [4, 15] seconds, got {requested_frames / fps:.3f}"
)
if not MINIMAX_H3_MIN_OUTPUT_SECONDS <= requested_frames / fps <= max_seconds:
raise OmniClientError(f"MiniMax H3 output duration must be {duration_range}, got {requested_frames / fps:.3f}")
num_frames = minimax_h3_align_frame_count(requested_frames)
height = getattr(sampling, "height", None)
@@ -220,7 +218,10 @@ def _resolve_task(
requested = extra_args.get("task")
if requested is not None:
return str(requested).lower()
if multi_modal_data.get("video") is not None or multi_modal_data.get("audio") is not None:
has_reference_audio = (
multi_modal_data.get("audio") is not None and extra_args.get("audio_mode", "native") != "lock_source"
)
if multi_modal_data.get("video") is not None or has_reference_audio:
return "ref2va"
if multi_modal_data.get("image") is not None:
return "fl2va"
@@ -428,14 +429,20 @@ def prepare_encoder_inputs(
audio_values = _audio_items(raw_audio)
diffusion_sampling_params = sampling
extra_args = getattr(diffusion_sampling_params, "extra_args", None) or {}
audio_mode = extra_args.get("audio_mode", "native")
if audio_mode not in ("native", "lock_source"):
raise OmniClientError("MiniMax H3 audio_mode must be native or lock_source")
lock_audio = audio_mode == "lock_source"
if lock_audio and len(audio_values) != 1:
raise OmniClientError("MiniMax H3 lock_source requires exactly one driving audio")
task = task if task is not None else _resolve_task(extra_args, multi_modal_data)
raw_images = load_minimax_h3_images(image_values) if image_values else []
if task == "t2va":
if raw_images or videos or audio_values:
if raw_images or videos or (audio_values and not lock_audio):
raise OmniClientError("t2va does not accept image, video, or audio conditions")
elif task == "fl2va":
if not raw_images or videos or audio_values:
if not raw_images or videos or (audio_values and not lock_audio):
raise OmniClientError("fl2va requires image conditions only")
if len(raw_images) > 2:
raise OmniClientError("fl2va accepts at most first and last images")
@@ -479,6 +486,12 @@ def prepare_encoder_inputs(
video_edit_mask = None
if audio_edit_mask is not None and bool(torch.all(audio_edit_mask == 1.0).item()):
audio_edit_mask = None
if resolve_long_video_mode(extra_args, task) == "continuation" and (
video_edit_mask is not None or audio_edit_mask is not None
):
raise OmniClientError("MiniMax H3 continuation does not support latent-mask editing; use long_video_mode=full")
if lock_audio and audio_edit_mask is not None:
raise OmniClientError("MiniMax H3 audio_mode=lock_source cannot be combined with audio latent-mask editing")
if video_edit_mask is not None and source_video is None:
raise OmniClientError("MiniMax H3 video_noise_mask requires source_video")
if audio_edit_mask is not None and source_audio is None and source_video is None:
@@ -555,7 +568,7 @@ def prepare_encoder_inputs(
audio_index += 1
condition_labels.append(("audio", audio_index))
condition_labels.append(("video", video_index))
for _ in audio_values:
for _ in [] if lock_audio else audio_values:
audio_index += 1
condition_labels.append(("audio", audio_index))
@@ -601,10 +614,15 @@ def prepare_encoder_inputs(
raise OmniClientError("MiniMax H3 edit audio must not be empty")
audio_edit = (waveform, int(sample_rate))
if raw_audio is not None:
if raw_audio is not None and not lock_audio:
validate_reference_audio_files(raw_audio)
standalone_audios = _load_audios(raw_audio) if raw_audio is not None else []
validate_reference_audio_waveforms(standalone_audios)
if lock_audio:
waveform, rate = standalone_audios[0]
if rate <= 0 or waveform.ndim not in (1, 2) or waveform.numel() == 0 or not torch.isfinite(waveform).all():
raise OmniClientError("MiniMax H3 driving audio must be a finite non-empty waveform with a positive rate")
else:
validate_reference_audio_waveforms(standalone_audios)
media_input = MiniMaxH3EncoderMediaInput(
task=task,
height=height,
@@ -617,6 +635,7 @@ def prepare_encoder_inputs(
video_audios=tuple(video_audio_inputs),
audios=tuple((waveform.float().contiguous(), int(sample_rate)) for waveform, sample_rate in standalone_audios),
keyframe_frame_indices=tuple(keyframe_indices),
audio_mode=audio_mode,
video_edit=video_edit,
video_edit_mask=video_edit_mask,
audio_edit=audio_edit,
@@ -630,7 +649,8 @@ def prepare_encoder_inputs(
embedded_audio_count = sum(item is not None for item in media_input.video_audios)
# Video soundtracks and standalone references have separate 15-second budgets.
validate_reference_audio_waveforms(audio_inputs[:embedded_audio_count])
validate_reference_audio_waveforms(audio_inputs[embedded_audio_count:])
if not lock_audio:
validate_reference_audio_waveforms(audio_inputs[embedded_audio_count:])
return PreparedEncoderInputs(
prompt=text,
@@ -723,10 +743,11 @@ def encode_media(
int(source_audio_t),
target_audio_t=media.audio_t,
)
if audio_lengths:
if any(length < 80 or length > 600 for length in audio_lengths):
reference_lengths = audio_lengths[:-1] if media.audio_mode == "lock_source" else audio_lengths
if reference_lengths:
if any(length < 80 or length > 600 for length in reference_lengths):
raise ValueError("MiniMax H3 audio references must each be between 2 and 15 seconds")
if sum(audio_lengths[:embedded_audio_count]) > 600 or sum(audio_lengths[embedded_audio_count:]) > 600:
if sum(reference_lengths[:embedded_audio_count]) > 600 or sum(reference_lengths[embedded_audio_count:]) > 600:
raise ValueError("MiniMax H3 audio references must be at most 15 seconds in total")
ref_blocks: list[dict[str, Any]] = []
@@ -750,7 +771,7 @@ def encode_media(
"kind": "audio",
"ref_audio_t": int(length),
}
for length in audio_lengths[embedded_audio_count:]
for length in reference_lengths[embedded_audio_count:]
)
video_edit_mask = None
if video_edit_clean_rows is not None:
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared long-video request limits for encoder and diffusion entry points."""
from collections.abc import Mapping
from typing import Any
from vllm_omni.errors import OmniClientError
MINIMAX_H3_MAX_FULL_SECONDS = 30.0
MINIMAX_H3_MAX_CONTINUATION_SECONDS = 300.0
def resolve_long_video_mode(extra: Mapping[str, Any], task: str) -> str:
long_video = extra.get("long_video", False)
if not isinstance(long_video, bool):
raise OmniClientError("MiniMax H3 long_video must be a boolean")
mode = extra.get("long_video_mode", "continuation" if long_video and task == "ref2va" else "full")
if mode not in ("full", "continuation"):
raise OmniClientError("MiniMax H3 long_video_mode must be full or continuation")
if mode == "continuation" and task != "ref2va":
raise OmniClientError("MiniMax H3 continuation requires Ref2VA request execution (not step execution)")
return mode
def max_output_seconds(extra: Mapping[str, Any], task: str) -> float:
mode = resolve_long_video_mode(extra, task)
if not extra.get("long_video", False):
return 15.0
return MINIMAX_H3_MAX_CONTINUATION_SECONDS if mode == "continuation" else MINIMAX_H3_MAX_FULL_SECONDS
def validate_encoded_frame_limit(extra: Mapping[str, Any], task: str, num_frames: int) -> None:
"""Apply the same cap to externally encoded requests, allowing grid rounding."""
seconds = max_output_seconds(extra, task)
requested_max = int(seconds * 24)
aligned_max = ((requested_max - 5 + 16) // 17) * 17 + 5
if num_frames > aligned_max:
raise OmniClientError(
f"MiniMax H3 output duration exceeds the {seconds:g}-second request limit "
f"({aligned_max} frames after alignment); got {num_frames} frames"
)