[Model] Add MiniMax-H3 long-video latent continuation with driving audio (#7838)
Signed-off-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
17 files changed
+1089
-50
No files matched your search
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Bounded H3 windows with synchronized latent-tail guides and global AV RoPE positions.
|
||||
|
||||
The guide/discard/append algorithm follows ComfyUI-Minimax-H3-Continuation:
|
||||
https://github.com/ttulttul/ComfyUI-Minimax-H3-Continuation
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.model_executor.models.minimax_h3.long_video import resolve_long_video_mode
|
||||
|
||||
from .packed_tokens import minimax_h3_pack_audio_latent, minimax_h3_patchify_video_latent
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _video_t(frames: int) -> int:
|
||||
return (frames - 5) // 17 * 5 + 2
|
||||
|
||||
|
||||
def _audio_t(frame_boundary: int) -> int:
|
||||
return round(frame_boundary * 5 / 3)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContinuationWindow:
|
||||
start: int
|
||||
end: int
|
||||
overlap: int
|
||||
|
||||
@property
|
||||
def audio_start(self) -> int:
|
||||
return _audio_t(self.start)
|
||||
|
||||
@property
|
||||
def audio_end(self) -> int:
|
||||
return _audio_t(self.end)
|
||||
|
||||
@property
|
||||
def overlap_audio_t(self) -> int:
|
||||
return _audio_t(self.start + self.overlap) - self.audio_start
|
||||
|
||||
|
||||
def resolve_continuation(extra: Mapping[str, Any], *, task: str, step_execution: bool) -> tuple[int, int] | None:
|
||||
mode = resolve_long_video_mode(extra, task)
|
||||
if mode == "full":
|
||||
return None
|
||||
if task != "ref2va" or step_execution:
|
||||
raise OmniClientError("MiniMax H3 continuation requires Ref2VA request execution (not step execution)")
|
||||
window = extra.get("continuation_window_frames", 277)
|
||||
overlap = extra.get("continuation_overlap_frames", 22)
|
||||
for name, value in (("window", window), ("overlap", overlap)):
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 5 or value % 17 != 5:
|
||||
raise OmniClientError(f"MiniMax H3 continuation {name} must be an integer on the 17n+5 frame grid")
|
||||
if not 107 <= window <= 345 or overlap >= window:
|
||||
raise OmniClientError("MiniMax H3 continuation window must be 107..345 frames and exceed its overlap")
|
||||
return window, overlap
|
||||
|
||||
|
||||
def plan_continuation_windows(total_frames: int, window_frames: int, overlap_frames: int) -> list[ContinuationWindow]:
|
||||
if total_frames < 5 or total_frames % 17 != 5:
|
||||
raise ValueError("total_frames must satisfy 17n+5")
|
||||
if not 5 <= overlap_frames < window_frames or window_frames % 17 != 5 or overlap_frames % 17 != 5:
|
||||
raise ValueError("window and overlap must satisfy 17n+5, with 5 <= overlap < window")
|
||||
end = min(total_frames, window_frames)
|
||||
windows = [ContinuationWindow(0, end, 0)]
|
||||
while end < total_frames:
|
||||
start = end - overlap_frames
|
||||
end = min(total_frames, start + window_frames)
|
||||
windows.append(ContinuationWindow(start, end, overlap_frames))
|
||||
return windows
|
||||
|
||||
|
||||
def diffuse_continuation(
|
||||
diffuse: Callable[..., tuple[torch.Tensor, torch.Tensor]],
|
||||
kwargs: dict[str, Any],
|
||||
*,
|
||||
window_frames: int,
|
||||
overlap_frames: int,
|
||||
text_conditioning: Sequence[tuple[torch.Tensor, torch.Tensor]] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Denoise fresh windows; retain old latents and append only new suffixes.
|
||||
|
||||
Guides are extra condition rows sharing the new target's temporal origin,
|
||||
not a masked target prefix. Audio boundaries refer to the cumulative frame
|
||||
timeline, preventing per-window rounding from accumulating A/V drift.
|
||||
Each window shifts temporal media positions onto that same global timeline
|
||||
before RoPE is evaluated; text and static image references remain fixed.
|
||||
"""
|
||||
windows = plan_continuation_windows(kwargs["num_frames"], window_frames, overlap_frames)
|
||||
if text_conditioning is not None and len(text_conditioning) != len(windows):
|
||||
raise OmniClientError("MiniMax H3 requires one text conditioning per continuation window")
|
||||
# Different prompts have different prefix lengths. Keep the media clock
|
||||
# anchored after the longest prefix so text length cannot shift AV time.
|
||||
media_time_origin = max(item[0].shape[0] for item in text_conditioning) if text_conditioning else None
|
||||
source_rows = kwargs.get("locked_audio_rows")
|
||||
source = None if source_rows is None else source_rows.reshape(2, kwargs["audio_t"], 32)
|
||||
video = audio = None
|
||||
for index, window in enumerate(windows):
|
||||
frames = window.end - window.start
|
||||
args = {
|
||||
**kwargs,
|
||||
"num_frames": frames,
|
||||
"latent_t": _video_t(frames),
|
||||
"audio_t": window.audio_end - window.audio_start,
|
||||
# Keep fractional RoPE units; audio slice indices alone are rounded.
|
||||
"temporal_offset": window.start * (40.0 / 24.0),
|
||||
}
|
||||
if text_conditioning is not None:
|
||||
args["text_embeddings"], args["text_tags"] = text_conditioning[index]
|
||||
args["media_time_origin"] = media_time_origin
|
||||
if source is not None:
|
||||
args["locked_audio_rows"] = source[:, window.audio_start : window.audio_end].reshape(-1, 32)
|
||||
overlap_v = _video_t(window.overlap) if window.overlap else 0
|
||||
if video is not None and audio is not None:
|
||||
guide_video = minimax_h3_patchify_video_latent(video[:, :, -overlap_v:], patch_size=(1, 2, 2))
|
||||
guide_audio = minimax_h3_pack_audio_latent(audio[..., -window.overlap_audio_t :])
|
||||
for name, guide in (("visual_condition", guide_video), ("audio_condition", guide_audio)):
|
||||
existing = kwargs.get(name)
|
||||
args[name] = guide if existing is None else torch.cat((existing, guide), dim=0)
|
||||
args["visual_condition_shapes"] = [
|
||||
*(kwargs.get("visual_condition_shapes") or []),
|
||||
(overlap_v, kwargs["latent_h"], kwargs["latent_w"]),
|
||||
]
|
||||
args["audio_condition_lengths"] = [*(kwargs.get("audio_condition_lengths") or []), window.overlap_audio_t]
|
||||
args["ref_blocks"] = [
|
||||
*(kwargs.get("ref_blocks") or []),
|
||||
{
|
||||
"kind": "latent_guide",
|
||||
"latent_t": overlap_v,
|
||||
"latent_h": kwargs["latent_h"],
|
||||
"latent_w": kwargs["latent_w"],
|
||||
"ref_audio_t": window.overlap_audio_t,
|
||||
},
|
||||
]
|
||||
logger.info(
|
||||
"MiniMax H3 continuation window %d/%d: frames [%d, %d), hidden overlap %d, temporal offset %.6f",
|
||||
index + 1,
|
||||
len(windows),
|
||||
window.start,
|
||||
window.end,
|
||||
window.overlap,
|
||||
args["temporal_offset"],
|
||||
)
|
||||
sampled_video, sampled_audio = diffuse(**args)
|
||||
video = sampled_video if video is None else torch.cat((video, sampled_video[:, :, overlap_v:]), dim=2)
|
||||
audio = (
|
||||
sampled_audio if audio is None else torch.cat((audio, sampled_audio[..., window.overlap_audio_t :]), dim=-1)
|
||||
)
|
||||
assert video is not None and audio is not None
|
||||
if video.shape[2] != kwargs["latent_t"] or audio.shape[-1] != kwargs["audio_t"]:
|
||||
raise RuntimeError("MiniMax H3 continuation produced inconsistent cumulative AV lengths")
|
||||
return video, audio
|
||||
@@ -87,6 +87,7 @@ class MiniMaxH3DenoiseBranch:
|
||||
raise ValueError(f"token_tags length {int(token_tags.view(-1).shape[0])} != seq_len {seq_len}")
|
||||
cu = packed["cu_seqlens"].to(torch.int32)
|
||||
self.device = device
|
||||
self.locked_audio_rows: torch.Tensor | None = None
|
||||
# ``used_len`` is the real (non-padding) document length; step-mode
|
||||
# batching concatenates layouts and needs both bounds as Python ints so
|
||||
# it can rebuild ``cu_seqlens`` without a device sync per step.
|
||||
@@ -245,7 +246,7 @@ class MiniMaxH3DenoiseBranch:
|
||||
self.audio_pos_dev,
|
||||
self.audio_update_mask_dev,
|
||||
audio_target_timesteps,
|
||||
t_audio,
|
||||
t_audio if self.locked_audio_rows is None else 1.0,
|
||||
audio_ref_cond_timestep,
|
||||
),
|
||||
)
|
||||
@@ -444,7 +445,7 @@ def minimax_h3_denoise_loop(
|
||||
audio_rows[audio_update], x0_audio, sigma_curr=s_a, sigma_next=s_a_next
|
||||
)
|
||||
audio_rows = audio_rows.clone()
|
||||
audio_rows[audio_update] = new_audio
|
||||
audio_rows[audio_update] = new_audio if positive.locked_audio_rows is None else positive.locked_audio_rows
|
||||
if audio_anchor is not None:
|
||||
audio_rows[~audio_update] = audio_anchor # per-step audio ref reset
|
||||
if on_step is not None:
|
||||
|
||||
@@ -14,6 +14,7 @@ Builder rules:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
import numpy as np
|
||||
@@ -345,6 +346,8 @@ def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
ref_blocks: Sequence[Mapping[str, object]],
|
||||
audio_channel: int = 2,
|
||||
seq_len: int | None = None,
|
||||
temporal_offset: float = 0.0,
|
||||
media_time_origin: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""General ref2va-family packed layout.
|
||||
|
||||
@@ -353,12 +356,22 @@ def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
- ``{"kind": "audio", "ref_audio_t": T}``
|
||||
- ``{"kind": "video"|"video_audio", "ref_audio_t": T,
|
||||
"latent_t": RT, "latent_h": RH, "latent_w": RW}``
|
||||
- Internal ``latent_guide``: a final AV condition block on the target's
|
||||
spatial grid and temporal origin, without advancing the reference clock.
|
||||
|
||||
Video-bearing blocks pack their audio rows immediately before their video
|
||||
rows; both share the same temporal origin and advance by the longer of the
|
||||
audio and video spans. Standalone audio advances the target origin by its
|
||||
own T, and image blocks advance it by one integer slot.
|
||||
|
||||
``temporal_offset`` is the window origin in 40-Hz RoPE units. It shifts
|
||||
target AV, reference AV and latent guides together, leaving text, static
|
||||
images, padding and all spatial coordinates unchanged.
|
||||
"""
|
||||
if not math.isfinite(temporal_offset) or temporal_offset < 0:
|
||||
raise ValueError("temporal_offset must be finite and non-negative")
|
||||
if media_time_origin is not None and media_time_origin < text_len:
|
||||
raise ValueError("media_time_origin must follow the text prefix")
|
||||
if not isinstance(ref_blocks, Sequence) or isinstance(ref_blocks, (str, bytes)):
|
||||
raise ValueError("ref_blocks must be a sequence")
|
||||
|
||||
@@ -383,11 +396,15 @@ def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
rows = rt * audio_channel
|
||||
item = {"kind": kind, "ref_audio_t": rt, "audio_rows": rows}
|
||||
ref_audio_rows += rows
|
||||
elif kind in ("video", "video_audio"):
|
||||
elif kind in ("video", "video_audio", "latent_guide"):
|
||||
rt = _positive_int(raw, "ref_audio_t", path, allow_zero=True)
|
||||
vt = _positive_int(raw, "latent_t", path)
|
||||
vh = _positive_int(raw, "latent_h", path)
|
||||
vw = _positive_int(raw, "latent_w", path)
|
||||
if kind == "latent_guide" and (
|
||||
index != len(ref_blocks) - 1 or (vh, vw) != (latent_h, latent_w) or vt > latent_t or rt > audio_t
|
||||
):
|
||||
raise ValueError("latent_guide must be last, match the target canvas, and fit its AV lengths")
|
||||
frame_rows = (vh // _PATCH_H) * (vw // _PATCH_W)
|
||||
audio_rows = rt * audio_channel
|
||||
video_rows = vt * frame_rows
|
||||
@@ -447,7 +464,7 @@ def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
# device-to-host synchronization.
|
||||
video_spans: list[dict[str, object]] = []
|
||||
for item in block_slices:
|
||||
if item["kind"] in ("video", "video_audio"):
|
||||
if item["kind"] in ("video", "video_audio", "latent_guide"):
|
||||
visual_sl = item["visual_sl"]
|
||||
assert isinstance(visual_sl, slice)
|
||||
video_spans.append(
|
||||
@@ -485,7 +502,7 @@ def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
hh, ww = torch.meshgrid(h_grid, w_grid, indexing="ij")
|
||||
target_frame = torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1)
|
||||
|
||||
t_cursor = float(text_len)
|
||||
t_cursor = float(text_len if media_time_origin is None else media_time_origin)
|
||||
for item in block_slices:
|
||||
kind = str(item["kind"])
|
||||
if kind == "image":
|
||||
@@ -567,7 +584,8 @@ def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
rv_g[:, :, 0] = _video_t_grid(vt, t_cursor)[:, None]
|
||||
rv_g[:, :, 1:] = rv_frame[None]
|
||||
g[visual_sl] = rv_g.reshape(-1, 3)
|
||||
t_cursor += max(float(ref_t), _video_t_span(vt))
|
||||
if kind != "latent_guide":
|
||||
t_cursor += max(float(ref_t), _video_t_span(vt))
|
||||
|
||||
input_ids[audio_sl] = MINIMAX_H3_AUDIO_ID
|
||||
input_ids[audio_sl.start] = MINIMAX_H3_AUDIO_FIRST_ID
|
||||
@@ -591,6 +609,13 @@ def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
video_g[:, :, 1:] = target_frame[None]
|
||||
g[video_sl] = video_g.reshape(-1, 3)
|
||||
|
||||
if temporal_offset:
|
||||
g[audio_mask, 0] += temporal_offset
|
||||
g[video_sl, 0] += temporal_offset
|
||||
for item in block_slices:
|
||||
if item["kind"] in ("video", "video_audio", "latent_guide"):
|
||||
g[item["visual_sl"], 0] += temporal_offset
|
||||
|
||||
target_img_pos = _range_for_slice(video_sl)
|
||||
target_audio_pos = _range_for_slice(audio_sl)
|
||||
img_pos = _cat_ranges(ref_img_pos_parts + [target_img_pos])
|
||||
|
||||
@@ -81,6 +81,7 @@ from vllm_omni.model_executor.models.minimax_h3.encoder_processing import (
|
||||
encode_media,
|
||||
prepare_encoder_inputs,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.long_video import validate_encoded_frame_limit
|
||||
from vllm_omni.model_executor.models.minimax_h3.preprocessing import build_minimax_h3_presentation
|
||||
from vllm_omni.model_executor.models.minimax_h3.reference_video import (
|
||||
MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY,
|
||||
@@ -99,6 +100,7 @@ from .condition_noise import (
|
||||
minimax_h3_audio_cond_noise_aug_rows,
|
||||
minimax_h3_imgvid_cond_noise_aug_rows,
|
||||
)
|
||||
from .continuation import diffuse_continuation, plan_continuation_windows, resolve_continuation
|
||||
from .denoise_loop import (
|
||||
MiniMaxH3DenoiseBranch,
|
||||
minimax_h3_denoise_loop,
|
||||
@@ -267,6 +269,7 @@ _MINIMAX_H3_DENOISE_INPUT_KEYS = (
|
||||
"audio_condition_lengths",
|
||||
"keyframe_frame_indices",
|
||||
"pad_seq_len",
|
||||
"locked_audio_rows",
|
||||
"video_edit_clean_rows",
|
||||
"video_edit_mask_rows",
|
||||
"video_edit_restore_mask_rows",
|
||||
@@ -1177,6 +1180,7 @@ class MiniMaxH3Pipeline(
|
||||
requested: str | None,
|
||||
multi_modal_data: dict[str, Any] | None = None,
|
||||
*,
|
||||
audio_mode: str = "native",
|
||||
turbo_spec: TurboSpec | None = None,
|
||||
has_native_lora: bool = False,
|
||||
) -> str:
|
||||
@@ -1186,7 +1190,9 @@ class MiniMaxH3Pipeline(
|
||||
# historical implicit default even for image-only references.
|
||||
if self.partition == "ref2va":
|
||||
requested = "ref2va"
|
||||
elif multi_modal_data.get("video") is not None or multi_modal_data.get("audio") is not None:
|
||||
elif multi_modal_data.get("video") is not None or (
|
||||
multi_modal_data.get("audio") is not None and audio_mode != "lock_source"
|
||||
):
|
||||
requested = "ref2va"
|
||||
elif multi_modal_data.get("image") is not None:
|
||||
requested = "fl2va"
|
||||
@@ -1567,6 +1573,9 @@ class MiniMaxH3Pipeline(
|
||||
audio_condition_lengths: list[int] | None = None,
|
||||
keyframe_frame_indices: list[int] | None = None,
|
||||
pad_seq_len: int | None = None,
|
||||
locked_audio_rows: torch.Tensor | None = None,
|
||||
temporal_offset: float = 0.0,
|
||||
media_time_origin: int | None = None,
|
||||
video_edit_clean_rows: torch.Tensor | None = None,
|
||||
video_edit_mask_rows: torch.Tensor | None = None,
|
||||
video_edit_restore_mask_rows: torch.Tensor | None = None,
|
||||
@@ -1603,6 +1612,8 @@ class MiniMaxH3Pipeline(
|
||||
audio_t=audio_t,
|
||||
ref_blocks=ref_blocks,
|
||||
seq_len=pad_seq_len,
|
||||
temporal_offset=temporal_offset,
|
||||
media_time_origin=media_time_origin,
|
||||
)
|
||||
else:
|
||||
packed = minimax_h3_packed_sequence(
|
||||
@@ -1660,6 +1671,13 @@ class MiniMaxH3Pipeline(
|
||||
full_video[branch.update_mask] = initial_video
|
||||
initial_video = full_video
|
||||
|
||||
if locked_audio_rows is not None:
|
||||
expected = (2 * audio_t, 32)
|
||||
if tuple(locked_audio_rows.shape) != expected:
|
||||
raise OmniClientError(f"MiniMax H3 driving audio rows must have shape {expected}")
|
||||
branch.locked_audio_rows = locked_audio_rows.to(device=self.device, dtype=torch.float32)
|
||||
initial_audio = branch.locked_audio_rows.cpu().clone()
|
||||
|
||||
audio_anchor = audio_condition
|
||||
if audio_anchor is not None:
|
||||
condition_audio_t = audio_condition_lengths
|
||||
@@ -1800,6 +1818,9 @@ class MiniMaxH3Pipeline(
|
||||
audio_condition_lengths: list[int] | None = None,
|
||||
keyframe_frame_indices: list[int] | None = None,
|
||||
pad_seq_len: int | None = None,
|
||||
locked_audio_rows: torch.Tensor | None = None,
|
||||
temporal_offset: float = 0.0,
|
||||
media_time_origin: int | None = None,
|
||||
video_edit_clean_rows: torch.Tensor | None = None,
|
||||
video_edit_mask_rows: torch.Tensor | None = None,
|
||||
video_edit_restore_mask_rows: torch.Tensor | None = None,
|
||||
@@ -1830,6 +1851,9 @@ class MiniMaxH3Pipeline(
|
||||
audio_condition_lengths=audio_condition_lengths,
|
||||
keyframe_frame_indices=keyframe_frame_indices,
|
||||
pad_seq_len=pad_seq_len,
|
||||
locked_audio_rows=locked_audio_rows,
|
||||
temporal_offset=temporal_offset,
|
||||
media_time_origin=media_time_origin,
|
||||
video_edit_clean_rows=video_edit_clean_rows,
|
||||
video_edit_mask_rows=video_edit_mask_rows,
|
||||
video_edit_restore_mask_rows=video_edit_restore_mask_rows,
|
||||
@@ -2160,8 +2184,14 @@ class MiniMaxH3Pipeline(
|
||||
sampling: Any,
|
||||
*,
|
||||
require_external_text: bool = False,
|
||||
) -> MiniMaxH3EncoderConditioning:
|
||||
) -> tuple[MiniMaxH3EncoderConditioning, list[tuple[torch.Tensor, torch.Tensor]] | None]:
|
||||
"""Prepare media once and encode either shared text or each window's text.
|
||||
|
||||
Returns initial conditioning and optional request-owned window embeddings.
|
||||
Invalid continuation prompts are broadcast before any encoder collective.
|
||||
"""
|
||||
group, rank, world_size = _dit_rank_world()
|
||||
prompts = (sampling.extra_args or {}).get("continuation_prompts")
|
||||
prepared = text_conditioning = None
|
||||
error = None
|
||||
if rank == 0:
|
||||
@@ -2172,6 +2202,7 @@ class MiniMaxH3Pipeline(
|
||||
task = self._resolve_task(
|
||||
(sampling.extra_args or {}).get("task"),
|
||||
multi_modal_data,
|
||||
audio_mode=(sampling.extra_args or {}).get("audio_mode", "native"),
|
||||
turbo_spec=turbo_spec,
|
||||
has_native_lora=has_native_lora,
|
||||
)
|
||||
@@ -2190,6 +2221,27 @@ class MiniMaxH3Pipeline(
|
||||
task=task,
|
||||
prepared_reference_videos=self._extract_prepared_reference_videos(raw_prompt),
|
||||
)
|
||||
if prompts is not None:
|
||||
continuation = resolve_continuation(
|
||||
sampling.extra_args or {},
|
||||
task=task,
|
||||
step_execution=bool(getattr(self.od_config, "step_execution", False)),
|
||||
)
|
||||
if continuation is None or not self.load_text_encoder:
|
||||
raise OmniClientError(
|
||||
"continuation_prompts requires continuation mode with a local text encoder"
|
||||
)
|
||||
window, overlap = continuation
|
||||
count = len(plan_continuation_windows(prepared.media.num_frames, window, overlap))
|
||||
if (
|
||||
not isinstance(prompts, list)
|
||||
or len(prompts) != count
|
||||
or any(not isinstance(prompt, str) or not prompt.strip() for prompt in prompts)
|
||||
):
|
||||
raise OmniClientError(f"continuation_prompts must contain exactly {count} non-empty strings")
|
||||
prepared = replace(prepared, prompt=prompts[0])
|
||||
# External text belongs to the main prompt, not the first window.
|
||||
text_conditioning = None
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
_broadcast_rank0_exception(error)
|
||||
@@ -2207,21 +2259,33 @@ class MiniMaxH3Pipeline(
|
||||
)
|
||||
else:
|
||||
hidden, tags = self.encode_prompt(prepared)
|
||||
window_text = None
|
||||
if prompts is not None:
|
||||
window_text = [(hidden, tags)]
|
||||
for index, prompt in enumerate(prompts[1:], start=2):
|
||||
logger.info("MiniMax H3 encoding continuation prompt %d/%d", index, len(prompts))
|
||||
window_text.append(self.encode_prompt(replace(prepared, prompt=prompt) if rank == 0 else None))
|
||||
media = self._encode_local_media(prepared.media if prepared is not None else None)
|
||||
return MiniMaxH3EncoderConditioning.from_components(MiniMaxH3TextConditioning(hidden, tags), media)
|
||||
return MiniMaxH3EncoderConditioning.from_components(MiniMaxH3TextConditioning(hidden, tags), media), window_text
|
||||
|
||||
def _prepare_request_inputs(self, raw_prompt: Any, sampling: Any) -> dict[str, Any]:
|
||||
if self.load_text_encoder:
|
||||
conditioning = self._prepare_local_conditioning(raw_prompt, sampling)
|
||||
elif self.load_vae_encoder:
|
||||
conditioning = self._prepare_local_conditioning(
|
||||
if (getattr(sampling, "extra_args", None) or {}).get(
|
||||
"continuation_prompts"
|
||||
) is not None and not self.load_text_encoder:
|
||||
raise OmniClientError("continuation_prompts requires continuation mode with a local text encoder")
|
||||
window_text = None
|
||||
if self.load_text_encoder or self.load_vae_encoder:
|
||||
conditioning, window_text = self._prepare_local_conditioning(
|
||||
raw_prompt,
|
||||
sampling,
|
||||
require_external_text=True,
|
||||
require_external_text=not self.load_text_encoder,
|
||||
)
|
||||
else:
|
||||
conditioning = self._extract_encoder_conditioning(raw_prompt)
|
||||
return self._prepare_encoder_conditioning_inputs(conditioning, sampling)
|
||||
context = self._prepare_encoder_conditioning_inputs(conditioning, sampling)
|
||||
if window_text is not None:
|
||||
context["continuation_text_conditioning"] = window_text
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _extract_encoder_conditioning(prompt: Any) -> MiniMaxH3EncoderConditioning:
|
||||
@@ -2242,6 +2306,18 @@ class MiniMaxH3Pipeline(
|
||||
sampling: Any,
|
||||
) -> dict[str, Any]:
|
||||
extra = sampling.extra_args or {}
|
||||
continuation = resolve_continuation(
|
||||
extra, task=conditioning.task, step_execution=bool(getattr(self.od_config, "step_execution", False))
|
||||
)
|
||||
validate_encoded_frame_limit(extra, conditioning.task, conditioning.num_frames)
|
||||
if continuation is not None and (
|
||||
conditioning.video_edit_clean_rows is not None or conditioning.audio_edit_clean_rows is not None
|
||||
):
|
||||
raise OmniClientError(
|
||||
"MiniMax H3 continuation does not support latent-mask editing; use long_video_mode=full"
|
||||
)
|
||||
if extra.get("audio_mode") == "lock_source" and conditioning.audio_edit_clean_rows is not None:
|
||||
raise OmniClientError("MiniMax H3 audio_mode=lock_source cannot be combined with audio latent-mask editing")
|
||||
requested_task = extra.get("task")
|
||||
if requested_task is not None and str(requested_task).lower() != conditioning.task:
|
||||
raise OmniClientError(
|
||||
@@ -2285,6 +2361,22 @@ class MiniMaxH3Pipeline(
|
||||
|
||||
visual_shapes = list(conditioning.visual_condition_shapes) or None
|
||||
audio_lengths = list(conditioning.audio_condition_lengths) or None
|
||||
audio_condition = conditioning.audio_condition
|
||||
locked_audio_rows = None
|
||||
if extra.get("audio_mode", "native") == "lock_source":
|
||||
if audio_condition is None or not audio_lengths:
|
||||
raise OmniClientError("MiniMax H3 lock_source requires encoded driving audio")
|
||||
drive_t = audio_lengths[-1]
|
||||
drive = audio_condition[-2 * drive_t :].reshape(2, drive_t, 32)
|
||||
# Pad/crop each stereo channel independently; flattening first would
|
||||
# shift the right channel when the source and target lengths differ.
|
||||
fitted = drive.new_zeros((2, conditioning.audio_t, 32))
|
||||
count = min(drive_t, conditioning.audio_t)
|
||||
fitted[:, :count] = drive[:, :count]
|
||||
locked_audio_rows = fitted.reshape(-1, 32).to(device=self.device)
|
||||
audio_condition = audio_condition[: -2 * drive_t]
|
||||
audio_condition = audio_condition if audio_condition.numel() else None
|
||||
audio_lengths = audio_lengths[:-1] or None
|
||||
|
||||
latent_edit: dict[str, torch.Tensor | None] = {
|
||||
"video_edit_clean_rows": None,
|
||||
@@ -2336,8 +2428,11 @@ class MiniMaxH3Pipeline(
|
||||
num_inference_steps=num_steps,
|
||||
extra_args=extra,
|
||||
)
|
||||
if continuation is not None and quality_plan.cache_dit is not None:
|
||||
raise OmniClientError("MiniMax H3 continuation requires uncached denoising; set quality=lossless")
|
||||
self._cache_dit_runtime.prepare(quality_plan.cache_dit)
|
||||
return {
|
||||
"continuation": continuation,
|
||||
"task": task,
|
||||
"height": conditioning.height,
|
||||
"width": conditioning.width,
|
||||
@@ -2354,15 +2449,12 @@ class MiniMaxH3Pipeline(
|
||||
else None
|
||||
),
|
||||
"visual_condition_shape": visual_shapes[0] if visual_shapes and len(visual_shapes) == 1 else None,
|
||||
"audio_condition": (
|
||||
conditioning.audio_condition.to(device=self.device)
|
||||
if conditioning.audio_condition is not None
|
||||
else None
|
||||
),
|
||||
"audio_condition": (audio_condition.to(device=self.device) if audio_condition is not None else None),
|
||||
"ref_audio_t": audio_lengths[0] if audio_lengths and len(audio_lengths) == 1 else None,
|
||||
"ref_blocks": list(conditioning.ref_blocks) or None,
|
||||
"visual_condition_shapes": visual_shapes,
|
||||
"audio_condition_lengths": audio_lengths,
|
||||
"locked_audio_rows": locked_audio_rows,
|
||||
"keyframe_frame_indices": list(conditioning.keyframe_frame_indices) or None,
|
||||
"pad_seq_len": _resolve_pad_seq_len(extra.get("pad_seq_len")),
|
||||
"seed": int(sampling.seed if sampling.seed is not None else 42),
|
||||
@@ -2403,7 +2495,18 @@ class MiniMaxH3Pipeline(
|
||||
videos = []
|
||||
audios = []
|
||||
for output_seed in _minimax_h3_output_seeds(context["seed"], num_outputs):
|
||||
video_latent, audio_latent = self.diffuse(**{**denoise_kwargs, "seed": output_seed})
|
||||
output_kwargs = {**denoise_kwargs, "seed": output_seed}
|
||||
if context.get("continuation") is None:
|
||||
video_latent, audio_latent = self.diffuse(**output_kwargs)
|
||||
else:
|
||||
window_frames, overlap_frames = context["continuation"]
|
||||
video_latent, audio_latent = diffuse_continuation(
|
||||
self.diffuse,
|
||||
output_kwargs,
|
||||
window_frames=window_frames,
|
||||
overlap_frames=overlap_frames,
|
||||
text_conditioning=context.get("continuation_text_conditioning"),
|
||||
)
|
||||
if context["preencode_mp4"]:
|
||||
videos.append(
|
||||
self.decode_to_mp4(
|
||||
@@ -2765,7 +2868,7 @@ class MiniMaxH3Pipeline(
|
||||
sigma_next=schedule["sigma_audio_next"],
|
||||
)
|
||||
audio_rows = audio_rows.clone()
|
||||
audio_rows[audio_update] = new_audio
|
||||
audio_rows[audio_update] = new_audio if branch.locked_audio_rows is None else branch.locked_audio_rows
|
||||
if audio_anchor is not None:
|
||||
audio_rows[~audio_update] = audio_anchor # per-step audio ref reset
|
||||
|
||||
|
||||
Reference in new issue
Block a user