[Model] Add MiniMax-H3 long-video latent continuation with driving audio (#7838)

Signed-off-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
汪志鹏 authored and GitHub committed 2026-09-20 23:44:33 +08:00
1 parent 16734bbdf3
commit 139a47a578
17 files changed
+1089 -50

No files matched your search

@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Bounded H3 windows with synchronized latent-tail guides and global AV RoPE positions.
The guide/discard/append algorithm follows ComfyUI-Minimax-H3-Continuation:
https://github.com/ttulttul/ComfyUI-Minimax-H3-Continuation
"""
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import Any
import torch
from vllm.logger import init_logger
from vllm_omni.errors import OmniClientError
from vllm_omni.model_executor.models.minimax_h3.long_video import resolve_long_video_mode
from .packed_tokens import minimax_h3_pack_audio_latent, minimax_h3_patchify_video_latent
logger = init_logger(__name__)
def _video_t(frames: int) -> int:
return (frames - 5) // 17 * 5 + 2
def _audio_t(frame_boundary: int) -> int:
return round(frame_boundary * 5 / 3)
@dataclass(frozen=True)
class ContinuationWindow:
start: int
end: int
overlap: int
@property
def audio_start(self) -> int:
return _audio_t(self.start)
@property
def audio_end(self) -> int:
return _audio_t(self.end)
@property
def overlap_audio_t(self) -> int:
return _audio_t(self.start + self.overlap) - self.audio_start
def resolve_continuation(extra: Mapping[str, Any], *, task: str, step_execution: bool) -> tuple[int, int] | None:
mode = resolve_long_video_mode(extra, task)
if mode == "full":
return None
if task != "ref2va" or step_execution:
raise OmniClientError("MiniMax H3 continuation requires Ref2VA request execution (not step execution)")
window = extra.get("continuation_window_frames", 277)
overlap = extra.get("continuation_overlap_frames", 22)
for name, value in (("window", window), ("overlap", overlap)):
if isinstance(value, bool) or not isinstance(value, int) or value < 5 or value % 17 != 5:
raise OmniClientError(f"MiniMax H3 continuation {name} must be an integer on the 17n+5 frame grid")
if not 107 <= window <= 345 or overlap >= window:
raise OmniClientError("MiniMax H3 continuation window must be 107..345 frames and exceed its overlap")
return window, overlap
def plan_continuation_windows(total_frames: int, window_frames: int, overlap_frames: int) -> list[ContinuationWindow]:
if total_frames < 5 or total_frames % 17 != 5:
raise ValueError("total_frames must satisfy 17n+5")
if not 5 <= overlap_frames < window_frames or window_frames % 17 != 5 or overlap_frames % 17 != 5:
raise ValueError("window and overlap must satisfy 17n+5, with 5 <= overlap < window")
end = min(total_frames, window_frames)
windows = [ContinuationWindow(0, end, 0)]
while end < total_frames:
start = end - overlap_frames
end = min(total_frames, start + window_frames)
windows.append(ContinuationWindow(start, end, overlap_frames))
return windows
def diffuse_continuation(
diffuse: Callable[..., tuple[torch.Tensor, torch.Tensor]],
kwargs: dict[str, Any],
*,
window_frames: int,
overlap_frames: int,
text_conditioning: Sequence[tuple[torch.Tensor, torch.Tensor]] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Denoise fresh windows; retain old latents and append only new suffixes.
Guides are extra condition rows sharing the new target's temporal origin,
not a masked target prefix. Audio boundaries refer to the cumulative frame
timeline, preventing per-window rounding from accumulating A/V drift.
Each window shifts temporal media positions onto that same global timeline
before RoPE is evaluated; text and static image references remain fixed.
"""
windows = plan_continuation_windows(kwargs["num_frames"], window_frames, overlap_frames)
if text_conditioning is not None and len(text_conditioning) != len(windows):
raise OmniClientError("MiniMax H3 requires one text conditioning per continuation window")
# Different prompts have different prefix lengths. Keep the media clock
# anchored after the longest prefix so text length cannot shift AV time.
media_time_origin = max(item[0].shape[0] for item in text_conditioning) if text_conditioning else None
source_rows = kwargs.get("locked_audio_rows")
source = None if source_rows is None else source_rows.reshape(2, kwargs["audio_t"], 32)
video = audio = None
for index, window in enumerate(windows):
frames = window.end - window.start
args = {
**kwargs,
"num_frames": frames,
"latent_t": _video_t(frames),
"audio_t": window.audio_end - window.audio_start,
# Keep fractional RoPE units; audio slice indices alone are rounded.
"temporal_offset": window.start * (40.0 / 24.0),
}
if text_conditioning is not None:
args["text_embeddings"], args["text_tags"] = text_conditioning[index]
args["media_time_origin"] = media_time_origin
if source is not None:
args["locked_audio_rows"] = source[:, window.audio_start : window.audio_end].reshape(-1, 32)
overlap_v = _video_t(window.overlap) if window.overlap else 0
if video is not None and audio is not None:
guide_video = minimax_h3_patchify_video_latent(video[:, :, -overlap_v:], patch_size=(1, 2, 2))
guide_audio = minimax_h3_pack_audio_latent(audio[..., -window.overlap_audio_t :])
for name, guide in (("visual_condition", guide_video), ("audio_condition", guide_audio)):
existing = kwargs.get(name)
args[name] = guide if existing is None else torch.cat((existing, guide), dim=0)
args["visual_condition_shapes"] = [
*(kwargs.get("visual_condition_shapes") or []),
(overlap_v, kwargs["latent_h"], kwargs["latent_w"]),
]
args["audio_condition_lengths"] = [*(kwargs.get("audio_condition_lengths") or []), window.overlap_audio_t]
args["ref_blocks"] = [
*(kwargs.get("ref_blocks") or []),
{
"kind": "latent_guide",
"latent_t": overlap_v,
"latent_h": kwargs["latent_h"],
"latent_w": kwargs["latent_w"],
"ref_audio_t": window.overlap_audio_t,
},
]
logger.info(
"MiniMax H3 continuation window %d/%d: frames [%d, %d), hidden overlap %d, temporal offset %.6f",
index + 1,
len(windows),
window.start,
window.end,
window.overlap,
args["temporal_offset"],
)
sampled_video, sampled_audio = diffuse(**args)
video = sampled_video if video is None else torch.cat((video, sampled_video[:, :, overlap_v:]), dim=2)
audio = (
sampled_audio if audio is None else torch.cat((audio, sampled_audio[..., window.overlap_audio_t :]), dim=-1)
)
assert video is not None and audio is not None
if video.shape[2] != kwargs["latent_t"] or audio.shape[-1] != kwargs["audio_t"]:
raise RuntimeError("MiniMax H3 continuation produced inconsistent cumulative AV lengths")
return video, audio
@@ -87,6 +87,7 @@ class MiniMaxH3DenoiseBranch:
raise ValueError(f"token_tags length {int(token_tags.view(-1).shape[0])} != seq_len {seq_len}")
cu = packed["cu_seqlens"].to(torch.int32)
self.device = device
self.locked_audio_rows: torch.Tensor | None = None
# ``used_len`` is the real (non-padding) document length; step-mode
# batching concatenates layouts and needs both bounds as Python ints so
# it can rebuild ``cu_seqlens`` without a device sync per step.
@@ -245,7 +246,7 @@ class MiniMaxH3DenoiseBranch:
self.audio_pos_dev,
self.audio_update_mask_dev,
audio_target_timesteps,
t_audio,
t_audio if self.locked_audio_rows is None else 1.0,
audio_ref_cond_timestep,
),
)
@@ -444,7 +445,7 @@ def minimax_h3_denoise_loop(
audio_rows[audio_update], x0_audio, sigma_curr=s_a, sigma_next=s_a_next
)
audio_rows = audio_rows.clone()
audio_rows[audio_update] = new_audio
audio_rows[audio_update] = new_audio if positive.locked_audio_rows is None else positive.locked_audio_rows
if audio_anchor is not None:
audio_rows[~audio_update] = audio_anchor # per-step audio ref reset
if on_step is not None:
@@ -14,6 +14,7 @@ Builder rules:
from __future__ import annotations
import math
from collections.abc import Mapping, Sequence
import numpy as np
@@ -345,6 +346,8 @@ def minimax_h3_packed_sequence_ref2va_blocks(
ref_blocks: Sequence[Mapping[str, object]],
audio_channel: int = 2,
seq_len: int | None = None,
temporal_offset: float = 0.0,
media_time_origin: int | None = None,
) -> dict[str, object]:
"""General ref2va-family packed layout.
@@ -353,12 +356,22 @@ def minimax_h3_packed_sequence_ref2va_blocks(
- ``{"kind": "audio", "ref_audio_t": T}``
- ``{"kind": "video"|"video_audio", "ref_audio_t": T,
"latent_t": RT, "latent_h": RH, "latent_w": RW}``
- Internal ``latent_guide``: a final AV condition block on the target's
spatial grid and temporal origin, without advancing the reference clock.
Video-bearing blocks pack their audio rows immediately before their video
rows; both share the same temporal origin and advance by the longer of the
audio and video spans. Standalone audio advances the target origin by its
own T, and image blocks advance it by one integer slot.
``temporal_offset`` is the window origin in 40-Hz RoPE units. It shifts
target AV, reference AV and latent guides together, leaving text, static
images, padding and all spatial coordinates unchanged.
"""
if not math.isfinite(temporal_offset) or temporal_offset < 0:
raise ValueError("temporal_offset must be finite and non-negative")
if media_time_origin is not None and media_time_origin < text_len:
raise ValueError("media_time_origin must follow the text prefix")
if not isinstance(ref_blocks, Sequence) or isinstance(ref_blocks, (str, bytes)):
raise ValueError("ref_blocks must be a sequence")
@@ -383,11 +396,15 @@ def minimax_h3_packed_sequence_ref2va_blocks(
rows = rt * audio_channel
item = {"kind": kind, "ref_audio_t": rt, "audio_rows": rows}
ref_audio_rows += rows
elif kind in ("video", "video_audio"):
elif kind in ("video", "video_audio", "latent_guide"):
rt = _positive_int(raw, "ref_audio_t", path, allow_zero=True)
vt = _positive_int(raw, "latent_t", path)
vh = _positive_int(raw, "latent_h", path)
vw = _positive_int(raw, "latent_w", path)
if kind == "latent_guide" and (
index != len(ref_blocks) - 1 or (vh, vw) != (latent_h, latent_w) or vt > latent_t or rt > audio_t
):
raise ValueError("latent_guide must be last, match the target canvas, and fit its AV lengths")
frame_rows = (vh // _PATCH_H) * (vw // _PATCH_W)
audio_rows = rt * audio_channel
video_rows = vt * frame_rows
@@ -447,7 +464,7 @@ def minimax_h3_packed_sequence_ref2va_blocks(
# device-to-host synchronization.
video_spans: list[dict[str, object]] = []
for item in block_slices:
if item["kind"] in ("video", "video_audio"):
if item["kind"] in ("video", "video_audio", "latent_guide"):
visual_sl = item["visual_sl"]
assert isinstance(visual_sl, slice)
video_spans.append(
@@ -485,7 +502,7 @@ def minimax_h3_packed_sequence_ref2va_blocks(
hh, ww = torch.meshgrid(h_grid, w_grid, indexing="ij")
target_frame = torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1)
t_cursor = float(text_len)
t_cursor = float(text_len if media_time_origin is None else media_time_origin)
for item in block_slices:
kind = str(item["kind"])
if kind == "image":
@@ -567,7 +584,8 @@ def minimax_h3_packed_sequence_ref2va_blocks(
rv_g[:, :, 0] = _video_t_grid(vt, t_cursor)[:, None]
rv_g[:, :, 1:] = rv_frame[None]
g[visual_sl] = rv_g.reshape(-1, 3)
t_cursor += max(float(ref_t), _video_t_span(vt))
if kind != "latent_guide":
t_cursor += max(float(ref_t), _video_t_span(vt))
input_ids[audio_sl] = MINIMAX_H3_AUDIO_ID
input_ids[audio_sl.start] = MINIMAX_H3_AUDIO_FIRST_ID
@@ -591,6 +609,13 @@ def minimax_h3_packed_sequence_ref2va_blocks(
video_g[:, :, 1:] = target_frame[None]
g[video_sl] = video_g.reshape(-1, 3)
if temporal_offset:
g[audio_mask, 0] += temporal_offset
g[video_sl, 0] += temporal_offset
for item in block_slices:
if item["kind"] in ("video", "video_audio", "latent_guide"):
g[item["visual_sl"], 0] += temporal_offset
target_img_pos = _range_for_slice(video_sl)
target_audio_pos = _range_for_slice(audio_sl)
img_pos = _cat_ranges(ref_img_pos_parts + [target_img_pos])
@@ -81,6 +81,7 @@ from vllm_omni.model_executor.models.minimax_h3.encoder_processing import (
encode_media,
prepare_encoder_inputs,
)
from vllm_omni.model_executor.models.minimax_h3.long_video import validate_encoded_frame_limit
from vllm_omni.model_executor.models.minimax_h3.preprocessing import build_minimax_h3_presentation
from vllm_omni.model_executor.models.minimax_h3.reference_video import (
MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY,
@@ -99,6 +100,7 @@ from .condition_noise import (
minimax_h3_audio_cond_noise_aug_rows,
minimax_h3_imgvid_cond_noise_aug_rows,
)
from .continuation import diffuse_continuation, plan_continuation_windows, resolve_continuation
from .denoise_loop import (
MiniMaxH3DenoiseBranch,
minimax_h3_denoise_loop,
@@ -267,6 +269,7 @@ _MINIMAX_H3_DENOISE_INPUT_KEYS = (
"audio_condition_lengths",
"keyframe_frame_indices",
"pad_seq_len",
"locked_audio_rows",
"video_edit_clean_rows",
"video_edit_mask_rows",
"video_edit_restore_mask_rows",
@@ -1177,6 +1180,7 @@ class MiniMaxH3Pipeline(
requested: str | None,
multi_modal_data: dict[str, Any] | None = None,
*,
audio_mode: str = "native",
turbo_spec: TurboSpec | None = None,
has_native_lora: bool = False,
) -> str:
@@ -1186,7 +1190,9 @@ class MiniMaxH3Pipeline(
# historical implicit default even for image-only references.
if self.partition == "ref2va":
requested = "ref2va"
elif multi_modal_data.get("video") is not None or multi_modal_data.get("audio") is not None:
elif multi_modal_data.get("video") is not None or (
multi_modal_data.get("audio") is not None and audio_mode != "lock_source"
):
requested = "ref2va"
elif multi_modal_data.get("image") is not None:
requested = "fl2va"
@@ -1567,6 +1573,9 @@ class MiniMaxH3Pipeline(
audio_condition_lengths: list[int] | None = None,
keyframe_frame_indices: list[int] | None = None,
pad_seq_len: int | None = None,
locked_audio_rows: torch.Tensor | None = None,
temporal_offset: float = 0.0,
media_time_origin: int | None = None,
video_edit_clean_rows: torch.Tensor | None = None,
video_edit_mask_rows: torch.Tensor | None = None,
video_edit_restore_mask_rows: torch.Tensor | None = None,
@@ -1603,6 +1612,8 @@ class MiniMaxH3Pipeline(
audio_t=audio_t,
ref_blocks=ref_blocks,
seq_len=pad_seq_len,
temporal_offset=temporal_offset,
media_time_origin=media_time_origin,
)
else:
packed = minimax_h3_packed_sequence(
@@ -1660,6 +1671,13 @@ class MiniMaxH3Pipeline(
full_video[branch.update_mask] = initial_video
initial_video = full_video
if locked_audio_rows is not None:
expected = (2 * audio_t, 32)
if tuple(locked_audio_rows.shape) != expected:
raise OmniClientError(f"MiniMax H3 driving audio rows must have shape {expected}")
branch.locked_audio_rows = locked_audio_rows.to(device=self.device, dtype=torch.float32)
initial_audio = branch.locked_audio_rows.cpu().clone()
audio_anchor = audio_condition
if audio_anchor is not None:
condition_audio_t = audio_condition_lengths
@@ -1800,6 +1818,9 @@ class MiniMaxH3Pipeline(
audio_condition_lengths: list[int] | None = None,
keyframe_frame_indices: list[int] | None = None,
pad_seq_len: int | None = None,
locked_audio_rows: torch.Tensor | None = None,
temporal_offset: float = 0.0,
media_time_origin: int | None = None,
video_edit_clean_rows: torch.Tensor | None = None,
video_edit_mask_rows: torch.Tensor | None = None,
video_edit_restore_mask_rows: torch.Tensor | None = None,
@@ -1830,6 +1851,9 @@ class MiniMaxH3Pipeline(
audio_condition_lengths=audio_condition_lengths,
keyframe_frame_indices=keyframe_frame_indices,
pad_seq_len=pad_seq_len,
locked_audio_rows=locked_audio_rows,
temporal_offset=temporal_offset,
media_time_origin=media_time_origin,
video_edit_clean_rows=video_edit_clean_rows,
video_edit_mask_rows=video_edit_mask_rows,
video_edit_restore_mask_rows=video_edit_restore_mask_rows,
@@ -2160,8 +2184,14 @@ class MiniMaxH3Pipeline(
sampling: Any,
*,
require_external_text: bool = False,
) -> MiniMaxH3EncoderConditioning:
) -> tuple[MiniMaxH3EncoderConditioning, list[tuple[torch.Tensor, torch.Tensor]] | None]:
"""Prepare media once and encode either shared text or each window's text.
Returns initial conditioning and optional request-owned window embeddings.
Invalid continuation prompts are broadcast before any encoder collective.
"""
group, rank, world_size = _dit_rank_world()
prompts = (sampling.extra_args or {}).get("continuation_prompts")
prepared = text_conditioning = None
error = None
if rank == 0:
@@ -2172,6 +2202,7 @@ class MiniMaxH3Pipeline(
task = self._resolve_task(
(sampling.extra_args or {}).get("task"),
multi_modal_data,
audio_mode=(sampling.extra_args or {}).get("audio_mode", "native"),
turbo_spec=turbo_spec,
has_native_lora=has_native_lora,
)
@@ -2190,6 +2221,27 @@ class MiniMaxH3Pipeline(
task=task,
prepared_reference_videos=self._extract_prepared_reference_videos(raw_prompt),
)
if prompts is not None:
continuation = resolve_continuation(
sampling.extra_args or {},
task=task,
step_execution=bool(getattr(self.od_config, "step_execution", False)),
)
if continuation is None or not self.load_text_encoder:
raise OmniClientError(
"continuation_prompts requires continuation mode with a local text encoder"
)
window, overlap = continuation
count = len(plan_continuation_windows(prepared.media.num_frames, window, overlap))
if (
not isinstance(prompts, list)
or len(prompts) != count
or any(not isinstance(prompt, str) or not prompt.strip() for prompt in prompts)
):
raise OmniClientError(f"continuation_prompts must contain exactly {count} non-empty strings")
prepared = replace(prepared, prompt=prompts[0])
# External text belongs to the main prompt, not the first window.
text_conditioning = None
except Exception as exc:
error = exc
_broadcast_rank0_exception(error)
@@ -2207,21 +2259,33 @@ class MiniMaxH3Pipeline(
)
else:
hidden, tags = self.encode_prompt(prepared)
window_text = None
if prompts is not None:
window_text = [(hidden, tags)]
for index, prompt in enumerate(prompts[1:], start=2):
logger.info("MiniMax H3 encoding continuation prompt %d/%d", index, len(prompts))
window_text.append(self.encode_prompt(replace(prepared, prompt=prompt) if rank == 0 else None))
media = self._encode_local_media(prepared.media if prepared is not None else None)
return MiniMaxH3EncoderConditioning.from_components(MiniMaxH3TextConditioning(hidden, tags), media)
return MiniMaxH3EncoderConditioning.from_components(MiniMaxH3TextConditioning(hidden, tags), media), window_text
def _prepare_request_inputs(self, raw_prompt: Any, sampling: Any) -> dict[str, Any]:
if self.load_text_encoder:
conditioning = self._prepare_local_conditioning(raw_prompt, sampling)
elif self.load_vae_encoder:
conditioning = self._prepare_local_conditioning(
if (getattr(sampling, "extra_args", None) or {}).get(
"continuation_prompts"
) is not None and not self.load_text_encoder:
raise OmniClientError("continuation_prompts requires continuation mode with a local text encoder")
window_text = None
if self.load_text_encoder or self.load_vae_encoder:
conditioning, window_text = self._prepare_local_conditioning(
raw_prompt,
sampling,
require_external_text=True,
require_external_text=not self.load_text_encoder,
)
else:
conditioning = self._extract_encoder_conditioning(raw_prompt)
return self._prepare_encoder_conditioning_inputs(conditioning, sampling)
context = self._prepare_encoder_conditioning_inputs(conditioning, sampling)
if window_text is not None:
context["continuation_text_conditioning"] = window_text
return context
@staticmethod
def _extract_encoder_conditioning(prompt: Any) -> MiniMaxH3EncoderConditioning:
@@ -2242,6 +2306,18 @@ class MiniMaxH3Pipeline(
sampling: Any,
) -> dict[str, Any]:
extra = sampling.extra_args or {}
continuation = resolve_continuation(
extra, task=conditioning.task, step_execution=bool(getattr(self.od_config, "step_execution", False))
)
validate_encoded_frame_limit(extra, conditioning.task, conditioning.num_frames)
if continuation is not None and (
conditioning.video_edit_clean_rows is not None or conditioning.audio_edit_clean_rows is not None
):
raise OmniClientError(
"MiniMax H3 continuation does not support latent-mask editing; use long_video_mode=full"
)
if extra.get("audio_mode") == "lock_source" and conditioning.audio_edit_clean_rows is not None:
raise OmniClientError("MiniMax H3 audio_mode=lock_source cannot be combined with audio latent-mask editing")
requested_task = extra.get("task")
if requested_task is not None and str(requested_task).lower() != conditioning.task:
raise OmniClientError(
@@ -2285,6 +2361,22 @@ class MiniMaxH3Pipeline(
visual_shapes = list(conditioning.visual_condition_shapes) or None
audio_lengths = list(conditioning.audio_condition_lengths) or None
audio_condition = conditioning.audio_condition
locked_audio_rows = None
if extra.get("audio_mode", "native") == "lock_source":
if audio_condition is None or not audio_lengths:
raise OmniClientError("MiniMax H3 lock_source requires encoded driving audio")
drive_t = audio_lengths[-1]
drive = audio_condition[-2 * drive_t :].reshape(2, drive_t, 32)
# Pad/crop each stereo channel independently; flattening first would
# shift the right channel when the source and target lengths differ.
fitted = drive.new_zeros((2, conditioning.audio_t, 32))
count = min(drive_t, conditioning.audio_t)
fitted[:, :count] = drive[:, :count]
locked_audio_rows = fitted.reshape(-1, 32).to(device=self.device)
audio_condition = audio_condition[: -2 * drive_t]
audio_condition = audio_condition if audio_condition.numel() else None
audio_lengths = audio_lengths[:-1] or None
latent_edit: dict[str, torch.Tensor | None] = {
"video_edit_clean_rows": None,
@@ -2336,8 +2428,11 @@ class MiniMaxH3Pipeline(
num_inference_steps=num_steps,
extra_args=extra,
)
if continuation is not None and quality_plan.cache_dit is not None:
raise OmniClientError("MiniMax H3 continuation requires uncached denoising; set quality=lossless")
self._cache_dit_runtime.prepare(quality_plan.cache_dit)
return {
"continuation": continuation,
"task": task,
"height": conditioning.height,
"width": conditioning.width,
@@ -2354,15 +2449,12 @@ class MiniMaxH3Pipeline(
else None
),
"visual_condition_shape": visual_shapes[0] if visual_shapes and len(visual_shapes) == 1 else None,
"audio_condition": (
conditioning.audio_condition.to(device=self.device)
if conditioning.audio_condition is not None
else None
),
"audio_condition": (audio_condition.to(device=self.device) if audio_condition is not None else None),
"ref_audio_t": audio_lengths[0] if audio_lengths and len(audio_lengths) == 1 else None,
"ref_blocks": list(conditioning.ref_blocks) or None,
"visual_condition_shapes": visual_shapes,
"audio_condition_lengths": audio_lengths,
"locked_audio_rows": locked_audio_rows,
"keyframe_frame_indices": list(conditioning.keyframe_frame_indices) or None,
"pad_seq_len": _resolve_pad_seq_len(extra.get("pad_seq_len")),
"seed": int(sampling.seed if sampling.seed is not None else 42),
@@ -2403,7 +2495,18 @@ class MiniMaxH3Pipeline(
videos = []
audios = []
for output_seed in _minimax_h3_output_seeds(context["seed"], num_outputs):
video_latent, audio_latent = self.diffuse(**{**denoise_kwargs, "seed": output_seed})
output_kwargs = {**denoise_kwargs, "seed": output_seed}
if context.get("continuation") is None:
video_latent, audio_latent = self.diffuse(**output_kwargs)
else:
window_frames, overlap_frames = context["continuation"]
video_latent, audio_latent = diffuse_continuation(
self.diffuse,
output_kwargs,
window_frames=window_frames,
overlap_frames=overlap_frames,
text_conditioning=context.get("continuation_text_conditioning"),
)
if context["preencode_mp4"]:
videos.append(
self.decode_to_mp4(
@@ -2765,7 +2868,7 @@ class MiniMaxH3Pipeline(
sigma_next=schedule["sigma_audio_next"],
)
audio_rows = audio_rows.clone()
audio_rows[audio_update] = new_audio
audio_rows[audio_update] = new_audio if branch.locked_audio_rows is None else branch.locked_audio_rows
if audio_anchor is not None:
audio_rows[~audio_update] = audio_anchor # per-step audio ref reset