From 139a47a578e01eb7cd4a0c3d35a02eba4b4ded6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Sun, 20 Sep 2026 23:44:33 +0800 Subject: [PATCH] [Model] Add MiniMax-H3 long-video latent continuation with driving audio (#7838) Signed-off-by: princepride --- recipes/MiniMaxAI/MiniMax-H3.md | 148 ++++++++++- .../layers/test_fused_qk_norm_rope.py | 17 ++ .../minimax_h3/test_minimax_h3_collectives.py | 3 +- .../test_minimax_h3_continuation.py | 231 ++++++++++++++++++ .../minimax_h3/test_minimax_h3_contract.py | 45 ++++ .../test_minimax_h3_latent_edit_pipeline.py | 1 + .../test_minimax_h3_long_video_limits.py | 143 +++++++++++ .../test_minimax_h3_step_execution.py | 38 ++- .../models/test_minimax_h3_encoder_payload.py | 62 +++++ .../diffusion/layers/fused_qk_norm_rope.py | 6 +- .../models/minimax_h3/continuation.py | 162 ++++++++++++ .../models/minimax_h3/denoise_loop.py | 5 +- .../models/minimax_h3/packed_sequence.py | 33 ++- .../models/minimax_h3/pipeline_minimax_h3.py | 135 ++++++++-- .../models/minimax_h3/conditioning.py | 3 + .../models/minimax_h3/encoder_processing.py | 65 +++-- .../models/minimax_h3/long_video.py | 42 ++++ 17 files changed, 1089 insertions(+), 50 deletions(-) create mode 100644 tests/diffusion/models/minimax_h3/test_minimax_h3_continuation.py create mode 100644 tests/diffusion/models/minimax_h3/test_minimax_h3_long_video_limits.py create mode 100644 vllm_omni/diffusion/models/minimax_h3/continuation.py create mode 100644 vllm_omni/model_executor/models/minimax_h3/long_video.py diff --git a/recipes/MiniMaxAI/MiniMax-H3.md b/recipes/MiniMaxAI/MiniMax-H3.md index 934e882f7..9eac85ece 100644 --- a/recipes/MiniMaxAI/MiniMax-H3.md +++ b/recipes/MiniMaxAI/MiniMax-H3.md @@ -825,7 +825,7 @@ at most 15 seconds combined. | FL2VA | first image, last image, or ordered first+last images | at most 2 images; `frame_indices` is `[0]`, `[-1]`, or `[0,-1]` | | Ref2VA | image-only, image+image, image+video, video+audio, and mixed image/video/audio | images ≤9, videos ≤3, audios ≤3, total references ≤12; audio requires a visual reference | -The H3 output contract is 4–15 seconds at 24 FPS, stereo 32 kHz audio, and a +The default H3 output contract is 4–15 seconds at 24 FPS, stereo 32 kHz audio, and a 32-pixel canvas multiple. T2VA requires one named output ratio from `21:9`, `16:9`, `4:3`, `1:1`, `3:4`, or `9:16`. FL2VA always follows the first input image's ratio and ignores a generic `aspect_ratio` override. Ref2VA defaults to @@ -836,6 +836,152 @@ default. `short_edge` controls the 768-pixel canvas and must be `768`. outputs; the synchronous raw-MP4 endpoint returns the first output when more than one is requested. +## Long video and driving audio + +Set `extra_params.long_video=true` to opt into durations above 15 seconds. +Ref2VA long-video requests now default to latent continuation with global +temporal RoPE positions (A+B), described below. Set `long_video_mode=full` +explicitly to sample the entire target latent at once for comparison. Both modes +retain the native `17n+5` video-frame grid, `5n+2` video-latent grid, and 40 Hz +audio latents. A request for 75 seconds becomes 1,807 frames (75.292 seconds at +24 FPS). In full mode, attention cost and activation memory grow with the full +sequence length. Reference-video and reference-audio limits remain unchanged. +Other task types retain full-mode behavior. Requests are limited to 30 seconds +in full mode and 300 seconds in continuation mode; without `long_video=true`, +the limit remains 15 seconds. The limits cover duration aliases and explicit +`num_frames`, and externally encoded requests are checked again before diffusion. +Native frame-grid alignment can round the output slightly above the requested +limit. These are request sanity bounds, not a guarantee that a given resolution +fits GPU memory. Full mode still allocates the whole sequence, and continuation +retains cumulative latents and reference inputs. + +Use `audio_mode=lock_source` with one `audio_reference` to drive generation +with a soundtrack, rather than treating it as a short reference. The encoder +places that waveform in the target audio latent, crops or zero-pads its two +channels independently to the output duration, and keeps those rows clean +(timestep 1) throughout sampling. This mode does not insert an `