[Model] Add MiniMax-H3 long-video latent continuation with driving audio (#7838)

Signed-off-by: princepride <wangzhipeng628@gmail.com>
This commit is contained in:
汪志鹏 authored and GitHub committed 2026-09-20 23:44:33 +08:00
1 parent 16734bbdf3
commit 139a47a578
17 files changed
+1089 -50

No files matched your search

@@ -527,3 +527,65 @@ def test_encoder_releases_workspace_after_cpu_payload(monkeypatch) -> None:
assert empty_cache_calls == 1
assert output.multimodal_outputs is not None
assert output.multimodal_outputs["embed"]["embedding"][0].device.type == "cpu"
def test_driving_audio_is_not_a_short_reference_and_survives_stage_transport():
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing as processing
sampling = OmniDiffusionSamplingParams(
width=64,
height=64,
fps=24,
extra_args={
"aspect_ratio": "1:1",
"duration": 20,
"long_video": True,
"audio_mode": "lock_source",
},
)
prepared = processing.prepare_encoder_inputs(
{"prompt": "A singer", "multi_modal_data": {"audio": (torch.zeros(75 * 800), 800)}}, sampling
)
media = MiniMaxH3EncoderMediaInput.from_mm_tensors(prepared.media.to_mm_tensors(), prepared.media.to_metadata())
assert media.task == "t2va"
assert media.audio_mode == "lock_source"
assert prepared.condition_labels == []
class FakeAudioVAE:
def encode_waveform(self, waveform: torch.Tensor, sample_rate: int) -> tuple[torch.Tensor, int]:
return torch.ones(6000, 32), 3000
audio_vae = FakeAudioVAE()
conditioning = processing.encode_media(media, video_vae=None, audio_vae=audio_vae, emit_conditioning=True)
assert conditioning.audio_condition_lengths == (3000,)
assert conditioning.ref_blocks == ()
def test_driving_audio_does_not_override_fl2va_task_inference():
from PIL import Image
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing as processing
sampling = OmniDiffusionSamplingParams(
width=64,
height=64,
fps=24,
num_frames=124,
extra_args={"audio_mode": "lock_source"},
)
prepared = processing.prepare_encoder_inputs(
{
"prompt": "A singer",
"multi_modal_data": {
"image": Image.new("RGB", (256, 256)),
"audio": (torch.zeros(5 * 800), 800),
},
},
sampling,
)
assert prepared.media.task == "fl2va"
assert prepared.media.audio_mode == "lock_source"
assert prepared.condition_labels == [("image", 1)]