[4/N] Unify worker-side MP4 encoding and wire Wan into it (#7048)
Signed-off-by: specture724 <specture724@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
12 files changed
+1177
-232
No files matched your search
@@ -2852,7 +2852,7 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
|
||||
class FakeEncoder:
|
||||
instances = []
|
||||
instances: list["FakeEncoder"] = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.pushes = []
|
||||
@@ -2873,6 +2873,8 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
|
||||
return torch.zeros(1, 1, 2)
|
||||
|
||||
class FakeVideoVAE:
|
||||
chunk_value_range = (0.0, 1.0)
|
||||
|
||||
def decode_with_chunks(self, latent, *, on_chunk):
|
||||
for value in (0.0, 0.25, 0.5):
|
||||
on_chunk(torch.full((1, 3, 3, 2, 2), value))
|
||||
@@ -2884,7 +2886,7 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
|
||||
pipeline.device = torch.device("cpu")
|
||||
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.utils.media_utils.ChunkedMP4Encoder",
|
||||
"vllm_omni.diffusion.utils.chunked_video.ChunkedMP4Encoder",
|
||||
FakeEncoder,
|
||||
)
|
||||
|
||||
@@ -2904,12 +2906,81 @@ def test_decode_to_mp4_batches_consumer_transfers(monkeypatch):
|
||||
assert encoder.pushes[1].shape == (3, 2, 2, 3)
|
||||
|
||||
|
||||
def _peer_rank_preencode_pipeline(monkeypatch):
|
||||
"""An H3 pipeline whose video VAE is a distributed rank that owns no output.
|
||||
|
||||
Every rank of the VAE group must call ``decode_with_chunks`` to keep the
|
||||
temporal collectives in lockstep, but only the owner receives chunks, so a
|
||||
peer's callback is never invoked.
|
||||
"""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
|
||||
class FakeAudioVAE:
|
||||
def decode_latent(self, latent):
|
||||
return torch.zeros(1, 1, 2)
|
||||
|
||||
class PeerRankVideoVAE:
|
||||
chunk_value_range = (0.0, 1.0)
|
||||
|
||||
def decode_with_chunks(self, latent, *, on_chunk):
|
||||
del latent, on_chunk
|
||||
|
||||
pipeline = object.__new__(mod.MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.audio_vae = FakeAudioVAE()
|
||||
pipeline.video_vae = PeerRankVideoVAE()
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.od_config = SimpleNamespace()
|
||||
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_peer_vae_rank_returns_no_preencoded_output_instead_of_failing(monkeypatch):
|
||||
"""A rank that owns no output must not turn its silence into a request error."""
|
||||
pipeline = _peer_rank_preencode_pipeline(monkeypatch)
|
||||
|
||||
output = pipeline.decode_to_mp4(torch.zeros(1), torch.zeros(1), height=2, width=2)
|
||||
|
||||
assert output == b""
|
||||
|
||||
|
||||
def test_peer_vae_rank_post_decode_reaches_post_processing(monkeypatch):
|
||||
"""The real H3 post_decode chain, not just the shared consumer, tolerates a peer rank."""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
pipeline = _peer_rank_preencode_pipeline(monkeypatch)
|
||||
monkeypatch.setattr(pipeline, "_unpack_denoised_rows", lambda *args, **kwargs: (torch.zeros(1), torch.zeros(1)))
|
||||
state = StepRequestState(
|
||||
request_id="peer-rank",
|
||||
sampling=SimpleNamespace(num_outputs_per_prompt=1),
|
||||
prompt="a prompt",
|
||||
)
|
||||
state.latents = torch.zeros(1)
|
||||
state.extra[mod._STEP_BRANCH] = object()
|
||||
state.extra[mod._STEP_AUDIO_ROWS] = torch.zeros(1)
|
||||
state.extra[mod._STEP_SHAPE] = {
|
||||
"latent_t": 1,
|
||||
"latent_h": 1,
|
||||
"latent_w": 1,
|
||||
"audio_t": 1,
|
||||
"height": 2,
|
||||
"width": 2,
|
||||
"preencode_mp4": True,
|
||||
}
|
||||
|
||||
output = pipeline.post_decode(state)
|
||||
|
||||
assert output.output == (b"", None)
|
||||
assert mod._minimax_h3_post_process(output.output)["video"] == [b""]
|
||||
|
||||
|
||||
def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatch):
|
||||
"""A client's encoder options must survive the worker-side pre-encode path."""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
|
||||
class FakeEncoder:
|
||||
instances = []
|
||||
instances: list["FakeEncoder"] = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
@@ -2929,6 +3000,8 @@ def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatc
|
||||
return torch.zeros(1, 1, 2)
|
||||
|
||||
class FakeVideoVAE:
|
||||
chunk_value_range = (0.0, 1.0)
|
||||
|
||||
def decode_with_chunks(self, latent, *, on_chunk):
|
||||
on_chunk(torch.zeros(1, 3, 1, 2, 2))
|
||||
|
||||
@@ -2938,7 +3011,7 @@ def test_request_video_codec_options_reach_the_preencoded_mp4_encoder(monkeypatc
|
||||
pipeline.video_vae = FakeVideoVAE()
|
||||
pipeline.device = torch.device("cpu")
|
||||
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_uses_manual_component_offload", lambda self, component: False)
|
||||
monkeypatch.setattr("vllm_omni.diffusion.utils.media_utils.ChunkedMP4Encoder", FakeEncoder)
|
||||
monkeypatch.setattr("vllm_omni.diffusion.utils.chunked_video.ChunkedMP4Encoder", FakeEncoder)
|
||||
|
||||
pipeline.decode_to_mp4(
|
||||
torch.zeros(1),
|
||||
|
||||
Reference in new issue
Block a user