[Model][MiniMax-H3] Support released VAE chunks directly
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
520e058b95
commit
b220df8397
3 files changed
+1059
-130
No files matched your search
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import multiprocessing as mp
|
import multiprocessing as mp
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -14,6 +15,11 @@ import torch
|
|||||||
import torch.distributed as dist
|
import torch.distributed as dist
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
|
|
||||||
|
import vllm_omni.diffusion.models.minimax_h3.temporal_chunks as temporal_chunks_module
|
||||||
|
import vllm_omni.diffusion.models.minimax_h3.vae as vae_module
|
||||||
|
from vllm_omni.diffusion.models.minimax_h3.temporal_chunks import (
|
||||||
|
MINIMAX_H3_RELEASED_BUNDLE_SHA256,
|
||||||
|
)
|
||||||
from vllm_omni.diffusion.models.minimax_h3.vae import (
|
from vllm_omni.diffusion.models.minimax_h3.vae import (
|
||||||
MiniMaxH3ChunkCallbackPeerError,
|
MiniMaxH3ChunkCallbackPeerError,
|
||||||
MiniMaxH3ChunkedDecodeUnsupportedError,
|
MiniMaxH3ChunkedDecodeUnsupportedError,
|
||||||
@@ -29,81 +35,145 @@ class _IdentityProcessor:
|
|||||||
return tensor
|
return tensor
|
||||||
|
|
||||||
|
|
||||||
class _FakeVendorTemporalModel(nn.Module):
|
class _FakeCompatTemporalModel(nn.Module):
|
||||||
temporal_chunk_callback_api_version = 1
|
use_3d_conv = True
|
||||||
|
clip_length = 17
|
||||||
|
tokens_chunk_size = 5
|
||||||
|
token_overlap = 2
|
||||||
|
token_drop = 3
|
||||||
|
vae_ratio_t = 4
|
||||||
|
frame_pre_padding = 3
|
||||||
|
frame_overlap = 5
|
||||||
|
isolated_first_frame = False
|
||||||
|
isolated_last_frame = False
|
||||||
vae_ratio = 1
|
vae_ratio = 1
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
|
latent_t: int,
|
||||||
*,
|
*,
|
||||||
collective: bool = False,
|
collective: bool = False,
|
||||||
metadata_mode: str = "valid",
|
tile_count: int = 2,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
values = torch.linspace(0, 1, 124).view(1, 1, 124, 1, 1)
|
|
||||||
self.expected = values.expand(1, 3, -1, 1, 1).contiguous()
|
|
||||||
self.processor = _IdentityProcessor()
|
|
||||||
self.parallel_tiling = collective
|
self.parallel_tiling = collective
|
||||||
self.collective = collective
|
self.collective = collective
|
||||||
self.metadata_mode = metadata_mode
|
self.tile_count = tile_count
|
||||||
|
pseudo_tokens = latent_t + self.token_drop
|
||||||
|
remainder = pseudo_tokens % self.tokens_chunk_size
|
||||||
|
self.pad_tokens = 0 if remainder == 0 else self.tokens_chunk_size - remainder
|
||||||
|
self.num_chunks = (pseudo_tokens + self.pad_tokens) // self.tokens_chunk_size - 1
|
||||||
|
ideal_frames = self.num_chunks * 17 + 5
|
||||||
|
output_frames = {
|
||||||
|
33: 111,
|
||||||
|
34: 115,
|
||||||
|
35: 119,
|
||||||
|
36: 120,
|
||||||
|
}.get(latent_t, ideal_frames)
|
||||||
|
values = torch.linspace(0, 1, ideal_frames).view(1, 1, ideal_frames, 1, 1)
|
||||||
|
self.ideal = values.expand(1, 3, -1, 1, 1).contiguous()
|
||||||
|
self.expected = self.ideal[:, :, :output_frames].clone()
|
||||||
|
self.processor = _IdentityProcessor()
|
||||||
self.decode_base_calls = 0
|
self.decode_base_calls = 0
|
||||||
self.vendor_callback_calls = 0
|
self.adaptive_decode_calls = 0
|
||||||
|
self.blend_calls = 0
|
||||||
|
self.eval()
|
||||||
|
|
||||||
@staticmethod
|
def split_tiles(self, size: int, tiled: bool):
|
||||||
def split_tiles(size: int, tiled: bool):
|
|
||||||
del size, tiled
|
del size, tiled
|
||||||
return [0, 1], None, None
|
return list(range(self.tile_count)), None, None
|
||||||
|
|
||||||
def decode_base(
|
def decode_base(self, latent: torch.Tensor) -> torch.Tensor:
|
||||||
self,
|
|
||||||
latent: torch.Tensor,
|
|
||||||
*,
|
|
||||||
temporal_chunk_callback=None,
|
|
||||||
) -> torch.Tensor:
|
|
||||||
del latent
|
del latent
|
||||||
self.decode_base_calls += 1
|
self.decode_base_calls += 1
|
||||||
output = self.expected.clone()
|
return self.expected.clone()
|
||||||
if temporal_chunk_callback is None:
|
|
||||||
return output
|
|
||||||
|
|
||||||
frame_start = 0
|
def _adaptive_decode(self, latent: torch.Tensor) -> torch.Tensor:
|
||||||
for chunk_index, chunk_size in enumerate([17] * 7 + [5]):
|
del latent
|
||||||
if self.collective:
|
call_index = self.adaptive_decode_calls % self.num_chunks
|
||||||
collective_probe = torch.tensor([chunk_index], dtype=torch.int32)
|
self.adaptive_decode_calls += 1
|
||||||
dist.all_reduce(collective_probe)
|
if self.collective and self.parallel_tiling:
|
||||||
self.vendor_callback_calls += 1
|
collective_probe = torch.tensor([call_index], dtype=torch.int32)
|
||||||
metadata_index = chunk_index
|
dist.all_reduce(collective_probe)
|
||||||
metadata_total = 8
|
decoded = torch.full((1, 3, 28, 1, 1), -1.0)
|
||||||
metadata_start = frame_start
|
body_start = call_index * 17
|
||||||
is_final = chunk_index == 7
|
decoded[:, :, 3:20].copy_(self.ideal[:, :, body_start : body_start + 17])
|
||||||
if self.metadata_mode == "gap" and chunk_index == 1:
|
if call_index == self.num_chunks - 1:
|
||||||
metadata_start += 1
|
decoded[:, :, 23:28].copy_(self.ideal[:, :, -5:])
|
||||||
elif self.metadata_mode == "bad_total" and chunk_index == 1:
|
return decoded
|
||||||
metadata_total += 1
|
|
||||||
elif self.metadata_mode == "no_final":
|
def _decode_temporal_output_frame_plan(
|
||||||
is_final = False
|
self,
|
||||||
temporal_chunk_callback(
|
latent: torch.Tensor,
|
||||||
self.expected[:, :, frame_start : frame_start + chunk_size].clone(),
|
head: torch.Tensor | None,
|
||||||
chunk_index=metadata_index,
|
tail: torch.Tensor | None,
|
||||||
total_chunks=metadata_total,
|
num_chunks: int,
|
||||||
frame_start=metadata_start,
|
pad_tokens: int,
|
||||||
is_final=is_final,
|
) -> tuple[int, int, int]:
|
||||||
)
|
del latent
|
||||||
frame_start += chunk_size
|
assert head is None
|
||||||
return output
|
assert tail is None
|
||||||
|
assert num_chunks == self.num_chunks
|
||||||
|
assert pad_tokens == self.pad_tokens
|
||||||
|
total_frames = num_chunks * 17 + 5
|
||||||
|
pad_frames = total_frames - int(self.expected.shape[2])
|
||||||
|
return total_frames, pad_frames, int(self.expected.shape[2])
|
||||||
|
|
||||||
|
def _decode_temporal_pad_frames(
|
||||||
|
self,
|
||||||
|
latent: torch.Tensor,
|
||||||
|
pad_tokens: int,
|
||||||
|
) -> int:
|
||||||
|
del latent
|
||||||
|
assert pad_tokens == self.pad_tokens
|
||||||
|
return int(self.ideal.shape[2] - self.expected.shape[2])
|
||||||
|
|
||||||
|
def blend(
|
||||||
|
self,
|
||||||
|
previous: torch.Tensor,
|
||||||
|
current: torch.Tensor,
|
||||||
|
extent: int,
|
||||||
|
*,
|
||||||
|
dim: int,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
del previous
|
||||||
|
assert extent == self.frame_overlap
|
||||||
|
assert dim == -3
|
||||||
|
self.blend_calls += 1
|
||||||
|
return current
|
||||||
|
|
||||||
|
|
||||||
def _vae(
|
def _install_released_source_fingerprint() -> None:
|
||||||
|
temporal_chunks_module._source_bundle_sha256 = lambda model: (
|
||||||
|
"fake/video_vae",
|
||||||
|
MINIMAX_H3_RELEASED_BUNDLE_SHA256,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _allow_released_source(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
temporal_chunks_module,
|
||||||
|
"_source_bundle_sha256",
|
||||||
|
lambda model: (
|
||||||
|
"fake/video_vae",
|
||||||
|
MINIMAX_H3_RELEASED_BUNDLE_SHA256,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _compat_vae(
|
||||||
|
latent_t: int,
|
||||||
*,
|
*,
|
||||||
parallel_size: int = 1,
|
parallel_size: int = 1,
|
||||||
collective: bool = False,
|
collective: bool = False,
|
||||||
metadata_mode: str = "valid",
|
tile_count: int = 2,
|
||||||
) -> tuple[MiniMaxH3VideoVAE, _FakeVendorTemporalModel]:
|
) -> tuple[MiniMaxH3VideoVAE, _FakeCompatTemporalModel]:
|
||||||
vae = object.__new__(MiniMaxH3VideoVAE)
|
vae = object.__new__(MiniMaxH3VideoVAE)
|
||||||
nn.Module.__init__(vae)
|
nn.Module.__init__(vae)
|
||||||
model = _FakeVendorTemporalModel(
|
model = _FakeCompatTemporalModel(
|
||||||
|
latent_t,
|
||||||
collective=collective,
|
collective=collective,
|
||||||
metadata_mode=metadata_mode,
|
tile_count=tile_count,
|
||||||
)
|
)
|
||||||
vae.model = model
|
vae.model = model
|
||||||
vae.config_dict = {
|
vae.config_dict = {
|
||||||
@@ -115,22 +185,57 @@ def _vae(
|
|||||||
return vae, model
|
return vae, model
|
||||||
|
|
||||||
|
|
||||||
def _latent() -> torch.Tensor:
|
def _latent(latent_t: int = 37) -> torch.Tensor:
|
||||||
return torch.zeros(1, 3, 37, 1, 1)
|
return torch.zeros(1, 3, latent_t, 1, 1)
|
||||||
|
|
||||||
|
|
||||||
def test_default_decode_path_does_not_request_vendor_chunks() -> None:
|
def test_default_decode_path_does_not_request_chunks() -> None:
|
||||||
vae, model = _vae()
|
vae, model = _compat_vae(37)
|
||||||
|
|
||||||
output = vae.decode_latent(_latent())
|
output = vae.decode_latent(_latent())
|
||||||
|
|
||||||
torch.testing.assert_close(output, model.expected, rtol=0, atol=0)
|
torch.testing.assert_close(output, model.expected, rtol=0, atol=0)
|
||||||
assert model.decode_base_calls == 1
|
assert model.decode_base_calls == 1
|
||||||
assert model.vendor_callback_calls == 0
|
assert model.adaptive_decode_calls == 0
|
||||||
|
|
||||||
|
|
||||||
def test_callback_chunks_are_owned_and_reconstruct_the_full_decode() -> None:
|
def test_default_decode_bypasses_compatibility_source_checks(
|
||||||
vae, model = _vae()
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
temporal_chunks_module,
|
||||||
|
"_source_bundle_sha256",
|
||||||
|
lambda model: pytest.fail("default decode fingerprinted remote code"),
|
||||||
|
)
|
||||||
|
|
||||||
|
output = vae.decode_latent(_latent())
|
||||||
|
|
||||||
|
torch.testing.assert_close(output, model.expected, rtol=0, atol=0)
|
||||||
|
assert model.decode_base_calls == 1
|
||||||
|
assert model.adaptive_decode_calls == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("latent_t", "expected_chunk_sizes"),
|
||||||
|
[
|
||||||
|
(37, [17] * 7 + [5]),
|
||||||
|
(62, [17] * 12 + [5]),
|
||||||
|
(72, [17] * 14 + [5]),
|
||||||
|
(107, [17] * 21 + [5]),
|
||||||
|
(36, [17] * 7 + [1]),
|
||||||
|
(35, [17] * 7),
|
||||||
|
(34, [17] * 6 + [13]),
|
||||||
|
(33, [17] * 6 + [9]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_released_compatibility_chunks_reconstruct_full_decode(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
latent_t: int,
|
||||||
|
expected_chunk_sizes: list[int],
|
||||||
|
) -> None:
|
||||||
|
_allow_released_source(monkeypatch)
|
||||||
|
vae, model = _compat_vae(latent_t)
|
||||||
chunks: list[torch.Tensor] = []
|
chunks: list[torch.Tensor] = []
|
||||||
snapshots: list[torch.Tensor] = []
|
snapshots: list[torch.Tensor] = []
|
||||||
metadata: list[tuple[int, int, int, bool]] = []
|
metadata: list[tuple[int, int, int, bool]] = []
|
||||||
@@ -150,18 +255,35 @@ def test_callback_chunks_are_owned_and_reconstruct_the_full_decode() -> None:
|
|||||||
snapshots.append(frames.clone())
|
snapshots.append(frames.clone())
|
||||||
metadata.append((chunk_index, total_chunks, frame_start, is_final))
|
metadata.append((chunk_index, total_chunks, frame_start, is_final))
|
||||||
|
|
||||||
output = vae.decode_latent_with_chunks(_latent(), consume)
|
output = vae.decode_latent_with_chunks(_latent(latent_t), consume)
|
||||||
|
|
||||||
torch.testing.assert_close(output, model.expected, rtol=0, atol=0)
|
torch.testing.assert_close(output, model.expected, rtol=0, atol=0)
|
||||||
torch.testing.assert_close(torch.cat(chunks, dim=2), output, rtol=0, atol=0)
|
torch.testing.assert_close(torch.cat(chunks, dim=2), output, rtol=0, atol=0)
|
||||||
|
assert [int(chunk.shape[2]) for chunk in chunks] == expected_chunk_sizes
|
||||||
|
expected_metadata = []
|
||||||
|
frame_start = 0
|
||||||
|
for chunk_index, chunk_size in enumerate(expected_chunk_sizes):
|
||||||
|
expected_metadata.append(
|
||||||
|
(
|
||||||
|
chunk_index,
|
||||||
|
len(expected_chunk_sizes),
|
||||||
|
frame_start,
|
||||||
|
chunk_index == len(expected_chunk_sizes) - 1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
frame_start += chunk_size
|
||||||
|
assert metadata == expected_metadata
|
||||||
for chunk, snapshot in zip(chunks, snapshots, strict=True):
|
for chunk, snapshot in zip(chunks, snapshots, strict=True):
|
||||||
torch.testing.assert_close(chunk, snapshot, rtol=0, atol=0)
|
torch.testing.assert_close(chunk, snapshot, rtol=0, atol=0)
|
||||||
assert len({chunk.untyped_storage().data_ptr() for chunk in chunks}) == len(chunks)
|
assert len({chunk.untyped_storage().data_ptr() for chunk in chunks}) == len(chunks)
|
||||||
assert metadata == [(index, 8, index * 17, index == 7) for index in range(8)]
|
assert model.adaptive_decode_calls == model.num_chunks
|
||||||
|
|
||||||
|
|
||||||
def test_callback_mutation_does_not_change_returned_full_output() -> None:
|
def test_compat_callback_mutation_does_not_change_returned_full_output(
|
||||||
vae, model = _vae()
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
_allow_released_source(monkeypatch)
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
|
||||||
def mutate(frames: torch.Tensor, **metadata: Any) -> None:
|
def mutate(frames: torch.Tensor, **metadata: Any) -> None:
|
||||||
del metadata
|
del metadata
|
||||||
@@ -172,29 +294,77 @@ def test_callback_mutation_does_not_change_returned_full_output() -> None:
|
|||||||
torch.testing.assert_close(output, model.expected, rtol=0, atol=0)
|
torch.testing.assert_close(output, model.expected, rtol=0, atol=0)
|
||||||
|
|
||||||
|
|
||||||
def test_callback_failure_is_deferred_until_vendor_decode_finishes() -> None:
|
def test_compat_source_fingerprint_is_cached_after_success(
|
||||||
vae, model = _vae()
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
callback_calls = 0
|
) -> None:
|
||||||
|
calls = 0
|
||||||
|
|
||||||
def fail(*args, **kwargs) -> None:
|
def fingerprint(model):
|
||||||
nonlocal callback_calls
|
nonlocal calls
|
||||||
del args, kwargs
|
del model
|
||||||
callback_calls += 1
|
calls += 1
|
||||||
raise LookupError("sink failed")
|
return "fake/video_vae", MINIMAX_H3_RELEASED_BUNDLE_SHA256
|
||||||
|
|
||||||
with pytest.raises(LookupError, match="sink failed"):
|
monkeypatch.setattr(
|
||||||
vae.decode_latent_with_chunks(_latent(), fail)
|
temporal_chunks_module,
|
||||||
|
"_source_bundle_sha256",
|
||||||
|
fingerprint,
|
||||||
|
)
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
|
||||||
assert callback_calls == 1
|
for _ in range(2):
|
||||||
assert model.vendor_callback_calls == 8
|
output = vae.decode_latent_with_chunks(
|
||||||
torch.testing.assert_close(vae.decode_latent(_latent()), model.expected, rtol=0, atol=0)
|
_latent(),
|
||||||
|
lambda *args, **kwargs: None,
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(output, model.expected, rtol=0, atol=0)
|
||||||
|
|
||||||
|
assert calls == 1
|
||||||
|
|
||||||
|
|
||||||
def test_terminal_callback_failure_preserves_the_original_exception() -> None:
|
def test_compat_callback_publishes_after_temporal_blending(
|
||||||
vae, model = _vae()
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
seen: list[tuple[int, int, int, bool]] = []
|
) -> None:
|
||||||
|
_allow_released_source(monkeypatch)
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
chunks: list[torch.Tensor] = []
|
||||||
|
|
||||||
def fail_final(
|
def mark_overlap(
|
||||||
|
previous: torch.Tensor,
|
||||||
|
current: torch.Tensor,
|
||||||
|
extent: int,
|
||||||
|
*,
|
||||||
|
dim: int,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
del previous
|
||||||
|
assert extent == 5
|
||||||
|
assert dim == -3
|
||||||
|
blended = current.clone()
|
||||||
|
blended[:, :, :extent].fill_(0.25)
|
||||||
|
return blended
|
||||||
|
|
||||||
|
monkeypatch.setattr(model, "blend", mark_overlap)
|
||||||
|
output = vae.decode_latent_with_chunks(
|
||||||
|
_latent(),
|
||||||
|
lambda frames, **metadata: chunks.append(frames.clone()),
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_overlap = torch.full_like(chunks[1][:, :, :5], 0.25)
|
||||||
|
torch.testing.assert_close(chunks[1][:, :, :5], expected_overlap, rtol=0, atol=0)
|
||||||
|
torch.testing.assert_close(output[:, :, 17:22], expected_overlap, rtol=0, atol=0)
|
||||||
|
torch.testing.assert_close(torch.cat(chunks, dim=2), output, rtol=0, atol=0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("fail_final", [False, True])
|
||||||
|
def test_compat_callback_failure_finishes_decode_and_recovers(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
fail_final: bool,
|
||||||
|
) -> None:
|
||||||
|
_allow_released_source(monkeypatch)
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
seen: list[tuple[int, bool]] = []
|
||||||
|
|
||||||
|
def fail(
|
||||||
frames: torch.Tensor,
|
frames: torch.Tensor,
|
||||||
*,
|
*,
|
||||||
chunk_index: int,
|
chunk_index: int,
|
||||||
@@ -202,48 +372,165 @@ def test_terminal_callback_failure_preserves_the_original_exception() -> None:
|
|||||||
frame_start: int,
|
frame_start: int,
|
||||||
is_final: bool,
|
is_final: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
del frames
|
del frames, total_chunks, frame_start
|
||||||
seen.append((chunk_index, total_chunks, frame_start, is_final))
|
seen.append((chunk_index, is_final))
|
||||||
if is_final:
|
if is_final == fail_final:
|
||||||
raise LookupError("final sink failed")
|
raise LookupError("compat sink failed")
|
||||||
|
|
||||||
with pytest.raises(LookupError, match="final sink failed"):
|
with pytest.raises(LookupError, match="compat sink failed"):
|
||||||
vae.decode_latent_with_chunks(_latent(), fail_final)
|
vae.decode_latent_with_chunks(_latent(), fail)
|
||||||
|
|
||||||
assert seen == [(index, 8, index * 17, index == 7) for index in range(8)]
|
assert model.adaptive_decode_calls == model.num_chunks
|
||||||
assert model.vendor_callback_calls == 8
|
if fail_final:
|
||||||
|
assert seen == [(index, index == 7) for index in range(8)]
|
||||||
|
else:
|
||||||
|
assert seen == [(0, False)]
|
||||||
|
|
||||||
|
recovered_chunks: list[torch.Tensor] = []
|
||||||
|
recovered = vae.decode_latent_with_chunks(
|
||||||
|
_latent(),
|
||||||
|
lambda frames, **metadata: recovered_chunks.append(frames),
|
||||||
|
)
|
||||||
|
torch.testing.assert_close(recovered, model.expected, rtol=0, atol=0)
|
||||||
|
torch.testing.assert_close(
|
||||||
|
torch.cat(recovered_chunks, dim=2),
|
||||||
|
model.expected,
|
||||||
|
rtol=0,
|
||||||
|
atol=0,
|
||||||
|
)
|
||||||
|
assert model.adaptive_decode_calls == model.num_chunks * 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("drift", ["changed", "missing"])
|
||||||
|
def test_compat_source_drift_fails_closed_before_decode(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
drift: str,
|
||||||
|
) -> None:
|
||||||
|
_allow_released_source(monkeypatch)
|
||||||
|
if drift == "changed":
|
||||||
|
monkeypatch.setattr(
|
||||||
|
temporal_chunks_module,
|
||||||
|
"_source_bundle_sha256",
|
||||||
|
lambda model: ("fake/video_vae", "0" * 64),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
temporal_chunks_module,
|
||||||
|
"_source_bundle_sha256",
|
||||||
|
lambda model: (None, None),
|
||||||
|
)
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
MiniMaxH3ChunkedDecodeUnsupportedError,
|
||||||
|
match="42ed227e",
|
||||||
|
):
|
||||||
|
vae.decode_latent_with_chunks(_latent(), lambda *args, **kwargs: None)
|
||||||
|
|
||||||
|
assert model.adaptive_decode_calls == 0
|
||||||
|
torch.testing.assert_close(vae.decode_latent(_latent()), model.expected, rtol=0, atol=0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_uninspectable_compat_source_maps_to_unsupported(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
temporal_chunks_module.inspect,
|
||||||
|
"getsourcefile",
|
||||||
|
lambda object_type: (_ for _ in ()).throw(TypeError("no source")),
|
||||||
|
)
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
|
||||||
|
with pytest.raises(MiniMaxH3ChunkedDecodeUnsupportedError, match="bundle=None"):
|
||||||
|
vae.decode_latent_with_chunks(_latent(), lambda *args, **kwargs: None)
|
||||||
|
|
||||||
|
assert model.adaptive_decode_calls == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_bundle_manifest_is_deterministic_and_excludes_init(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
package = tmp_path / "video_vae"
|
||||||
|
package.mkdir()
|
||||||
|
(package / "z.py").write_text("z = 1\n", encoding="utf-8")
|
||||||
|
(package / "klvae.py").write_text("k = 2\n", encoding="utf-8")
|
||||||
|
(package / "a.py").write_text("a = 3\n", encoding="utf-8")
|
||||||
|
(package / "__init__.py").write_text("ignored = 1\n", encoding="utf-8")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
temporal_chunks_module.inspect,
|
||||||
|
"getsourcefile",
|
||||||
|
lambda object_type: str(package / "klvae.py"),
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = hashlib.sha256()
|
||||||
|
for filename in ("a.py", "klvae.py", "z.py"):
|
||||||
|
expected.update(filename.encode())
|
||||||
|
expected.update(b"\0")
|
||||||
|
expected.update((package / filename).read_bytes())
|
||||||
|
expected.update(b"\0")
|
||||||
|
_, first = temporal_chunks_module._source_bundle_sha256(nn.Linear(1, 1))
|
||||||
|
assert first == expected.hexdigest()
|
||||||
|
|
||||||
|
(package / "__init__.py").write_text("ignored = 2\n", encoding="utf-8")
|
||||||
|
_, after_init = temporal_chunks_module._source_bundle_sha256(nn.Linear(1, 1))
|
||||||
|
assert after_init == first
|
||||||
|
(package / "a.py").write_text("a = 4\n", encoding="utf-8")
|
||||||
|
_, after_change = temporal_chunks_module._source_bundle_sha256(nn.Linear(1, 1))
|
||||||
|
assert after_change != first
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("metadata_mode", "message"),
|
("attribute", "value"),
|
||||||
[
|
[
|
||||||
("gap", "discontinuous chunk metadata"),
|
("tokens_chunk_size", 6),
|
||||||
("bad_total", "discontinuous chunk metadata"),
|
("token_overlap", 1),
|
||||||
("no_final", "discontinuous chunk metadata"),
|
("isolated_first_frame", True),
|
||||||
|
("isolated_last_frame", True),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_adapter_rejects_a_broken_vendor_callback_contract(
|
def test_compat_config_drift_fails_closed(
|
||||||
metadata_mode: str,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
message: str,
|
attribute: str,
|
||||||
|
value: object,
|
||||||
) -> None:
|
) -> None:
|
||||||
vae, model = _vae(metadata_mode=metadata_mode)
|
_allow_released_source(monkeypatch)
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
setattr(model, attribute, value)
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match=message):
|
with pytest.raises(
|
||||||
vae.decode_latent_with_chunks(_latent(), lambda *args, **kwargs: None)
|
MiniMaxH3ChunkedDecodeUnsupportedError,
|
||||||
|
match="released temporal configuration",
|
||||||
assert model.vendor_callback_calls == 8
|
):
|
||||||
|
|
||||||
|
|
||||||
def test_callback_request_fails_closed_without_the_vendor_api() -> None:
|
|
||||||
vae, model = _vae()
|
|
||||||
model.temporal_chunk_callback_api_version = -1
|
|
||||||
|
|
||||||
with pytest.raises(MiniMaxH3ChunkedDecodeUnsupportedError, match="official MiniMax"):
|
|
||||||
vae.decode_latent_with_chunks(_latent(), lambda *args, **kwargs: None)
|
vae.decode_latent_with_chunks(_latent(), lambda *args, **kwargs: None)
|
||||||
|
|
||||||
|
assert model.adaptive_decode_calls == 0
|
||||||
torch.testing.assert_close(vae.decode_latent(_latent()), model.expected, rtol=0, atol=0)
|
torch.testing.assert_close(vae.decode_latent(_latent()), model.expected, rtol=0, atol=0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("invalid", ["rank", "empty_time", "training", "non_3d"])
|
||||||
|
def test_compat_structural_invalids_fail_before_decode(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
invalid: str,
|
||||||
|
) -> None:
|
||||||
|
_allow_released_source(monkeypatch)
|
||||||
|
vae, model = _compat_vae(37)
|
||||||
|
latent = _latent()
|
||||||
|
if invalid == "rank":
|
||||||
|
latent = latent[:, :, 0]
|
||||||
|
elif invalid == "empty_time":
|
||||||
|
latent = latent[:, :, :0]
|
||||||
|
elif invalid == "training":
|
||||||
|
model.train()
|
||||||
|
else:
|
||||||
|
model.use_3d_conv = False
|
||||||
|
|
||||||
|
with pytest.raises(MiniMaxH3ChunkedDecodeUnsupportedError):
|
||||||
|
vae.decode_latent_with_chunks(latent, lambda *args, **kwargs: None)
|
||||||
|
|
||||||
|
assert model.adaptive_decode_calls == 0
|
||||||
|
|
||||||
|
|
||||||
def _distributed_worker(
|
def _distributed_worker(
|
||||||
rank: int,
|
rank: int,
|
||||||
init_file: str,
|
init_file: str,
|
||||||
@@ -256,11 +543,105 @@ def _distributed_worker(
|
|||||||
world_size=2,
|
world_size=2,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
vae, model = _vae(parallel_size=2, collective=True)
|
vae_module.get_data_parallel_world_size = lambda: 1
|
||||||
vae._native_parallel_state = MethodType(
|
vae_module.model_parallel_is_initialized = lambda: True
|
||||||
lambda self: {"sp_process_group": dist.group.WORLD},
|
|
||||||
vae,
|
def attach_group(vae: MiniMaxH3VideoVAE) -> None:
|
||||||
|
vae._native_parallel_state = MethodType(
|
||||||
|
lambda self: {"sp_process_group": dist.group.WORLD},
|
||||||
|
vae,
|
||||||
|
)
|
||||||
|
|
||||||
|
_install_released_source_fingerprint()
|
||||||
|
source_vae, source_model = _compat_vae(
|
||||||
|
37,
|
||||||
|
parallel_size=2,
|
||||||
|
collective=True,
|
||||||
)
|
)
|
||||||
|
attach_group(source_vae)
|
||||||
|
if rank == 1:
|
||||||
|
temporal_chunks_module._source_bundle_sha256 = lambda model: (
|
||||||
|
"fake/video_vae",
|
||||||
|
"0" * 64,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
source_vae.decode_latent_with_chunks(
|
||||||
|
_latent(),
|
||||||
|
(lambda *args, **kwargs: None) if rank == 0 else None,
|
||||||
|
)
|
||||||
|
except BaseException as exc: # noqa: BLE001
|
||||||
|
source_mismatch_type = type(exc).__name__
|
||||||
|
else:
|
||||||
|
source_mismatch_type = "none"
|
||||||
|
source_decode_calls = source_model.adaptive_decode_calls
|
||||||
|
|
||||||
|
_install_released_source_fingerprint()
|
||||||
|
planner_vae, planner_model = _compat_vae(
|
||||||
|
37,
|
||||||
|
parallel_size=2,
|
||||||
|
collective=True,
|
||||||
|
)
|
||||||
|
attach_group(planner_vae)
|
||||||
|
if rank == 1:
|
||||||
|
|
||||||
|
def fail_plan(self, *args, **kwargs):
|
||||||
|
del self, args, kwargs
|
||||||
|
raise RuntimeError("rank-local planner failed")
|
||||||
|
|
||||||
|
setattr(
|
||||||
|
planner_model,
|
||||||
|
"_decode_temporal_output_frame_plan",
|
||||||
|
MethodType(fail_plan, planner_model),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
planner_vae.decode_latent_with_chunks(
|
||||||
|
_latent(),
|
||||||
|
(lambda *args, **kwargs: None) if rank == 0 else None,
|
||||||
|
)
|
||||||
|
except BaseException as exc: # noqa: BLE001
|
||||||
|
planner_failure_type = type(exc).__name__
|
||||||
|
else:
|
||||||
|
planner_failure_type = "none"
|
||||||
|
planner_decode_calls = planner_model.adaptive_decode_calls
|
||||||
|
|
||||||
|
dtype_vae, dtype_model = _compat_vae(
|
||||||
|
37,
|
||||||
|
parallel_size=2,
|
||||||
|
collective=True,
|
||||||
|
)
|
||||||
|
attach_group(dtype_vae)
|
||||||
|
temporal_chunks_module.resolve_minimax_h3_temporal_cat_dtype = (
|
||||||
|
(lambda model: torch.float16) if rank == 0 else (lambda model: None)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
dtype_vae.decode_latent_with_chunks(
|
||||||
|
_latent(),
|
||||||
|
(lambda *args, **kwargs: None) if rank == 0 else None,
|
||||||
|
)
|
||||||
|
except BaseException as exc: # noqa: BLE001
|
||||||
|
dtype_mismatch_type = type(exc).__name__
|
||||||
|
else:
|
||||||
|
dtype_mismatch_type = "none"
|
||||||
|
dtype_decode_calls = dtype_model.adaptive_decode_calls
|
||||||
|
temporal_chunks_module.resolve_minimax_h3_temporal_cat_dtype = lambda model: None
|
||||||
|
|
||||||
|
vae_module.get_data_parallel_world_size = lambda: 2
|
||||||
|
dp_vae, dp_model = _compat_vae(37, parallel_size=2, collective=True)
|
||||||
|
attach_group(dp_vae)
|
||||||
|
try:
|
||||||
|
dp_vae.decode_latent_with_chunks(
|
||||||
|
_latent(),
|
||||||
|
(lambda *args, **kwargs: None) if rank == 0 else None,
|
||||||
|
)
|
||||||
|
except BaseException as exc: # noqa: BLE001
|
||||||
|
dp_vae_type = type(exc).__name__
|
||||||
|
else:
|
||||||
|
dp_vae_type = "none"
|
||||||
|
dp_decode_calls = dp_model.adaptive_decode_calls
|
||||||
|
vae_module.get_data_parallel_world_size = lambda: 1
|
||||||
|
|
||||||
|
vae, model = _compat_vae(37, parallel_size=2, collective=True)
|
||||||
|
attach_group(vae)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
vae.decode_latent_with_chunks(
|
vae.decode_latent_with_chunks(
|
||||||
@@ -298,7 +679,26 @@ def _distributed_worker(
|
|||||||
succeed if rank == 0 else None,
|
succeed if rank == 0 else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
local_vae, local_model = _vae(parallel_size=1)
|
fallback_vae, fallback_model = _compat_vae(
|
||||||
|
37,
|
||||||
|
parallel_size=2,
|
||||||
|
collective=True,
|
||||||
|
tile_count=1,
|
||||||
|
)
|
||||||
|
attach_group(fallback_vae)
|
||||||
|
fallback_calls = 0
|
||||||
|
|
||||||
|
def consume_fallback(*args, **kwargs) -> None:
|
||||||
|
nonlocal fallback_calls
|
||||||
|
del args, kwargs
|
||||||
|
fallback_calls += 1
|
||||||
|
|
||||||
|
fallback_output = fallback_vae.decode_latent_with_chunks(
|
||||||
|
_latent(),
|
||||||
|
consume_fallback if rank == 0 else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
local_vae, local_model = _compat_vae(37, parallel_size=1)
|
||||||
local_calls = 0
|
local_calls = 0
|
||||||
|
|
||||||
def consume_local(*args, **kwargs) -> None:
|
def consume_local(*args, **kwargs) -> None:
|
||||||
@@ -316,11 +716,26 @@ def _distributed_worker(
|
|||||||
result_queue.put(
|
result_queue.put(
|
||||||
{
|
{
|
||||||
"rank": rank,
|
"rank": rank,
|
||||||
|
"source_mismatch_type": source_mismatch_type,
|
||||||
|
"source_decode_calls": source_decode_calls,
|
||||||
|
"planner_failure_type": planner_failure_type,
|
||||||
|
"planner_decode_calls": planner_decode_calls,
|
||||||
|
"dtype_mismatch_type": dtype_mismatch_type,
|
||||||
|
"dtype_decode_calls": dtype_decode_calls,
|
||||||
|
"dp_vae_type": dp_vae_type,
|
||||||
|
"dp_decode_calls": dp_decode_calls,
|
||||||
"invalid_owner_type": invalid_owner_type,
|
"invalid_owner_type": invalid_owner_type,
|
||||||
"failure_type": failure_type,
|
"failure_type": failure_type,
|
||||||
"vendor_calls": model.vendor_callback_calls,
|
"adaptive_calls": model.adaptive_decode_calls,
|
||||||
"success_calls": success_calls,
|
"success_calls": success_calls,
|
||||||
"output_matches": torch.equal(output, model.expected),
|
"output_matches": torch.equal(output, model.expected),
|
||||||
|
"fallback_calls": fallback_calls,
|
||||||
|
"fallback_decode_calls": fallback_model.adaptive_decode_calls,
|
||||||
|
"fallback_output_matches": torch.equal(
|
||||||
|
fallback_output,
|
||||||
|
fallback_model.expected,
|
||||||
|
),
|
||||||
|
"fallback_tiling_restored": fallback_model.parallel_tiling,
|
||||||
"local_calls": local_calls,
|
"local_calls": local_calls,
|
||||||
"local_output_matches": torch.equal(local_output, local_model.expected),
|
"local_output_matches": torch.equal(local_output, local_model.expected),
|
||||||
"recovery": int(recovery.item()),
|
"recovery": int(recovery.item()),
|
||||||
@@ -367,13 +782,37 @@ def test_two_rank_callback_failure_is_symmetric_and_group_recovers(
|
|||||||
"LookupError",
|
"LookupError",
|
||||||
MiniMaxH3ChunkCallbackPeerError.__name__,
|
MiniMaxH3ChunkCallbackPeerError.__name__,
|
||||||
]
|
]
|
||||||
|
assert [result["source_mismatch_type"] for result in results] == [
|
||||||
|
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||||
|
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||||
|
]
|
||||||
|
assert [result["source_decode_calls"] for result in results] == [0, 0]
|
||||||
|
assert [result["planner_failure_type"] for result in results] == [
|
||||||
|
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||||
|
"RuntimeError",
|
||||||
|
]
|
||||||
|
assert [result["planner_decode_calls"] for result in results] == [0, 0]
|
||||||
|
assert [result["dtype_mismatch_type"] for result in results] == [
|
||||||
|
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||||
|
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||||
|
]
|
||||||
|
assert [result["dtype_decode_calls"] for result in results] == [0, 0]
|
||||||
|
assert [result["dp_vae_type"] for result in results] == [
|
||||||
|
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||||
|
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||||
|
]
|
||||||
|
assert [result["dp_decode_calls"] for result in results] == [0, 0]
|
||||||
assert [result["invalid_owner_type"] for result in results] == [
|
assert [result["invalid_owner_type"] for result in results] == [
|
||||||
"ValueError",
|
"ValueError",
|
||||||
"ValueError",
|
"ValueError",
|
||||||
]
|
]
|
||||||
assert [result["vendor_calls"] for result in results] == [16, 16]
|
assert [result["adaptive_calls"] for result in results] == [14, 14]
|
||||||
assert [result["success_calls"] for result in results] == [8, 0]
|
assert [result["success_calls"] for result in results] == [8, 0]
|
||||||
assert all(result["output_matches"] for result in results)
|
assert all(result["output_matches"] for result in results)
|
||||||
|
assert [result["fallback_calls"] for result in results] == [8, 0]
|
||||||
|
assert [result["fallback_decode_calls"] for result in results] == [7, 7]
|
||||||
|
assert all(result["fallback_output_matches"] for result in results)
|
||||||
|
assert all(result["fallback_tiling_restored"] for result in results)
|
||||||
# With VAE parallelism disabled, the caller still designates one request owner.
|
# With VAE parallelism disabled, the caller still designates one request owner.
|
||||||
assert [result["local_calls"] for result in results] == [8, 0]
|
assert [result["local_calls"] for result in results] == [8, 0]
|
||||||
assert all(result["local_output_matches"] for result in results)
|
assert all(result["local_output_matches"] for result in results)
|
||||||
|
|||||||
@@ -0,0 +1,408 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
|
"""Source-gated temporal chunk compatibility for the released MiniMax H3 VAE."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib
|
||||||
|
import inspect
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
import torch
|
||||||
|
import torch.nn as nn
|
||||||
|
|
||||||
|
MINIMAX_H3_RELEASED_REVISION = "42ed227ee7df40d41602854ae760620d6eb651fe"
|
||||||
|
# Compatibility pins detect upstream code drift; they are not a trust or
|
||||||
|
# security boundary for remote code execution.
|
||||||
|
MINIMAX_H3_RELEASED_BUNDLE_SHA256 = "30e64afbaa940696bf8bfe2c86003a4b1666c84b22fa3aa12403a3e6a03f705d"
|
||||||
|
|
||||||
|
_RELEASED_TEMPORAL_CONFIG = {
|
||||||
|
"clip_length": 17,
|
||||||
|
"tokens_chunk_size": 5,
|
||||||
|
"token_overlap": 2,
|
||||||
|
"token_drop": 3,
|
||||||
|
"vae_ratio_t": 4,
|
||||||
|
"frame_pre_padding": 3,
|
||||||
|
"frame_overlap": 5,
|
||||||
|
"isolated_first_frame": False,
|
||||||
|
"isolated_last_frame": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MiniMaxH3TemporalChunkCompatibilityError(RuntimeError):
|
||||||
|
"""Raised when remote H3 code cannot use the local compatibility path."""
|
||||||
|
|
||||||
|
|
||||||
|
class _RawTemporalChunkCallback(Protocol):
|
||||||
|
def __call__(
|
||||||
|
self,
|
||||||
|
frames: torch.Tensor,
|
||||||
|
*,
|
||||||
|
chunk_index: int,
|
||||||
|
total_chunks: int,
|
||||||
|
frame_start: int,
|
||||||
|
is_final: bool,
|
||||||
|
) -> None: ...
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class MiniMaxH3TemporalDecodePlan:
|
||||||
|
latent: torch.Tensor
|
||||||
|
num_chunks: int
|
||||||
|
total_frames: int
|
||||||
|
pad_frames: int
|
||||||
|
output_frames: int
|
||||||
|
chunk_frames: tuple[int, ...]
|
||||||
|
temporal_dtype: torch.dtype | None
|
||||||
|
|
||||||
|
|
||||||
|
class _RawChunkAssembly:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
callback: _RawTemporalChunkCallback,
|
||||||
|
plan: MiniMaxH3TemporalDecodePlan,
|
||||||
|
) -> None:
|
||||||
|
self.callback = callback
|
||||||
|
self.plan = plan
|
||||||
|
self.frames: torch.Tensor | None = None
|
||||||
|
self.write_pos = 0
|
||||||
|
self.logical_frames = 0
|
||||||
|
self.dropped_frames = 0
|
||||||
|
self.chunk_index = 0
|
||||||
|
self.pending_final: tuple[torch.Tensor, int, int] | None = None
|
||||||
|
self.contract_error: RuntimeError | None = None
|
||||||
|
|
||||||
|
def write(self, part: torch.Tensor) -> None:
|
||||||
|
part_frames = int(part.shape[2])
|
||||||
|
if part_frames <= 0:
|
||||||
|
return
|
||||||
|
self.logical_frames += part_frames
|
||||||
|
remaining = self.plan.output_frames - self.write_pos
|
||||||
|
copy_frames = min(part_frames, max(0, remaining))
|
||||||
|
if copy_frames <= 0:
|
||||||
|
self.dropped_frames += part_frames
|
||||||
|
return
|
||||||
|
if self.chunk_index >= len(self.plan.chunk_frames):
|
||||||
|
self.contract_error = RuntimeError("MiniMax-H3 compatibility decode emitted an extra chunk")
|
||||||
|
else:
|
||||||
|
expected_frames = self.plan.chunk_frames[self.chunk_index]
|
||||||
|
if copy_frames != expected_frames and self.contract_error is None:
|
||||||
|
self.contract_error = RuntimeError(
|
||||||
|
"MiniMax-H3 compatibility chunk size mismatch: "
|
||||||
|
f"chunk={self.chunk_index}, frames={copy_frames}/{expected_frames}"
|
||||||
|
)
|
||||||
|
|
||||||
|
emitted = part[:, :, :copy_frames]
|
||||||
|
if self.frames is None:
|
||||||
|
output_shape = list(emitted.shape)
|
||||||
|
output_shape[2] = self.plan.output_frames
|
||||||
|
self.frames = torch.empty(
|
||||||
|
output_shape,
|
||||||
|
dtype=emitted.dtype,
|
||||||
|
device=emitted.device,
|
||||||
|
)
|
||||||
|
frame_start = self.write_pos
|
||||||
|
frame_stop = frame_start + copy_frames
|
||||||
|
self.frames[:, :, frame_start:frame_stop].copy_(emitted)
|
||||||
|
if frame_stop == self.plan.output_frames:
|
||||||
|
self.pending_final = (emitted, self.chunk_index, frame_start)
|
||||||
|
elif self.contract_error is None:
|
||||||
|
self.callback(
|
||||||
|
emitted,
|
||||||
|
chunk_index=self.chunk_index,
|
||||||
|
total_chunks=len(self.plan.chunk_frames),
|
||||||
|
frame_start=frame_start,
|
||||||
|
is_final=False,
|
||||||
|
)
|
||||||
|
self.chunk_index += 1
|
||||||
|
self.write_pos = frame_stop
|
||||||
|
self.dropped_frames += part_frames - copy_frames
|
||||||
|
|
||||||
|
def finish(self) -> torch.Tensor:
|
||||||
|
if self.contract_error is not None:
|
||||||
|
raise self.contract_error
|
||||||
|
if self.frames is None or self.pending_final is None:
|
||||||
|
raise RuntimeError("MiniMax-H3 compatibility decode produced no terminal chunk")
|
||||||
|
if (
|
||||||
|
self.logical_frames != self.plan.total_frames
|
||||||
|
or self.dropped_frames != self.plan.pad_frames
|
||||||
|
or self.write_pos != self.plan.output_frames
|
||||||
|
or self.chunk_index != len(self.plan.chunk_frames)
|
||||||
|
):
|
||||||
|
raise RuntimeError(
|
||||||
|
"MiniMax-H3 compatibility frame plan mismatch: "
|
||||||
|
f"logical={self.logical_frames}/{self.plan.total_frames}, "
|
||||||
|
f"dropped={self.dropped_frames}/{self.plan.pad_frames}, "
|
||||||
|
f"written={self.write_pos}/{self.plan.output_frames}, "
|
||||||
|
f"chunks={self.chunk_index}/{len(self.plan.chunk_frames)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
final_frames, final_index, final_start = self.pending_final
|
||||||
|
self.callback(
|
||||||
|
final_frames,
|
||||||
|
chunk_index=final_index,
|
||||||
|
total_chunks=len(self.plan.chunk_frames),
|
||||||
|
frame_start=final_start,
|
||||||
|
is_final=True,
|
||||||
|
)
|
||||||
|
return self.frames
|
||||||
|
|
||||||
|
|
||||||
|
def _source_bundle_sha256(model: nn.Module) -> tuple[str | None, str | None]:
|
||||||
|
try:
|
||||||
|
source = inspect.getsourcefile(type(model))
|
||||||
|
except TypeError:
|
||||||
|
return None, None
|
||||||
|
if source is None:
|
||||||
|
return None, None
|
||||||
|
package_dir = Path(source).parent
|
||||||
|
files = sorted(path for path in package_dir.glob("*.py") if path.name != "__init__.py")
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
try:
|
||||||
|
for path in files:
|
||||||
|
digest.update(path.name.encode())
|
||||||
|
digest.update(b"\0")
|
||||||
|
digest.update(path.read_bytes())
|
||||||
|
digest.update(b"\0")
|
||||||
|
except OSError:
|
||||||
|
return str(package_dir), None
|
||||||
|
return str(package_dir), digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_released_sources(model: nn.Module) -> None:
|
||||||
|
source_dir, actual = _source_bundle_sha256(model)
|
||||||
|
if actual != MINIMAX_H3_RELEASED_BUNDLE_SHA256:
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(
|
||||||
|
"The vLLM-owned MiniMax-H3 chunk adapter supports only remote code "
|
||||||
|
f"from MiniMaxAI/MiniMax-H3@{MINIMAX_H3_RELEASED_REVISION}: "
|
||||||
|
f"bundle={actual!r} ({source_dir}). "
|
||||||
|
"The normal complete-output decode remains available."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_released_config(model: nn.Module) -> None:
|
||||||
|
actual = {name: getattr(model, name, None) for name in _RELEASED_TEMPORAL_CONFIG}
|
||||||
|
if actual != _RELEASED_TEMPORAL_CONFIG:
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(
|
||||||
|
"The vLLM-owned MiniMax-H3 chunk adapter requires the released temporal "
|
||||||
|
f"configuration {_RELEASED_TEMPORAL_CONFIG}, found {actual}. "
|
||||||
|
"The normal complete-output decode remains available."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_compat_contract(model: nn.Module, latent: torch.Tensor) -> None:
|
||||||
|
if latent.ndim != 5 or int(latent.shape[2]) <= 0:
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(
|
||||||
|
f"MiniMax-H3 video latent must have positive rank-5 time, got {tuple(latent.shape)}"
|
||||||
|
)
|
||||||
|
if not bool(getattr(model, "use_3d_conv", False)):
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError("MiniMax-H3 temporal chunk decode requires the 3D decoder")
|
||||||
|
if bool(getattr(model, "training", False)):
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError("MiniMax-H3 temporal chunk decode is inference-only")
|
||||||
|
required_methods = (
|
||||||
|
"_adaptive_decode",
|
||||||
|
"_decode_temporal_output_frame_plan",
|
||||||
|
"_decode_temporal_pad_frames",
|
||||||
|
"blend",
|
||||||
|
)
|
||||||
|
missing = [name for name in required_methods if not callable(getattr(model, name, None))]
|
||||||
|
if missing:
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(
|
||||||
|
"Loaded MiniMax-H3 VAE lacks required released-code methods: " + ", ".join(missing)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _truncate_chunk_frames(
|
||||||
|
logical_chunk_frames: list[int],
|
||||||
|
output_frames: int,
|
||||||
|
) -> tuple[int, ...]:
|
||||||
|
remaining = output_frames
|
||||||
|
result: list[int] = []
|
||||||
|
for logical_frames in logical_chunk_frames:
|
||||||
|
emitted_frames = min(logical_frames, max(0, remaining))
|
||||||
|
if emitted_frames > 0:
|
||||||
|
result.append(emitted_frames)
|
||||||
|
remaining -= emitted_frames
|
||||||
|
if remaining != 0:
|
||||||
|
raise RuntimeError(f"MiniMax-H3 compatibility chunk plan cannot cover output: remaining={remaining}")
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_decode_plan(
|
||||||
|
model: nn.Module,
|
||||||
|
latent: torch.Tensor,
|
||||||
|
temporal_dtype: torch.dtype | None,
|
||||||
|
) -> MiniMaxH3TemporalDecodePlan:
|
||||||
|
pseudo_total_tokens = int(latent.shape[2]) + int(model.token_drop) # type: ignore[attr-defined]
|
||||||
|
tokens_chunk_size = int(model.tokens_chunk_size) # type: ignore[attr-defined]
|
||||||
|
remainder = pseudo_total_tokens % tokens_chunk_size
|
||||||
|
pad_tokens = 0 if remainder == 0 else tokens_chunk_size - remainder
|
||||||
|
pseudo_total_tokens += pad_tokens
|
||||||
|
num_chunks = pseudo_total_tokens // tokens_chunk_size - int(model.token_drop > 0) # type: ignore[attr-defined]
|
||||||
|
if num_chunks <= 0:
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(
|
||||||
|
f"MiniMax-H3 temporal chunk plan is empty for latent T={latent.shape[2]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
temporal_latent = latent
|
||||||
|
if pad_tokens:
|
||||||
|
pad = temporal_latent[:, :, -1:].repeat(1, 1, pad_tokens, 1, 1)
|
||||||
|
temporal_latent = torch.cat((temporal_latent, pad), dim=2)
|
||||||
|
total_frames, pad_frames, output_frames = model._decode_temporal_output_frame_plan( # type: ignore[attr-defined]
|
||||||
|
temporal_latent,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
num_chunks,
|
||||||
|
pad_tokens,
|
||||||
|
)
|
||||||
|
if output_frames <= 0:
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(
|
||||||
|
f"MiniMax-H3 temporal chunk plan has {output_frames} output frames"
|
||||||
|
)
|
||||||
|
|
||||||
|
chunk_decoded_frames = tokens_chunk_size * int(model.vae_ratio_t) # type: ignore[attr-defined]
|
||||||
|
split_count = int(model.token_drop > 0) + 1 # type: ignore[attr-defined]
|
||||||
|
logical_chunk_frames: list[int] = []
|
||||||
|
final_overlap_frames = 0
|
||||||
|
for temporal_index in range(num_chunks):
|
||||||
|
token_start = temporal_index * tokens_chunk_size
|
||||||
|
token_stop = token_start + tokens_chunk_size + int(model.token_overlap) # type: ignore[attr-defined]
|
||||||
|
clip_tokens = max(
|
||||||
|
0,
|
||||||
|
min(token_stop, int(temporal_latent.shape[2])) - min(token_start, int(temporal_latent.shape[2])),
|
||||||
|
)
|
||||||
|
clip_frames = clip_tokens * int(model.vae_ratio_t) # type: ignore[attr-defined]
|
||||||
|
for split_index in range(split_count):
|
||||||
|
frame_start = split_index * chunk_decoded_frames
|
||||||
|
frame_stop = min(frame_start + chunk_decoded_frames, clip_frames)
|
||||||
|
part_frames = max(
|
||||||
|
0,
|
||||||
|
frame_stop - frame_start - int(model.frame_pre_padding), # type: ignore[attr-defined]
|
||||||
|
)
|
||||||
|
if split_index == 0:
|
||||||
|
if part_frames > 0:
|
||||||
|
logical_chunk_frames.append(part_frames)
|
||||||
|
else:
|
||||||
|
final_overlap_frames = part_frames
|
||||||
|
if final_overlap_frames > 0:
|
||||||
|
logical_chunk_frames.append(final_overlap_frames)
|
||||||
|
if sum(logical_chunk_frames) != int(total_frames):
|
||||||
|
raise RuntimeError(
|
||||||
|
"MiniMax-H3 compatibility planner disagrees with released remote code: "
|
||||||
|
f"frames={sum(logical_chunk_frames)}/{total_frames}"
|
||||||
|
)
|
||||||
|
chunk_frames = _truncate_chunk_frames(logical_chunk_frames, int(output_frames))
|
||||||
|
return MiniMaxH3TemporalDecodePlan(
|
||||||
|
latent=temporal_latent,
|
||||||
|
num_chunks=num_chunks,
|
||||||
|
total_frames=int(total_frames),
|
||||||
|
pad_frames=int(pad_frames),
|
||||||
|
output_frames=int(output_frames),
|
||||||
|
chunk_frames=chunk_frames,
|
||||||
|
temporal_dtype=temporal_dtype,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_minimax_h3_temporal_cat_dtype(model: nn.Module) -> torch.dtype | None:
|
||||||
|
module = importlib.import_module(type(model).__module__)
|
||||||
|
resolver = getattr(module, "_resolve_temporal_cat_dtype", None)
|
||||||
|
if resolver is None:
|
||||||
|
return None
|
||||||
|
if not callable(resolver):
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(
|
||||||
|
"Loaded MiniMax-H3 VAE has a non-callable temporal dtype resolver"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
dtype = resolver()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(str(exc)) from None
|
||||||
|
if dtype is not None and not isinstance(dtype, torch.dtype):
|
||||||
|
raise MiniMaxH3TemporalChunkCompatibilityError(
|
||||||
|
f"Loaded MiniMax-H3 VAE returned invalid temporal dtype {dtype!r}"
|
||||||
|
)
|
||||||
|
return dtype
|
||||||
|
|
||||||
|
|
||||||
|
def decode_minimax_h3_temporal_chunks_compat(
|
||||||
|
model: nn.Module,
|
||||||
|
plan: MiniMaxH3TemporalDecodePlan,
|
||||||
|
callback: _RawTemporalChunkCallback,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""Mirror the released Apache-2.0 H3 temporal assembler with callbacks.
|
||||||
|
|
||||||
|
Keep this implementation and the source/config fingerprints above in sync
|
||||||
|
with ``MiniMaxAI/MiniMax-H3@42ed227e``. Unknown remote code fails closed
|
||||||
|
before this function is called.
|
||||||
|
"""
|
||||||
|
|
||||||
|
chunk_decoded_frames = int(model.tokens_chunk_size) * int(model.vae_ratio_t) # type: ignore[attr-defined]
|
||||||
|
split_count = int(model.token_drop > 0) + 1 # type: ignore[attr-defined]
|
||||||
|
assembly = _RawChunkAssembly(callback=callback, plan=plan)
|
||||||
|
overlap: torch.Tensor | None = None
|
||||||
|
|
||||||
|
for temporal_index in range(plan.num_chunks):
|
||||||
|
token_start = temporal_index * int(model.tokens_chunk_size) # type: ignore[attr-defined]
|
||||||
|
token_stop = token_start + int(model.tokens_chunk_size) + int(model.token_overlap) # type: ignore[attr-defined]
|
||||||
|
chunk_latent = plan.latent[:, :, token_start:token_stop]
|
||||||
|
decoded = model._adaptive_decode(chunk_latent) # type: ignore[attr-defined]
|
||||||
|
if plan.temporal_dtype is not None and decoded.dtype != plan.temporal_dtype:
|
||||||
|
decoded = decoded.to(plan.temporal_dtype)
|
||||||
|
if decoded.device != plan.latent.device:
|
||||||
|
decoded = decoded.to(plan.latent.device)
|
||||||
|
|
||||||
|
for split_index in range(split_count):
|
||||||
|
frame_start = split_index * chunk_decoded_frames
|
||||||
|
frame_stop = min(frame_start + chunk_decoded_frames, int(decoded.shape[2]))
|
||||||
|
part = decoded[:, :, frame_start:frame_stop]
|
||||||
|
part = part[:, :, int(model.frame_pre_padding) :] # type: ignore[attr-defined]
|
||||||
|
if split_index == 0:
|
||||||
|
if overlap is not None:
|
||||||
|
part = model.blend( # type: ignore[attr-defined]
|
||||||
|
overlap,
|
||||||
|
part,
|
||||||
|
int(model.frame_overlap), # type: ignore[attr-defined]
|
||||||
|
dim=-3,
|
||||||
|
)
|
||||||
|
overlap = None
|
||||||
|
assembly.write(part)
|
||||||
|
else:
|
||||||
|
overlap = part.contiguous()
|
||||||
|
del part
|
||||||
|
|
||||||
|
if temporal_index == plan.num_chunks - 1 and overlap is not None:
|
||||||
|
assembly.write(overlap)
|
||||||
|
overlap = None
|
||||||
|
del decoded, chunk_latent
|
||||||
|
|
||||||
|
return assembly.finish()
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_minimax_h3_temporal_chunks_compat(
|
||||||
|
model: nn.Module,
|
||||||
|
latent: torch.Tensor,
|
||||||
|
*,
|
||||||
|
validate_sources: bool = True,
|
||||||
|
) -> MiniMaxH3TemporalDecodePlan:
|
||||||
|
"""Build a validated plan without entering decoder collectives."""
|
||||||
|
|
||||||
|
if validate_sources:
|
||||||
|
_validate_released_sources(model)
|
||||||
|
_validate_released_config(model)
|
||||||
|
_validate_compat_contract(model, latent)
|
||||||
|
temporal_dtype = resolve_minimax_h3_temporal_cat_dtype(model)
|
||||||
|
return _build_decode_plan(model, latent, temporal_dtype)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"MINIMAX_H3_RELEASED_REVISION",
|
||||||
|
"MINIMAX_H3_RELEASED_BUNDLE_SHA256",
|
||||||
|
"MiniMaxH3TemporalDecodePlan",
|
||||||
|
"MiniMaxH3TemporalChunkCompatibilityError",
|
||||||
|
"decode_minimax_h3_temporal_chunks_compat",
|
||||||
|
"prepare_minimax_h3_temporal_chunks_compat",
|
||||||
|
"resolve_minimax_h3_temporal_cat_dtype",
|
||||||
|
]
|
||||||
@@ -21,7 +21,11 @@ from vllm.logger import init_logger
|
|||||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
||||||
DistributedVaeMixin,
|
DistributedVaeMixin,
|
||||||
)
|
)
|
||||||
from vllm_omni.diffusion.distributed.parallel_state import get_world_group
|
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||||
|
get_data_parallel_world_size,
|
||||||
|
get_world_group,
|
||||||
|
model_parallel_is_initialized,
|
||||||
|
)
|
||||||
from vllm_omni.diffusion.offloader.module_residency import (
|
from vllm_omni.diffusion.offloader.module_residency import (
|
||||||
BoundedAllocatorCache,
|
BoundedAllocatorCache,
|
||||||
PinnedModuleStager,
|
PinnedModuleStager,
|
||||||
@@ -29,6 +33,12 @@ from vllm_omni.diffusion.offloader.module_residency import (
|
|||||||
|
|
||||||
from .ops import install_h3_vae_optimizations
|
from .ops import install_h3_vae_optimizations
|
||||||
from .packed_tokens import minimax_h3_patchify_video_latent
|
from .packed_tokens import minimax_h3_patchify_video_latent
|
||||||
|
from .temporal_chunks import (
|
||||||
|
MiniMaxH3TemporalChunkCompatibilityError,
|
||||||
|
MiniMaxH3TemporalDecodePlan,
|
||||||
|
decode_minimax_h3_temporal_chunks_compat,
|
||||||
|
prepare_minimax_h3_temporal_chunks_compat,
|
||||||
|
)
|
||||||
|
|
||||||
MINIMAX_H3_KEYFRAME_ENCODE_SEED = 42
|
MINIMAX_H3_KEYFRAME_ENCODE_SEED = 42
|
||||||
MINIMAX_H3_AUDIO_SAMPLE_RATE = 32000
|
MINIMAX_H3_AUDIO_SAMPLE_RATE = 32000
|
||||||
@@ -60,7 +70,7 @@ class MiniMaxH3VideoChunkCallback(Protocol):
|
|||||||
|
|
||||||
|
|
||||||
class MiniMaxH3ChunkedDecodeUnsupportedError(RuntimeError):
|
class MiniMaxH3ChunkedDecodeUnsupportedError(RuntimeError):
|
||||||
"""Raised when the loaded MiniMax H3 VAE lacks the vendor chunk API."""
|
"""Raised when loaded MiniMax H3 code cannot honor the chunk contract."""
|
||||||
|
|
||||||
|
|
||||||
class MiniMaxH3ChunkCallbackPeerError(RuntimeError):
|
class MiniMaxH3ChunkCallbackPeerError(RuntimeError):
|
||||||
@@ -338,6 +348,10 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
def _chunk_process_group(self) -> dist.ProcessGroup | None:
|
def _chunk_process_group(self) -> dist.ProcessGroup | None:
|
||||||
if not self.is_distributed_enabled():
|
if not self.is_distributed_enabled():
|
||||||
return None
|
return None
|
||||||
|
if model_parallel_is_initialized() and get_data_parallel_world_size() > 1:
|
||||||
|
raise MiniMaxH3ChunkedDecodeUnsupportedError(
|
||||||
|
"MiniMax-H3 temporal chunks do not yet support combining data parallelism with VAE patch parallelism"
|
||||||
|
)
|
||||||
group = self._native_parallel_state().get("sp_process_group")
|
group = self._native_parallel_state().get("sp_process_group")
|
||||||
if group is None or dist.get_world_size(group) != self.parallel_size:
|
if group is None or dist.get_world_size(group) != self.parallel_size:
|
||||||
raise RuntimeError("MiniMax-H3 VAE chunk decode has an invalid spatial-parallel group")
|
raise RuntimeError("MiniMax-H3 VAE chunk decode has an invalid spatial-parallel group")
|
||||||
@@ -395,14 +409,48 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
raise callback_error
|
raise callback_error
|
||||||
raise MiniMaxH3ChunkCallbackPeerError("MiniMax-H3 video chunk callback failed on this VAE group's output rank")
|
raise MiniMaxH3ChunkCallbackPeerError("MiniMax-H3 video chunk callback failed on this VAE group's output rank")
|
||||||
|
|
||||||
def _validate_vendor_chunk_api(self) -> None:
|
@staticmethod
|
||||||
version = getattr(self.model, "temporal_chunk_callback_api_version", None)
|
def _synchronize_chunk_preflight_error(
|
||||||
if version != 1:
|
preflight_error: Exception | None,
|
||||||
raise MiniMaxH3ChunkedDecodeUnsupportedError(
|
temporal_dtype: torch.dtype | None,
|
||||||
"Loaded MiniMax-H3 VAE does not provide temporal_chunk_callback API v1 "
|
*,
|
||||||
f"(found {version!r}); install a model snapshot containing the official "
|
group: dist.ProcessGroup | None,
|
||||||
"MiniMax temporal chunk callback"
|
device: torch.device,
|
||||||
|
) -> None:
|
||||||
|
if group is None:
|
||||||
|
if preflight_error is not None:
|
||||||
|
raise preflight_error
|
||||||
|
return
|
||||||
|
failed = torch.tensor(
|
||||||
|
[int(preflight_error is not None)],
|
||||||
|
dtype=torch.int32,
|
||||||
|
device=device,
|
||||||
|
)
|
||||||
|
dist.all_reduce(failed, op=dist.ReduceOp.MAX, group=group)
|
||||||
|
if int(failed.item()) == 0:
|
||||||
|
dtype_codes = {
|
||||||
|
None: 0,
|
||||||
|
torch.float16: 1,
|
||||||
|
torch.bfloat16: 2,
|
||||||
|
torch.float32: 3,
|
||||||
|
}
|
||||||
|
code = torch.tensor(
|
||||||
|
[dtype_codes.get(temporal_dtype, -1)],
|
||||||
|
dtype=torch.int32,
|
||||||
|
device=device,
|
||||||
)
|
)
|
||||||
|
lowest = code.clone()
|
||||||
|
highest = code.clone()
|
||||||
|
dist.all_reduce(lowest, op=dist.ReduceOp.MIN, group=group)
|
||||||
|
dist.all_reduce(highest, op=dist.ReduceOp.MAX, group=group)
|
||||||
|
if int(lowest.item()) == int(highest.item()) and int(lowest.item()) >= 0:
|
||||||
|
return
|
||||||
|
raise MiniMaxH3ChunkedDecodeUnsupportedError(
|
||||||
|
"MiniMax-H3 VAE ranks selected incompatible temporal concat dtypes"
|
||||||
|
)
|
||||||
|
if preflight_error is not None:
|
||||||
|
raise preflight_error
|
||||||
|
raise MiniMaxH3ChunkedDecodeUnsupportedError("A peer VAE rank rejected the MiniMax-H3 temporal chunk preflight")
|
||||||
|
|
||||||
@torch.inference_mode()
|
@torch.inference_mode()
|
||||||
def encode_image(self, image: Image.Image) -> torch.Tensor:
|
def encode_image(self, image: Image.Image) -> torch.Tensor:
|
||||||
@@ -506,11 +554,11 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
latent: torch.Tensor,
|
latent: torch.Tensor,
|
||||||
chunk_callback: MiniMaxH3VideoChunkCallback | None,
|
chunk_callback: MiniMaxH3VideoChunkCallback | None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""Decode a video while publishing vendor-committed temporal chunks.
|
"""Decode a video while publishing committed temporal chunks.
|
||||||
|
|
||||||
Every rank collaborating in the configured H3 VAE tile group must call
|
Every rank collaborating in the configured H3 VAE tile group must call
|
||||||
this method, with ``chunk_callback`` supplied only on group rank 0. A
|
this method, with ``chunk_callback`` supplied only on group rank 0. A
|
||||||
callback failure is deferred until the vendor decoder finishes its
|
callback failure is deferred until the compatibility decoder finishes its
|
||||||
remaining collectives, then propagated to every group rank. With
|
remaining collectives, then propagated to every group rank. With
|
||||||
``parallel_size=1`` the runtime designates each request's output owner
|
``parallel_size=1`` the runtime designates each request's output owner
|
||||||
by supplying a callback there and ``None`` on replica peers.
|
by supplying a callback there and ``None`` on replica peers.
|
||||||
@@ -522,8 +570,41 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
requested-size crop as the complete H3 pipeline output.
|
requested-size crop as the complete H3 pipeline output.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
self._validate_vendor_chunk_api()
|
|
||||||
group = self._chunk_process_group()
|
group = self._chunk_process_group()
|
||||||
|
plan: MiniMaxH3TemporalDecodePlan | None = None
|
||||||
|
preflight_error: Exception | None = None
|
||||||
|
if latent.ndim != 5:
|
||||||
|
preflight_error = MiniMaxH3ChunkedDecodeUnsupportedError(
|
||||||
|
f"MiniMax-H3 video latent must be rank 5, got {tuple(latent.shape)}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
denormalized = self._denormalize_latent(latent)
|
||||||
|
plan = prepare_minimax_h3_temporal_chunks_compat(
|
||||||
|
self.model,
|
||||||
|
denormalized,
|
||||||
|
validate_sources=not getattr(
|
||||||
|
self,
|
||||||
|
"_minimax_h3_chunk_sources_validated",
|
||||||
|
False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self._minimax_h3_chunk_sources_validated = True
|
||||||
|
del denormalized
|
||||||
|
except MiniMaxH3TemporalChunkCompatibilityError as exc:
|
||||||
|
preflight_error = MiniMaxH3ChunkedDecodeUnsupportedError(str(exc))
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
# Every VAE rank must reach the preflight reduction even when
|
||||||
|
# one node's private planner or environment fails locally.
|
||||||
|
exc.__traceback__ = None
|
||||||
|
preflight_error = exc
|
||||||
|
self._synchronize_chunk_preflight_error(
|
||||||
|
preflight_error,
|
||||||
|
plan.temporal_dtype if plan is not None else None,
|
||||||
|
group=group,
|
||||||
|
device=latent.device,
|
||||||
|
)
|
||||||
|
assert plan is not None
|
||||||
is_output_rank = self._is_chunk_output_rank(
|
is_output_rank = self._is_chunk_output_rank(
|
||||||
chunk_callback,
|
chunk_callback,
|
||||||
group=group,
|
group=group,
|
||||||
@@ -560,7 +641,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
or is_final != (chunk_index + 1 == total_chunks)
|
or is_final != (chunk_index + 1 == total_chunks)
|
||||||
):
|
):
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"MiniMax-H3 vendor callback emitted discontinuous chunk metadata: "
|
"MiniMax-H3 temporal backend emitted discontinuous chunk metadata: "
|
||||||
f"index={chunk_index}/{next_chunk_index}, "
|
f"index={chunk_index}/{next_chunk_index}, "
|
||||||
f"total={total_chunks}/{expected_total_chunks}, "
|
f"total={total_chunks}/{expected_total_chunks}, "
|
||||||
f"frame_start={frame_start}/{next_frame_start}, "
|
f"frame_start={frame_start}/{next_frame_start}, "
|
||||||
@@ -570,7 +651,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
frames = frames.clone(memory_format=torch.contiguous_format)
|
frames = frames.clone(memory_format=torch.contiguous_format)
|
||||||
frame_count = int(frames.shape[2])
|
frame_count = int(frames.shape[2])
|
||||||
if frame_count <= 0:
|
if frame_count <= 0:
|
||||||
raise RuntimeError("MiniMax-H3 vendor callback emitted an empty chunk")
|
raise RuntimeError("MiniMax-H3 temporal backend emitted an empty chunk")
|
||||||
next_chunk_index += 1
|
next_chunk_index += 1
|
||||||
next_frame_start += frame_count
|
next_frame_start += frame_count
|
||||||
saw_final = is_final
|
saw_final = is_final
|
||||||
@@ -589,9 +670,10 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
callback_error = exc
|
callback_error = exc
|
||||||
|
|
||||||
with self._decode_tiling_context(latent):
|
with self._decode_tiling_context(latent):
|
||||||
decoded = self.model.decode_base(
|
decoded = decode_minimax_h3_temporal_chunks_compat(
|
||||||
self._denormalize_latent(latent),
|
self.model,
|
||||||
temporal_chunk_callback=publish,
|
plan,
|
||||||
|
publish,
|
||||||
)
|
)
|
||||||
frames = self._normalize_decoded_frames(decoded)
|
frames = self._normalize_decoded_frames(decoded)
|
||||||
if (
|
if (
|
||||||
@@ -600,7 +682,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
and (not saw_final or next_chunk_index != expected_total_chunks or next_frame_start != int(frames.shape[2]))
|
and (not saw_final or next_chunk_index != expected_total_chunks or next_frame_start != int(frames.shape[2]))
|
||||||
):
|
):
|
||||||
callback_error = RuntimeError(
|
callback_error = RuntimeError(
|
||||||
"MiniMax-H3 vendor callback did not reconstruct the full decode: "
|
"MiniMax-H3 temporal backend did not reconstruct the full decode: "
|
||||||
f"final={saw_final}, chunks={next_chunk_index}/{expected_total_chunks}, "
|
f"final={saw_final}, chunks={next_chunk_index}/{expected_total_chunks}, "
|
||||||
f"frames={next_frame_start}/{frames.shape[2]}"
|
f"frames={next_frame_start}/{frames.shape[2]}"
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in new issue
Block a user