diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_chunks.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_chunks.py index 1d57a7c77..fa51cef52 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_chunks.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_chunks.py @@ -402,6 +402,29 @@ def test_compat_callback_failure_finishes_decode_and_recovers( assert model.adaptive_decode_calls == model.num_chunks * 2 +def test_native_decoder_failure_propagates_on_single_rank( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _allow_released_source(monkeypatch) + vae, model = _compat_vae(37) + callback_calls = 0 + + def fail_decode(latent: torch.Tensor) -> torch.Tensor: + del latent + raise LookupError("native decoder failed") + + def consume(*args, **kwargs) -> None: + nonlocal callback_calls + del args, kwargs + callback_calls += 1 + + monkeypatch.setattr(model, "_adaptive_decode", fail_decode) + with pytest.raises(LookupError, match="native decoder failed"): + vae.decode_latent_with_chunks(_latent(), consume) + + assert callback_calls == 0 + + @pytest.mark.parametrize("drift", ["changed", "missing"]) def test_compat_source_drift_fails_closed_before_decode( monkeypatch: pytest.MonkeyPatch, @@ -746,6 +769,98 @@ def _distributed_worker( dist.destroy_process_group() +def _wrong_owner_worker( + rank: int, + init_file: str, + result_queue: Any, +) -> None: + dist.init_process_group( + "gloo", + init_method=f"file://{init_file}", + rank=rank, + world_size=3, + ) + try: + vae_module.get_data_parallel_world_size = lambda: 1 + vae_module.model_parallel_is_initialized = lambda: True + _install_released_source_fingerprint() + vae, model = _compat_vae( + 37, + parallel_size=3, + collective=True, + tile_count=3, + ) + vae._native_parallel_state = MethodType( + lambda self: {"sp_process_group": dist.group.WORLD}, + vae, + ) + + try: + vae.decode_latent_with_chunks( + _latent(), + (lambda *args, **kwargs: None) if rank == 1 else None, + ) + except BaseException as exc: # noqa: BLE001 + failure_type = type(exc).__name__ + else: + failure_type = "none" + + recovery = torch.tensor([rank + 1], dtype=torch.int32) + dist.all_reduce(recovery) + result_queue.put( + { + "rank": rank, + "failure_type": failure_type, + "adaptive_calls": model.adaptive_decode_calls, + "recovery": int(recovery.item()), + } + ) + finally: + dist.destroy_process_group() + + +@pytest.mark.parallel +def test_three_rank_wrong_callback_owner_is_symmetric_and_group_recovers( + tmp_path: Path, +) -> None: + context = mp.get_context("spawn") + result_queue = context.Queue() + init_file = str(tmp_path / "gloo-wrong-owner-init") + processes = [ + context.Process( + target=_wrong_owner_worker, + args=(rank, init_file, result_queue), + ) + for rank in range(3) + ] + for process in processes: + process.start() + deadline = time.monotonic() + 90 + for process in processes: + process.join(timeout=max(0, deadline - time.monotonic())) + for process in processes: + if process.is_alive(): + process.join(timeout=1) + hung = [process for process in processes if process.is_alive()] + for process in hung: + process.terminate() + process.join(timeout=5) + assert not hung, "three-rank callback-owner validation deadlocked" + assert [process.exitcode for process in processes] == [0, 0, 0] + + results = sorted( + [result_queue.get(timeout=5) for _ in range(3)], + key=lambda result: result["rank"], + ) + assert [result["failure_type"] for result in results] == [ + "ValueError", + "ValueError", + "ValueError", + ] + assert [result["adaptive_calls"] for result in results] == [0, 0, 0] + assert [result["recovery"] for result in results] == [6, 6, 6] + + @pytest.mark.parallel def test_two_rank_callback_failure_is_symmetric_and_group_recovers( tmp_path: Path, diff --git a/vllm_omni/diffusion/models/minimax_h3/chunked_decode.py b/vllm_omni/diffusion/models/minimax_h3/chunked_decode.py index 70b6cfe59..a91884017 100644 --- a/vllm_omni/diffusion/models/minimax_h3/chunked_decode.py +++ b/vllm_omni/diffusion/models/minimax_h3/chunked_decode.py @@ -2,8 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Model-local coordination for MiniMax H3 temporal chunk decoding. -This boundary owns released-code compatibility and VAE-group failure semantics. -Cropping, representation conversion, transport, and encoding remain runtime +This boundary owns released-code compatibility and reconciles preflight and +callback failures at fixed VAE-group rendezvous points. Cropping, +representation conversion, transport, and encoding remain runtime responsibilities. """ @@ -19,31 +20,24 @@ import torch.nn as nn from .temporal_chunks import ( MiniMaxH3TemporalChunkCompatibilityError, MiniMaxH3TemporalDecodePlan, + _TemporalChunkCallbackSignature, decode_minimax_h3_temporal_chunks_compat, prepare_minimax_h3_temporal_chunks_compat, ) -class MiniMaxH3VideoChunkCallback(Protocol): +class MiniMaxH3VideoChunkCallback(_TemporalChunkCallbackSignature, Protocol): """Consume one ordered, temporally committed H3 video chunk. ``frames`` is an owned, contiguous float32 RGB tensor in ``BCTHW`` layout and the ``[0, 1]`` value range. The callback runs synchronously on the producer's current device stream. Before reading it on another stream, the consumer must establish ordering from the producer stream (for example via - an event) and retain the allocation for that stream's lifetime. + an event) and retain the allocation for that stream's lifetime. The + consumer may retain or mutate the tensor; it does not alias the returned + complete decode. """ - def __call__( - self, - frames: torch.Tensor, - *, - chunk_index: int, - total_chunks: int, - frame_start: int, - is_final: bool, - ) -> None: ... - class MiniMaxH3ChunkedDecodeUnsupportedError(RuntimeError): """Raised when loaded MiniMax H3 code cannot honor the chunk contract.""" @@ -77,6 +71,12 @@ class _MiniMaxH3ChunkDecodeCoordinator: The coordinator is a plain Python object. It caches only successful source validation; request callbacks, tensors, metadata, and process groups remain local to :meth:`decode`. + + Errors raised inside the checkpoint decoder after native collectives begin + cannot be reconciled here: another rank may already be blocked inside that + collective. Resolving such a failure requires native collective handling or + supervisor-level timeout and worker-group teardown; the process group cannot + be assumed reusable. """ def __init__(self) -> None: @@ -93,14 +93,16 @@ class _MiniMaxH3ChunkDecodeCoordinator: return chunk_callback is not None group_rank = dist.get_rank(group) - callback_owners = torch.tensor( - [int(chunk_callback is not None)], - dtype=torch.int32, + # A valid configuration has one contribution: rank 0 contributes 1. + # A non-zero owner contributes at least 2, and multiple owners sum to + # at least 3, so every rank makes the same decision from one reduction. + callback_owner = torch.tensor( + [group_rank + 1 if chunk_callback is not None else 0], + dtype=torch.int64, device=device, ) - dist.all_reduce(callback_owners, group=group) - valid_owner = int(callback_owners.item()) == 1 and ((group_rank == 0) == (chunk_callback is not None)) - if not valid_owner: + dist.all_reduce(callback_owner, op=dist.ReduceOp.SUM, group=group) + if int(callback_owner.item()) != 1: raise ValueError("MiniMax-H3 chunk decode requires a callback on exactly VAE group rank 0") return group_rank == 0 @@ -282,6 +284,11 @@ class _MiniMaxH3ChunkDecodeCoordinator: exc.__traceback__ = None callback_error = exc + # Do not catch and reduce exceptions from this region. A peer may + # already be blocked in one of the checkpoint's native tile + # collectives, in which case a new adapter-level collective would also + # deadlock. A native backend or supervisor must handle timeout and + # worker-group teardown. with host._decode_tiling_context(latent): decoded = decode_minimax_h3_temporal_chunks_compat( host.model, diff --git a/vllm_omni/diffusion/models/minimax_h3/temporal_chunks.py b/vllm_omni/diffusion/models/minimax_h3/temporal_chunks.py index 27f6d3457..dbe81c3dd 100644 --- a/vllm_omni/diffusion/models/minimax_h3/temporal_chunks.py +++ b/vllm_omni/diffusion/models/minimax_h3/temporal_chunks.py @@ -36,10 +36,13 @@ class MiniMaxH3TemporalChunkCompatibilityError(RuntimeError): """Raised when remote H3 code cannot use the local compatibility path.""" -class _RawTemporalChunkCallback(Protocol): +class _TemporalChunkCallbackSignature(Protocol): + """Shared callable shape; each producer defines tensor semantics.""" + def __call__( self, frames: torch.Tensor, + /, *, chunk_index: int, total_chunks: int, @@ -63,7 +66,7 @@ class _RawChunkAssembly: def __init__( self, *, - callback: _RawTemporalChunkCallback, + callback: _TemporalChunkCallbackSignature, plan: MiniMaxH3TemporalDecodePlan, ) -> None: self.callback = callback @@ -330,13 +333,16 @@ def resolve_minimax_h3_temporal_cat_dtype(model: nn.Module) -> torch.dtype | Non def decode_minimax_h3_temporal_chunks_compat( model: nn.Module, plan: MiniMaxH3TemporalDecodePlan, - callback: _RawTemporalChunkCallback, + callback: _TemporalChunkCallbackSignature, ) -> torch.Tensor: """Mirror the released Apache-2.0 H3 temporal assembler with callbacks. Keep this implementation and the source/config fingerprints above in sync with ``MiniMaxAI/MiniMax-H3@42ed227e``. Unknown remote code fails closed before this function is called. + + Callback tensors are borrowed decoder-domain views and must not be retained + or mutated. The high-level adapter owns normalization and storage isolation. """ chunk_decoded_frames = int(model.tokens_chunk_size) * int(model.vae_ratio_t) # type: ignore[attr-defined]