diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_chunks.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_chunks.py index fa51cef52..6994c3432 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_chunks.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_vae_chunks.py @@ -769,6 +769,55 @@ def _distributed_worker( dist.destroy_process_group() +def _mismatched_plan_worker( + rank: int, + init_file: str, + result_queue: Any, +) -> None: + dist.init_process_group( + "gloo", + init_method=f"file://{init_file}", + rank=rank, + world_size=2, + ) + try: + vae_module.get_data_parallel_world_size = lambda: 1 + vae_module.model_parallel_is_initialized = lambda: True + _install_released_source_fingerprint() + + vae, model = _compat_vae( + 37 if rank == 0 else 62, + parallel_size=2, + collective=True, + ) + vae._native_parallel_state = MethodType( + lambda self: {"sp_process_group": dist.group.WORLD}, + vae, + ) + try: + vae.decode_latent_with_chunks( + _latent(37 if rank == 0 else 62), + (lambda *args, **kwargs: None) if rank == 0 else None, + ) + except BaseException as exc: # noqa: BLE001 + failure_type = type(exc).__name__ + else: + failure_type = "none" + + recovery = torch.tensor([rank + 1], dtype=torch.int32) + dist.all_reduce(recovery) + result_queue.put( + { + "rank": rank, + "failure_type": failure_type, + "adaptive_calls": model.adaptive_decode_calls, + "recovery": int(recovery.item()), + } + ) + finally: + dist.destroy_process_group() + + def _wrong_owner_worker( rank: int, init_file: str, @@ -861,6 +910,47 @@ def test_three_rank_wrong_callback_owner_is_symmetric_and_group_recovers( assert [result["recovery"] for result in results] == [6, 6, 6] +@pytest.mark.parallel +def test_two_rank_mismatched_decode_plans_fail_before_decoder_collectives( + tmp_path: Path, +) -> None: + context = mp.get_context("spawn") + result_queue = context.Queue() + init_file = str(tmp_path / "gloo-mismatched-plan-init") + processes = [ + context.Process( + target=_mismatched_plan_worker, + args=(rank, init_file, result_queue), + ) + for rank in range(2) + ] + for process in processes: + process.start() + deadline = time.monotonic() + 90 + for process in processes: + process.join(timeout=max(0, deadline - time.monotonic())) + for process in processes: + if process.is_alive(): + process.join(timeout=1) + hung = [process for process in processes if process.is_alive()] + for process in hung: + process.terminate() + process.join(timeout=5) + assert not hung, "mismatched MiniMax-H3 decode plans deadlocked" + assert [process.exitcode for process in processes] == [0, 0] + + results = sorted( + [result_queue.get(timeout=5) for _ in range(2)], + key=lambda result: result["rank"], + ) + assert [result["failure_type"] for result in results] == [ + MiniMaxH3ChunkedDecodeUnsupportedError.__name__, + MiniMaxH3ChunkedDecodeUnsupportedError.__name__, + ] + assert [result["adaptive_calls"] for result in results] == [0, 0] + assert [result["recovery"] for result in results] == [3, 3] + + @pytest.mark.parallel def test_two_rank_callback_failure_is_symmetric_and_group_recovers( tmp_path: Path, diff --git a/vllm_omni/diffusion/models/minimax_h3/chunked_decode.py b/vllm_omni/diffusion/models/minimax_h3/chunked_decode.py index a91884017..88cf5395a 100644 --- a/vllm_omni/diffusion/models/minimax_h3/chunked_decode.py +++ b/vllm_omni/diffusion/models/minimax_h3/chunked_decode.py @@ -82,6 +82,37 @@ class _MiniMaxH3ChunkDecodeCoordinator: def __init__(self) -> None: self._sources_validated = False + @staticmethod + def _build_execution_signature( + latent: torch.Tensor, + plan: MiniMaxH3TemporalDecodePlan, + ) -> tuple[int, ...]: + """Describe every shape and plan field used by the temporal schedule.""" + + dtype_codes: dict[torch.dtype | None, int] = { + None: 0, + torch.float16: 1, + torch.bfloat16: 2, + torch.float32: 3, + torch.float64: 4, + } + return ( + # Keep the layout versioned so adding a field cannot silently make + # an older and newer worker appear compatible. + 1, + *[int(size) for size in latent.shape], + dtype_codes.get(latent.dtype, -1), + *[int(size) for size in plan.latent.shape], + dtype_codes.get(plan.latent.dtype, -1), + int(plan.num_chunks), + int(plan.total_frames), + int(plan.pad_frames), + int(plan.output_frames), + len(plan.chunk_frames), + *[int(frame_count) for frame_count in plan.chunk_frames], + dtype_codes.get(plan.temporal_dtype, -1), + ) + @staticmethod def _is_output_rank( chunk_callback: MiniMaxH3VideoChunkCallback | None, @@ -140,6 +171,7 @@ class _MiniMaxH3ChunkDecodeCoordinator: def _synchronize_preflight_error( preflight_error: Exception | None, temporal_dtype: torch.dtype | None, + execution_signature: tuple[int, ...] | None, *, group: dist.ProcessGroup | None, device: torch.device, @@ -171,6 +203,44 @@ class _MiniMaxH3ChunkDecodeCoordinator: dist.all_reduce(lowest, op=dist.ReduceOp.MIN, group=group) dist.all_reduce(highest, op=dist.ReduceOp.MAX, group=group) if int(lowest.item()) == int(highest.item()) and int(lowest.item()) >= 0: + if execution_signature is None: + raise RuntimeError("MiniMax-H3 temporal decode preflight produced no execution signature") + + # The plan length is shape-dependent, so first agree on a + # common tensor size and then compare the complete signatures. + # This keeps all ranks in tensor collectives and catches a + # schedule mismatch before entering the decoder's collectives. + signature_length = torch.tensor( + [len(execution_signature)], + dtype=torch.int64, + device=device, + ) + max_signature_length = signature_length.clone() + dist.all_reduce( + max_signature_length, + op=dist.ReduceOp.MAX, + group=group, + ) + padded_signature = torch.full( + (int(max_signature_length.item()),), + -2, + dtype=torch.int64, + device=device, + ) + padded_signature[: len(execution_signature)] = torch.tensor( + execution_signature, + dtype=torch.int64, + device=device, + ) + gathered_signatures = [ + torch.empty_like(padded_signature) + for _ in range(dist.get_world_size(group)) + ] + dist.all_gather(gathered_signatures, padded_signature, group=group) + if any(not torch.equal(padded_signature, peer) for peer in gathered_signatures): + raise MiniMaxH3ChunkedDecodeUnsupportedError( + "MiniMax-H3 VAE ranks built incompatible temporal decode plans" + ) return raise MiniMaxH3ChunkedDecodeUnsupportedError( "MiniMax-H3 VAE ranks selected incompatible temporal concat dtypes" @@ -215,6 +285,7 @@ class _MiniMaxH3ChunkDecodeCoordinator: self._synchronize_preflight_error( preflight_error, plan.temporal_dtype if plan is not None else None, + self._build_execution_signature(latent, plan) if plan is not None else None, group=group, device=latent.device, )