[Bugfix][MiniMax-H3] Reconcile chunk callback ownership

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
hsliuustc0106 committed 2026-09-01 23:11:29 +08:00
1 parent f04ff76c54
commit 0b5b0979d8
3 files changed
+151 -23

No files matched your search

@@ -402,6 +402,29 @@ def test_compat_callback_failure_finishes_decode_and_recovers(
assert model.adaptive_decode_calls == model.num_chunks * 2
def test_native_decoder_failure_propagates_on_single_rank(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_allow_released_source(monkeypatch)
vae, model = _compat_vae(37)
callback_calls = 0
def fail_decode(latent: torch.Tensor) -> torch.Tensor:
del latent
raise LookupError("native decoder failed")
def consume(*args, **kwargs) -> None:
nonlocal callback_calls
del args, kwargs
callback_calls += 1
monkeypatch.setattr(model, "_adaptive_decode", fail_decode)
with pytest.raises(LookupError, match="native decoder failed"):
vae.decode_latent_with_chunks(_latent(), consume)
assert callback_calls == 0
@pytest.mark.parametrize("drift", ["changed", "missing"])
def test_compat_source_drift_fails_closed_before_decode(
monkeypatch: pytest.MonkeyPatch,
@@ -746,6 +769,98 @@ def _distributed_worker(
dist.destroy_process_group()
def _wrong_owner_worker(
rank: int,
init_file: str,
result_queue: Any,
) -> None:
dist.init_process_group(
"gloo",
init_method=f"file://{init_file}",
rank=rank,
world_size=3,
)
try:
vae_module.get_data_parallel_world_size = lambda: 1
vae_module.model_parallel_is_initialized = lambda: True
_install_released_source_fingerprint()
vae, model = _compat_vae(
37,
parallel_size=3,
collective=True,
tile_count=3,
)
vae._native_parallel_state = MethodType(
lambda self: {"sp_process_group": dist.group.WORLD},
vae,
)
try:
vae.decode_latent_with_chunks(
_latent(),
(lambda *args, **kwargs: None) if rank == 1 else None,
)
except BaseException as exc: # noqa: BLE001
failure_type = type(exc).__name__
else:
failure_type = "none"
recovery = torch.tensor([rank + 1], dtype=torch.int32)
dist.all_reduce(recovery)
result_queue.put(
{
"rank": rank,
"failure_type": failure_type,
"adaptive_calls": model.adaptive_decode_calls,
"recovery": int(recovery.item()),
}
)
finally:
dist.destroy_process_group()
@pytest.mark.parallel
def test_three_rank_wrong_callback_owner_is_symmetric_and_group_recovers(
tmp_path: Path,
) -> None:
context = mp.get_context("spawn")
result_queue = context.Queue()
init_file = str(tmp_path / "gloo-wrong-owner-init")
processes = [
context.Process(
target=_wrong_owner_worker,
args=(rank, init_file, result_queue),
)
for rank in range(3)
]
for process in processes:
process.start()
deadline = time.monotonic() + 90
for process in processes:
process.join(timeout=max(0, deadline - time.monotonic()))
for process in processes:
if process.is_alive():
process.join(timeout=1)
hung = [process for process in processes if process.is_alive()]
for process in hung:
process.terminate()
process.join(timeout=5)
assert not hung, "three-rank callback-owner validation deadlocked"
assert [process.exitcode for process in processes] == [0, 0, 0]
results = sorted(
[result_queue.get(timeout=5) for _ in range(3)],
key=lambda result: result["rank"],
)
assert [result["failure_type"] for result in results] == [
"ValueError",
"ValueError",
"ValueError",
]
assert [result["adaptive_calls"] for result in results] == [0, 0, 0]
assert [result["recovery"] for result in results] == [6, 6, 6]
@pytest.mark.parallel
def test_two_rank_callback_failure_is_symmetric_and_group_recovers(
tmp_path: Path,
@@ -2,8 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Model-local coordination for MiniMax H3 temporal chunk decoding.
This boundary owns released-code compatibility and VAE-group failure semantics.
Cropping, representation conversion, transport, and encoding remain runtime
This boundary owns released-code compatibility and reconciles preflight and
callback failures at fixed VAE-group rendezvous points. Cropping,
representation conversion, transport, and encoding remain runtime
responsibilities.
"""
@@ -19,31 +20,24 @@ import torch.nn as nn
from .temporal_chunks import (
MiniMaxH3TemporalChunkCompatibilityError,
MiniMaxH3TemporalDecodePlan,
_TemporalChunkCallbackSignature,
decode_minimax_h3_temporal_chunks_compat,
prepare_minimax_h3_temporal_chunks_compat,
)
class MiniMaxH3VideoChunkCallback(Protocol):
class MiniMaxH3VideoChunkCallback(_TemporalChunkCallbackSignature, Protocol):
"""Consume one ordered, temporally committed H3 video chunk.
``frames`` is an owned, contiguous float32 RGB tensor in ``BCTHW`` layout
and the ``[0, 1]`` value range. The callback runs synchronously on the
producer's current device stream. Before reading it on another stream, the
consumer must establish ordering from the producer stream (for example via
an event) and retain the allocation for that stream's lifetime.
an event) and retain the allocation for that stream's lifetime. The
consumer may retain or mutate the tensor; it does not alias the returned
complete decode.
"""
def __call__(
self,
frames: torch.Tensor,
*,
chunk_index: int,
total_chunks: int,
frame_start: int,
is_final: bool,
) -> None: ...
class MiniMaxH3ChunkedDecodeUnsupportedError(RuntimeError):
"""Raised when loaded MiniMax H3 code cannot honor the chunk contract."""
@@ -77,6 +71,12 @@ class _MiniMaxH3ChunkDecodeCoordinator:
The coordinator is a plain Python object. It caches only successful source
validation; request callbacks, tensors, metadata, and process groups remain
local to :meth:`decode`.
Errors raised inside the checkpoint decoder after native collectives begin
cannot be reconciled here: another rank may already be blocked inside that
collective. Resolving such a failure requires native collective handling or
supervisor-level timeout and worker-group teardown; the process group cannot
be assumed reusable.
"""
def __init__(self) -> None:
@@ -93,14 +93,16 @@ class _MiniMaxH3ChunkDecodeCoordinator:
return chunk_callback is not None
group_rank = dist.get_rank(group)
callback_owners = torch.tensor(
[int(chunk_callback is not None)],
dtype=torch.int32,
# A valid configuration has one contribution: rank 0 contributes 1.
# A non-zero owner contributes at least 2, and multiple owners sum to
# at least 3, so every rank makes the same decision from one reduction.
callback_owner = torch.tensor(
[group_rank + 1 if chunk_callback is not None else 0],
dtype=torch.int64,
device=device,
)
dist.all_reduce(callback_owners, group=group)
valid_owner = int(callback_owners.item()) == 1 and ((group_rank == 0) == (chunk_callback is not None))
if not valid_owner:
dist.all_reduce(callback_owner, op=dist.ReduceOp.SUM, group=group)
if int(callback_owner.item()) != 1:
raise ValueError("MiniMax-H3 chunk decode requires a callback on exactly VAE group rank 0")
return group_rank == 0
@@ -282,6 +284,11 @@ class _MiniMaxH3ChunkDecodeCoordinator:
exc.__traceback__ = None
callback_error = exc
# Do not catch and reduce exceptions from this region. A peer may
# already be blocked in one of the checkpoint's native tile
# collectives, in which case a new adapter-level collective would also
# deadlock. A native backend or supervisor must handle timeout and
# worker-group teardown.
with host._decode_tiling_context(latent):
decoded = decode_minimax_h3_temporal_chunks_compat(
host.model,
@@ -36,10 +36,13 @@ class MiniMaxH3TemporalChunkCompatibilityError(RuntimeError):
"""Raised when remote H3 code cannot use the local compatibility path."""
class _RawTemporalChunkCallback(Protocol):
class _TemporalChunkCallbackSignature(Protocol):
"""Shared callable shape; each producer defines tensor semantics."""
def __call__(
self,
frames: torch.Tensor,
/,
*,
chunk_index: int,
total_chunks: int,
@@ -63,7 +66,7 @@ class _RawChunkAssembly:
def __init__(
self,
*,
callback: _RawTemporalChunkCallback,
callback: _TemporalChunkCallbackSignature,
plan: MiniMaxH3TemporalDecodePlan,
) -> None:
self.callback = callback
@@ -330,13 +333,16 @@ def resolve_minimax_h3_temporal_cat_dtype(model: nn.Module) -> torch.dtype | Non
def decode_minimax_h3_temporal_chunks_compat(
model: nn.Module,
plan: MiniMaxH3TemporalDecodePlan,
callback: _RawTemporalChunkCallback,
callback: _TemporalChunkCallbackSignature,
) -> torch.Tensor:
"""Mirror the released Apache-2.0 H3 temporal assembler with callbacks.
Keep this implementation and the source/config fingerprints above in sync
with ``MiniMaxAI/MiniMax-H3@42ed227e``. Unknown remote code fails closed
before this function is called.
Callback tensors are borrowed decoder-domain views and must not be retained
or mutated. The high-level adapter owns normalization and storage isolation.
"""
chunk_decoded_frames = int(model.tokens_chunk_size) * int(model.vae_ratio_t) # type: ignore[attr-defined]