[Bugfix][MiniMax-H3] Reconcile chunk callback ownership
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
f04ff76c54
commit
0b5b0979d8
3 files changed
+151
-23
No files matched your search
@@ -402,6 +402,29 @@ def test_compat_callback_failure_finishes_decode_and_recovers(
|
||||
assert model.adaptive_decode_calls == model.num_chunks * 2
|
||||
|
||||
|
||||
def test_native_decoder_failure_propagates_on_single_rank(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_allow_released_source(monkeypatch)
|
||||
vae, model = _compat_vae(37)
|
||||
callback_calls = 0
|
||||
|
||||
def fail_decode(latent: torch.Tensor) -> torch.Tensor:
|
||||
del latent
|
||||
raise LookupError("native decoder failed")
|
||||
|
||||
def consume(*args, **kwargs) -> None:
|
||||
nonlocal callback_calls
|
||||
del args, kwargs
|
||||
callback_calls += 1
|
||||
|
||||
monkeypatch.setattr(model, "_adaptive_decode", fail_decode)
|
||||
with pytest.raises(LookupError, match="native decoder failed"):
|
||||
vae.decode_latent_with_chunks(_latent(), consume)
|
||||
|
||||
assert callback_calls == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drift", ["changed", "missing"])
|
||||
def test_compat_source_drift_fails_closed_before_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -746,6 +769,98 @@ def _distributed_worker(
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _wrong_owner_worker(
|
||||
rank: int,
|
||||
init_file: str,
|
||||
result_queue: Any,
|
||||
) -> None:
|
||||
dist.init_process_group(
|
||||
"gloo",
|
||||
init_method=f"file://{init_file}",
|
||||
rank=rank,
|
||||
world_size=3,
|
||||
)
|
||||
try:
|
||||
vae_module.get_data_parallel_world_size = lambda: 1
|
||||
vae_module.model_parallel_is_initialized = lambda: True
|
||||
_install_released_source_fingerprint()
|
||||
vae, model = _compat_vae(
|
||||
37,
|
||||
parallel_size=3,
|
||||
collective=True,
|
||||
tile_count=3,
|
||||
)
|
||||
vae._native_parallel_state = MethodType(
|
||||
lambda self: {"sp_process_group": dist.group.WORLD},
|
||||
vae,
|
||||
)
|
||||
|
||||
try:
|
||||
vae.decode_latent_with_chunks(
|
||||
_latent(),
|
||||
(lambda *args, **kwargs: None) if rank == 1 else None,
|
||||
)
|
||||
except BaseException as exc: # noqa: BLE001
|
||||
failure_type = type(exc).__name__
|
||||
else:
|
||||
failure_type = "none"
|
||||
|
||||
recovery = torch.tensor([rank + 1], dtype=torch.int32)
|
||||
dist.all_reduce(recovery)
|
||||
result_queue.put(
|
||||
{
|
||||
"rank": rank,
|
||||
"failure_type": failure_type,
|
||||
"adaptive_calls": model.adaptive_decode_calls,
|
||||
"recovery": int(recovery.item()),
|
||||
}
|
||||
)
|
||||
finally:
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parallel
|
||||
def test_three_rank_wrong_callback_owner_is_symmetric_and_group_recovers(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
context = mp.get_context("spawn")
|
||||
result_queue = context.Queue()
|
||||
init_file = str(tmp_path / "gloo-wrong-owner-init")
|
||||
processes = [
|
||||
context.Process(
|
||||
target=_wrong_owner_worker,
|
||||
args=(rank, init_file, result_queue),
|
||||
)
|
||||
for rank in range(3)
|
||||
]
|
||||
for process in processes:
|
||||
process.start()
|
||||
deadline = time.monotonic() + 90
|
||||
for process in processes:
|
||||
process.join(timeout=max(0, deadline - time.monotonic()))
|
||||
for process in processes:
|
||||
if process.is_alive():
|
||||
process.join(timeout=1)
|
||||
hung = [process for process in processes if process.is_alive()]
|
||||
for process in hung:
|
||||
process.terminate()
|
||||
process.join(timeout=5)
|
||||
assert not hung, "three-rank callback-owner validation deadlocked"
|
||||
assert [process.exitcode for process in processes] == [0, 0, 0]
|
||||
|
||||
results = sorted(
|
||||
[result_queue.get(timeout=5) for _ in range(3)],
|
||||
key=lambda result: result["rank"],
|
||||
)
|
||||
assert [result["failure_type"] for result in results] == [
|
||||
"ValueError",
|
||||
"ValueError",
|
||||
"ValueError",
|
||||
]
|
||||
assert [result["adaptive_calls"] for result in results] == [0, 0, 0]
|
||||
assert [result["recovery"] for result in results] == [6, 6, 6]
|
||||
|
||||
|
||||
@pytest.mark.parallel
|
||||
def test_two_rank_callback_failure_is_symmetric_and_group_recovers(
|
||||
tmp_path: Path,
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Model-local coordination for MiniMax H3 temporal chunk decoding.
|
||||
|
||||
This boundary owns released-code compatibility and VAE-group failure semantics.
|
||||
Cropping, representation conversion, transport, and encoding remain runtime
|
||||
This boundary owns released-code compatibility and reconciles preflight and
|
||||
callback failures at fixed VAE-group rendezvous points. Cropping,
|
||||
representation conversion, transport, and encoding remain runtime
|
||||
responsibilities.
|
||||
"""
|
||||
|
||||
@@ -19,31 +20,24 @@ import torch.nn as nn
|
||||
from .temporal_chunks import (
|
||||
MiniMaxH3TemporalChunkCompatibilityError,
|
||||
MiniMaxH3TemporalDecodePlan,
|
||||
_TemporalChunkCallbackSignature,
|
||||
decode_minimax_h3_temporal_chunks_compat,
|
||||
prepare_minimax_h3_temporal_chunks_compat,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxH3VideoChunkCallback(Protocol):
|
||||
class MiniMaxH3VideoChunkCallback(_TemporalChunkCallbackSignature, Protocol):
|
||||
"""Consume one ordered, temporally committed H3 video chunk.
|
||||
|
||||
``frames`` is an owned, contiguous float32 RGB tensor in ``BCTHW`` layout
|
||||
and the ``[0, 1]`` value range. The callback runs synchronously on the
|
||||
producer's current device stream. Before reading it on another stream, the
|
||||
consumer must establish ordering from the producer stream (for example via
|
||||
an event) and retain the allocation for that stream's lifetime.
|
||||
an event) and retain the allocation for that stream's lifetime. The
|
||||
consumer may retain or mutate the tensor; it does not alias the returned
|
||||
complete decode.
|
||||
"""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
frames: torch.Tensor,
|
||||
*,
|
||||
chunk_index: int,
|
||||
total_chunks: int,
|
||||
frame_start: int,
|
||||
is_final: bool,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class MiniMaxH3ChunkedDecodeUnsupportedError(RuntimeError):
|
||||
"""Raised when loaded MiniMax H3 code cannot honor the chunk contract."""
|
||||
@@ -77,6 +71,12 @@ class _MiniMaxH3ChunkDecodeCoordinator:
|
||||
The coordinator is a plain Python object. It caches only successful source
|
||||
validation; request callbacks, tensors, metadata, and process groups remain
|
||||
local to :meth:`decode`.
|
||||
|
||||
Errors raised inside the checkpoint decoder after native collectives begin
|
||||
cannot be reconciled here: another rank may already be blocked inside that
|
||||
collective. Resolving such a failure requires native collective handling or
|
||||
supervisor-level timeout and worker-group teardown; the process group cannot
|
||||
be assumed reusable.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
@@ -93,14 +93,16 @@ class _MiniMaxH3ChunkDecodeCoordinator:
|
||||
return chunk_callback is not None
|
||||
|
||||
group_rank = dist.get_rank(group)
|
||||
callback_owners = torch.tensor(
|
||||
[int(chunk_callback is not None)],
|
||||
dtype=torch.int32,
|
||||
# A valid configuration has one contribution: rank 0 contributes 1.
|
||||
# A non-zero owner contributes at least 2, and multiple owners sum to
|
||||
# at least 3, so every rank makes the same decision from one reduction.
|
||||
callback_owner = torch.tensor(
|
||||
[group_rank + 1 if chunk_callback is not None else 0],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
dist.all_reduce(callback_owners, group=group)
|
||||
valid_owner = int(callback_owners.item()) == 1 and ((group_rank == 0) == (chunk_callback is not None))
|
||||
if not valid_owner:
|
||||
dist.all_reduce(callback_owner, op=dist.ReduceOp.SUM, group=group)
|
||||
if int(callback_owner.item()) != 1:
|
||||
raise ValueError("MiniMax-H3 chunk decode requires a callback on exactly VAE group rank 0")
|
||||
return group_rank == 0
|
||||
|
||||
@@ -282,6 +284,11 @@ class _MiniMaxH3ChunkDecodeCoordinator:
|
||||
exc.__traceback__ = None
|
||||
callback_error = exc
|
||||
|
||||
# Do not catch and reduce exceptions from this region. A peer may
|
||||
# already be blocked in one of the checkpoint's native tile
|
||||
# collectives, in which case a new adapter-level collective would also
|
||||
# deadlock. A native backend or supervisor must handle timeout and
|
||||
# worker-group teardown.
|
||||
with host._decode_tiling_context(latent):
|
||||
decoded = decode_minimax_h3_temporal_chunks_compat(
|
||||
host.model,
|
||||
|
||||
@@ -36,10 +36,13 @@ class MiniMaxH3TemporalChunkCompatibilityError(RuntimeError):
|
||||
"""Raised when remote H3 code cannot use the local compatibility path."""
|
||||
|
||||
|
||||
class _RawTemporalChunkCallback(Protocol):
|
||||
class _TemporalChunkCallbackSignature(Protocol):
|
||||
"""Shared callable shape; each producer defines tensor semantics."""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
frames: torch.Tensor,
|
||||
/,
|
||||
*,
|
||||
chunk_index: int,
|
||||
total_chunks: int,
|
||||
@@ -63,7 +66,7 @@ class _RawChunkAssembly:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
callback: _RawTemporalChunkCallback,
|
||||
callback: _TemporalChunkCallbackSignature,
|
||||
plan: MiniMaxH3TemporalDecodePlan,
|
||||
) -> None:
|
||||
self.callback = callback
|
||||
@@ -330,13 +333,16 @@ def resolve_minimax_h3_temporal_cat_dtype(model: nn.Module) -> torch.dtype | Non
|
||||
def decode_minimax_h3_temporal_chunks_compat(
|
||||
model: nn.Module,
|
||||
plan: MiniMaxH3TemporalDecodePlan,
|
||||
callback: _RawTemporalChunkCallback,
|
||||
callback: _TemporalChunkCallbackSignature,
|
||||
) -> torch.Tensor:
|
||||
"""Mirror the released Apache-2.0 H3 temporal assembler with callbacks.
|
||||
|
||||
Keep this implementation and the source/config fingerprints above in sync
|
||||
with ``MiniMaxAI/MiniMax-H3@42ed227e``. Unknown remote code fails closed
|
||||
before this function is called.
|
||||
|
||||
Callback tensors are borrowed decoder-domain views and must not be retained
|
||||
or mutated. The high-level adapter owns normalization and storage isolation.
|
||||
"""
|
||||
|
||||
chunk_decoded_frames = int(model.tokens_chunk_size) * int(model.vae_ratio_t) # type: ignore[attr-defined]
|
||||
|
||||
Reference in new issue
Block a user