[Bugfix][MiniMax-H3] Reconcile temporal decode plans across VAE ranks
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com> (cherry picked from commit ac4ec9c0fcf3394e8476d3d2d913d7c86f5df8e3)
This commit is contained in:
1 parent
ff2e33a329
commit
f9c9de8d16
2 files changed
+161
No files matched your search
@@ -769,6 +769,55 @@ def _distributed_worker(
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _mismatched_plan_worker(
|
||||
rank: int,
|
||||
init_file: str,
|
||||
result_queue: Any,
|
||||
) -> None:
|
||||
dist.init_process_group(
|
||||
"gloo",
|
||||
init_method=f"file://{init_file}",
|
||||
rank=rank,
|
||||
world_size=2,
|
||||
)
|
||||
try:
|
||||
vae_module.get_data_parallel_world_size = lambda: 1
|
||||
vae_module.model_parallel_is_initialized = lambda: True
|
||||
_install_released_source_fingerprint()
|
||||
|
||||
vae, model = _compat_vae(
|
||||
37 if rank == 0 else 62,
|
||||
parallel_size=2,
|
||||
collective=True,
|
||||
)
|
||||
vae._native_parallel_state = MethodType(
|
||||
lambda self: {"sp_process_group": dist.group.WORLD},
|
||||
vae,
|
||||
)
|
||||
try:
|
||||
vae.decode_latent_with_chunks(
|
||||
_latent(37 if rank == 0 else 62),
|
||||
(lambda *args, **kwargs: None) if rank == 0 else None,
|
||||
)
|
||||
except BaseException as exc: # noqa: BLE001
|
||||
failure_type = type(exc).__name__
|
||||
else:
|
||||
failure_type = "none"
|
||||
|
||||
recovery = torch.tensor([rank + 1], dtype=torch.int32)
|
||||
dist.all_reduce(recovery)
|
||||
result_queue.put(
|
||||
{
|
||||
"rank": rank,
|
||||
"failure_type": failure_type,
|
||||
"adaptive_calls": model.adaptive_decode_calls,
|
||||
"recovery": int(recovery.item()),
|
||||
}
|
||||
)
|
||||
finally:
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _wrong_owner_worker(
|
||||
rank: int,
|
||||
init_file: str,
|
||||
@@ -861,6 +910,47 @@ def test_three_rank_wrong_callback_owner_is_symmetric_and_group_recovers(
|
||||
assert [result["recovery"] for result in results] == [6, 6, 6]
|
||||
|
||||
|
||||
@pytest.mark.parallel
|
||||
def test_two_rank_mismatched_decode_plans_fail_before_decoder_collectives(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
context = mp.get_context("spawn")
|
||||
result_queue = context.Queue()
|
||||
init_file = str(tmp_path / "gloo-mismatched-plan-init")
|
||||
processes = [
|
||||
context.Process(
|
||||
target=_mismatched_plan_worker,
|
||||
args=(rank, init_file, result_queue),
|
||||
)
|
||||
for rank in range(2)
|
||||
]
|
||||
for process in processes:
|
||||
process.start()
|
||||
deadline = time.monotonic() + 90
|
||||
for process in processes:
|
||||
process.join(timeout=max(0, deadline - time.monotonic()))
|
||||
for process in processes:
|
||||
if process.is_alive():
|
||||
process.join(timeout=1)
|
||||
hung = [process for process in processes if process.is_alive()]
|
||||
for process in hung:
|
||||
process.terminate()
|
||||
process.join(timeout=5)
|
||||
assert not hung, "mismatched MiniMax-H3 decode plans deadlocked"
|
||||
assert [process.exitcode for process in processes] == [0, 0]
|
||||
|
||||
results = sorted(
|
||||
[result_queue.get(timeout=5) for _ in range(2)],
|
||||
key=lambda result: result["rank"],
|
||||
)
|
||||
assert [result["failure_type"] for result in results] == [
|
||||
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||
MiniMaxH3ChunkedDecodeUnsupportedError.__name__,
|
||||
]
|
||||
assert [result["adaptive_calls"] for result in results] == [0, 0]
|
||||
assert [result["recovery"] for result in results] == [3, 3]
|
||||
|
||||
|
||||
@pytest.mark.parallel
|
||||
def test_two_rank_callback_failure_is_symmetric_and_group_recovers(
|
||||
tmp_path: Path,
|
||||
|
||||
@@ -82,6 +82,37 @@ class _MiniMaxH3ChunkDecodeCoordinator:
|
||||
def __init__(self) -> None:
|
||||
self._sources_validated = False
|
||||
|
||||
@staticmethod
|
||||
def _build_execution_signature(
|
||||
latent: torch.Tensor,
|
||||
plan: MiniMaxH3TemporalDecodePlan,
|
||||
) -> tuple[int, ...]:
|
||||
"""Describe every shape and plan field used by the temporal schedule."""
|
||||
|
||||
dtype_codes: dict[torch.dtype | None, int] = {
|
||||
None: 0,
|
||||
torch.float16: 1,
|
||||
torch.bfloat16: 2,
|
||||
torch.float32: 3,
|
||||
torch.float64: 4,
|
||||
}
|
||||
return (
|
||||
# Keep the layout versioned so adding a field cannot silently make
|
||||
# an older and newer worker appear compatible.
|
||||
1,
|
||||
*[int(size) for size in latent.shape],
|
||||
dtype_codes.get(latent.dtype, -1),
|
||||
*[int(size) for size in plan.latent.shape],
|
||||
dtype_codes.get(plan.latent.dtype, -1),
|
||||
int(plan.num_chunks),
|
||||
int(plan.total_frames),
|
||||
int(plan.pad_frames),
|
||||
int(plan.output_frames),
|
||||
len(plan.chunk_frames),
|
||||
*[int(frame_count) for frame_count in plan.chunk_frames],
|
||||
dtype_codes.get(plan.temporal_dtype, -1),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_output_rank(
|
||||
chunk_callback: MiniMaxH3VideoChunkCallback | None,
|
||||
@@ -140,6 +171,7 @@ class _MiniMaxH3ChunkDecodeCoordinator:
|
||||
def _synchronize_preflight_error(
|
||||
preflight_error: Exception | None,
|
||||
temporal_dtype: torch.dtype | None,
|
||||
execution_signature: tuple[int, ...] | None,
|
||||
*,
|
||||
group: dist.ProcessGroup | None,
|
||||
device: torch.device,
|
||||
@@ -171,6 +203,44 @@ class _MiniMaxH3ChunkDecodeCoordinator:
|
||||
dist.all_reduce(lowest, op=dist.ReduceOp.MIN, group=group)
|
||||
dist.all_reduce(highest, op=dist.ReduceOp.MAX, group=group)
|
||||
if int(lowest.item()) == int(highest.item()) and int(lowest.item()) >= 0:
|
||||
if execution_signature is None:
|
||||
raise RuntimeError("MiniMax-H3 temporal decode preflight produced no execution signature")
|
||||
|
||||
# The plan length is shape-dependent, so first agree on a
|
||||
# common tensor size and then compare the complete signatures.
|
||||
# This keeps all ranks in tensor collectives and catches a
|
||||
# schedule mismatch before entering the decoder's collectives.
|
||||
signature_length = torch.tensor(
|
||||
[len(execution_signature)],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
max_signature_length = signature_length.clone()
|
||||
dist.all_reduce(
|
||||
max_signature_length,
|
||||
op=dist.ReduceOp.MAX,
|
||||
group=group,
|
||||
)
|
||||
padded_signature = torch.full(
|
||||
(int(max_signature_length.item()),),
|
||||
-2,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
padded_signature[: len(execution_signature)] = torch.tensor(
|
||||
execution_signature,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
gathered_signatures = [
|
||||
torch.empty_like(padded_signature)
|
||||
for _ in range(dist.get_world_size(group))
|
||||
]
|
||||
dist.all_gather(gathered_signatures, padded_signature, group=group)
|
||||
if any(not torch.equal(padded_signature, peer) for peer in gathered_signatures):
|
||||
raise MiniMaxH3ChunkedDecodeUnsupportedError(
|
||||
"MiniMax-H3 VAE ranks built incompatible temporal decode plans"
|
||||
)
|
||||
return
|
||||
raise MiniMaxH3ChunkedDecodeUnsupportedError(
|
||||
"MiniMax-H3 VAE ranks selected incompatible temporal concat dtypes"
|
||||
@@ -215,6 +285,7 @@ class _MiniMaxH3ChunkDecodeCoordinator:
|
||||
self._synchronize_preflight_error(
|
||||
preflight_error,
|
||||
plan.temporal_dtype if plan is not None else None,
|
||||
self._build_execution_signature(latent, plan) if plan is not None else None,
|
||||
group=group,
|
||||
device=latent.device,
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user