[Bugfix][MiniMax-H3] Keep keyframe VAE encoding rank-local

Signed-off-by: david6666666 <530634352@qq.com>
This commit is contained in:
david6666666 committed 2026-09-18 01:20:13 +00:00
1 parent f9253ae1ad
commit 3b84cc4a7f
5 files changed
+95 -45

No files matched your search

@@ -43,7 +43,7 @@ def _collective_worker(rank, rendezvous, backend):
assert value.item() == 2
def encode_image(self, image):
self.collective()
assert rank == 0
assert image.size == (32, 32)
return torch.ones(1, 96)
@@ -115,6 +115,23 @@ def _collective_worker(rank, rendezvous, backend):
torch.testing.assert_close(conditioning.audio_condition.cpu(), torch.full((160, 32), 3.0))
assert conditioning.audio_condition_lengths == (80,)
image_media = (
MiniMaxH3EncoderMediaInput(
task="fl2va",
height=32,
width=32,
num_frames=96,
latent_t=29,
audio_t=160,
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
)
if rank == 0
else None
)
image_conditioning = model._encode_local_media(image_media)
torch.testing.assert_close(image_conditioning.visual_condition.cpu(), torch.ones(1, 96))
assert image_conditioning.visual_condition_shapes == ((1, 2, 2),)
model.video_vae.fail = True
with pytest.raises(OmniClientError, match="codec failed on rank 1"):
model._encode_local_media(media)
@@ -11,18 +11,26 @@ if HAS_TRITON:
@triton.jit
def _qk_rms_norm_rope_exact_kernel(
x_ptr,
q_ptr,
k_ptr,
cos_ptr,
sin_ptr,
output_ptr,
x_stride_token,
x_stride_head,
x_stride_dim,
q_output_ptr,
k_output_ptr,
q_stride_token,
q_stride_head,
q_stride_dim,
k_stride_token,
k_stride_head,
k_stride_dim,
rope_stride_token,
rope_stride_dim,
output_stride_token,
output_stride_head,
output_stride_dim,
q_output_stride_token,
q_output_stride_head,
q_output_stride_dim,
k_output_stride_token,
k_output_stride_head,
k_output_stride_dim,
num_heads: tl.constexpr,
head_dim: tl.constexpr,
rotary_dim: tl.constexpr,
@@ -31,12 +39,15 @@ if HAS_TRITON:
):
token = tl.program_id(0)
head_group = tl.program_id(1)
qk_domain = tl.program_id(2)
heads = head_group * heads_per_program + tl.arange(0, heads_per_program)
dims = tl.arange(0, head_dim)
mask = heads[:, None] < num_heads
offsets = token * x_stride_token + heads[:, None] * x_stride_head + dims[None, :] * x_stride_dim
x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
q_offsets = token * q_stride_token + heads[:, None] * q_stride_head + dims[None, :] * q_stride_dim
k_offsets = token * k_stride_token + heads[:, None] * k_stride_head + dims[None, :] * k_stride_dim
x_ptrs = tl.where(qk_domain == 0, q_ptr + q_offsets, k_ptr + k_offsets)
x = tl.load(x_ptrs, mask=mask, other=0.0).to(tl.float32)
inv_rms = tl.rsqrt(tl.sum(x * x, axis=1) / head_dim + eps)
normalized = (x * inv_rms[:, None]).to(tl.float16)
@@ -46,8 +57,10 @@ if HAS_TRITON:
dims + rotary_half,
tl.where(dims < rotary_dim, dims - rotary_half, dims),
)
pair_offsets = token * x_stride_token + heads[:, None] * x_stride_head + pair_dims[None, :] * x_stride_dim
pair_x = tl.load(x_ptr + pair_offsets, mask=mask, other=0.0).to(tl.float32)
q_pair_offsets = token * q_stride_token + heads[:, None] * q_stride_head + pair_dims[None, :] * q_stride_dim
k_pair_offsets = token * k_stride_token + heads[:, None] * k_stride_head + pair_dims[None, :] * k_stride_dim
pair_x_ptrs = tl.where(qk_domain == 0, q_ptr + q_pair_offsets, k_ptr + k_pair_offsets)
pair_x = tl.load(pair_x_ptrs, mask=mask, other=0.0).to(tl.float32)
pair_normalized = (pair_x * inv_rms[:, None]).to(tl.float16)
rope_mask = dims < rotary_dim
@@ -79,10 +92,18 @@ if HAS_TRITON:
)
output = tl.where(rope_mask[None, :], rotated, normalized)
output_offsets = (
token * output_stride_token + heads[:, None] * output_stride_head + dims[None, :] * output_stride_dim
q_output_offsets = (
token * q_output_stride_token + heads[:, None] * q_output_stride_head + dims[None, :] * q_output_stride_dim
)
tl.store(output_ptr + output_offsets, output, mask=mask)
k_output_offsets = (
token * k_output_stride_token + heads[:, None] * k_output_stride_head + dims[None, :] * k_output_stride_dim
)
output_ptrs = tl.where(
qk_domain == 0,
q_output_ptr + q_output_offsets,
k_output_ptr + k_output_offsets,
)
tl.store(output_ptrs, output, mask=mask)
def _supported_inputs(
@@ -143,15 +164,11 @@ def try_qk_norm_rope_exact(
sin = sin.reshape(tokens, sin.shape[-1])
q_output = torch.empty_like(q)
k_output = torch.empty_like(k)
# This launch layout is part of the exactness contract on the validated
# SM90 and SM103 targets: changing the warp-to-row mapping changes the
# FP32 RMS reduction order.
# Keep Q and K in separate launches so each kernel has fixed input and
# output pointers. A combined qk-domain launch introduces runtime pointer
# selection; on SM90, its generated arithmetic differs enough to
# accumulate through the 50-step I2VA denoising loop.
# The first two launch dimensions are part of the exactness contract on
# the validated SM90 and SM103 targets: changing the warp-to-row mapping
# changes the FP32 RMS reduction order.
heads_per_program = 8
grid = (tokens, triton.cdiv(heads, heads_per_program))
grid = (tokens, triton.cdiv(heads, heads_per_program), 2)
launch_args = {
"num_heads": heads,
"head_dim": head_dim,
@@ -162,29 +179,22 @@ def try_qk_norm_rope_exact(
}
_qk_rms_norm_rope_exact_kernel[grid](
q,
cos,
sin,
q_output,
q.stride(0),
q.stride(1),
q.stride(2),
cos.stride(0),
cos.stride(1),
q_output.stride(0),
q_output.stride(1),
q_output.stride(2),
**launch_args,
)
_qk_rms_norm_rope_exact_kernel[grid](
k,
cos,
sin,
q_output,
k_output,
q.stride(0),
q.stride(1),
q.stride(2),
k.stride(0),
k.stride(1),
k.stride(2),
cos.stride(0),
cos.stride(1),
q_output.stride(0),
q_output.stride(1),
q_output.stride(2),
k_output.stride(0),
k_output.stride(1),
k_output.stride(2),
@@ -1953,7 +1953,22 @@ class MiniMaxH3Pipeline(
def _encode_local_media(self, media: MiniMaxH3EncoderMediaInput | None) -> MiniMaxH3EncoderMediaConditioning:
group, rank, world_size = _dit_rank_world()
distributed_video = self.video_vae.is_distributed_enabled()
# Keep image/keyframe encoding on the output rank. The legacy
# single-stage path encoded images locally and only used the tiled VAE
# collective for reference videos. Treating every visual input as a
# distributed video after the unified encoder path was introduced
# changes the H100 keyframe latent and therefore the I2VA trajectory.
distributed_vae = self.video_vae.is_distributed_enabled()
distributed_video = False
if distributed_vae:
has_video = torch.tensor(
[int(media is not None and bool(media.videos))],
dtype=torch.long,
device=self.device,
)
if world_size > 1:
dist.broadcast(has_video, src=0, group=group)
distributed_video = bool(has_video.item())
if distributed_video:
media = self._distribute_media_inputs(media)
conditioning = None
@@ -1966,6 +1981,7 @@ class MiniMaxH3Pipeline(
video_vae=self.video_vae,
audio_vae=self.audio_vae,
emit_conditioning=rank == 0,
encode_images=rank == 0,
component_scope=self._component_on_device,
)
except ValueError as exc:
+7 -2
View File
@@ -602,8 +602,13 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
if getattr(self, "decode_only", False):
raise RuntimeError("MiniMax H3 decode-only video VAE cannot encode images")
previous_parallel = self.model.parallel_tiling
if int(getattr(self, "parallel_size", 1)) <= 1:
parallel_size = int(getattr(self, "parallel_size", 1))
if parallel_size <= 1:
self.model.parallel_tiling = False
# Keyframe conditioning follows the official single-image contract.
# Do not let it enter the tiled VAE collective even when reference
# videos use the same VAE instance with patch parallelism enabled.
tiling_context = self._rank_local_tiling() if parallel_size > 1 else nullcontext()
parameter = next(self.parameters())
previous_dtype = parameter.dtype
if previous_dtype != torch.float32:
@@ -611,7 +616,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
devices = [parameter.device] if parameter.device.type != "cpu" else []
try:
with (
self._encoder_tiling_context(image.height, image.width),
tiling_context,
torch.random.fork_rng(
devices=devices,
device_type=parameter.device.type,
@@ -481,6 +481,7 @@ def encode_media(
video_vae: Any,
audio_vae: Any,
emit_conditioning: bool,
encode_images: bool = True,
component_scope: Callable[[Any], AbstractContextManager[Any]] = nullcontext,
) -> MiniMaxH3EncoderMediaConditioning | None:
"""Run codecs on participating ranks; only the output rank runs audio.
@@ -495,10 +496,11 @@ def encode_media(
if video_vae is None:
raise RuntimeError("MiniMax H3 video VAE is not resident on this rank")
with component_scope(video_vae):
for value in media.images:
image = _image_from_tensor(value)
visual_rows.append(video_vae.encode_image(image))
visual_shapes.append((1, image.height // 16, image.width // 16))
if encode_images:
for value in media.images:
image = _image_from_tensor(value)
visual_rows.append(video_vae.encode_image(image))
visual_shapes.append((1, image.height // 16, image.width // 16))
for value in media.videos:
frames = np.asarray(value.detach().cpu().to(torch.uint8).contiguous().numpy())
rows, shape = video_vae.encode_video(frames)