From 3b84cc4a7fa86836d79b5e373082129c92716b6f Mon Sep 17 00:00:00 2001 From: david6666666 <530634352@qq.com> Date: Fri, 18 Sep 2026 01:20:13 +0000 Subject: [PATCH] [Bugfix][MiniMax-H3] Keep keyframe VAE encoding rank-local Signed-off-by: david6666666 <530634352@qq.com> --- .../minimax_h3/test_minimax_h3_collectives.py | 19 ++++- .../models/minimax_h3/ops/vae/qk_norm_rope.py | 84 +++++++++++-------- .../models/minimax_h3/pipeline_minimax_h3.py | 18 +++- vllm_omni/diffusion/models/minimax_h3/vae.py | 9 +- .../models/minimax_h3/encoder_processing.py | 10 ++- 5 files changed, 95 insertions(+), 45 deletions(-) diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_collectives.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_collectives.py index 9d44bf445..10aaf83cb 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_collectives.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_collectives.py @@ -43,7 +43,7 @@ def _collective_worker(rank, rendezvous, backend): assert value.item() == 2 def encode_image(self, image): - self.collective() + assert rank == 0 assert image.size == (32, 32) return torch.ones(1, 96) @@ -115,6 +115,23 @@ def _collective_worker(rank, rendezvous, backend): torch.testing.assert_close(conditioning.audio_condition.cpu(), torch.full((160, 32), 3.0)) assert conditioning.audio_condition_lengths == (80,) + image_media = ( + MiniMaxH3EncoderMediaInput( + task="fl2va", + height=32, + width=32, + num_frames=96, + latent_t=29, + audio_t=160, + images=(torch.zeros(32, 32, 3, dtype=torch.uint8),), + ) + if rank == 0 + else None + ) + image_conditioning = model._encode_local_media(image_media) + torch.testing.assert_close(image_conditioning.visual_condition.cpu(), torch.ones(1, 96)) + assert image_conditioning.visual_condition_shapes == ((1, 2, 2),) + model.video_vae.fail = True with pytest.raises(OmniClientError, match="codec failed on rank 1"): model._encode_local_media(media) diff --git a/vllm_omni/diffusion/models/minimax_h3/ops/vae/qk_norm_rope.py b/vllm_omni/diffusion/models/minimax_h3/ops/vae/qk_norm_rope.py index 4e433b108..f964c1577 100644 --- a/vllm_omni/diffusion/models/minimax_h3/ops/vae/qk_norm_rope.py +++ b/vllm_omni/diffusion/models/minimax_h3/ops/vae/qk_norm_rope.py @@ -11,18 +11,26 @@ if HAS_TRITON: @triton.jit def _qk_rms_norm_rope_exact_kernel( - x_ptr, + q_ptr, + k_ptr, cos_ptr, sin_ptr, - output_ptr, - x_stride_token, - x_stride_head, - x_stride_dim, + q_output_ptr, + k_output_ptr, + q_stride_token, + q_stride_head, + q_stride_dim, + k_stride_token, + k_stride_head, + k_stride_dim, rope_stride_token, rope_stride_dim, - output_stride_token, - output_stride_head, - output_stride_dim, + q_output_stride_token, + q_output_stride_head, + q_output_stride_dim, + k_output_stride_token, + k_output_stride_head, + k_output_stride_dim, num_heads: tl.constexpr, head_dim: tl.constexpr, rotary_dim: tl.constexpr, @@ -31,12 +39,15 @@ if HAS_TRITON: ): token = tl.program_id(0) head_group = tl.program_id(1) + qk_domain = tl.program_id(2) heads = head_group * heads_per_program + tl.arange(0, heads_per_program) dims = tl.arange(0, head_dim) mask = heads[:, None] < num_heads - offsets = token * x_stride_token + heads[:, None] * x_stride_head + dims[None, :] * x_stride_dim - x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32) + q_offsets = token * q_stride_token + heads[:, None] * q_stride_head + dims[None, :] * q_stride_dim + k_offsets = token * k_stride_token + heads[:, None] * k_stride_head + dims[None, :] * k_stride_dim + x_ptrs = tl.where(qk_domain == 0, q_ptr + q_offsets, k_ptr + k_offsets) + x = tl.load(x_ptrs, mask=mask, other=0.0).to(tl.float32) inv_rms = tl.rsqrt(tl.sum(x * x, axis=1) / head_dim + eps) normalized = (x * inv_rms[:, None]).to(tl.float16) @@ -46,8 +57,10 @@ if HAS_TRITON: dims + rotary_half, tl.where(dims < rotary_dim, dims - rotary_half, dims), ) - pair_offsets = token * x_stride_token + heads[:, None] * x_stride_head + pair_dims[None, :] * x_stride_dim - pair_x = tl.load(x_ptr + pair_offsets, mask=mask, other=0.0).to(tl.float32) + q_pair_offsets = token * q_stride_token + heads[:, None] * q_stride_head + pair_dims[None, :] * q_stride_dim + k_pair_offsets = token * k_stride_token + heads[:, None] * k_stride_head + pair_dims[None, :] * k_stride_dim + pair_x_ptrs = tl.where(qk_domain == 0, q_ptr + q_pair_offsets, k_ptr + k_pair_offsets) + pair_x = tl.load(pair_x_ptrs, mask=mask, other=0.0).to(tl.float32) pair_normalized = (pair_x * inv_rms[:, None]).to(tl.float16) rope_mask = dims < rotary_dim @@ -79,10 +92,18 @@ if HAS_TRITON: ) output = tl.where(rope_mask[None, :], rotated, normalized) - output_offsets = ( - token * output_stride_token + heads[:, None] * output_stride_head + dims[None, :] * output_stride_dim + q_output_offsets = ( + token * q_output_stride_token + heads[:, None] * q_output_stride_head + dims[None, :] * q_output_stride_dim ) - tl.store(output_ptr + output_offsets, output, mask=mask) + k_output_offsets = ( + token * k_output_stride_token + heads[:, None] * k_output_stride_head + dims[None, :] * k_output_stride_dim + ) + output_ptrs = tl.where( + qk_domain == 0, + q_output_ptr + q_output_offsets, + k_output_ptr + k_output_offsets, + ) + tl.store(output_ptrs, output, mask=mask) def _supported_inputs( @@ -143,15 +164,11 @@ def try_qk_norm_rope_exact( sin = sin.reshape(tokens, sin.shape[-1]) q_output = torch.empty_like(q) k_output = torch.empty_like(k) - # This launch layout is part of the exactness contract on the validated - # SM90 and SM103 targets: changing the warp-to-row mapping changes the - # FP32 RMS reduction order. - # Keep Q and K in separate launches so each kernel has fixed input and - # output pointers. A combined qk-domain launch introduces runtime pointer - # selection; on SM90, its generated arithmetic differs enough to - # accumulate through the 50-step I2VA denoising loop. + # The first two launch dimensions are part of the exactness contract on + # the validated SM90 and SM103 targets: changing the warp-to-row mapping + # changes the FP32 RMS reduction order. heads_per_program = 8 - grid = (tokens, triton.cdiv(heads, heads_per_program)) + grid = (tokens, triton.cdiv(heads, heads_per_program), 2) launch_args = { "num_heads": heads, "head_dim": head_dim, @@ -162,29 +179,22 @@ def try_qk_norm_rope_exact( } _qk_rms_norm_rope_exact_kernel[grid]( q, - cos, - sin, - q_output, - q.stride(0), - q.stride(1), - q.stride(2), - cos.stride(0), - cos.stride(1), - q_output.stride(0), - q_output.stride(1), - q_output.stride(2), - **launch_args, - ) - _qk_rms_norm_rope_exact_kernel[grid]( k, cos, sin, + q_output, k_output, + q.stride(0), + q.stride(1), + q.stride(2), k.stride(0), k.stride(1), k.stride(2), cos.stride(0), cos.stride(1), + q_output.stride(0), + q_output.stride(1), + q_output.stride(2), k_output.stride(0), k_output.stride(1), k_output.stride(2), diff --git a/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py b/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py index 60d6f4125..256e4d222 100644 --- a/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py +++ b/vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py @@ -1953,7 +1953,22 @@ class MiniMaxH3Pipeline( def _encode_local_media(self, media: MiniMaxH3EncoderMediaInput | None) -> MiniMaxH3EncoderMediaConditioning: group, rank, world_size = _dit_rank_world() - distributed_video = self.video_vae.is_distributed_enabled() + # Keep image/keyframe encoding on the output rank. The legacy + # single-stage path encoded images locally and only used the tiled VAE + # collective for reference videos. Treating every visual input as a + # distributed video after the unified encoder path was introduced + # changes the H100 keyframe latent and therefore the I2VA trajectory. + distributed_vae = self.video_vae.is_distributed_enabled() + distributed_video = False + if distributed_vae: + has_video = torch.tensor( + [int(media is not None and bool(media.videos))], + dtype=torch.long, + device=self.device, + ) + if world_size > 1: + dist.broadcast(has_video, src=0, group=group) + distributed_video = bool(has_video.item()) if distributed_video: media = self._distribute_media_inputs(media) conditioning = None @@ -1966,6 +1981,7 @@ class MiniMaxH3Pipeline( video_vae=self.video_vae, audio_vae=self.audio_vae, emit_conditioning=rank == 0, + encode_images=rank == 0, component_scope=self._component_on_device, ) except ValueError as exc: diff --git a/vllm_omni/diffusion/models/minimax_h3/vae.py b/vllm_omni/diffusion/models/minimax_h3/vae.py index d02649229..124a21e44 100644 --- a/vllm_omni/diffusion/models/minimax_h3/vae.py +++ b/vllm_omni/diffusion/models/minimax_h3/vae.py @@ -602,8 +602,13 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin): if getattr(self, "decode_only", False): raise RuntimeError("MiniMax H3 decode-only video VAE cannot encode images") previous_parallel = self.model.parallel_tiling - if int(getattr(self, "parallel_size", 1)) <= 1: + parallel_size = int(getattr(self, "parallel_size", 1)) + if parallel_size <= 1: self.model.parallel_tiling = False + # Keyframe conditioning follows the official single-image contract. + # Do not let it enter the tiled VAE collective even when reference + # videos use the same VAE instance with patch parallelism enabled. + tiling_context = self._rank_local_tiling() if parallel_size > 1 else nullcontext() parameter = next(self.parameters()) previous_dtype = parameter.dtype if previous_dtype != torch.float32: @@ -611,7 +616,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin): devices = [parameter.device] if parameter.device.type != "cpu" else [] try: with ( - self._encoder_tiling_context(image.height, image.width), + tiling_context, torch.random.fork_rng( devices=devices, device_type=parameter.device.type, diff --git a/vllm_omni/model_executor/models/minimax_h3/encoder_processing.py b/vllm_omni/model_executor/models/minimax_h3/encoder_processing.py index 2d6998b7a..0411e7fd8 100644 --- a/vllm_omni/model_executor/models/minimax_h3/encoder_processing.py +++ b/vllm_omni/model_executor/models/minimax_h3/encoder_processing.py @@ -481,6 +481,7 @@ def encode_media( video_vae: Any, audio_vae: Any, emit_conditioning: bool, + encode_images: bool = True, component_scope: Callable[[Any], AbstractContextManager[Any]] = nullcontext, ) -> MiniMaxH3EncoderMediaConditioning | None: """Run codecs on participating ranks; only the output rank runs audio. @@ -495,10 +496,11 @@ def encode_media( if video_vae is None: raise RuntimeError("MiniMax H3 video VAE is not resident on this rank") with component_scope(video_vae): - for value in media.images: - image = _image_from_tensor(value) - visual_rows.append(video_vae.encode_image(image)) - visual_shapes.append((1, image.height // 16, image.width // 16)) + if encode_images: + for value in media.images: + image = _image_from_tensor(value) + visual_rows.append(video_vae.encode_image(image)) + visual_shapes.append((1, image.height // 16, image.width // 16)) for value in media.videos: frames = np.asarray(value.detach().cpu().to(torch.uint8).contiguous().numpy()) rows, shape = video_vae.encode_video(frames)