[Bugfix][MiniMax-H3] Keep keyframe VAE encoding rank-local
Signed-off-by: david6666666 <530634352@qq.com>
This commit is contained in:
1 parent
f9253ae1ad
commit
3b84cc4a7f
5 files changed
+95
-45
No files matched your search
@@ -43,7 +43,7 @@ def _collective_worker(rank, rendezvous, backend):
|
||||
assert value.item() == 2
|
||||
|
||||
def encode_image(self, image):
|
||||
self.collective()
|
||||
assert rank == 0
|
||||
assert image.size == (32, 32)
|
||||
return torch.ones(1, 96)
|
||||
|
||||
@@ -115,6 +115,23 @@ def _collective_worker(rank, rendezvous, backend):
|
||||
torch.testing.assert_close(conditioning.audio_condition.cpu(), torch.full((160, 32), 3.0))
|
||||
assert conditioning.audio_condition_lengths == (80,)
|
||||
|
||||
image_media = (
|
||||
MiniMaxH3EncoderMediaInput(
|
||||
task="fl2va",
|
||||
height=32,
|
||||
width=32,
|
||||
num_frames=96,
|
||||
latent_t=29,
|
||||
audio_t=160,
|
||||
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
|
||||
)
|
||||
if rank == 0
|
||||
else None
|
||||
)
|
||||
image_conditioning = model._encode_local_media(image_media)
|
||||
torch.testing.assert_close(image_conditioning.visual_condition.cpu(), torch.ones(1, 96))
|
||||
assert image_conditioning.visual_condition_shapes == ((1, 2, 2),)
|
||||
|
||||
model.video_vae.fail = True
|
||||
with pytest.raises(OmniClientError, match="codec failed on rank 1"):
|
||||
model._encode_local_media(media)
|
||||
|
||||
@@ -11,18 +11,26 @@ if HAS_TRITON:
|
||||
|
||||
@triton.jit
|
||||
def _qk_rms_norm_rope_exact_kernel(
|
||||
x_ptr,
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
cos_ptr,
|
||||
sin_ptr,
|
||||
output_ptr,
|
||||
x_stride_token,
|
||||
x_stride_head,
|
||||
x_stride_dim,
|
||||
q_output_ptr,
|
||||
k_output_ptr,
|
||||
q_stride_token,
|
||||
q_stride_head,
|
||||
q_stride_dim,
|
||||
k_stride_token,
|
||||
k_stride_head,
|
||||
k_stride_dim,
|
||||
rope_stride_token,
|
||||
rope_stride_dim,
|
||||
output_stride_token,
|
||||
output_stride_head,
|
||||
output_stride_dim,
|
||||
q_output_stride_token,
|
||||
q_output_stride_head,
|
||||
q_output_stride_dim,
|
||||
k_output_stride_token,
|
||||
k_output_stride_head,
|
||||
k_output_stride_dim,
|
||||
num_heads: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
rotary_dim: tl.constexpr,
|
||||
@@ -31,12 +39,15 @@ if HAS_TRITON:
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
head_group = tl.program_id(1)
|
||||
qk_domain = tl.program_id(2)
|
||||
heads = head_group * heads_per_program + tl.arange(0, heads_per_program)
|
||||
dims = tl.arange(0, head_dim)
|
||||
mask = heads[:, None] < num_heads
|
||||
|
||||
offsets = token * x_stride_token + heads[:, None] * x_stride_head + dims[None, :] * x_stride_dim
|
||||
x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
|
||||
q_offsets = token * q_stride_token + heads[:, None] * q_stride_head + dims[None, :] * q_stride_dim
|
||||
k_offsets = token * k_stride_token + heads[:, None] * k_stride_head + dims[None, :] * k_stride_dim
|
||||
x_ptrs = tl.where(qk_domain == 0, q_ptr + q_offsets, k_ptr + k_offsets)
|
||||
x = tl.load(x_ptrs, mask=mask, other=0.0).to(tl.float32)
|
||||
inv_rms = tl.rsqrt(tl.sum(x * x, axis=1) / head_dim + eps)
|
||||
normalized = (x * inv_rms[:, None]).to(tl.float16)
|
||||
|
||||
@@ -46,8 +57,10 @@ if HAS_TRITON:
|
||||
dims + rotary_half,
|
||||
tl.where(dims < rotary_dim, dims - rotary_half, dims),
|
||||
)
|
||||
pair_offsets = token * x_stride_token + heads[:, None] * x_stride_head + pair_dims[None, :] * x_stride_dim
|
||||
pair_x = tl.load(x_ptr + pair_offsets, mask=mask, other=0.0).to(tl.float32)
|
||||
q_pair_offsets = token * q_stride_token + heads[:, None] * q_stride_head + pair_dims[None, :] * q_stride_dim
|
||||
k_pair_offsets = token * k_stride_token + heads[:, None] * k_stride_head + pair_dims[None, :] * k_stride_dim
|
||||
pair_x_ptrs = tl.where(qk_domain == 0, q_ptr + q_pair_offsets, k_ptr + k_pair_offsets)
|
||||
pair_x = tl.load(pair_x_ptrs, mask=mask, other=0.0).to(tl.float32)
|
||||
pair_normalized = (pair_x * inv_rms[:, None]).to(tl.float16)
|
||||
|
||||
rope_mask = dims < rotary_dim
|
||||
@@ -79,10 +92,18 @@ if HAS_TRITON:
|
||||
)
|
||||
output = tl.where(rope_mask[None, :], rotated, normalized)
|
||||
|
||||
output_offsets = (
|
||||
token * output_stride_token + heads[:, None] * output_stride_head + dims[None, :] * output_stride_dim
|
||||
q_output_offsets = (
|
||||
token * q_output_stride_token + heads[:, None] * q_output_stride_head + dims[None, :] * q_output_stride_dim
|
||||
)
|
||||
tl.store(output_ptr + output_offsets, output, mask=mask)
|
||||
k_output_offsets = (
|
||||
token * k_output_stride_token + heads[:, None] * k_output_stride_head + dims[None, :] * k_output_stride_dim
|
||||
)
|
||||
output_ptrs = tl.where(
|
||||
qk_domain == 0,
|
||||
q_output_ptr + q_output_offsets,
|
||||
k_output_ptr + k_output_offsets,
|
||||
)
|
||||
tl.store(output_ptrs, output, mask=mask)
|
||||
|
||||
|
||||
def _supported_inputs(
|
||||
@@ -143,15 +164,11 @@ def try_qk_norm_rope_exact(
|
||||
sin = sin.reshape(tokens, sin.shape[-1])
|
||||
q_output = torch.empty_like(q)
|
||||
k_output = torch.empty_like(k)
|
||||
# This launch layout is part of the exactness contract on the validated
|
||||
# SM90 and SM103 targets: changing the warp-to-row mapping changes the
|
||||
# FP32 RMS reduction order.
|
||||
# Keep Q and K in separate launches so each kernel has fixed input and
|
||||
# output pointers. A combined qk-domain launch introduces runtime pointer
|
||||
# selection; on SM90, its generated arithmetic differs enough to
|
||||
# accumulate through the 50-step I2VA denoising loop.
|
||||
# The first two launch dimensions are part of the exactness contract on
|
||||
# the validated SM90 and SM103 targets: changing the warp-to-row mapping
|
||||
# changes the FP32 RMS reduction order.
|
||||
heads_per_program = 8
|
||||
grid = (tokens, triton.cdiv(heads, heads_per_program))
|
||||
grid = (tokens, triton.cdiv(heads, heads_per_program), 2)
|
||||
launch_args = {
|
||||
"num_heads": heads,
|
||||
"head_dim": head_dim,
|
||||
@@ -162,29 +179,22 @@ def try_qk_norm_rope_exact(
|
||||
}
|
||||
_qk_rms_norm_rope_exact_kernel[grid](
|
||||
q,
|
||||
cos,
|
||||
sin,
|
||||
q_output,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
cos.stride(0),
|
||||
cos.stride(1),
|
||||
q_output.stride(0),
|
||||
q_output.stride(1),
|
||||
q_output.stride(2),
|
||||
**launch_args,
|
||||
)
|
||||
_qk_rms_norm_rope_exact_kernel[grid](
|
||||
k,
|
||||
cos,
|
||||
sin,
|
||||
q_output,
|
||||
k_output,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
k.stride(0),
|
||||
k.stride(1),
|
||||
k.stride(2),
|
||||
cos.stride(0),
|
||||
cos.stride(1),
|
||||
q_output.stride(0),
|
||||
q_output.stride(1),
|
||||
q_output.stride(2),
|
||||
k_output.stride(0),
|
||||
k_output.stride(1),
|
||||
k_output.stride(2),
|
||||
|
||||
@@ -1953,7 +1953,22 @@ class MiniMaxH3Pipeline(
|
||||
|
||||
def _encode_local_media(self, media: MiniMaxH3EncoderMediaInput | None) -> MiniMaxH3EncoderMediaConditioning:
|
||||
group, rank, world_size = _dit_rank_world()
|
||||
distributed_video = self.video_vae.is_distributed_enabled()
|
||||
# Keep image/keyframe encoding on the output rank. The legacy
|
||||
# single-stage path encoded images locally and only used the tiled VAE
|
||||
# collective for reference videos. Treating every visual input as a
|
||||
# distributed video after the unified encoder path was introduced
|
||||
# changes the H100 keyframe latent and therefore the I2VA trajectory.
|
||||
distributed_vae = self.video_vae.is_distributed_enabled()
|
||||
distributed_video = False
|
||||
if distributed_vae:
|
||||
has_video = torch.tensor(
|
||||
[int(media is not None and bool(media.videos))],
|
||||
dtype=torch.long,
|
||||
device=self.device,
|
||||
)
|
||||
if world_size > 1:
|
||||
dist.broadcast(has_video, src=0, group=group)
|
||||
distributed_video = bool(has_video.item())
|
||||
if distributed_video:
|
||||
media = self._distribute_media_inputs(media)
|
||||
conditioning = None
|
||||
@@ -1966,6 +1981,7 @@ class MiniMaxH3Pipeline(
|
||||
video_vae=self.video_vae,
|
||||
audio_vae=self.audio_vae,
|
||||
emit_conditioning=rank == 0,
|
||||
encode_images=rank == 0,
|
||||
component_scope=self._component_on_device,
|
||||
)
|
||||
except ValueError as exc:
|
||||
|
||||
@@ -602,8 +602,13 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
if getattr(self, "decode_only", False):
|
||||
raise RuntimeError("MiniMax H3 decode-only video VAE cannot encode images")
|
||||
previous_parallel = self.model.parallel_tiling
|
||||
if int(getattr(self, "parallel_size", 1)) <= 1:
|
||||
parallel_size = int(getattr(self, "parallel_size", 1))
|
||||
if parallel_size <= 1:
|
||||
self.model.parallel_tiling = False
|
||||
# Keyframe conditioning follows the official single-image contract.
|
||||
# Do not let it enter the tiled VAE collective even when reference
|
||||
# videos use the same VAE instance with patch parallelism enabled.
|
||||
tiling_context = self._rank_local_tiling() if parallel_size > 1 else nullcontext()
|
||||
parameter = next(self.parameters())
|
||||
previous_dtype = parameter.dtype
|
||||
if previous_dtype != torch.float32:
|
||||
@@ -611,7 +616,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
devices = [parameter.device] if parameter.device.type != "cpu" else []
|
||||
try:
|
||||
with (
|
||||
self._encoder_tiling_context(image.height, image.width),
|
||||
tiling_context,
|
||||
torch.random.fork_rng(
|
||||
devices=devices,
|
||||
device_type=parameter.device.type,
|
||||
|
||||
@@ -481,6 +481,7 @@ def encode_media(
|
||||
video_vae: Any,
|
||||
audio_vae: Any,
|
||||
emit_conditioning: bool,
|
||||
encode_images: bool = True,
|
||||
component_scope: Callable[[Any], AbstractContextManager[Any]] = nullcontext,
|
||||
) -> MiniMaxH3EncoderMediaConditioning | None:
|
||||
"""Run codecs on participating ranks; only the output rank runs audio.
|
||||
@@ -495,10 +496,11 @@ def encode_media(
|
||||
if video_vae is None:
|
||||
raise RuntimeError("MiniMax H3 video VAE is not resident on this rank")
|
||||
with component_scope(video_vae):
|
||||
for value in media.images:
|
||||
image = _image_from_tensor(value)
|
||||
visual_rows.append(video_vae.encode_image(image))
|
||||
visual_shapes.append((1, image.height // 16, image.width // 16))
|
||||
if encode_images:
|
||||
for value in media.images:
|
||||
image = _image_from_tensor(value)
|
||||
visual_rows.append(video_vae.encode_image(image))
|
||||
visual_shapes.append((1, image.height // 16, image.width // 16))
|
||||
for value in media.videos:
|
||||
frames = np.asarray(value.detach().cpu().to(torch.uint8).contiguous().numpy())
|
||||
rows, shape = video_vae.encode_video(frames)
|
||||
|
||||
Reference in new issue
Block a user