[Attention] Refine TRTLLM attention support for MiniMax H3 (#5779)
Signed-off-by: Bo Li <22713281+bobboli@users.noreply.github.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: lishunyang12 <lishunyang12@163.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
11 files changed
+370
-33
No files matched your search
@@ -149,7 +149,8 @@ def minimax_h3_denoise_loop(
|
||||
device: torch.device,
|
||||
imgvid_cond_noise_aug_for_inference: float = MINIMAX_H3_IMGVID_COND_TIMESTEP,
|
||||
audio_cond_noise_aug_for_inference: float = MINIMAX_H3_AUDIO_REF_COND_TIMESTEP,
|
||||
on_step: Callable[[int, torch.Tensor, torch.Tensor], None] | None = None,
|
||||
on_step_end: Callable[[int, torch.Tensor, torch.Tensor], None] | None = None,
|
||||
on_step_start: Callable[[int, float, float], None] | None = None,
|
||||
step_profiler: Callable[[int], AbstractContextManager] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the full denoise loop; returns final (video_rows, audio_rows).
|
||||
@@ -206,6 +207,9 @@ def minimax_h3_denoise_loop(
|
||||
set_forward_context_denoise_step_idx(step)
|
||||
s_v, s_v_next = sigmas_video[step], sigmas_video[step + 1]
|
||||
s_a, s_a_next = sigmas_audio[step], sigmas_audio[step + 1]
|
||||
if on_step_start is not None:
|
||||
# Attention gates use the scheduler-style descending timestep.
|
||||
on_step_start(step, s_v, s_a)
|
||||
t_v, t_a = 1.0 - s_v, 1.0 - s_a
|
||||
imgvid_cond_t = max(t_v, float(imgvid_cond_noise_aug_for_inference))
|
||||
audio_ref_cond_t = max(t_a, float(audio_cond_noise_aug_for_inference))
|
||||
@@ -246,8 +250,8 @@ def minimax_h3_denoise_loop(
|
||||
audio_rows[audio_update] = new_audio
|
||||
if audio_anchor is not None:
|
||||
audio_rows[~audio_update] = audio_anchor # per-step audio ref reset
|
||||
if on_step is not None:
|
||||
on_step(step, video_rows, audio_rows)
|
||||
if on_step_end is not None:
|
||||
on_step_end(step, video_rows, audio_rows)
|
||||
|
||||
set_forward_context_denoise_step_idx(None)
|
||||
return video_rows, audio_rows
|
||||
|
||||
@@ -311,6 +311,8 @@ class MiniMaxH3Attention(nn.Module):
|
||||
quant_config: QuantizationConfig | None,
|
||||
*,
|
||||
prefix: str,
|
||||
role: str = "self",
|
||||
role_category: str | None = None,
|
||||
skip_sequence_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -351,8 +353,9 @@ class MiniMaxH3Attention(nn.Module):
|
||||
causal=False,
|
||||
# Packed rows reach the impl as [B, S, N, D].
|
||||
qkv_layout="BSND",
|
||||
role=role,
|
||||
role_category=role_category,
|
||||
skip_sequence_parallel=skip_sequence_parallel,
|
||||
role="self",
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
@@ -582,6 +585,8 @@ class MiniMaxH3TokenRefinerBlock(nn.Module):
|
||||
arch,
|
||||
quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
role="minimax_h3.token_refiner",
|
||||
role_category="self",
|
||||
skip_sequence_parallel=True,
|
||||
)
|
||||
self.mlp = MiniMaxH3MLP(
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
init_world_group,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.forward_context import DenoiseProgressMixin
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import (
|
||||
DiffusersPipelineLoader,
|
||||
)
|
||||
@@ -446,6 +447,7 @@ class _SingleRankEncoderGroup:
|
||||
|
||||
class MiniMaxH3Pipeline(
|
||||
nn.Module,
|
||||
DenoiseProgressMixin,
|
||||
ProgressBarMixin,
|
||||
DiffusionPipelineProfilerMixin,
|
||||
SupportImageInput,
|
||||
@@ -1371,7 +1373,11 @@ class MiniMaxH3Pipeline(
|
||||
device=self.device,
|
||||
imgvid_cond_noise_aug_for_inference=(MINIMAX_H3_IMGVID_COND_TIMESTEP),
|
||||
audio_cond_noise_aug_for_inference=(MINIMAX_H3_AUDIO_REF_COND_TIMESTEP),
|
||||
on_step=lambda step, video, audio: progress.update(),
|
||||
on_step_start=lambda step, video_sigma, audio_sigma: self.record_denoise_step(
|
||||
step,
|
||||
normalized_timestep=video_sigma,
|
||||
),
|
||||
on_step_end=lambda step, video, audio: progress.update(),
|
||||
)
|
||||
|
||||
target_video = video_rows[branch.update_mask_dev]
|
||||
|
||||
Reference in new issue
Block a user