[Attention] Refine TRTLLM attention support for MiniMax H3 (#5779)
Signed-off-by: Bo Li <22713281+bobboli@users.noreply.github.com> Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: lishunyang12 <lishunyang12@163.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
11 files changed
+370
-33
No files matched your search
@@ -180,6 +180,7 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
prefix: str = "",
|
||||
qkv_layout: str | None = None,
|
||||
backend_kwargs: dict | None = None,
|
||||
role: str = "self",
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
self.num_heads = num_heads
|
||||
@@ -187,6 +188,7 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
self.softmax_scale = softmax_scale
|
||||
self.causal = causal
|
||||
self.num_kv_heads = num_kv_heads if num_kv_heads is not None else num_heads
|
||||
self.role = role
|
||||
|
||||
self.skip = SkipSoftmaxConfig.from_backend_kwargs(backend_kwargs)
|
||||
self._warned_missing_timestep = False
|
||||
@@ -255,8 +257,16 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata | None = None,
|
||||
) -> torch.Tensor:
|
||||
extra = getattr(attn_metadata, "extra", {}) if attn_metadata is not None else {}
|
||||
packed_keys = ("cu_seqlens_q", "cu_seqlens_k", "max_seqlen_q", "max_seqlen_k")
|
||||
present_packed_keys = [key for key in packed_keys if key in extra]
|
||||
if present_packed_keys and len(present_packed_keys) != len(packed_keys):
|
||||
missing = sorted(set(packed_keys) - set(present_packed_keys))
|
||||
raise ValueError(f"Incomplete packed TRTLLM attention metadata; missing {missing}")
|
||||
has_packed_metadata = len(present_packed_keys) == len(packed_keys)
|
||||
|
||||
attn_mask = getattr(attn_metadata, "attn_mask", None) if attn_metadata is not None else None
|
||||
if attn_mask is not None:
|
||||
if attn_mask is not None and not has_packed_metadata:
|
||||
raise ValueError(
|
||||
"TRTLLM_ATTN does not support attn_mask (Wan sets one only under SP with "
|
||||
"mask_sp_padding=True and a non-divisible seq len). Either set "
|
||||
@@ -270,30 +280,96 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
"another backend via --diffusion-attention-backend."
|
||||
)
|
||||
|
||||
batch, q_len, num_q_heads, head_dim = query.shape
|
||||
physical_batch, q_len, num_q_heads, head_dim = query.shape
|
||||
kv_len, num_kv_heads = key.shape[1], key.shape[2]
|
||||
|
||||
device = query.device
|
||||
|
||||
q = query.reshape(batch * q_len, num_q_heads, head_dim).contiguous()
|
||||
k = key.reshape(batch * kv_len, num_kv_heads, head_dim).contiguous()
|
||||
v = value.reshape(batch * kv_len, num_kv_heads, head_dim).contiguous()
|
||||
q = query.reshape(physical_batch * q_len, num_q_heads, head_dim).contiguous()
|
||||
k = key.reshape(physical_batch * kv_len, num_kv_heads, head_dim).contiguous()
|
||||
v = value.reshape(physical_batch * kv_len, num_kv_heads, head_dim).contiguous()
|
||||
output_tokens = q.shape[0]
|
||||
|
||||
seq_lens = torch.full((batch,), kv_len, dtype=torch.int32, device=device)
|
||||
cu_seq_lens_q = torch.arange(0, (batch + 1) * q_len, step=q_len, dtype=torch.int32, device=device)
|
||||
cu_seq_lens_kv = torch.arange(0, (batch + 1) * kv_len, step=kv_len, dtype=torch.int32, device=device)
|
||||
if has_packed_metadata:
|
||||
cu_seq_lens_q = extra["cu_seqlens_q"]
|
||||
cu_seq_lens_kv = extra["cu_seqlens_k"]
|
||||
if cu_seq_lens_q.ndim != 1 or cu_seq_lens_kv.ndim != 1:
|
||||
raise ValueError("Packed TRTLLM attention cu_seqlens tensors must be one-dimensional")
|
||||
if cu_seq_lens_q.numel() != cu_seq_lens_kv.numel() or cu_seq_lens_q.numel() < 2:
|
||||
raise ValueError("Packed TRTLLM attention requires matching non-empty Q and KV sequence batches")
|
||||
if int(cu_seq_lens_q[-1].item()) != q.shape[0] or int(cu_seq_lens_kv[-1].item()) != k.shape[0]:
|
||||
raise ValueError("Packed TRTLLM attention cu_seqlens must cover all Q/K/V tokens")
|
||||
|
||||
# Drop trailing empty sequences retained by packed producers when
|
||||
# alignment adds no padding tokens.
|
||||
while (
|
||||
cu_seq_lens_q.numel() > 2
|
||||
and int(cu_seq_lens_q[-1].item()) == int(cu_seq_lens_q[-2].item())
|
||||
and int(cu_seq_lens_kv[-1].item()) == int(cu_seq_lens_kv[-2].item())
|
||||
):
|
||||
cu_seq_lens_q = cu_seq_lens_q[:-1]
|
||||
cu_seq_lens_kv = cu_seq_lens_kv[:-1]
|
||||
|
||||
# A packed prefix mask denotes structural suffix padding, not another
|
||||
# document. Exclude it before quantization and attention; in particular,
|
||||
# SAGE block quantization must not span the valid/padding boundary.
|
||||
if attn_mask is not None:
|
||||
if q.shape[0] != k.shape[0] or attn_mask.numel() != q.shape[0]:
|
||||
raise ValueError("Packed TRTLLM prefix masks require equal Q/K lengths")
|
||||
flat_mask = attn_mask.reshape(-1).to(dtype=torch.bool)
|
||||
valid_tokens = int(flat_mask.sum().item())
|
||||
expected_mask = torch.arange(flat_mask.numel(), device=device) < valid_tokens
|
||||
if not torch.equal(flat_mask, expected_mask):
|
||||
raise ValueError("TRTLLM_ATTN only supports prefix-valid packed attention masks")
|
||||
|
||||
q_boundary = (cu_seq_lens_q == valid_tokens).nonzero(as_tuple=False)
|
||||
kv_boundary = (cu_seq_lens_kv == valid_tokens).nonzero(as_tuple=False)
|
||||
if q_boundary.numel() != 1 or kv_boundary.numel() != 1:
|
||||
raise ValueError("Packed TRTLLM prefix length must be a Q/K sequence boundary")
|
||||
q_end = int(q_boundary.item()) + 1
|
||||
kv_end = int(kv_boundary.item()) + 1
|
||||
if q_end != kv_end:
|
||||
raise ValueError("Packed TRTLLM prefix masks require matching Q/K sequence batches")
|
||||
|
||||
q = q[:valid_tokens]
|
||||
k = k[:valid_tokens]
|
||||
v = v[:valid_tokens]
|
||||
cu_seq_lens_q = cu_seq_lens_q[:q_end].contiguous()
|
||||
cu_seq_lens_kv = cu_seq_lens_kv[:kv_end].contiguous()
|
||||
|
||||
batch = cu_seq_lens_q.numel() - 1
|
||||
seq_lens = (cu_seq_lens_kv[1:] - cu_seq_lens_kv[:-1]).to(dtype=torch.int32).contiguous()
|
||||
max_q_len = int(extra["max_seqlen_q"])
|
||||
max_kv_len = int(extra["max_seqlen_k"])
|
||||
else:
|
||||
batch = physical_batch
|
||||
seq_lens = torch.full((batch,), kv_len, dtype=torch.int32, device=device)
|
||||
cu_seq_lens_q = torch.arange(0, (batch + 1) * q_len, step=q_len, dtype=torch.int32, device=device)
|
||||
cu_seq_lens_kv = torch.arange(0, (batch + 1) * kv_len, step=kv_len, dtype=torch.int32, device=device)
|
||||
max_q_len = q_len
|
||||
max_kv_len = kv_len
|
||||
workspace = self._get_workspace(device)
|
||||
|
||||
bmm1_scale = self.softmax_scale
|
||||
bmm2_scale = 1.0
|
||||
|
||||
_skip_factor = self._resolve_skip_factor(kv_len)
|
||||
_skip_factor = self._resolve_skip_factor(max_kv_len)
|
||||
|
||||
# SAGE kwargs are only understood by newer FlashInfer builds; pass them exclusively when
|
||||
# SAGE quant is active (which already requires the kernel, checked at init) so the dense
|
||||
# path stays compatible with older builds that lack these parameters.
|
||||
sage_kwargs: dict = {}
|
||||
if self.quant.enabled:
|
||||
# The SAGE kernel requires every KV sequence to contain at least one full
|
||||
# quantization block. Small auxiliary attention sites use the dense kernel.
|
||||
use_sage = self.quant.enabled and bool(torch.all(seq_lens >= self.quant.k_block_size).item())
|
||||
if self.quant.enabled and not use_sage:
|
||||
message = (
|
||||
f"TRTLLM_ATTN SAGE quantization is configured for attention role {self.role!r}, but at least one "
|
||||
f"KV sequence is shorter than k_block_size={self.quant.k_block_size}. Falling back to dense "
|
||||
"attention for this input."
|
||||
)
|
||||
logger.warning_once(message)
|
||||
if use_sage:
|
||||
q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)
|
||||
sage_kwargs["sage_attn_sfs"] = sage_attn_sfs
|
||||
sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes
|
||||
@@ -304,8 +380,8 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
value=v,
|
||||
workspace_buffer=workspace,
|
||||
seq_lens=seq_lens,
|
||||
max_q_len=q_len,
|
||||
max_kv_len=kv_len,
|
||||
max_q_len=max_q_len,
|
||||
max_kv_len=max_kv_len,
|
||||
bmm1_scale=bmm1_scale,
|
||||
bmm2_scale=bmm2_scale,
|
||||
o_sf_scale=-1.0,
|
||||
@@ -319,4 +395,12 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
skip_softmax_threshold_scale_factor=_skip_factor,
|
||||
**sage_kwargs,
|
||||
)
|
||||
return out.reshape(batch, q_len, num_q_heads, head_dim)
|
||||
if out.shape[0] != output_tokens:
|
||||
padded_out = torch.zeros(
|
||||
(output_tokens, num_q_heads, head_dim),
|
||||
dtype=out.dtype,
|
||||
device=out.device,
|
||||
)
|
||||
padded_out[: out.shape[0]] = out
|
||||
out = padded_out
|
||||
return out.reshape(physical_batch, q_len, num_q_heads, head_dim)
|
||||
@@ -89,6 +89,15 @@ class Attention(nn.Module):
|
||||
role_category=role_category,
|
||||
allow_trtllm_default=allow_trtllm_default,
|
||||
)
|
||||
parallel_config = getattr(config, "parallel_config", None)
|
||||
allgather_degree = getattr(parallel_config, "allgather_degree", 1)
|
||||
# TODO: Move AllGather-KV compatibility into an AttentionBackend capability
|
||||
# so validation does not depend on backend names.
|
||||
if not skip_sequence_parallel and allgather_degree > 1 and attn_backend_cls.get_name() == "TRTLLM_ATTN":
|
||||
raise ValueError(
|
||||
"TRTLLM_ATTN does not support AllGather-KV sequence parallelism. "
|
||||
"Set --allgather-degree 1 or select another diffusion attention backend."
|
||||
)
|
||||
if spec is not None:
|
||||
backend_kwargs = spec.backend_kwargs()
|
||||
self.backend_pref = spec.backend
|
||||
@@ -107,6 +116,7 @@ class Attention(nn.Module):
|
||||
qkv_layout=qkv_layout,
|
||||
prefix=prefix,
|
||||
backend_kwargs=backend_kwargs,
|
||||
role=role,
|
||||
)
|
||||
# Instantiate fallback backend for float32 support
|
||||
self.sdpa_fallback = SDPABackend.get_impl_cls()(
|
||||
|
||||
@@ -1434,6 +1434,10 @@ class SkipSoftmaxSpec:
|
||||
if self.target_sparsity is not None and self.threshold is not None:
|
||||
raise ValueError("skip_softmax: set either target_sparsity or threshold, not both.")
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.threshold is not None or self.target_sparsity is not None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttnQuantSpec:
|
||||
|
||||
@@ -221,9 +221,15 @@ class DenoiseProgressMixin:
|
||||
step_idx: int | None,
|
||||
timestep=None,
|
||||
scheduler=None,
|
||||
normalized_timestep: float | None = None,
|
||||
) -> None:
|
||||
set_forward_context_denoise_step_idx(step_idx)
|
||||
if timestep is None or _forward_context is None:
|
||||
if _forward_context is None:
|
||||
return
|
||||
if normalized_timestep is not None:
|
||||
_forward_context.denoise_timestep = float(normalized_timestep)
|
||||
return
|
||||
if timestep is None:
|
||||
return
|
||||
scheduler = scheduler if scheduler is not None else getattr(self, "scheduler", None)
|
||||
ntt = getattr(getattr(scheduler, "config", None), "num_train_timesteps", None)
|
||||
|
||||
@@ -41,6 +41,9 @@ _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = {
|
||||
supports_multimodal_inputs=True,
|
||||
max_multimodal_image_inputs=9,
|
||||
supports_mixed_reference_inputs=True,
|
||||
# H3 represents alignment padding as a second packed sequence. The
|
||||
# packed TRTLLM backend consumes cu_seqlens and isolates that padding.
|
||||
attention_mask_free=True,
|
||||
),
|
||||
"WanPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
||||
"WanImageToVideoPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
||||
|
||||
@@ -149,7 +149,8 @@ def minimax_h3_denoise_loop(
|
||||
device: torch.device,
|
||||
imgvid_cond_noise_aug_for_inference: float = MINIMAX_H3_IMGVID_COND_TIMESTEP,
|
||||
audio_cond_noise_aug_for_inference: float = MINIMAX_H3_AUDIO_REF_COND_TIMESTEP,
|
||||
on_step: Callable[[int, torch.Tensor, torch.Tensor], None] | None = None,
|
||||
on_step_end: Callable[[int, torch.Tensor, torch.Tensor], None] | None = None,
|
||||
on_step_start: Callable[[int, float, float], None] | None = None,
|
||||
step_profiler: Callable[[int], AbstractContextManager] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the full denoise loop; returns final (video_rows, audio_rows).
|
||||
@@ -206,6 +207,9 @@ def minimax_h3_denoise_loop(
|
||||
set_forward_context_denoise_step_idx(step)
|
||||
s_v, s_v_next = sigmas_video[step], sigmas_video[step + 1]
|
||||
s_a, s_a_next = sigmas_audio[step], sigmas_audio[step + 1]
|
||||
if on_step_start is not None:
|
||||
# Attention gates use the scheduler-style descending timestep.
|
||||
on_step_start(step, s_v, s_a)
|
||||
t_v, t_a = 1.0 - s_v, 1.0 - s_a
|
||||
imgvid_cond_t = max(t_v, float(imgvid_cond_noise_aug_for_inference))
|
||||
audio_ref_cond_t = max(t_a, float(audio_cond_noise_aug_for_inference))
|
||||
@@ -246,8 +250,8 @@ def minimax_h3_denoise_loop(
|
||||
audio_rows[audio_update] = new_audio
|
||||
if audio_anchor is not None:
|
||||
audio_rows[~audio_update] = audio_anchor # per-step audio ref reset
|
||||
if on_step is not None:
|
||||
on_step(step, video_rows, audio_rows)
|
||||
if on_step_end is not None:
|
||||
on_step_end(step, video_rows, audio_rows)
|
||||
|
||||
set_forward_context_denoise_step_idx(None)
|
||||
return video_rows, audio_rows
|
||||
|
||||
@@ -311,6 +311,8 @@ class MiniMaxH3Attention(nn.Module):
|
||||
quant_config: QuantizationConfig | None,
|
||||
*,
|
||||
prefix: str,
|
||||
role: str = "self",
|
||||
role_category: str | None = None,
|
||||
skip_sequence_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -351,8 +353,9 @@ class MiniMaxH3Attention(nn.Module):
|
||||
causal=False,
|
||||
# Packed rows reach the impl as [B, S, N, D].
|
||||
qkv_layout="BSND",
|
||||
role=role,
|
||||
role_category=role_category,
|
||||
skip_sequence_parallel=skip_sequence_parallel,
|
||||
role="self",
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
@@ -582,6 +585,8 @@ class MiniMaxH3TokenRefinerBlock(nn.Module):
|
||||
arch,
|
||||
quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
role="minimax_h3.token_refiner",
|
||||
role_category="self",
|
||||
skip_sequence_parallel=True,
|
||||
)
|
||||
self.mlp = MiniMaxH3MLP(
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
init_world_group,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.forward_context import DenoiseProgressMixin
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import (
|
||||
DiffusersPipelineLoader,
|
||||
)
|
||||
@@ -446,6 +447,7 @@ class _SingleRankEncoderGroup:
|
||||
|
||||
class MiniMaxH3Pipeline(
|
||||
nn.Module,
|
||||
DenoiseProgressMixin,
|
||||
ProgressBarMixin,
|
||||
DiffusionPipelineProfilerMixin,
|
||||
SupportImageInput,
|
||||
@@ -1371,7 +1373,11 @@ class MiniMaxH3Pipeline(
|
||||
device=self.device,
|
||||
imgvid_cond_noise_aug_for_inference=(MINIMAX_H3_IMGVID_COND_TIMESTEP),
|
||||
audio_cond_noise_aug_for_inference=(MINIMAX_H3_AUDIO_REF_COND_TIMESTEP),
|
||||
on_step=lambda step, video, audio: progress.update(),
|
||||
on_step_start=lambda step, video_sigma, audio_sigma: self.record_denoise_step(
|
||||
step,
|
||||
normalized_timestep=video_sigma,
|
||||
),
|
||||
on_step_end=lambda step, video, audio: progress.update(),
|
||||
)
|
||||
|
||||
target_video = video_rows[branch.update_mask_dev]
|
||||
|
||||
Reference in new issue
Block a user