[perf] Add mask-free TRTLLM packed-padding path (#6542)
Signed-off-by: Bo Li <22713281+bobboli@users.noreply.github.com>
This commit is contained in:
1 parent
b454e34966
commit
51b7565f33
7 files changed
+314
-123
No files matched your search
@@ -1,12 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Batched multi-request packing for MiniMax H3 step-wise execution.
|
||||
|
||||
Request mode forwards exactly one packed sequence per denoise step. Step mode
|
||||
(continuous batching) may hold several requests at once, so this module
|
||||
concatenates their packed layouts into a single sequence and rebuilds
|
||||
``cu_seqlens`` with one document per request plus that request's
|
||||
alignment-padding tail. Attention therefore never crosses a request boundary,
|
||||
while every request keeps its own RoPE coordinates, token tags, and timesteps.
|
||||
``cu_seqlens`` with one document per request plus any nonempty alignment-padding
|
||||
tail. Attention therefore never crosses a request boundary, while every request
|
||||
keeps its own RoPE coordinates, token tags, and timesteps.
|
||||
A batch of one is not rebuilt at all -- it delegates to the request-mode layout.
|
||||
|
||||
Row order is the request order, so the concatenated video velocity returned by
|
||||
@@ -93,12 +94,13 @@ def minimax_h3_batched_forward_kwargs(
|
||||
timesteps[audio_pos[branch.audio_update_mask_dev]] = float(t_audio[index])
|
||||
timesteps[audio_pos[~branch.audio_update_mask_dev]] = float(audio_ref_cond_timesteps[index])
|
||||
|
||||
# One document for the request's real rows and one for its padding tail,
|
||||
# matching the single-request layout. The tail bound is emitted even when
|
||||
# the 64-row alignment left it empty, so that a request always
|
||||
# contributes exactly two bounds regardless of its length.
|
||||
cu_bounds.append(seq_offset + branch.used_len)
|
||||
cu_bounds.append(seq_offset + branch.seq_len)
|
||||
# Empty documents are omitted because not every varlen kernel accepts
|
||||
# repeated interior boundaries.
|
||||
real_end = seq_offset + branch.used_len
|
||||
padded_end = seq_offset + branch.seq_len
|
||||
cu_bounds.append(real_end)
|
||||
if padded_end != real_end:
|
||||
cu_bounds.append(padded_end)
|
||||
max_seqlen = max(max_seqlen, branch.used_len, branch.seq_len - branch.used_len)
|
||||
|
||||
text_offset += branch.text_len
|
||||
|
||||
@@ -27,7 +27,11 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
PackedPaddingMetadata,
|
||||
VideoTokenLayout,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.attention.ops.minimax_h3_modulation import (
|
||||
indexed_gate,
|
||||
@@ -472,6 +476,7 @@ class MiniMaxH3Attention(nn.Module):
|
||||
f"max_seqlen must be within the packed sequence, got {max_seqlen} for length {packed_total}"
|
||||
)
|
||||
attn_mask = None
|
||||
mask_free_packed_padding = False
|
||||
if num_requests > 1:
|
||||
# A step-mode batch packs one document per request, so its valid
|
||||
# rows are block-diagonal rather than a prefix: neither a KV prefix
|
||||
@@ -498,14 +503,25 @@ class MiniMaxH3Attention(nn.Module):
|
||||
# supports_packed_mask_free: backend consumes the packed metadata
|
||||
# without ever reading attn_mask (CUDA packed varlen, NPU
|
||||
# npu_attn_varlen opt-in with its own fallback rebuild).
|
||||
no_mask = not getattr(self.attention, "use_ring", False) and (
|
||||
self.attention.attn_backend.supports_prefix_kv_slicing
|
||||
or self.attention.attn_backend.supports_packed_mask_free()
|
||||
use_ring = getattr(self.attention, "use_ring", False)
|
||||
mask_free_packed_padding = not use_ring and self.attention.attn_backend.supports_packed_mask_free()
|
||||
no_mask = not use_ring and (
|
||||
self.attention.attn_backend.supports_prefix_kv_slicing or mask_free_packed_padding
|
||||
)
|
||||
if used < packed_total and not no_mask:
|
||||
attn_mask = torch.arange(packed_total, device=q.device)[None] < used
|
||||
metadata = AttentionMetadata(
|
||||
attn_mask=attn_mask,
|
||||
packed_padding=(
|
||||
PackedPaddingMetadata(
|
||||
q_length=used,
|
||||
kv_length=used,
|
||||
cu_seqlens_q=cu_seqlens[:2],
|
||||
cu_seqlens_k=cu_seqlens[:2],
|
||||
)
|
||||
if mask_free_packed_padding
|
||||
else None
|
||||
),
|
||||
extra={
|
||||
"cu_seqlens_q": cu_seqlens,
|
||||
"cu_seqlens_k": cu_seqlens,
|
||||
@@ -575,9 +591,9 @@ class MiniMaxH3Attention(nn.Module):
|
||||
self.q_norm.variance_epsilon,
|
||||
)
|
||||
|
||||
# Each request contributes a document for its rows plus one for its
|
||||
# alignment padding. Local/Ulysses backends unpad it, while Ring keeps
|
||||
# aligned rows for fixed-size P2P buffers.
|
||||
# Each request contributes a document for its rows plus one for any
|
||||
# nonempty alignment padding. Local/Ulysses backends unpad it, while
|
||||
# Ring keeps aligned rows for fixed-size P2P buffers.
|
||||
out = self._run_packed_attention(
|
||||
q,
|
||||
k,
|
||||
|
||||
Reference in new issue
Block a user