[perf] Add mask-free TRTLLM packed-padding path (#6542)

Signed-off-by: Bo Li <22713281+bobboli@users.noreply.github.com>
This commit is contained in:
Bo Li authored and GitHub committed 2026-08-25 07:45:21 +00:00
1 parent b454e34966
commit 51b7565f33
7 files changed
+314 -123

No files matched your search

@@ -1,12 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Batched multi-request packing for MiniMax H3 step-wise execution.
Request mode forwards exactly one packed sequence per denoise step. Step mode
(continuous batching) may hold several requests at once, so this module
concatenates their packed layouts into a single sequence and rebuilds
``cu_seqlens`` with one document per request plus that request's
alignment-padding tail. Attention therefore never crosses a request boundary,
while every request keeps its own RoPE coordinates, token tags, and timesteps.
``cu_seqlens`` with one document per request plus any nonempty alignment-padding
tail. Attention therefore never crosses a request boundary, while every request
keeps its own RoPE coordinates, token tags, and timesteps.
A batch of one is not rebuilt at all -- it delegates to the request-mode layout.
Row order is the request order, so the concatenated video velocity returned by
@@ -93,12 +94,13 @@ def minimax_h3_batched_forward_kwargs(
timesteps[audio_pos[branch.audio_update_mask_dev]] = float(t_audio[index])
timesteps[audio_pos[~branch.audio_update_mask_dev]] = float(audio_ref_cond_timesteps[index])
# One document for the request's real rows and one for its padding tail,
# matching the single-request layout. The tail bound is emitted even when
# the 64-row alignment left it empty, so that a request always
# contributes exactly two bounds regardless of its length.
cu_bounds.append(seq_offset + branch.used_len)
cu_bounds.append(seq_offset + branch.seq_len)
# Empty documents are omitted because not every varlen kernel accepts
# repeated interior boundaries.
real_end = seq_offset + branch.used_len
padded_end = seq_offset + branch.seq_len
cu_bounds.append(real_end)
if padded_end != real_end:
cu_bounds.append(padded_end)
max_seqlen = max(max_seqlen, branch.used_len, branch.seq_len - branch.used_len)
text_offset += branch.text_len
@@ -27,7 +27,11 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata, VideoTokenLayout
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionMetadata,
PackedPaddingMetadata,
VideoTokenLayout,
)
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.attention.ops.minimax_h3_modulation import (
indexed_gate,
@@ -472,6 +476,7 @@ class MiniMaxH3Attention(nn.Module):
f"max_seqlen must be within the packed sequence, got {max_seqlen} for length {packed_total}"
)
attn_mask = None
mask_free_packed_padding = False
if num_requests > 1:
# A step-mode batch packs one document per request, so its valid
# rows are block-diagonal rather than a prefix: neither a KV prefix
@@ -498,14 +503,25 @@ class MiniMaxH3Attention(nn.Module):
# supports_packed_mask_free: backend consumes the packed metadata
# without ever reading attn_mask (CUDA packed varlen, NPU
# npu_attn_varlen opt-in with its own fallback rebuild).
no_mask = not getattr(self.attention, "use_ring", False) and (
self.attention.attn_backend.supports_prefix_kv_slicing
or self.attention.attn_backend.supports_packed_mask_free()
use_ring = getattr(self.attention, "use_ring", False)
mask_free_packed_padding = not use_ring and self.attention.attn_backend.supports_packed_mask_free()
no_mask = not use_ring and (
self.attention.attn_backend.supports_prefix_kv_slicing or mask_free_packed_padding
)
if used < packed_total and not no_mask:
attn_mask = torch.arange(packed_total, device=q.device)[None] < used
metadata = AttentionMetadata(
attn_mask=attn_mask,
packed_padding=(
PackedPaddingMetadata(
q_length=used,
kv_length=used,
cu_seqlens_q=cu_seqlens[:2],
cu_seqlens_k=cu_seqlens[:2],
)
if mask_free_packed_padding
else None
),
extra={
"cu_seqlens_q": cu_seqlens,
"cu_seqlens_k": cu_seqlens,
@@ -575,9 +591,9 @@ class MiniMaxH3Attention(nn.Module):
self.q_norm.variance_epsilon,
)
# Each request contributes a document for its rows plus one for its
# alignment padding. Local/Ulysses backends unpad it, while Ring keeps
# aligned rows for fixed-size P2P buffers.
# Each request contributes a document for its rows plus one for any
# nonempty alignment padding. Local/Ulysses backends unpad it, while
# Ring keeps aligned rows for fixed-size P2P buffers.
out = self._run_packed_attention(
q,
k,