[Kernel][Boogu-Image] Fuse Q/K RMSNorm + interleaved RoPE via fused_qk_norm_rope (#6982)

Signed-off-by: Qihan Kang <rollykanggg@gmail.com>
This commit is contained in:
KANG Qihan authored and GitHub committed 2026-09-13 09:39:27 +08:00
1 parent 7e520f9cc4
commit f7d9deb45a
8 files changed
+703 -53

No files matched your search

@@ -31,10 +31,17 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.layers.fused_qk_norm_rope import (
_fused_cuda_supported,
fused_qk_norm_rope,
fused_qk_norm_rope_min_tokens,
)
from vllm_omni.diffusion.models.utils import make_attention_mask
from vllm_omni.platforms import current_omni_platform
@@ -43,7 +50,10 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
RotaryEmbedding = tuple[torch.Tensor, torch.Tensor]
# (cos, sin) — optionally (cos, sin, packed_table) where packed_table is the
# fp32 [tokens, head_dim] = [cos(theta) | sin(theta)] layout the fused
# qk-norm+RoPE op consumes; the eager path ignores the third element.
RotaryEmbedding = tuple[torch.Tensor, ...]
RotaryFrequencyTables = list[RotaryEmbedding]
@@ -56,9 +66,12 @@ def apply_rotary_emb(x: torch.Tensor, rotary_emb: RotaryEmbedding) -> torch.Tens
Args:
x: Query or key tensor of shape [B, S, H, D].
rotary_emb: Cosine and sine tensors, each shaped [B, S, D].
rotary_emb: Cosine and sine tensors, each shaped [B, S, D]. The tuple
may carry a third element (the packed table for the fused
qk-norm+RoPE path); it is ignored here, hence the indexed access
instead of tuple unpacking.
"""
freqs_cos, freqs_sin = rotary_emb
freqs_cos, freqs_sin = rotary_emb[0], rotary_emb[1]
freqs_cos = freqs_cos.unsqueeze(2)
freqs_sin = freqs_sin.unsqueeze(2)
@@ -73,6 +86,87 @@ def apply_rotary_emb(x: torch.Tensor, rotary_emb: RotaryEmbedding) -> torch.Tens
return x_out.type_as(x)
# The fused qk-norm+RoPE path is always on where the fast path exists; on
# other platforms the eager chain below is the behaviour-identical fallback.
_FUSED_QK_NORM_ROPE = HAS_TRITON and current_platform.is_cuda()
# Boogu's default for the fused-path token gate: fuse only when a rotary table
# spans at least this many positions (counted as B*S). Below the crossover a
# request is host-bound and the fused path's per-call Python launch (custom-op
# impl + Triton launcher) can cost more end-to-end than the kernel saves.
# Measured on one H200: on the pre-#6871 main, 512^2 (1.1k tokens) paid
# +191 ms/request and the crossover sat at ~2,350 tokens; on the current main
# the 512^2 family measures as noise (+/-25 ms, sign-inconsistent) while
# >=768^2 (2.3k tokens) gains. The crossover is host-dependent — re-benchmark
# on your hardware and override with VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS
# (0 = always fuse); see fused_qk_norm_rope_min_tokens().
_FUSED_MIN_TOKENS = 2048
def _with_packed_rope_table(rotary_emb: RotaryEmbedding) -> RotaryEmbedding:
"""Append the packed fp32 table the fused qk-norm+RoPE op consumes.
``freqs_cos``/``freqs_sin`` are theta-repeat-interleaved ``[B, S, D]``
(``cos[2i] == cos[2i+1] == cos(theta_i)``), so taking every even column
yields the theta-width halves and the packed table is
``[B*S, D] = [cos(theta) | sin(theta)]``. Tables spanning fewer positions
(counted as ``B*S``) than the token gate — Boogu's ``_FUSED_MIN_TOKENS``
default, overridable through ``VLLM_OMNI_FUSED_QK_NORM_ROPE_MIN_TOKENS``
(``0`` = always fuse) — are returned unchanged: without the third element
every consumer stays on the eager chain (this also keeps short tables like
the context refiner's off the fused path).
"""
freqs_cos, freqs_sin = rotary_emb[0], rotary_emb[1]
if freqs_cos.shape[0] * freqs_cos.shape[1] < fused_qk_norm_rope_min_tokens(_FUSED_MIN_TOKENS):
return rotary_emb
packed = torch.cat((freqs_cos[..., ::2], freqs_sin[..., ::2]), dim=-1)
return freqs_cos, freqs_sin, packed.reshape(-1, freqs_cos.shape[-1]).float()
def _qk_norm_rope(
query: torch.Tensor,
key: torch.Tensor,
norm_q,
norm_k,
rotary_emb: RotaryEmbedding | None,
dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-head Q/K RMSNorm + RoPE + cast, fused into one kernel launch when
a packed rope table is present; otherwise the original eager chain."""
# Gate on the op's own fast-path predicate: when the fused CUDA kernel
# cannot run (dtype, device, or geometry outside even
# rotary_dim <= head_dim <= 256), the original chain below must be used
# bit-exactly rather than the op's internal eager fallback, whose rounding
# differs from vLLM's RMSNorm by one ulp on a fraction of elements.
if (
rotary_emb is not None
and len(rotary_emb) > 2
and rotary_emb[2] is not None
and _fused_cuda_supported(query, key, query.shape[-1], query.shape[-1], interleaved=True)
):
batch, seq_len, num_heads, head_dim = query.shape
num_kv_heads = key.shape[2]
fused_q, fused_k = fused_qk_norm_rope(
query.view(batch * seq_len, num_heads, head_dim),
key.view(batch * seq_len, num_kv_heads, head_dim),
norm_q.weight,
norm_k.weight,
rotary_emb[2],
norm_q.variance_epsilon,
interleaved=True,
)
return (
fused_q.view(batch, seq_len, num_heads, head_dim),
fused_k.view(batch, seq_len, num_kv_heads, head_dim),
)
query = norm_q(query)
key = norm_k(key)
if rotary_emb is not None:
query = apply_rotary_emb(query, rotary_emb)
key = apply_rotary_emb(key, rotary_emb)
return query.to(dtype), key.to(dtype)
def swiglu(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
return F.silu(x.float(), inplace=False).to(x.dtype) * y
@@ -559,13 +653,7 @@ class BooguImageSelfAttention(nn.Module):
key = key.unflatten(-1, (self.num_local_kv_heads, self.head_dim))
value = value.unflatten(-1, (self.num_local_kv_heads, self.head_dim))
query = self.norm_q(query)
key = self.norm_k(key)
if rotary_emb is not None:
query = apply_rotary_emb(query, rotary_emb)
key = apply_rotary_emb(key, rotary_emb)
query, key = query.to(dtype), key.to(dtype)
query, key = _qk_norm_rope(query, key, self.norm_q, self.norm_k, rotary_emb, dtype)
attn_metadata = AttentionMetadata(attn_mask=attention_mask) if attention_mask is not None else None
attn_output = self.attn(query, key, value, attn_metadata)
@@ -711,13 +799,7 @@ class BooguImageJointAttention(nn.Module):
key = key.view(batch_size, -1, self.num_local_kv_heads, self.head_dim)
value = value.view(batch_size, -1, self.num_local_kv_heads, self.head_dim)
query = self.norm_q(query)
key = self.norm_k(key)
if rotary_emb is not None:
query = apply_rotary_emb(query, rotary_emb)
key = apply_rotary_emb(key, rotary_emb)
query, key = query.to(dtype), key.to(dtype)
query, key = _qk_norm_rope(query, key, self.norm_q, self.norm_k, rotary_emb, dtype)
attn_metadata = AttentionMetadata(attn_mask=joint_attention_mask) if joint_attention_mask is not None else None
attn_output = self.attn(query, key, value, attn_metadata)
@@ -1449,6 +1531,11 @@ class BooguImageTransformer2DModel(nn.Module):
shift += ref_img_len
idx += 1
if _FUSED_QK_NORM_ROPE:
# The rebuilt per-reference-image tuple needs its own packed
# table; the forward-level tuples do not flow into this batch.
batch_ref_img_rotary_emb = _with_packed_rope_table(batch_ref_img_rotary_emb)
for layer in self.ref_image_refiner:
batch_ref_image_hidden_states = layer(
batch_ref_image_hidden_states, batch_ref_img_mask, batch_ref_img_rotary_emb, batch_temb
@@ -1534,6 +1621,17 @@ class BooguImageTransformer2DModel(nn.Module):
device,
)
if _FUSED_QK_NORM_ROPE:
# One packed [cos|sin] table per rotary embedding that reaches an
# attention, built once per forward; the tuples grow a third
# element that the fused path consumes and the eager path ignores.
# (ref_img_rotary_emb is packed inside img_patch_embed_and_refine,
# on the rebuilt per-reference-image batch tuple.)
context_rotary_emb = _with_packed_rope_table(context_rotary_emb)
noise_rotary_emb = _with_packed_rope_table(noise_rotary_emb)
rotary_emb = _with_packed_rope_table(rotary_emb)
combined_img_rotary_emb = _with_packed_rope_table(combined_img_rotary_emb)
# Context refinement.
for layer in self.context_refiner:
instruction_hidden_states = layer(instruction_hidden_states, instruction_attention_mask, context_rotary_emb)