[Perf] Fuse AdaLayerNorm CUDA eager path with Triton (#8231)
Signed-off-by: GUOGUO <xwdong1998@163.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
182b77da4c
commit
ee9ab9ea88
8 files changed
+1255
-14
No files matched your search
@@ -1,3 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from collections.abc import Callable
|
||||
from importlib.util import find_spec
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -5,6 +9,8 @@ import torch
|
||||
import torch.nn as nn
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.triton_utils import HAS_TRITON as _HAS_TRITON
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from vllm_omni.diffusion.layers.custom_op import CustomOp
|
||||
from vllm_omni.diffusion.layers.norm import LayerNorm
|
||||
@@ -16,6 +22,282 @@ logger = init_logger(__name__)
|
||||
|
||||
_HAS_MINDIESD = find_spec("mindiesd") is not None
|
||||
|
||||
# Fast-path bound: one Triton program covers the whole normalized dimension, so
|
||||
# BLOCK_C = next_power_of_2(C) grows with the hidden size. The diffusion models
|
||||
# consuming AdaLayerNorm use hidden sizes 2240 (Sana-WM), 3072/3584 (Wan2.2,
|
||||
# Qwen-Image) and 5120 (Wan2.2 A14B), i.e. BLOCK_C up to 8192, which compiles
|
||||
# and runs efficiently on H200 (measured; C=5120 -> 30-111 us depending on L).
|
||||
# Beyond that, masked-lane waste and register pressure grow without a measured
|
||||
# benefit, so the fast path falls back to forward_native.
|
||||
_MAX_BLOCK_C = 8192
|
||||
|
||||
_ADALN_CONFIGS: dict = {}
|
||||
_ADALN_DTYPES = (torch.bfloat16, torch.float16, torch.float32)
|
||||
# Runtime configs whose Triton compile/launch failed synchronously; they fall
|
||||
# back to forward_native and are not retried (mirrors the LTX2
|
||||
# residual-AdaLN failed-key cache in residual_adaln.py).
|
||||
_FAILED_ADALN_KEYS: set = set()
|
||||
_adaln_fused_forward: Callable[["AdaLayerNorm", torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor | None] | None
|
||||
|
||||
|
||||
if _HAS_TRITON:
|
||||
|
||||
@triton.jit
|
||||
def _adaln_scale_shift_layernorm_kernel(
|
||||
x_ptr,
|
||||
scale_ptr,
|
||||
shift_ptr,
|
||||
out_ptr,
|
||||
weight_ptr,
|
||||
bias_ptr,
|
||||
eps,
|
||||
channels,
|
||||
seq_len,
|
||||
scale_row_stride,
|
||||
shift_row_stride,
|
||||
has_weight: tl.constexpr,
|
||||
has_bias: tl.constexpr,
|
||||
is_half: tl.constexpr,
|
||||
per_sample_scale: tl.constexpr,
|
||||
per_sample_shift: tl.constexpr,
|
||||
block_c: tl.constexpr,
|
||||
):
|
||||
"""One program per LayerNorm row: out = ln(x) * (1 + scale) + shift.
|
||||
|
||||
Rows enumerate contiguous (B, L, C) or (B, F, S, C) input.
|
||||
sample_idx = row // seq_len selects a sample, or a flattened
|
||||
batch/frame group with seq_len = S. Per-group modulation is
|
||||
(B, 1, C) or (B, F, 1, C); explicit row strides support projection
|
||||
chunk views. Shared modulation addresses one channel vector.
|
||||
(B, C) at B > 1 is a native-path broadcast error and never reaches
|
||||
this kernel.
|
||||
|
||||
Mean/variance use a shift-invariant two-pass: the row is centered on
|
||||
its first element (x0) before the fp32 tree sum, so for inputs with a
|
||||
large constant offset and small variance the summation and the
|
||||
x - mean cancellation stay accurate where a plain fp32 sum of
|
||||
large-offset values loses the low-order bits (the reason
|
||||
fused_adaptive_group_norm_silu uses Welford/Chan). Reductions
|
||||
accumulate in fp32 (matches the golden path, which computes
|
||||
F.layer_norm on x.float()). For half-precision outputs the golden
|
||||
chain rounds after every torch op (LN result, 1+scale, product, sum),
|
||||
so the kernel replicates exactly those roundings - keeping the output
|
||||
bit-faithful to the frozen semantics. fp32 outputs have no
|
||||
intermediate rounding in the golden path and use a plain fp32 chain.
|
||||
block_c covers the whole normalized dim (masked); it is never
|
||||
autotuned. Non-contiguous last-dim inputs never reach this kernel
|
||||
(routed to forward_native upstream).
|
||||
"""
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
sample_idx = row // seq_len
|
||||
cols = tl.arange(0, block_c)
|
||||
mask = cols < channels
|
||||
x = tl.load(x_ptr + row * channels + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
# Center on the first element before summing: for x ~ offset + noise
|
||||
# this keeps every value entering the tree sum at noise magnitude.
|
||||
x0 = tl.load(x_ptr + row * channels).to(tl.float32)
|
||||
xs = tl.where(mask, x - x0, 0.0)
|
||||
mean_xs = tl.sum(xs, axis=0) / channels
|
||||
# xm = xs - mean_xs is exactly x - (x0 + mean_xs): the row mean enters
|
||||
# only through this centering, and the shift keeps it accurate.
|
||||
xm = tl.where(mask, xs - mean_xs, 0.0)
|
||||
var = tl.sum(xm * xm, axis=0) / channels
|
||||
xn = xm * tl.rsqrt(var + eps)
|
||||
if has_weight:
|
||||
xn = xn * tl.load(weight_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
if has_bias:
|
||||
xn = xn + tl.load(bias_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
if per_sample_scale:
|
||||
s = tl.load(scale_ptr + sample_idx * scale_row_stride + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
else:
|
||||
s = tl.load(scale_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
if per_sample_shift:
|
||||
sh = tl.load(shift_ptr + sample_idx * shift_row_stride + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
else:
|
||||
sh = tl.load(shift_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
out_dtype = out_ptr.dtype.element_ty
|
||||
if is_half:
|
||||
# Golden bf16/fp16 chain: round after each op, exactly as the
|
||||
# sequence of torch elementwise kernels does (fp32 internal math).
|
||||
ln16 = xn.to(out_dtype)
|
||||
t1 = (1.0 + s).to(out_dtype)
|
||||
t2 = (ln16.to(tl.float32) * t1.to(tl.float32)).to(out_dtype)
|
||||
y = (t2.to(tl.float32) + sh).to(out_dtype)
|
||||
else:
|
||||
y = xn * (1.0 + s) + sh
|
||||
# Preserve the input shape and store output rows contiguously.
|
||||
tl.store(out_ptr + row * channels + cols, y.to(out_dtype), mask=mask)
|
||||
|
||||
def _adaln_modulation_mode(t: torch.Tensor, x: torch.Tensor):
|
||||
"""Classify scale/shift against 3D/4D x for the fused kernel.
|
||||
|
||||
"shared" every leading dim is 1: (C,), (1, C), (1, 1, C)
|
||||
"per_sample" (B, 1, C) or (B, F, 1, C): one row per sample/frame
|
||||
None anything else - native torch broadcasting on the
|
||||
forward_native path handles it (or raises, for the
|
||||
frozen B > 1 (B, C) RuntimeError)
|
||||
"""
|
||||
shape = t.shape
|
||||
if t.ndim < 1 or t.ndim > x.ndim or shape[-1] != x.shape[-1]:
|
||||
return None
|
||||
if all(d == 1 for d in shape[:-1]):
|
||||
return "shared"
|
||||
if x.ndim == 3 and t.ndim == 3 and shape == (x.shape[0], 1, x.shape[-1]):
|
||||
return "per_sample"
|
||||
if x.ndim == 4 and t.ndim == 4 and shape == (x.shape[0], x.shape[1], 1, x.shape[-1]):
|
||||
# Batch/frame groups must be addressable with one row stride, including chunk views.
|
||||
if shape[0] > 1 and shape[1] > 1 and t.stride(0) != shape[1] * t.stride(1):
|
||||
return None
|
||||
return "per_sample"
|
||||
return None
|
||||
|
||||
def _adaln_param_matches(x: torch.Tensor, p: torch.Tensor) -> bool:
|
||||
return (
|
||||
p.dtype is x.dtype
|
||||
and p.is_cuda
|
||||
and p.device == x.device
|
||||
and p.ndim == 1
|
||||
and p.shape[0] == x.shape[-1]
|
||||
and p.is_contiguous()
|
||||
)
|
||||
|
||||
def _adaln_inputs_supported(module: "AdaLayerNorm", x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor):
|
||||
"""Strict fast-path eligibility (mirrors the LTX2 residual-AdaLN
|
||||
guard pattern): every tensor the kernel dereferences must be a dense
|
||||
CUDA tensor on x's device with a supported layout and dtype; anything
|
||||
else falls back to forward_native, which reproduces torch semantics
|
||||
exactly - including the frozen B > 1 (B, C) RuntimeError. Raw pointers
|
||||
go to one kernel, which does no cross-device checking. Runs native
|
||||
under torch.compile so regional compilation is not disrupted."""
|
||||
if torch.compiler.is_compiling():
|
||||
return None
|
||||
if (
|
||||
not _HAS_TRITON
|
||||
or x.device.type != "cuda"
|
||||
or x.ndim not in (3, 4)
|
||||
or x.numel() == 0
|
||||
or not x.is_contiguous()
|
||||
):
|
||||
return None
|
||||
if x.dtype not in _ADALN_DTYPES:
|
||||
return None
|
||||
if scale.dtype is not x.dtype or shift.dtype is not x.dtype:
|
||||
return None
|
||||
if not (scale.is_cuda and shift.is_cuda):
|
||||
return None
|
||||
if scale.device != x.device or shift.device != x.device:
|
||||
return None
|
||||
mode_s = _adaln_modulation_mode(scale, x)
|
||||
mode_h = _adaln_modulation_mode(shift, x)
|
||||
if mode_s is None or mode_h is None:
|
||||
return None
|
||||
# The kernel addresses the modulation rows densely per channel and
|
||||
# strides per sample: the last dim must be unit-stride; other dims
|
||||
# are handled by the explicit row stride (chunk views supported).
|
||||
# 0-dim (scalar) modulation is classified None above and falls back.
|
||||
if scale.stride(-1) != 1 or shift.stride(-1) != 1:
|
||||
return None
|
||||
if x.data_ptr() % 16 or scale.data_ptr() % 16 or shift.data_ptr() % 16:
|
||||
return None
|
||||
channels = x.shape[-1]
|
||||
ln = module.layernorm
|
||||
if tuple(ln.normalized_shape) != (channels,):
|
||||
return None
|
||||
if torch.is_grad_enabled() and any(
|
||||
t is not None and t.requires_grad for t in (x, scale, shift, ln.weight, ln.bias)
|
||||
):
|
||||
return None
|
||||
block_c = triton.next_power_of_2(channels)
|
||||
if block_c > _MAX_BLOCK_C:
|
||||
return None
|
||||
for p in (ln.weight, ln.bias):
|
||||
if p is not None and not _adaln_param_matches(x, p):
|
||||
return None
|
||||
return channels, block_c, mode_s == "per_sample", mode_h == "per_sample"
|
||||
|
||||
def _adaln_fused_forward_impl(module: "AdaLayerNorm", x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor):
|
||||
"""Fused fast path. Returns the output tensor, or None when the inputs
|
||||
are unsupported or the optimized path failed synchronously (the caller
|
||||
then uses forward_native). The runtime fallback is best-effort: it
|
||||
catches synchronous Triton compile/launch errors, not asynchronous
|
||||
CUDA execution faults."""
|
||||
supported = _adaln_inputs_supported(module, x, scale, shift)
|
||||
if supported is None:
|
||||
return None
|
||||
channels, block_c, per_sample_scale, per_sample_shift = supported
|
||||
seq_len = x.shape[-2]
|
||||
rows = x.numel() // channels
|
||||
out = torch.empty(x.shape, dtype=x.dtype, device=x.device)
|
||||
ln = module.layernorm
|
||||
weight = ln.weight
|
||||
bias = ln.bias
|
||||
has_weight = weight is not None
|
||||
has_bias = bias is not None
|
||||
is_half = x.dtype is not torch.float32
|
||||
cfg = _ADALN_CONFIGS.get(block_c)
|
||||
if cfg is None:
|
||||
num_warps = 4 if block_c <= 1024 else (8 if block_c <= 4096 else 16)
|
||||
cfg = (block_c, num_warps)
|
||||
_ADALN_CONFIGS[block_c] = cfg
|
||||
# Step across frames when F > 1, otherwise across batches.
|
||||
scale_row_stride = scale.stride(1) if scale.ndim == 4 and scale.shape[1] > 1 else scale.stride(0)
|
||||
shift_row_stride = shift.stride(1) if shift.ndim == 4 and shift.shape[1] > 1 else shift.stride(0)
|
||||
# Dummy pointer args for disabled branches: never dereferenced because
|
||||
# the loads are constexpr-pruned when has_weight/has_bias is False.
|
||||
args = (
|
||||
x,
|
||||
scale,
|
||||
shift,
|
||||
out,
|
||||
weight if has_weight else scale,
|
||||
bias if has_bias else scale,
|
||||
module.eps,
|
||||
channels,
|
||||
seq_len,
|
||||
scale_row_stride,
|
||||
shift_row_stride,
|
||||
has_weight,
|
||||
has_bias,
|
||||
is_half,
|
||||
per_sample_scale,
|
||||
per_sample_shift,
|
||||
block_c,
|
||||
)
|
||||
# The key carries every field that selects the compiled binary
|
||||
# (dtype covers is_half; per-sample flags and affine presence select
|
||||
# constexpr variants; eps is a runtime scalar). A failure in one
|
||||
# variant must not disable the others.
|
||||
runtime_key = (
|
||||
x.device.index,
|
||||
channels,
|
||||
x.dtype,
|
||||
has_weight,
|
||||
has_bias,
|
||||
per_sample_scale,
|
||||
per_sample_shift,
|
||||
)
|
||||
if runtime_key in _FAILED_ADALN_KEYS:
|
||||
return None
|
||||
try:
|
||||
_adaln_scale_shift_layernorm_kernel[(rows,)](*args, num_warps=cfg[1])
|
||||
except Exception as exc: # noqa: BLE001 - fail closed after optimized-path failure
|
||||
_FAILED_ADALN_KEYS.add(runtime_key)
|
||||
logger.warning(
|
||||
"Disabling the AdaLayerNorm fused fast path on %s after failure: %s",
|
||||
x.device,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
return out
|
||||
|
||||
_adaln_fused_forward = _adaln_fused_forward_impl
|
||||
|
||||
|
||||
else:
|
||||
# No Triton: the fused fast path is unavailable; forward_cuda falls back
|
||||
# to forward_native below.
|
||||
_adaln_fused_forward = None
|
||||
|
||||
|
||||
class AdaLayerNorm(CustomOp):
|
||||
"""
|
||||
@@ -36,6 +318,11 @@ class AdaLayerNorm(CustomOp):
|
||||
scale: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if _adaln_fused_forward is None:
|
||||
return self.forward_native(x, scale, shift)
|
||||
out = _adaln_fused_forward(self, x, scale, shift)
|
||||
if out is not None:
|
||||
return out
|
||||
return self.forward_native(x, scale, shift)
|
||||
|
||||
def forward_hip(
|
||||
@@ -46,6 +333,14 @@ class AdaLayerNorm(CustomOp):
|
||||
) -> torch.Tensor:
|
||||
return self.forward_native(x, scale, shift)
|
||||
|
||||
def forward_musa(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.forward_native(x, scale, shift)
|
||||
|
||||
def forward_npu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
|
||||
Reference in new issue
Block a user