[Kernel] Add typed SAGE for TRTLLM_ATTN (#5509)

Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
SYLAR authored and GitHub committed 2026-07-31 23:11:39 +00:00
1 parent d33c905d80
commit 896f05e7e6
5 files changed
+301 -94

No files matched your search

@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import inspect
import math
from dataclasses import dataclass, replace
@@ -83,6 +85,66 @@ except Exception as e: # pragma: no cover - import guard
)
@functools.lru_cache(maxsize=1)
def _sage_kernel_available() -> bool:
if not HAS_FLASHINFER:
return False
try:
return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters
except (TypeError, ValueError):
return False
@functools.lru_cache(maxsize=1)
def _sage_quantize_fn():
try:
from flashinfer import trtllm_sage_attention_quantize
return trtllm_sage_attention_quantize
except Exception: # pragma: no cover
return None
_QK_QUANT_DTYPES = {
"int8": torch.int8,
"fp8_e4m3": torch.float8_e4m3fn,
}
@dataclass(frozen=True)
class QuantConfig:
dtype_qk: str | None = None
q_block_size: int = 1
k_block_size: int = 16
@classmethod
def from_backend_kwargs(cls, backend_kwargs: dict | None) -> "QuantConfig":
q = (backend_kwargs or {}).get("quant") or {}
return cls(
dtype_qk=q.get("dtype_qk"),
q_block_size=int(q.get("q_block_size", 1) or 1),
k_block_size=int(q.get("k_block_size", 16) or 16),
)
@property
def enabled(self) -> bool:
return self.dtype_qk is not None
def quantize(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, quantize_fn):
qk_quant_dtype = _QK_QUANT_DTYPES[self.dtype_qk]
q_q, k_q, v_q, q_sfs, k_sfs, v_sfs = quantize_fn(
q,
k,
v,
q_block_size=self.q_block_size,
k_block_size=self.k_block_size,
qk_quant_dtype=qk_quant_dtype,
)
sage_attn_sfs = (q_sfs, k_sfs, None, v_sfs)
num_elts_per_sage_attn_blk = (self.q_block_size, self.k_block_size, 0, 1)
return q_q, k_q, v_q, sage_attn_sfs, num_elts_per_sage_attn_blk
def _workspace_bytes() -> int:
import vllm.envs as envs
@@ -129,6 +191,30 @@ class TrtllmAttentionImpl(AttentionImpl):
self.skip = SkipSoftmaxConfig.from_backend_kwargs(backend_kwargs)
self._warned_missing_timestep = False
self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)
# Resolve the SAGE quantize fn once at init so the compiled forward path never calls the
# lru_cache-wrapped getter (which triggers a Dynamo graph break every step).
self._sage_quantize_fn = None
if self.quant.enabled:
if self.quant.dtype_qk not in _QK_QUANT_DTYPES:
raise RuntimeError(
f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got "
f"{self.quant.dtype_qk!r}. FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE."
)
if not _sage_kernel_available():
raise RuntimeError(
"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build does not "
"expose the trtllm-gen sage_attn_sfs kernel path. Install a FlashInfer build "
"that provides it, or remove the quant config."
)
self._sage_quantize_fn = _sage_quantize_fn()
if self._sage_quantize_fn is None:
raise RuntimeError(
"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build lacks "
"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "
"FlashInfer, or remove the quant config."
)
def set_layer_calibration(self, a: float, b: float) -> None:
self.skip = replace(self.skip, a=a, b=b)
@@ -203,6 +289,15 @@ class TrtllmAttentionImpl(AttentionImpl):
_skip_factor = self._resolve_skip_factor(kv_len)
# SAGE kwargs are only understood by newer FlashInfer builds; pass them exclusively when
# SAGE quant is active (which already requires the kernel, checked at init) so the dense
# path stays compatible with older builds that lack these parameters.
sage_kwargs: dict = {}
if self.quant.enabled:
q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)
sage_kwargs["sage_attn_sfs"] = sage_attn_sfs
sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes
out = trtllm_ragged_attention_deepseek(
query=q,
key=k,
@@ -222,5 +317,6 @@ class TrtllmAttentionImpl(AttentionImpl):
is_causal=self.causal,
return_lse=False,
skip_softmax_threshold_scale_factor=_skip_factor,
**sage_kwargs,
)
return out.reshape(batch, q_len, num_q_heads, head_dim)
+60 -16
View File
@@ -1419,44 +1419,88 @@ class SkipSoftmaxSpec:
raise ValueError("skip_softmax: set either target_sparsity or threshold, not both.")
@dataclass
class AttnQuantSpec:
dtype_qk: str | None = None
dtype_vo: str | None = None
q_block_size: int = 1
k_block_size: int = 16
flashinfer_backend: str | None = None
_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})
_VALID_BLOCK_SIZES = frozenset({1, 4, 16})
def __post_init__(self) -> None:
for name, v in (("dtype_qk", self.dtype_qk), ("dtype_vo", self.dtype_vo)):
if v is not None and v not in self._VALID_DTYPES:
raise ValueError(f"quant.{name}={v!r} unsupported; use one of {sorted(self._VALID_DTYPES)}.")
for name, v in (("q_block_size", self.q_block_size), ("k_block_size", self.k_block_size)):
if v not in self._VALID_BLOCK_SIZES:
raise ValueError(
f"quant.{name}={v!r} unsupported; kernels exist only for {sorted(self._VALID_BLOCK_SIZES)}."
)
@property
def enabled(self) -> bool:
return self.dtype_qk is not None or self.dtype_vo is not None
@dataclass
class AttentionSpec:
"""Specifies a backend and its typed backend-specific config for one attention role."""
backend: str
skip_softmax: SkipSoftmaxSpec | None = None
quant: AttnQuantSpec | None = None
skip_calibration: dict | None = field(default=None, repr=False)
def __post_init__(self) -> None:
if not isinstance(self.backend, str):
raise TypeError(f"Expected str for AttentionSpec.backend, got {type(self.backend)!r}")
self.skip_softmax = self._coerce_skip_softmax(self.skip_softmax)
self.skip_softmax = self._coerce(self.skip_softmax, SkipSoftmaxSpec, "skip_softmax")
self.quant = self._coerce(self.quant, AttnQuantSpec, "quant")
if self.skip_softmax is not None and self.backend.upper() != "TRTLLM_ATTN":
raise ValueError(
f"skip_softmax is only supported by the TRTLLM_ATTN backend, but backend={self.backend!r}. "
f"Remove skip_softmax or set backend to TRTLLM_ATTN."
"Remove skip_softmax or set backend to TRTLLM_ATTN."
)
if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):
raise ValueError(
f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but "
f"backend={self.backend!r}. Remove quant or set a supported backend."
)
@staticmethod
def _coerce_skip_softmax(value: Any) -> SkipSoftmaxSpec | None:
if value is None or isinstance(value, SkipSoftmaxSpec):
def _coerce(value: Any, cls: type, field_name: str) -> Any:
if value is None or isinstance(value, cls):
return value
if isinstance(value, Mapping):
return SkipSoftmaxSpec(**dict(value))
raise TypeError(f"Expected dict or SkipSoftmaxSpec for skip_softmax, got {type(value)!r}")
return cls(**dict(value))
raise TypeError(f"Expected dict or {cls.__name__} for {field_name}, got {type(value)!r}")
def backend_kwargs(self) -> dict[str, Any] | None:
"""Serialize typed backend config into the kwargs dict the backend impl consumes."""
if self.skip_softmax is None:
return None
ss = self.skip_softmax
kw: dict[str, Any] = {}
if ss.threshold is not None:
kw["skip_softmax_threshold"] = ss.threshold
if ss.target_sparsity is not None:
kw["target_sparsity"] = ss.target_sparsity
if ss.disabled_until_timestep:
kw["disabled_until_timestep"] = ss.disabled_until_timestep
if self.skip_softmax is not None:
ss = self.skip_softmax
if ss.threshold is not None:
kw["skip_softmax_threshold"] = ss.threshold
if ss.target_sparsity is not None:
kw["target_sparsity"] = ss.target_sparsity
if ss.disabled_until_timestep:
kw["disabled_until_timestep"] = ss.disabled_until_timestep
if self.quant is not None and self.quant.enabled:
q = self.quant
quant_kw: dict[str, Any] = {
"dtype_qk": q.dtype_qk,
"q_block_size": q.q_block_size,
"k_block_size": q.k_block_size,
}
if q.dtype_vo is not None:
quant_kw["dtype_vo"] = q.dtype_vo
if q.flashinfer_backend is not None:
quant_kw["flashinfer_backend"] = q.flashinfer_backend
kw["quant"] = quant_kw
return kw or None
@@ -1526,7 +1570,7 @@ class AttentionConfig:
normalized[role] = node
return
spec_keys = {"backend", "skip_softmax"}
spec_keys = {"backend", "skip_softmax", "quant"}
node_dict = dict(node)
node_keys = set(node_dict)
if node_keys & spec_keys: