[Kernel] Add typed SAGE for TRTLLM_ATTN (#5509)
Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
1 parent
d33c905d80
commit
896f05e7e6
5 files changed
+301
-94
No files matched your search
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
import math
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
@@ -83,6 +85,66 @@ except Exception as e: # pragma: no cover - import guard
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _sage_kernel_available() -> bool:
|
||||
if not HAS_FLASHINFER:
|
||||
return False
|
||||
try:
|
||||
return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _sage_quantize_fn():
|
||||
try:
|
||||
from flashinfer import trtllm_sage_attention_quantize
|
||||
|
||||
return trtllm_sage_attention_quantize
|
||||
except Exception: # pragma: no cover
|
||||
return None
|
||||
|
||||
|
||||
_QK_QUANT_DTYPES = {
|
||||
"int8": torch.int8,
|
||||
"fp8_e4m3": torch.float8_e4m3fn,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuantConfig:
|
||||
dtype_qk: str | None = None
|
||||
q_block_size: int = 1
|
||||
k_block_size: int = 16
|
||||
|
||||
@classmethod
|
||||
def from_backend_kwargs(cls, backend_kwargs: dict | None) -> "QuantConfig":
|
||||
q = (backend_kwargs or {}).get("quant") or {}
|
||||
return cls(
|
||||
dtype_qk=q.get("dtype_qk"),
|
||||
q_block_size=int(q.get("q_block_size", 1) or 1),
|
||||
k_block_size=int(q.get("k_block_size", 16) or 16),
|
||||
)
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.dtype_qk is not None
|
||||
|
||||
def quantize(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, quantize_fn):
|
||||
qk_quant_dtype = _QK_QUANT_DTYPES[self.dtype_qk]
|
||||
q_q, k_q, v_q, q_sfs, k_sfs, v_sfs = quantize_fn(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_block_size=self.q_block_size,
|
||||
k_block_size=self.k_block_size,
|
||||
qk_quant_dtype=qk_quant_dtype,
|
||||
)
|
||||
sage_attn_sfs = (q_sfs, k_sfs, None, v_sfs)
|
||||
num_elts_per_sage_attn_blk = (self.q_block_size, self.k_block_size, 0, 1)
|
||||
return q_q, k_q, v_q, sage_attn_sfs, num_elts_per_sage_attn_blk
|
||||
|
||||
|
||||
def _workspace_bytes() -> int:
|
||||
import vllm.envs as envs
|
||||
|
||||
@@ -129,6 +191,30 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
self.skip = SkipSoftmaxConfig.from_backend_kwargs(backend_kwargs)
|
||||
self._warned_missing_timestep = False
|
||||
|
||||
self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)
|
||||
# Resolve the SAGE quantize fn once at init so the compiled forward path never calls the
|
||||
# lru_cache-wrapped getter (which triggers a Dynamo graph break every step).
|
||||
self._sage_quantize_fn = None
|
||||
if self.quant.enabled:
|
||||
if self.quant.dtype_qk not in _QK_QUANT_DTYPES:
|
||||
raise RuntimeError(
|
||||
f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got "
|
||||
f"{self.quant.dtype_qk!r}. FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE."
|
||||
)
|
||||
if not _sage_kernel_available():
|
||||
raise RuntimeError(
|
||||
"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build does not "
|
||||
"expose the trtllm-gen sage_attn_sfs kernel path. Install a FlashInfer build "
|
||||
"that provides it, or remove the quant config."
|
||||
)
|
||||
self._sage_quantize_fn = _sage_quantize_fn()
|
||||
if self._sage_quantize_fn is None:
|
||||
raise RuntimeError(
|
||||
"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build lacks "
|
||||
"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "
|
||||
"FlashInfer, or remove the quant config."
|
||||
)
|
||||
|
||||
def set_layer_calibration(self, a: float, b: float) -> None:
|
||||
self.skip = replace(self.skip, a=a, b=b)
|
||||
|
||||
@@ -203,6 +289,15 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
|
||||
_skip_factor = self._resolve_skip_factor(kv_len)
|
||||
|
||||
# SAGE kwargs are only understood by newer FlashInfer builds; pass them exclusively when
|
||||
# SAGE quant is active (which already requires the kernel, checked at init) so the dense
|
||||
# path stays compatible with older builds that lack these parameters.
|
||||
sage_kwargs: dict = {}
|
||||
if self.quant.enabled:
|
||||
q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)
|
||||
sage_kwargs["sage_attn_sfs"] = sage_attn_sfs
|
||||
sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes
|
||||
|
||||
out = trtllm_ragged_attention_deepseek(
|
||||
query=q,
|
||||
key=k,
|
||||
@@ -222,5 +317,6 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
is_causal=self.causal,
|
||||
return_lse=False,
|
||||
skip_softmax_threshold_scale_factor=_skip_factor,
|
||||
**sage_kwargs,
|
||||
)
|
||||
return out.reshape(batch, q_len, num_q_heads, head_dim)
|
||||
+60
-16
@@ -1419,44 +1419,88 @@ class SkipSoftmaxSpec:
|
||||
raise ValueError("skip_softmax: set either target_sparsity or threshold, not both.")
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttnQuantSpec:
|
||||
dtype_qk: str | None = None
|
||||
dtype_vo: str | None = None
|
||||
q_block_size: int = 1
|
||||
k_block_size: int = 16
|
||||
flashinfer_backend: str | None = None
|
||||
|
||||
_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})
|
||||
_VALID_BLOCK_SIZES = frozenset({1, 4, 16})
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name, v in (("dtype_qk", self.dtype_qk), ("dtype_vo", self.dtype_vo)):
|
||||
if v is not None and v not in self._VALID_DTYPES:
|
||||
raise ValueError(f"quant.{name}={v!r} unsupported; use one of {sorted(self._VALID_DTYPES)}.")
|
||||
for name, v in (("q_block_size", self.q_block_size), ("k_block_size", self.k_block_size)):
|
||||
if v not in self._VALID_BLOCK_SIZES:
|
||||
raise ValueError(
|
||||
f"quant.{name}={v!r} unsupported; kernels exist only for {sorted(self._VALID_BLOCK_SIZES)}."
|
||||
)
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.dtype_qk is not None or self.dtype_vo is not None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttentionSpec:
|
||||
"""Specifies a backend and its typed backend-specific config for one attention role."""
|
||||
|
||||
backend: str
|
||||
skip_softmax: SkipSoftmaxSpec | None = None
|
||||
quant: AttnQuantSpec | None = None
|
||||
skip_calibration: dict | None = field(default=None, repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.backend, str):
|
||||
raise TypeError(f"Expected str for AttentionSpec.backend, got {type(self.backend)!r}")
|
||||
self.skip_softmax = self._coerce_skip_softmax(self.skip_softmax)
|
||||
self.skip_softmax = self._coerce(self.skip_softmax, SkipSoftmaxSpec, "skip_softmax")
|
||||
self.quant = self._coerce(self.quant, AttnQuantSpec, "quant")
|
||||
if self.skip_softmax is not None and self.backend.upper() != "TRTLLM_ATTN":
|
||||
raise ValueError(
|
||||
f"skip_softmax is only supported by the TRTLLM_ATTN backend, but backend={self.backend!r}. "
|
||||
f"Remove skip_softmax or set backend to TRTLLM_ATTN."
|
||||
"Remove skip_softmax or set backend to TRTLLM_ATTN."
|
||||
)
|
||||
if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):
|
||||
raise ValueError(
|
||||
f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but "
|
||||
f"backend={self.backend!r}. Remove quant or set a supported backend."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_skip_softmax(value: Any) -> SkipSoftmaxSpec | None:
|
||||
if value is None or isinstance(value, SkipSoftmaxSpec):
|
||||
def _coerce(value: Any, cls: type, field_name: str) -> Any:
|
||||
if value is None or isinstance(value, cls):
|
||||
return value
|
||||
if isinstance(value, Mapping):
|
||||
return SkipSoftmaxSpec(**dict(value))
|
||||
raise TypeError(f"Expected dict or SkipSoftmaxSpec for skip_softmax, got {type(value)!r}")
|
||||
return cls(**dict(value))
|
||||
raise TypeError(f"Expected dict or {cls.__name__} for {field_name}, got {type(value)!r}")
|
||||
|
||||
def backend_kwargs(self) -> dict[str, Any] | None:
|
||||
"""Serialize typed backend config into the kwargs dict the backend impl consumes."""
|
||||
if self.skip_softmax is None:
|
||||
return None
|
||||
ss = self.skip_softmax
|
||||
kw: dict[str, Any] = {}
|
||||
if ss.threshold is not None:
|
||||
kw["skip_softmax_threshold"] = ss.threshold
|
||||
if ss.target_sparsity is not None:
|
||||
kw["target_sparsity"] = ss.target_sparsity
|
||||
if ss.disabled_until_timestep:
|
||||
kw["disabled_until_timestep"] = ss.disabled_until_timestep
|
||||
if self.skip_softmax is not None:
|
||||
ss = self.skip_softmax
|
||||
if ss.threshold is not None:
|
||||
kw["skip_softmax_threshold"] = ss.threshold
|
||||
if ss.target_sparsity is not None:
|
||||
kw["target_sparsity"] = ss.target_sparsity
|
||||
if ss.disabled_until_timestep:
|
||||
kw["disabled_until_timestep"] = ss.disabled_until_timestep
|
||||
if self.quant is not None and self.quant.enabled:
|
||||
q = self.quant
|
||||
quant_kw: dict[str, Any] = {
|
||||
"dtype_qk": q.dtype_qk,
|
||||
"q_block_size": q.q_block_size,
|
||||
"k_block_size": q.k_block_size,
|
||||
}
|
||||
if q.dtype_vo is not None:
|
||||
quant_kw["dtype_vo"] = q.dtype_vo
|
||||
if q.flashinfer_backend is not None:
|
||||
quant_kw["flashinfer_backend"] = q.flashinfer_backend
|
||||
kw["quant"] = quant_kw
|
||||
return kw or None
|
||||
|
||||
|
||||
@@ -1526,7 +1570,7 @@ class AttentionConfig:
|
||||
normalized[role] = node
|
||||
return
|
||||
|
||||
spec_keys = {"backend", "skip_softmax"}
|
||||
spec_keys = {"backend", "skip_softmax", "quant"}
|
||||
node_dict = dict(node)
|
||||
node_keys = set(node_dict)
|
||||
if node_keys & spec_keys:
|
||||
|
||||
Reference in new issue
Block a user