[Kernel] Improve 2 backends importing FlashInfer: FLASHINFER_ATTN and TRTLLM_ATTN (#7431)

Signed-off-by: Ruqing Xu <7891482+xrq-phys@users.noreply.github.com>
This commit is contained in:
RuQing Xu authored and GitHub committed 2026-09-28 21:21:58 +00:00
1 parent d31dbc5b0a
commit 0173cb3748
7 files changed
+160 -104

No files matched your search

@@ -64,6 +64,7 @@ The lifecycle labels used below are:
| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
| --- | --- | --- | --- | --- |
| `DIFFUSION_ATTENTION_BACKEND` | Backend name or `auto`; default `auto` (platform selection) | Diffusion stages; read when `OmniDiffusionConfig` is constructed | `diffusion_attention_config.default` wins. An unknown backend fails during backend resolution. | Stable fallback |
| `DIFFUSION_ATTENTION_QUANT` | `<dtype_qk>:<dtype_vo>[:<q_block_size>:<k_block_size>]`; default unset | Diffusion stages; read when `OmniDiffusionConfig` is constructed | Used only when `DIFFUSION_ATTENTION_BACKEND` sets a non-`auto` backend and no explicit attention default exists. Quantization support varies by backend. Malformed fields raise `ValueError`. | Stable fallback |
| `DIFFUSION_CACHE_BACKEND` | `none`, `cache_dit`, `tea_cache`, `mag_cache`, `step_cache`, `stepcache`, or `step_cache_dit`; default `none` | Diffusion runner startup | Explicit `cache_backend` in config wins. Otherwise this name wins over the deprecated alias. Unsupported values raise `ValueError` during runner setup. | Stable fallback |
| `DIFFUSION_CACHE_ADAPTER` | Same values as `DIFFUSION_CACHE_BACKEND`; default `none` | Diffusion runner startup | Used only when neither explicit `cache_backend` nor `DIFFUSION_CACHE_BACKEND` is set. Unsupported values raise `ValueError`. | Deprecated; use `DIFFUSION_CACHE_BACKEND` |
| `OMNI_DIFFUSION_PROMPT_EMBED_CACHE` | Boolean: `1`, `true`, `yes`, `on`, `0`, `false`, `no`, or `off`; default disabled | Each diffusion runner; resolved during model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
@@ -77,6 +78,13 @@ Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
`SAGE_ATTN`, `FLASHINFER_ATTN`, and `TRTLLM_ATTN`. Platform support still
depends on the installed kernels and model path.
For `DIFFUSION_ATTENTION_QUANT`, the dtype fields accept `float16`,
`bfloat16`, `int8`, or `fp8_e4m3`; the selected backend may support only a
subset. For example, `DIFFUSION_ATTENTION_BACKEND=TRTLLM_ATTN` with
`DIFFUSION_ATTENTION_QUANT=int8:bfloat16:1:16` selects INT8 Q/K SageAttention.
Explicit `diffusion_attention_config.default` takes precedence over both
environment variables.
### Serving and runtime
| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
+2 -1
View File
@@ -164,7 +164,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
"""Make an inventory expansion an explicit review decision."""
category_counts = Counter(item.category for item in ENVIRONMENT_VARIABLE_INVENTORY.values())
assert category_counts == {
EnvironmentVariableCategory.PUBLIC_OMNI: 31,
EnvironmentVariableCategory.PUBLIC_OMNI: 32,
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 29,
EnvironmentVariableCategory.MODEL_SPECIFIC: 59,
@@ -190,6 +190,7 @@ def test_new_public_omni_names_use_project_prefix():
"""Grandfather legacy names without allowing more prefix exceptions."""
legacy_public_names = {
"DIFFUSION_ATTENTION_BACKEND",
"DIFFUSION_ATTENTION_QUANT",
"DIFFUSION_CACHE_ADAPTER",
"DIFFUSION_CACHE_BACKEND",
"OMNI_DIFFUSION_PROMPT_EMBED_CACHE",
+41 -32
View File
@@ -7,6 +7,7 @@ from unittest.mock import Mock
import pytest
import torch
import torch.nn.functional as F
from packaging.version import Version
from vllm_omni.diffusion.attention.backends import trtllm_attn as tg
from vllm_omni.diffusion.attention.backends.abstract import (
@@ -76,27 +77,51 @@ def test_quant_rejects_non_sage_dtype():
_impl(quant={"dtype_qk": "bfloat16"})
def test_quant_quantize_requires_flashinfer_routine(monkeypatch):
import vllm_omni.diffusion.attention.backends.trtllm_attn as mod
monkeypatch.setattr(mod, "_sage_kernel_available", lambda: True)
monkeypatch.setattr(mod, "_sage_quantize_fn", lambda: None)
with pytest.raises(RuntimeError, match="trtllm_sage_attention_quantize"):
_impl(quant={"dtype_qk": "int8"})
def test_quant_quantize_calls_routine_and_shapes_sfs():
from vllm_omni.diffusion.attention.backends.trtllm_attn import QuantConfig
captured: dict[str, object] = {}
def fake_quantize(q, k, v, q_block_size, k_block_size, qk_quant_dtype):
captured.update(q_block_size=q_block_size, k_block_size=k_block_size, qk_quant_dtype=qk_quant_dtype)
return "qq", "kq", "vq", "qsfs", "ksfs", "vsfs"
def fake_quantize(
q,
k,
v,
q_block_size,
k_block_size,
qk_quant_dtype,
smooth_k,
cum_seq_lens_q,
cum_seq_lens_kv,
):
captured.update(
q_block_size=q_block_size,
k_block_size=k_block_size,
qk_quant_dtype=qk_quant_dtype,
smooth_k=smooth_k,
cum_seq_lens_q=cum_seq_lens_q,
cum_seq_lens_kv=cum_seq_lens_kv,
)
return "qq", "kq", "vq", "qsfs", "ksfs", "vsfs", "k_mean"
q_q, k_q, v_q, sfs, blk = QuantConfig(dtype_qk="int8").quantize(object(), object(), object(), fake_quantize)
cu_seq_lens_q = object()
cu_seq_lens_kv = object()
q_q, k_q, v_q, sfs, blk = QuantConfig(dtype_qk="int8").quantize(
object(),
object(),
object(),
fake_quantize,
cu_seq_lens_q,
cu_seq_lens_kv,
)
assert (q_q, k_q, v_q, sfs, blk) == ("qq", "kq", "vq", ("qsfs", "ksfs", None, "vsfs"), (1, 16, 0, 1))
assert captured == {"q_block_size": 1, "k_block_size": 16, "qk_quant_dtype": torch.int8}
assert captured == {
"q_block_size": 1,
"k_block_size": 16,
"qk_quant_dtype": torch.int8,
"smooth_k": True,
"cum_seq_lens_q": cu_seq_lens_q,
"cum_seq_lens_kv": cu_seq_lens_kv,
}
def test_skip_factor_none_without_curve():
@@ -442,8 +467,8 @@ def test_bf16_packed_padding_matches_sdpa():
requires_sage = pytest.mark.skipif(
not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None),
reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)",
not _has_trtllm_attn() or Version(tg.flashinfer.__version__) < Version("0.6.18rc10"),
reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.18rc10",
)
@@ -493,22 +518,6 @@ def test_sage_packed_non_aligned_length_matches_sdpa():
assert torch.count_nonzero(out[:, used:]) == 0
@requires_sage
def test_sage_short_sequence_uses_dense_kernel():
torch.manual_seed(0)
b, s, h, d = 1, 14, 8, 128
scale = 1.0 / math.sqrt(d)
q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3))
impl = _impl(quant={"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16})
impl._sage_quantize_fn = lambda *args, **kwargs: pytest.fail("short sequence must not use SAGE")
out = impl.forward_cuda(q, k, v)
ref = _sdpa_ref(q, k, v, scale)
rel = (out.float() - ref).abs().mean() / ref.abs().mean()
assert torch.isfinite(out).all()
assert rel < 0.01, f"SAGE short-sequence dense fallback rel err {rel:.4f} too high"
@requires_trtllm_attn
def test_skip_tiny_threshold_matches_dense():
torch.manual_seed(0)
@@ -55,6 +55,7 @@ class EnvironmentVariableClassification:
# grandfathers the older public names below.
_PUBLIC_OMNI = (
"DIFFUSION_ATTENTION_BACKEND",
"DIFFUSION_ATTENTION_QUANT",
"DIFFUSION_CACHE_ADAPTER",
"DIFFUSION_CACHE_BACKEND",
"OMNI_DIFFUSION_PROMPT_EMBED_CACHE",
@@ -194,6 +194,22 @@ class FlashInferAttentionImpl(AttentionImpl):
raise ValueError(f"Unsupported {option_name}={dtype}; expected one of: {choices}")
return dtype
@torch.compiler.disable
@staticmethod
def _extract_scalar(scalar_tensor: torch.Tensor):
return scalar_tensor.item()
@staticmethod
def _per_tensor_quantize(
tensor: torch.Tensor,
to_dtype: torch.dtype,
) -> tuple[torch.Tensor, float]:
if torch.finfo(to_dtype).bits == 8:
scale_tensor = tensor.abs().amax().float().clamp_min(1e-6) / torch.finfo(to_dtype).max
tensor = (tensor.float() * torch.reciprocal(scale_tensor)).to(to_dtype)
return tensor, FlashInferAttentionImpl._extract_scalar(scale_tensor)
return tensor.to(to_dtype), 1.0
@staticmethod
def _pack_mask_for_flashinfer(
attn_mask: torch.Tensor, batch_size: int, qo_len: int, kv_len: int
@@ -311,11 +327,9 @@ class FlashInferAttentionImpl(AttentionImpl):
q = query.reshape(batch_size * qo_len, num_q_heads, head_dim_qk)
k = key.reshape(batch_size * kv_len, num_kv_heads, head_dim_k)
v = value.reshape(batch_size * kv_len, num_kv_heads, head_dim_vo)
if self.dtype_qk is not None:
q = q.to(self.dtype_qk)
k = k.to(self.dtype_qk)
if self.dtype_vo is not None:
v = v.to(self.dtype_vo)
q, q_scale = self._per_tensor_quantize(q, self.dtype_qk or q.dtype)
k, k_scale = self._per_tensor_quantize(k, self.dtype_qk or k.dtype)
v, v_scale = self._per_tensor_quantize(v, self.dtype_vo or v.dtype)
flat_mask = None
if custom_mask is not None:
@@ -343,7 +357,7 @@ class FlashInferAttentionImpl(AttentionImpl):
),
flat_mask,
)
out = self._run_wrapper(q, k, v)
out = self._run_wrapper(q, k, v, q_scale, k_scale, v_scale)
out = out.reshape(batch_size, qo_len, num_q_heads, head_dim_vo)
return out.to(query.dtype) if out.dtype != query.dtype else out
@@ -353,8 +367,28 @@ class FlashInferAttentionImpl(AttentionImpl):
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
q_scale: float,
k_scale: float,
v_scale: float,
) -> torch.Tensor:
return self._wrapper.run(query, key, value)
try:
return self._wrapper.run(
query,
key,
value,
q_scale=q_scale,
k_scale=k_scale,
v_scale=v_scale,
)
except NotImplementedError as _:
# Older FlashInfer does not route scales for CuTeDSL backends.
# Apply some scales manually.
if q_scale != 1.0 or k_scale != 1.0:
raise NotImplementedError("FlashInfer/CuTeDSL backend doesn't support quantizing QK into FP8 yet.")
out = self._wrapper.run(query, key, value)
if v_scale is not None and v_scale != 1.0:
out = out * v_scale
return out
def _sdpa_for_unsupported_mask(
self,
@@ -1,13 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import functools
import inspect
import math
from dataclasses import dataclass, replace
from typing import NamedTuple, cast
import torch
from packaging.version import InvalidVersion, Version
from vllm.logger import init_logger
from vllm_omni.diffusion.attention.backends.abstract import (
@@ -44,10 +43,10 @@ class SkipSoftmaxConfig:
return cls(
threshold=_validate_control(bk.get("skip_softmax_threshold"), "skip_softmax_threshold", 0.0, None),
target_sparsity=_validate_control(bk.get("target_sparsity"), "target_sparsity", 0.0, 1.0),
disabled_until_timestep=cast(
float,
_validate_control(bk.get("disabled_until_timestep", 0.0), "disabled_until_timestep", 0.0, 1.0),
),
disabled_until_timestep=_validate_control(
bk.get("disabled_until_timestep", 0.0), "disabled_until_timestep", 0.0, 1.0
)
or 0.0,
)
@property
@@ -77,6 +76,7 @@ class SkipSoftmaxConfig:
try:
import flashinfer
from flashinfer.prefill import trtllm_ragged_attention_deepseek
HAS_FLASHINFER = True
@@ -88,26 +88,6 @@ except Exception as e: # pragma: no cover - import guard
)
@functools.lru_cache(maxsize=1)
def _sage_kernel_available() -> bool:
if not HAS_FLASHINFER:
return False
try:
return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters
except (TypeError, ValueError):
return False
@functools.lru_cache(maxsize=1)
def _sage_quantize_fn():
try:
from flashinfer import trtllm_sage_attention_quantize
return trtllm_sage_attention_quantize
except Exception: # pragma: no cover
return None
_QK_QUANT_DTYPES = {
"int8": torch.int8,
"fp8_e4m3": torch.float8_e4m3fn,
@@ -133,15 +113,26 @@ class QuantConfig:
def enabled(self) -> bool:
return self.dtype_qk is not None
def quantize(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, quantize_fn):
def quantize(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
quantize_fn,
cu_seq_lens_q: torch.Tensor,
cu_seq_lens_kv: torch.Tensor,
):
qk_quant_dtype = _QK_QUANT_DTYPES[cast(str, self.dtype_qk)]
q_q, k_q, v_q, q_sfs, k_sfs, v_sfs = quantize_fn(
q_q, k_q, v_q, q_sfs, k_sfs, v_sfs, _ = quantize_fn(
q,
k,
v,
q_block_size=self.q_block_size,
k_block_size=self.k_block_size,
qk_quant_dtype=qk_quant_dtype,
smooth_k=True,
cum_seq_lens_q=cu_seq_lens_q,
cum_seq_lens_kv=cu_seq_lens_kv,
)
sage_attn_sfs = (q_sfs, k_sfs, None, v_sfs)
num_elts_per_sage_attn_blk = (self.q_block_size, self.k_block_size, 0, 1)
@@ -157,7 +148,6 @@ class _PackedLayout(NamedTuple):
cu_seqlens_kv: torch.Tensor
batch_size: int
seq_lens: torch.Tensor
known_min_kv_length: int | None = None
def _workspace_bytes() -> int:
@@ -216,9 +206,13 @@ class TrtllmAttentionImpl(AttentionImpl):
self.skip = SkipSoftmaxConfig.from_backend_kwargs(backend_kwargs)
self._warned_missing_timestep = False
self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)
# Resolve the SAGE quantize fn once at init so the compiled forward path never calls the
# lru_cache-wrapped getter (which triggers a Dynamo graph break every step).
# Override: SageAttention does not support causal attention
if causal:
self.quant = QuantConfig(dtype_qk=None, q_block_size=0, k_block_size=0)
else:
self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)
# Resolve the SAGE quantize fn once at init so the compiled forward path
# does not import it every step.
self._sage_quantize_fn = None
if self.quant.enabled:
if self.quant.dtype_qk not in _QK_QUANT_DTYPES:
@@ -226,19 +220,16 @@ class TrtllmAttentionImpl(AttentionImpl):
f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got "
f"{self.quant.dtype_qk!r}. FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE."
)
if not _sage_kernel_available():
try:
flashinfer_version = Version(flashinfer.__version__)
except (AttributeError, InvalidVersion):
flashinfer_version = None
if flashinfer_version is not None and flashinfer_version < Version("0.6.18rc10"):
raise RuntimeError(
"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build does not "
"expose the trtllm-gen sage_attn_sfs kernel path. Install a FlashInfer build "
"that provides it, or remove the quant config."
)
self._sage_quantize_fn = _sage_quantize_fn()
if self._sage_quantize_fn is None:
raise RuntimeError(
"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build lacks "
"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "
"FlashInfer, or remove the quant config."
f"FlashInfer {flashinfer_version} is too old for TRTLLM_ATTN quant (SAGE); "
"install flashinfer >= 0.6.18rc10."
)
self._sage_quantize_fn = flashinfer.trtllm_sage_attention_quantize
def set_layer_calibration(self, a: float, b: float) -> None:
self.skip = replace(self.skip, a=a, b=b)
@@ -330,7 +321,6 @@ class TrtllmAttentionImpl(AttentionImpl):
# Canonical packed-padding metadata is [0, valid_kv_tokens], so this slice
# is the one-element sequence-length view expected by the kernel.
seq_lens=cu_seq_lens_kv[1:],
known_min_kv_length=valid_kv_tokens,
)
@staticmethod
@@ -407,7 +397,6 @@ class TrtllmAttentionImpl(AttentionImpl):
v = value.reshape(physical_batch * kv_len, num_kv_heads, head_dim).contiguous()
output_tokens = q.shape[0]
known_min_kv_length: int | None = kv_len
if has_packed_metadata:
cu_seq_lens_q = extra["cu_seqlens_q"]
cu_seq_lens_kv = extra["cu_seqlens_k"]
@@ -444,7 +433,6 @@ class TrtllmAttentionImpl(AttentionImpl):
cu_seq_lens_kv = packed_layout.cu_seqlens_kv
batch = packed_layout.batch_size
seq_lens = packed_layout.seq_lens
known_min_kv_length = packed_layout.known_min_kv_length
max_q_len = int(extra["max_seqlen_q"])
max_kv_len = int(extra["max_seqlen_k"])
else:
@@ -465,24 +453,15 @@ class TrtllmAttentionImpl(AttentionImpl):
# SAGE quant is active (which already requires the kernel, checked at init) so the dense
# path stays compatible with older builds that lack these parameters.
sage_kwargs: dict = {}
# The SAGE kernel requires every KV sequence to contain at least one full
# quantization block. Small auxiliary attention sites use the dense kernel.
use_sage = False
if self.quant.enabled:
if known_min_kv_length is not None:
sage_lengths_supported = known_min_kv_length >= self.quant.k_block_size
else:
sage_lengths_supported = bool(torch.all(seq_lens >= self.quant.k_block_size).item())
use_sage = sage_lengths_supported
if self.quant.enabled and not use_sage:
message = (
f"TRTLLM_ATTN SAGE quantization is configured for attention role {self.role!r}, but at least one "
f"KV sequence is shorter than k_block_size={self.quant.k_block_size}. Falling back to dense "
"attention for this input."
q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(
q,
k,
v,
self._sage_quantize_fn,
cu_seq_lens_q,
cu_seq_lens_kv,
)
logger.warning_once(message)
if use_sage:
q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)
sage_kwargs["sage_attn_sfs"] = sage_attn_sfs
sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes
@@ -505,6 +484,7 @@ class TrtllmAttentionImpl(AttentionImpl):
is_causal=self.causal,
return_lse=False,
skip_softmax_threshold_scale_factor=_skip_factor,
skip_all_rows_active_check=True,
**sage_kwargs,
)
if out.shape[0] != output_tokens:
+25 -2
View File
@@ -2261,7 +2261,7 @@ def build_attention_config(
Called exactly once in ``OmniDiffusionConfig.__post_init__``.
Handles type-conversion **and** env-var fallback
(``DIFFUSION_ATTENTION_BACKEND``).
(``DIFFUSION_ATTENTION_BACKEND`` and ``DIFFUSION_ATTENTION_QUANT``).
"""
normalized = parse_attention_config(attention_config)
@@ -2275,7 +2275,30 @@ def build_attention_config(
if env_attention_backend.lower() == "auto":
return normalized
normalized.default = AttentionSpec(backend=env_attention_backend)
# If config file doesn't explicitly setup quantization specs, optionally set
# from the env-var.
quant = None
env_attention_quant = os.environ.get("DIFFUSION_ATTENTION_QUANT")
if env_attention_quant is not None:
fields = [field.strip() for field in env_attention_quant.split(":")]
if len(fields) not in (2, 4) or not all(fields):
raise ValueError(
"DIFFUSION_ATTENTION_QUANT must have the format <dtype_qk>:<dtype_vo>[:<q_block_size>:<k_block_size>]."
)
quant_kwargs: dict[str, Any] = {
"dtype_qk": fields[0],
"dtype_vo": fields[1],
}
if len(fields) == 4:
q_block_size, k_block_size = int(fields[2]), int(fields[3])
if (q_block_size, k_block_size) != (0, 0):
quant_kwargs.update(
q_block_size=q_block_size,
k_block_size=k_block_size,
)
quant = AttnQuantSpec(**quant_kwargs)
normalized.default = AttentionSpec(backend=env_attention_backend, quant=quant)
logger.info(
"Parsed attention config from DIFFUSION_ATTENTION_BACKEND '%s': default=%s, per_role=%s",
env_attention_backend,