[Kernel] Improve 2 backends importing FlashInfer: FLASHINFER_ATTN and TRTLLM_ATTN (#7431)
Signed-off-by: Ruqing Xu <7891482+xrq-phys@users.noreply.github.com>
This commit is contained in:
1 parent
d31dbc5b0a
commit
0173cb3748
7 files changed
+160
-104
No files matched your search
@@ -64,6 +64,7 @@ The lifecycle labels used below are:
|
||||
| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `DIFFUSION_ATTENTION_BACKEND` | Backend name or `auto`; default `auto` (platform selection) | Diffusion stages; read when `OmniDiffusionConfig` is constructed | `diffusion_attention_config.default` wins. An unknown backend fails during backend resolution. | Stable fallback |
|
||||
| `DIFFUSION_ATTENTION_QUANT` | `<dtype_qk>:<dtype_vo>[:<q_block_size>:<k_block_size>]`; default unset | Diffusion stages; read when `OmniDiffusionConfig` is constructed | Used only when `DIFFUSION_ATTENTION_BACKEND` sets a non-`auto` backend and no explicit attention default exists. Quantization support varies by backend. Malformed fields raise `ValueError`. | Stable fallback |
|
||||
| `DIFFUSION_CACHE_BACKEND` | `none`, `cache_dit`, `tea_cache`, `mag_cache`, `step_cache`, `stepcache`, or `step_cache_dit`; default `none` | Diffusion runner startup | Explicit `cache_backend` in config wins. Otherwise this name wins over the deprecated alias. Unsupported values raise `ValueError` during runner setup. | Stable fallback |
|
||||
| `DIFFUSION_CACHE_ADAPTER` | Same values as `DIFFUSION_CACHE_BACKEND`; default `none` | Diffusion runner startup | Used only when neither explicit `cache_backend` nor `DIFFUSION_CACHE_BACKEND` is set. Unsupported values raise `ValueError`. | Deprecated; use `DIFFUSION_CACHE_BACKEND` |
|
||||
| `OMNI_DIFFUSION_PROMPT_EMBED_CACHE` | Boolean: `1`, `true`, `yes`, `on`, `0`, `false`, `no`, or `off`; default disabled | Each diffusion runner; resolved during model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental |
|
||||
@@ -77,6 +78,13 @@ Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of
|
||||
`SAGE_ATTN`, `FLASHINFER_ATTN`, and `TRTLLM_ATTN`. Platform support still
|
||||
depends on the installed kernels and model path.
|
||||
|
||||
For `DIFFUSION_ATTENTION_QUANT`, the dtype fields accept `float16`,
|
||||
`bfloat16`, `int8`, or `fp8_e4m3`; the selected backend may support only a
|
||||
subset. For example, `DIFFUSION_ATTENTION_BACKEND=TRTLLM_ATTN` with
|
||||
`DIFFUSION_ATTENTION_QUANT=int8:bfloat16:1:16` selects INT8 Q/K SageAttention.
|
||||
Explicit `diffusion_attention_config.default` takes precedence over both
|
||||
environment variables.
|
||||
|
||||
### Serving and runtime
|
||||
|
||||
| Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle |
|
||||
|
||||
@@ -164,7 +164,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
"""Make an inventory expansion an explicit review decision."""
|
||||
category_counts = Counter(item.category for item in ENVIRONMENT_VARIABLE_INVENTORY.values())
|
||||
assert category_counts == {
|
||||
EnvironmentVariableCategory.PUBLIC_OMNI: 31,
|
||||
EnvironmentVariableCategory.PUBLIC_OMNI: 32,
|
||||
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 29,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 59,
|
||||
@@ -190,6 +190,7 @@ def test_new_public_omni_names_use_project_prefix():
|
||||
"""Grandfather legacy names without allowing more prefix exceptions."""
|
||||
legacy_public_names = {
|
||||
"DIFFUSION_ATTENTION_BACKEND",
|
||||
"DIFFUSION_ATTENTION_QUANT",
|
||||
"DIFFUSION_CACHE_ADAPTER",
|
||||
"DIFFUSION_CACHE_BACKEND",
|
||||
"OMNI_DIFFUSION_PROMPT_EMBED_CACHE",
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from packaging.version import Version
|
||||
|
||||
from vllm_omni.diffusion.attention.backends import trtllm_attn as tg
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
@@ -76,27 +77,51 @@ def test_quant_rejects_non_sage_dtype():
|
||||
_impl(quant={"dtype_qk": "bfloat16"})
|
||||
|
||||
|
||||
def test_quant_quantize_requires_flashinfer_routine(monkeypatch):
|
||||
import vllm_omni.diffusion.attention.backends.trtllm_attn as mod
|
||||
|
||||
monkeypatch.setattr(mod, "_sage_kernel_available", lambda: True)
|
||||
monkeypatch.setattr(mod, "_sage_quantize_fn", lambda: None)
|
||||
with pytest.raises(RuntimeError, match="trtllm_sage_attention_quantize"):
|
||||
_impl(quant={"dtype_qk": "int8"})
|
||||
|
||||
|
||||
def test_quant_quantize_calls_routine_and_shapes_sfs():
|
||||
from vllm_omni.diffusion.attention.backends.trtllm_attn import QuantConfig
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_quantize(q, k, v, q_block_size, k_block_size, qk_quant_dtype):
|
||||
captured.update(q_block_size=q_block_size, k_block_size=k_block_size, qk_quant_dtype=qk_quant_dtype)
|
||||
return "qq", "kq", "vq", "qsfs", "ksfs", "vsfs"
|
||||
def fake_quantize(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_block_size,
|
||||
k_block_size,
|
||||
qk_quant_dtype,
|
||||
smooth_k,
|
||||
cum_seq_lens_q,
|
||||
cum_seq_lens_kv,
|
||||
):
|
||||
captured.update(
|
||||
q_block_size=q_block_size,
|
||||
k_block_size=k_block_size,
|
||||
qk_quant_dtype=qk_quant_dtype,
|
||||
smooth_k=smooth_k,
|
||||
cum_seq_lens_q=cum_seq_lens_q,
|
||||
cum_seq_lens_kv=cum_seq_lens_kv,
|
||||
)
|
||||
return "qq", "kq", "vq", "qsfs", "ksfs", "vsfs", "k_mean"
|
||||
|
||||
q_q, k_q, v_q, sfs, blk = QuantConfig(dtype_qk="int8").quantize(object(), object(), object(), fake_quantize)
|
||||
cu_seq_lens_q = object()
|
||||
cu_seq_lens_kv = object()
|
||||
q_q, k_q, v_q, sfs, blk = QuantConfig(dtype_qk="int8").quantize(
|
||||
object(),
|
||||
object(),
|
||||
object(),
|
||||
fake_quantize,
|
||||
cu_seq_lens_q,
|
||||
cu_seq_lens_kv,
|
||||
)
|
||||
assert (q_q, k_q, v_q, sfs, blk) == ("qq", "kq", "vq", ("qsfs", "ksfs", None, "vsfs"), (1, 16, 0, 1))
|
||||
assert captured == {"q_block_size": 1, "k_block_size": 16, "qk_quant_dtype": torch.int8}
|
||||
assert captured == {
|
||||
"q_block_size": 1,
|
||||
"k_block_size": 16,
|
||||
"qk_quant_dtype": torch.int8,
|
||||
"smooth_k": True,
|
||||
"cum_seq_lens_q": cu_seq_lens_q,
|
||||
"cum_seq_lens_kv": cu_seq_lens_kv,
|
||||
}
|
||||
|
||||
|
||||
def test_skip_factor_none_without_curve():
|
||||
@@ -442,8 +467,8 @@ def test_bf16_packed_padding_matches_sdpa():
|
||||
|
||||
|
||||
requires_sage = pytest.mark.skipif(
|
||||
not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None),
|
||||
reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)",
|
||||
not _has_trtllm_attn() or Version(tg.flashinfer.__version__) < Version("0.6.18rc10"),
|
||||
reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.18rc10",
|
||||
)
|
||||
|
||||
|
||||
@@ -493,22 +518,6 @@ def test_sage_packed_non_aligned_length_matches_sdpa():
|
||||
assert torch.count_nonzero(out[:, used:]) == 0
|
||||
|
||||
|
||||
@requires_sage
|
||||
def test_sage_short_sequence_uses_dense_kernel():
|
||||
torch.manual_seed(0)
|
||||
b, s, h, d = 1, 14, 8, 128
|
||||
scale = 1.0 / math.sqrt(d)
|
||||
q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3))
|
||||
|
||||
impl = _impl(quant={"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16})
|
||||
impl._sage_quantize_fn = lambda *args, **kwargs: pytest.fail("short sequence must not use SAGE")
|
||||
out = impl.forward_cuda(q, k, v)
|
||||
ref = _sdpa_ref(q, k, v, scale)
|
||||
rel = (out.float() - ref).abs().mean() / ref.abs().mean()
|
||||
assert torch.isfinite(out).all()
|
||||
assert rel < 0.01, f"SAGE short-sequence dense fallback rel err {rel:.4f} too high"
|
||||
|
||||
|
||||
@requires_trtllm_attn
|
||||
def test_skip_tiny_threshold_matches_dense():
|
||||
torch.manual_seed(0)
|
||||
|
||||
@@ -55,6 +55,7 @@ class EnvironmentVariableClassification:
|
||||
# grandfathers the older public names below.
|
||||
_PUBLIC_OMNI = (
|
||||
"DIFFUSION_ATTENTION_BACKEND",
|
||||
"DIFFUSION_ATTENTION_QUANT",
|
||||
"DIFFUSION_CACHE_ADAPTER",
|
||||
"DIFFUSION_CACHE_BACKEND",
|
||||
"OMNI_DIFFUSION_PROMPT_EMBED_CACHE",
|
||||
|
||||
@@ -194,6 +194,22 @@ class FlashInferAttentionImpl(AttentionImpl):
|
||||
raise ValueError(f"Unsupported {option_name}={dtype}; expected one of: {choices}")
|
||||
return dtype
|
||||
|
||||
@torch.compiler.disable
|
||||
@staticmethod
|
||||
def _extract_scalar(scalar_tensor: torch.Tensor):
|
||||
return scalar_tensor.item()
|
||||
|
||||
@staticmethod
|
||||
def _per_tensor_quantize(
|
||||
tensor: torch.Tensor,
|
||||
to_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, float]:
|
||||
if torch.finfo(to_dtype).bits == 8:
|
||||
scale_tensor = tensor.abs().amax().float().clamp_min(1e-6) / torch.finfo(to_dtype).max
|
||||
tensor = (tensor.float() * torch.reciprocal(scale_tensor)).to(to_dtype)
|
||||
return tensor, FlashInferAttentionImpl._extract_scalar(scale_tensor)
|
||||
return tensor.to(to_dtype), 1.0
|
||||
|
||||
@staticmethod
|
||||
def _pack_mask_for_flashinfer(
|
||||
attn_mask: torch.Tensor, batch_size: int, qo_len: int, kv_len: int
|
||||
@@ -311,11 +327,9 @@ class FlashInferAttentionImpl(AttentionImpl):
|
||||
q = query.reshape(batch_size * qo_len, num_q_heads, head_dim_qk)
|
||||
k = key.reshape(batch_size * kv_len, num_kv_heads, head_dim_k)
|
||||
v = value.reshape(batch_size * kv_len, num_kv_heads, head_dim_vo)
|
||||
if self.dtype_qk is not None:
|
||||
q = q.to(self.dtype_qk)
|
||||
k = k.to(self.dtype_qk)
|
||||
if self.dtype_vo is not None:
|
||||
v = v.to(self.dtype_vo)
|
||||
q, q_scale = self._per_tensor_quantize(q, self.dtype_qk or q.dtype)
|
||||
k, k_scale = self._per_tensor_quantize(k, self.dtype_qk or k.dtype)
|
||||
v, v_scale = self._per_tensor_quantize(v, self.dtype_vo or v.dtype)
|
||||
|
||||
flat_mask = None
|
||||
if custom_mask is not None:
|
||||
@@ -343,7 +357,7 @@ class FlashInferAttentionImpl(AttentionImpl):
|
||||
),
|
||||
flat_mask,
|
||||
)
|
||||
out = self._run_wrapper(q, k, v)
|
||||
out = self._run_wrapper(q, k, v, q_scale, k_scale, v_scale)
|
||||
out = out.reshape(batch_size, qo_len, num_q_heads, head_dim_vo)
|
||||
return out.to(query.dtype) if out.dtype != query.dtype else out
|
||||
|
||||
@@ -353,8 +367,28 @@ class FlashInferAttentionImpl(AttentionImpl):
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
q_scale: float,
|
||||
k_scale: float,
|
||||
v_scale: float,
|
||||
) -> torch.Tensor:
|
||||
return self._wrapper.run(query, key, value)
|
||||
try:
|
||||
return self._wrapper.run(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
q_scale=q_scale,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
except NotImplementedError as _:
|
||||
# Older FlashInfer does not route scales for CuTeDSL backends.
|
||||
# Apply some scales manually.
|
||||
if q_scale != 1.0 or k_scale != 1.0:
|
||||
raise NotImplementedError("FlashInfer/CuTeDSL backend doesn't support quantizing QK into FP8 yet.")
|
||||
out = self._wrapper.run(query, key, value)
|
||||
if v_scale is not None and v_scale != 1.0:
|
||||
out = out * v_scale
|
||||
return out
|
||||
|
||||
def _sdpa_for_unsupported_mask(
|
||||
self,
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
import math
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import NamedTuple, cast
|
||||
|
||||
import torch
|
||||
from packaging.version import InvalidVersion, Version
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
@@ -44,10 +43,10 @@ class SkipSoftmaxConfig:
|
||||
return cls(
|
||||
threshold=_validate_control(bk.get("skip_softmax_threshold"), "skip_softmax_threshold", 0.0, None),
|
||||
target_sparsity=_validate_control(bk.get("target_sparsity"), "target_sparsity", 0.0, 1.0),
|
||||
disabled_until_timestep=cast(
|
||||
float,
|
||||
_validate_control(bk.get("disabled_until_timestep", 0.0), "disabled_until_timestep", 0.0, 1.0),
|
||||
),
|
||||
disabled_until_timestep=_validate_control(
|
||||
bk.get("disabled_until_timestep", 0.0), "disabled_until_timestep", 0.0, 1.0
|
||||
)
|
||||
or 0.0,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -77,6 +76,7 @@ class SkipSoftmaxConfig:
|
||||
|
||||
|
||||
try:
|
||||
import flashinfer
|
||||
from flashinfer.prefill import trtllm_ragged_attention_deepseek
|
||||
|
||||
HAS_FLASHINFER = True
|
||||
@@ -88,26 +88,6 @@ except Exception as e: # pragma: no cover - import guard
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _sage_kernel_available() -> bool:
|
||||
if not HAS_FLASHINFER:
|
||||
return False
|
||||
try:
|
||||
return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _sage_quantize_fn():
|
||||
try:
|
||||
from flashinfer import trtllm_sage_attention_quantize
|
||||
|
||||
return trtllm_sage_attention_quantize
|
||||
except Exception: # pragma: no cover
|
||||
return None
|
||||
|
||||
|
||||
_QK_QUANT_DTYPES = {
|
||||
"int8": torch.int8,
|
||||
"fp8_e4m3": torch.float8_e4m3fn,
|
||||
@@ -133,15 +113,26 @@ class QuantConfig:
|
||||
def enabled(self) -> bool:
|
||||
return self.dtype_qk is not None
|
||||
|
||||
def quantize(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, quantize_fn):
|
||||
def quantize(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
quantize_fn,
|
||||
cu_seq_lens_q: torch.Tensor,
|
||||
cu_seq_lens_kv: torch.Tensor,
|
||||
):
|
||||
qk_quant_dtype = _QK_QUANT_DTYPES[cast(str, self.dtype_qk)]
|
||||
q_q, k_q, v_q, q_sfs, k_sfs, v_sfs = quantize_fn(
|
||||
q_q, k_q, v_q, q_sfs, k_sfs, v_sfs, _ = quantize_fn(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
q_block_size=self.q_block_size,
|
||||
k_block_size=self.k_block_size,
|
||||
qk_quant_dtype=qk_quant_dtype,
|
||||
smooth_k=True,
|
||||
cum_seq_lens_q=cu_seq_lens_q,
|
||||
cum_seq_lens_kv=cu_seq_lens_kv,
|
||||
)
|
||||
sage_attn_sfs = (q_sfs, k_sfs, None, v_sfs)
|
||||
num_elts_per_sage_attn_blk = (self.q_block_size, self.k_block_size, 0, 1)
|
||||
@@ -157,7 +148,6 @@ class _PackedLayout(NamedTuple):
|
||||
cu_seqlens_kv: torch.Tensor
|
||||
batch_size: int
|
||||
seq_lens: torch.Tensor
|
||||
known_min_kv_length: int | None = None
|
||||
|
||||
|
||||
def _workspace_bytes() -> int:
|
||||
@@ -216,9 +206,13 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
self.skip = SkipSoftmaxConfig.from_backend_kwargs(backend_kwargs)
|
||||
self._warned_missing_timestep = False
|
||||
|
||||
self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)
|
||||
# Resolve the SAGE quantize fn once at init so the compiled forward path never calls the
|
||||
# lru_cache-wrapped getter (which triggers a Dynamo graph break every step).
|
||||
# Override: SageAttention does not support causal attention
|
||||
if causal:
|
||||
self.quant = QuantConfig(dtype_qk=None, q_block_size=0, k_block_size=0)
|
||||
else:
|
||||
self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)
|
||||
# Resolve the SAGE quantize fn once at init so the compiled forward path
|
||||
# does not import it every step.
|
||||
self._sage_quantize_fn = None
|
||||
if self.quant.enabled:
|
||||
if self.quant.dtype_qk not in _QK_QUANT_DTYPES:
|
||||
@@ -226,19 +220,16 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got "
|
||||
f"{self.quant.dtype_qk!r}. FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE."
|
||||
)
|
||||
if not _sage_kernel_available():
|
||||
try:
|
||||
flashinfer_version = Version(flashinfer.__version__)
|
||||
except (AttributeError, InvalidVersion):
|
||||
flashinfer_version = None
|
||||
if flashinfer_version is not None and flashinfer_version < Version("0.6.18rc10"):
|
||||
raise RuntimeError(
|
||||
"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build does not "
|
||||
"expose the trtllm-gen sage_attn_sfs kernel path. Install a FlashInfer build "
|
||||
"that provides it, or remove the quant config."
|
||||
)
|
||||
self._sage_quantize_fn = _sage_quantize_fn()
|
||||
if self._sage_quantize_fn is None:
|
||||
raise RuntimeError(
|
||||
"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build lacks "
|
||||
"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "
|
||||
"FlashInfer, or remove the quant config."
|
||||
f"FlashInfer {flashinfer_version} is too old for TRTLLM_ATTN quant (SAGE); "
|
||||
"install flashinfer >= 0.6.18rc10."
|
||||
)
|
||||
self._sage_quantize_fn = flashinfer.trtllm_sage_attention_quantize
|
||||
|
||||
def set_layer_calibration(self, a: float, b: float) -> None:
|
||||
self.skip = replace(self.skip, a=a, b=b)
|
||||
@@ -330,7 +321,6 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
# Canonical packed-padding metadata is [0, valid_kv_tokens], so this slice
|
||||
# is the one-element sequence-length view expected by the kernel.
|
||||
seq_lens=cu_seq_lens_kv[1:],
|
||||
known_min_kv_length=valid_kv_tokens,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -407,7 +397,6 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
v = value.reshape(physical_batch * kv_len, num_kv_heads, head_dim).contiguous()
|
||||
output_tokens = q.shape[0]
|
||||
|
||||
known_min_kv_length: int | None = kv_len
|
||||
if has_packed_metadata:
|
||||
cu_seq_lens_q = extra["cu_seqlens_q"]
|
||||
cu_seq_lens_kv = extra["cu_seqlens_k"]
|
||||
@@ -444,7 +433,6 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
cu_seq_lens_kv = packed_layout.cu_seqlens_kv
|
||||
batch = packed_layout.batch_size
|
||||
seq_lens = packed_layout.seq_lens
|
||||
known_min_kv_length = packed_layout.known_min_kv_length
|
||||
max_q_len = int(extra["max_seqlen_q"])
|
||||
max_kv_len = int(extra["max_seqlen_k"])
|
||||
else:
|
||||
@@ -465,24 +453,15 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
# SAGE quant is active (which already requires the kernel, checked at init) so the dense
|
||||
# path stays compatible with older builds that lack these parameters.
|
||||
sage_kwargs: dict = {}
|
||||
# The SAGE kernel requires every KV sequence to contain at least one full
|
||||
# quantization block. Small auxiliary attention sites use the dense kernel.
|
||||
use_sage = False
|
||||
if self.quant.enabled:
|
||||
if known_min_kv_length is not None:
|
||||
sage_lengths_supported = known_min_kv_length >= self.quant.k_block_size
|
||||
else:
|
||||
sage_lengths_supported = bool(torch.all(seq_lens >= self.quant.k_block_size).item())
|
||||
use_sage = sage_lengths_supported
|
||||
if self.quant.enabled and not use_sage:
|
||||
message = (
|
||||
f"TRTLLM_ATTN SAGE quantization is configured for attention role {self.role!r}, but at least one "
|
||||
f"KV sequence is shorter than k_block_size={self.quant.k_block_size}. Falling back to dense "
|
||||
"attention for this input."
|
||||
q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
self._sage_quantize_fn,
|
||||
cu_seq_lens_q,
|
||||
cu_seq_lens_kv,
|
||||
)
|
||||
logger.warning_once(message)
|
||||
if use_sage:
|
||||
q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)
|
||||
sage_kwargs["sage_attn_sfs"] = sage_attn_sfs
|
||||
sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes
|
||||
|
||||
@@ -505,6 +484,7 @@ class TrtllmAttentionImpl(AttentionImpl):
|
||||
is_causal=self.causal,
|
||||
return_lse=False,
|
||||
skip_softmax_threshold_scale_factor=_skip_factor,
|
||||
skip_all_rows_active_check=True,
|
||||
**sage_kwargs,
|
||||
)
|
||||
if out.shape[0] != output_tokens:
|
||||
|
||||
@@ -2261,7 +2261,7 @@ def build_attention_config(
|
||||
|
||||
Called exactly once in ``OmniDiffusionConfig.__post_init__``.
|
||||
Handles type-conversion **and** env-var fallback
|
||||
(``DIFFUSION_ATTENTION_BACKEND``).
|
||||
(``DIFFUSION_ATTENTION_BACKEND`` and ``DIFFUSION_ATTENTION_QUANT``).
|
||||
"""
|
||||
normalized = parse_attention_config(attention_config)
|
||||
|
||||
@@ -2275,7 +2275,30 @@ def build_attention_config(
|
||||
if env_attention_backend.lower() == "auto":
|
||||
return normalized
|
||||
|
||||
normalized.default = AttentionSpec(backend=env_attention_backend)
|
||||
# If config file doesn't explicitly setup quantization specs, optionally set
|
||||
# from the env-var.
|
||||
quant = None
|
||||
env_attention_quant = os.environ.get("DIFFUSION_ATTENTION_QUANT")
|
||||
if env_attention_quant is not None:
|
||||
fields = [field.strip() for field in env_attention_quant.split(":")]
|
||||
if len(fields) not in (2, 4) or not all(fields):
|
||||
raise ValueError(
|
||||
"DIFFUSION_ATTENTION_QUANT must have the format <dtype_qk>:<dtype_vo>[:<q_block_size>:<k_block_size>]."
|
||||
)
|
||||
quant_kwargs: dict[str, Any] = {
|
||||
"dtype_qk": fields[0],
|
||||
"dtype_vo": fields[1],
|
||||
}
|
||||
if len(fields) == 4:
|
||||
q_block_size, k_block_size = int(fields[2]), int(fields[3])
|
||||
if (q_block_size, k_block_size) != (0, 0):
|
||||
quant_kwargs.update(
|
||||
q_block_size=q_block_size,
|
||||
k_block_size=k_block_size,
|
||||
)
|
||||
quant = AttnQuantSpec(**quant_kwargs)
|
||||
|
||||
normalized.default = AttentionSpec(backend=env_attention_backend, quant=quant)
|
||||
logger.info(
|
||||
"Parsed attention config from DIFFUSION_ATTENTION_BACKEND '%s': default=%s, per_role=%s",
|
||||
env_attention_backend,
|
||||
|
||||
Reference in new issue
Block a user