diff --git a/docs/configuration/environment_variables.md b/docs/configuration/environment_variables.md index 15d61b9fe..039565030 100644 --- a/docs/configuration/environment_variables.md +++ b/docs/configuration/environment_variables.md @@ -64,6 +64,7 @@ The lifecycle labels used below are: | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle | | --- | --- | --- | --- | --- | | `DIFFUSION_ATTENTION_BACKEND` | Backend name or `auto`; default `auto` (platform selection) | Diffusion stages; read when `OmniDiffusionConfig` is constructed | `diffusion_attention_config.default` wins. An unknown backend fails during backend resolution. | Stable fallback | +| `DIFFUSION_ATTENTION_QUANT` | `:[::]`; default unset | Diffusion stages; read when `OmniDiffusionConfig` is constructed | Used only when `DIFFUSION_ATTENTION_BACKEND` sets a non-`auto` backend and no explicit attention default exists. Quantization support varies by backend. Malformed fields raise `ValueError`. | Stable fallback | | `DIFFUSION_CACHE_BACKEND` | `none`, `cache_dit`, `tea_cache`, `mag_cache`, `step_cache`, `stepcache`, or `step_cache_dit`; default `none` | Diffusion runner startup | Explicit `cache_backend` in config wins. Otherwise this name wins over the deprecated alias. Unsupported values raise `ValueError` during runner setup. | Stable fallback | | `DIFFUSION_CACHE_ADAPTER` | Same values as `DIFFUSION_CACHE_BACKEND`; default `none` | Diffusion runner startup | Used only when neither explicit `cache_backend` nor `DIFFUSION_CACHE_BACKEND` is set. Unsupported values raise `ValueError`. | Deprecated; use `DIFFUSION_CACHE_BACKEND` | | `OMNI_DIFFUSION_PROMPT_EMBED_CACHE` | Boolean: `1`, `true`, `yes`, `on`, `0`, `false`, `no`, or `off`; default disabled | Each diffusion runner; resolved during model setup | A recognized environment value overrides the explicit enable setting. An unrecognized value is ignored. | Experimental | @@ -77,6 +78,13 @@ Backend names for `DIFFUSION_ATTENTION_BACKEND` are the members of `SAGE_ATTN`, `FLASHINFER_ATTN`, and `TRTLLM_ATTN`. Platform support still depends on the installed kernels and model path. +For `DIFFUSION_ATTENTION_QUANT`, the dtype fields accept `float16`, +`bfloat16`, `int8`, or `fp8_e4m3`; the selected backend may support only a +subset. For example, `DIFFUSION_ATTENTION_BACKEND=TRTLLM_ATTN` with +`DIFFUSION_ATTENTION_QUANT=int8:bfloat16:1:16` selects INT8 Q/K SageAttention. +Explicit `diffusion_attention_config.default` takes precedence over both +environment variables. + ### Serving and runtime | Name | Type and default | Applies to and read time | Precedence and invalid values | Lifecycle | diff --git a/tests/config/test_environment_variables.py b/tests/config/test_environment_variables.py index dc87aa7be..6d3c4588a 100644 --- a/tests/config/test_environment_variables.py +++ b/tests/config/test_environment_variables.py @@ -164,7 +164,7 @@ def test_inventory_matches_reviewed_snapshot_counts(): """Make an inventory expansion an explicit review decision.""" category_counts = Counter(item.category for item in ENVIRONMENT_VARIABLE_INVENTORY.values()) assert category_counts == { - EnvironmentVariableCategory.PUBLIC_OMNI: 31, + EnvironmentVariableCategory.PUBLIC_OMNI: 32, EnvironmentVariableCategory.INHERITED_VLLM: 20, EnvironmentVariableCategory.PLATFORM_EXTERNAL: 29, EnvironmentVariableCategory.MODEL_SPECIFIC: 59, @@ -190,6 +190,7 @@ def test_new_public_omni_names_use_project_prefix(): """Grandfather legacy names without allowing more prefix exceptions.""" legacy_public_names = { "DIFFUSION_ATTENTION_BACKEND", + "DIFFUSION_ATTENTION_QUANT", "DIFFUSION_CACHE_ADAPTER", "DIFFUSION_CACHE_BACKEND", "OMNI_DIFFUSION_PROMPT_EMBED_CACHE", diff --git a/tests/diffusion/attention/test_trtllm_attn.py b/tests/diffusion/attention/test_trtllm_attn.py index 336f7d037..fad20e6be 100644 --- a/tests/diffusion/attention/test_trtllm_attn.py +++ b/tests/diffusion/attention/test_trtllm_attn.py @@ -7,6 +7,7 @@ from unittest.mock import Mock import pytest import torch import torch.nn.functional as F +from packaging.version import Version from vllm_omni.diffusion.attention.backends import trtllm_attn as tg from vllm_omni.diffusion.attention.backends.abstract import ( @@ -76,27 +77,51 @@ def test_quant_rejects_non_sage_dtype(): _impl(quant={"dtype_qk": "bfloat16"}) -def test_quant_quantize_requires_flashinfer_routine(monkeypatch): - import vllm_omni.diffusion.attention.backends.trtllm_attn as mod - - monkeypatch.setattr(mod, "_sage_kernel_available", lambda: True) - monkeypatch.setattr(mod, "_sage_quantize_fn", lambda: None) - with pytest.raises(RuntimeError, match="trtllm_sage_attention_quantize"): - _impl(quant={"dtype_qk": "int8"}) - - def test_quant_quantize_calls_routine_and_shapes_sfs(): from vllm_omni.diffusion.attention.backends.trtllm_attn import QuantConfig captured: dict[str, object] = {} - def fake_quantize(q, k, v, q_block_size, k_block_size, qk_quant_dtype): - captured.update(q_block_size=q_block_size, k_block_size=k_block_size, qk_quant_dtype=qk_quant_dtype) - return "qq", "kq", "vq", "qsfs", "ksfs", "vsfs" + def fake_quantize( + q, + k, + v, + q_block_size, + k_block_size, + qk_quant_dtype, + smooth_k, + cum_seq_lens_q, + cum_seq_lens_kv, + ): + captured.update( + q_block_size=q_block_size, + k_block_size=k_block_size, + qk_quant_dtype=qk_quant_dtype, + smooth_k=smooth_k, + cum_seq_lens_q=cum_seq_lens_q, + cum_seq_lens_kv=cum_seq_lens_kv, + ) + return "qq", "kq", "vq", "qsfs", "ksfs", "vsfs", "k_mean" - q_q, k_q, v_q, sfs, blk = QuantConfig(dtype_qk="int8").quantize(object(), object(), object(), fake_quantize) + cu_seq_lens_q = object() + cu_seq_lens_kv = object() + q_q, k_q, v_q, sfs, blk = QuantConfig(dtype_qk="int8").quantize( + object(), + object(), + object(), + fake_quantize, + cu_seq_lens_q, + cu_seq_lens_kv, + ) assert (q_q, k_q, v_q, sfs, blk) == ("qq", "kq", "vq", ("qsfs", "ksfs", None, "vsfs"), (1, 16, 0, 1)) - assert captured == {"q_block_size": 1, "k_block_size": 16, "qk_quant_dtype": torch.int8} + assert captured == { + "q_block_size": 1, + "k_block_size": 16, + "qk_quant_dtype": torch.int8, + "smooth_k": True, + "cum_seq_lens_q": cu_seq_lens_q, + "cum_seq_lens_kv": cu_seq_lens_kv, + } def test_skip_factor_none_without_curve(): @@ -442,8 +467,8 @@ def test_bf16_packed_padding_matches_sdpa(): requires_sage = pytest.mark.skipif( - not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None), - reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)", + not _has_trtllm_attn() or Version(tg.flashinfer.__version__) < Version("0.6.18rc10"), + reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.18rc10", ) @@ -493,22 +518,6 @@ def test_sage_packed_non_aligned_length_matches_sdpa(): assert torch.count_nonzero(out[:, used:]) == 0 -@requires_sage -def test_sage_short_sequence_uses_dense_kernel(): - torch.manual_seed(0) - b, s, h, d = 1, 14, 8, 128 - scale = 1.0 / math.sqrt(d) - q, k, v = (torch.randn(b, s, h, d, device="cuda", dtype=torch.bfloat16) for _ in range(3)) - - impl = _impl(quant={"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}) - impl._sage_quantize_fn = lambda *args, **kwargs: pytest.fail("short sequence must not use SAGE") - out = impl.forward_cuda(q, k, v) - ref = _sdpa_ref(q, k, v, scale) - rel = (out.float() - ref).abs().mean() / ref.abs().mean() - assert torch.isfinite(out).all() - assert rel < 0.01, f"SAGE short-sequence dense fallback rel err {rel:.4f} too high" - - @requires_trtllm_attn def test_skip_tiny_threshold_matches_dense(): torch.manual_seed(0) diff --git a/vllm_omni/config/environment_variable_inventory.py b/vllm_omni/config/environment_variable_inventory.py index ccdc2133e..69c1df1c3 100644 --- a/vllm_omni/config/environment_variable_inventory.py +++ b/vllm_omni/config/environment_variable_inventory.py @@ -55,6 +55,7 @@ class EnvironmentVariableClassification: # grandfathers the older public names below. _PUBLIC_OMNI = ( "DIFFUSION_ATTENTION_BACKEND", + "DIFFUSION_ATTENTION_QUANT", "DIFFUSION_CACHE_ADAPTER", "DIFFUSION_CACHE_BACKEND", "OMNI_DIFFUSION_PROMPT_EMBED_CACHE", diff --git a/vllm_omni/diffusion/attention/backends/flashinfer_attn.py b/vllm_omni/diffusion/attention/backends/flashinfer_attn.py index 33b4a819e..26aa0aa9b 100644 --- a/vllm_omni/diffusion/attention/backends/flashinfer_attn.py +++ b/vllm_omni/diffusion/attention/backends/flashinfer_attn.py @@ -194,6 +194,22 @@ class FlashInferAttentionImpl(AttentionImpl): raise ValueError(f"Unsupported {option_name}={dtype}; expected one of: {choices}") return dtype + @torch.compiler.disable + @staticmethod + def _extract_scalar(scalar_tensor: torch.Tensor): + return scalar_tensor.item() + + @staticmethod + def _per_tensor_quantize( + tensor: torch.Tensor, + to_dtype: torch.dtype, + ) -> tuple[torch.Tensor, float]: + if torch.finfo(to_dtype).bits == 8: + scale_tensor = tensor.abs().amax().float().clamp_min(1e-6) / torch.finfo(to_dtype).max + tensor = (tensor.float() * torch.reciprocal(scale_tensor)).to(to_dtype) + return tensor, FlashInferAttentionImpl._extract_scalar(scale_tensor) + return tensor.to(to_dtype), 1.0 + @staticmethod def _pack_mask_for_flashinfer( attn_mask: torch.Tensor, batch_size: int, qo_len: int, kv_len: int @@ -311,11 +327,9 @@ class FlashInferAttentionImpl(AttentionImpl): q = query.reshape(batch_size * qo_len, num_q_heads, head_dim_qk) k = key.reshape(batch_size * kv_len, num_kv_heads, head_dim_k) v = value.reshape(batch_size * kv_len, num_kv_heads, head_dim_vo) - if self.dtype_qk is not None: - q = q.to(self.dtype_qk) - k = k.to(self.dtype_qk) - if self.dtype_vo is not None: - v = v.to(self.dtype_vo) + q, q_scale = self._per_tensor_quantize(q, self.dtype_qk or q.dtype) + k, k_scale = self._per_tensor_quantize(k, self.dtype_qk or k.dtype) + v, v_scale = self._per_tensor_quantize(v, self.dtype_vo or v.dtype) flat_mask = None if custom_mask is not None: @@ -343,7 +357,7 @@ class FlashInferAttentionImpl(AttentionImpl): ), flat_mask, ) - out = self._run_wrapper(q, k, v) + out = self._run_wrapper(q, k, v, q_scale, k_scale, v_scale) out = out.reshape(batch_size, qo_len, num_q_heads, head_dim_vo) return out.to(query.dtype) if out.dtype != query.dtype else out @@ -353,8 +367,28 @@ class FlashInferAttentionImpl(AttentionImpl): query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, + q_scale: float, + k_scale: float, + v_scale: float, ) -> torch.Tensor: - return self._wrapper.run(query, key, value) + try: + return self._wrapper.run( + query, + key, + value, + q_scale=q_scale, + k_scale=k_scale, + v_scale=v_scale, + ) + except NotImplementedError as _: + # Older FlashInfer does not route scales for CuTeDSL backends. + # Apply some scales manually. + if q_scale != 1.0 or k_scale != 1.0: + raise NotImplementedError("FlashInfer/CuTeDSL backend doesn't support quantizing QK into FP8 yet.") + out = self._wrapper.run(query, key, value) + if v_scale is not None and v_scale != 1.0: + out = out * v_scale + return out def _sdpa_for_unsupported_mask( self, diff --git a/vllm_omni/diffusion/attention/backends/trtllm_attn.py b/vllm_omni/diffusion/attention/backends/trtllm_attn.py index 967e81c6e..c9257f988 100644 --- a/vllm_omni/diffusion/attention/backends/trtllm_attn.py +++ b/vllm_omni/diffusion/attention/backends/trtllm_attn.py @@ -1,13 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project -import functools -import inspect import math from dataclasses import dataclass, replace from typing import NamedTuple, cast import torch +from packaging.version import InvalidVersion, Version from vllm.logger import init_logger from vllm_omni.diffusion.attention.backends.abstract import ( @@ -44,10 +43,10 @@ class SkipSoftmaxConfig: return cls( threshold=_validate_control(bk.get("skip_softmax_threshold"), "skip_softmax_threshold", 0.0, None), target_sparsity=_validate_control(bk.get("target_sparsity"), "target_sparsity", 0.0, 1.0), - disabled_until_timestep=cast( - float, - _validate_control(bk.get("disabled_until_timestep", 0.0), "disabled_until_timestep", 0.0, 1.0), - ), + disabled_until_timestep=_validate_control( + bk.get("disabled_until_timestep", 0.0), "disabled_until_timestep", 0.0, 1.0 + ) + or 0.0, ) @property @@ -77,6 +76,7 @@ class SkipSoftmaxConfig: try: + import flashinfer from flashinfer.prefill import trtllm_ragged_attention_deepseek HAS_FLASHINFER = True @@ -88,26 +88,6 @@ except Exception as e: # pragma: no cover - import guard ) -@functools.lru_cache(maxsize=1) -def _sage_kernel_available() -> bool: - if not HAS_FLASHINFER: - return False - try: - return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters - except (TypeError, ValueError): - return False - - -@functools.lru_cache(maxsize=1) -def _sage_quantize_fn(): - try: - from flashinfer import trtllm_sage_attention_quantize - - return trtllm_sage_attention_quantize - except Exception: # pragma: no cover - return None - - _QK_QUANT_DTYPES = { "int8": torch.int8, "fp8_e4m3": torch.float8_e4m3fn, @@ -133,15 +113,26 @@ class QuantConfig: def enabled(self) -> bool: return self.dtype_qk is not None - def quantize(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, quantize_fn): + def quantize( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + quantize_fn, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_kv: torch.Tensor, + ): qk_quant_dtype = _QK_QUANT_DTYPES[cast(str, self.dtype_qk)] - q_q, k_q, v_q, q_sfs, k_sfs, v_sfs = quantize_fn( + q_q, k_q, v_q, q_sfs, k_sfs, v_sfs, _ = quantize_fn( q, k, v, q_block_size=self.q_block_size, k_block_size=self.k_block_size, qk_quant_dtype=qk_quant_dtype, + smooth_k=True, + cum_seq_lens_q=cu_seq_lens_q, + cum_seq_lens_kv=cu_seq_lens_kv, ) sage_attn_sfs = (q_sfs, k_sfs, None, v_sfs) num_elts_per_sage_attn_blk = (self.q_block_size, self.k_block_size, 0, 1) @@ -157,7 +148,6 @@ class _PackedLayout(NamedTuple): cu_seqlens_kv: torch.Tensor batch_size: int seq_lens: torch.Tensor - known_min_kv_length: int | None = None def _workspace_bytes() -> int: @@ -216,9 +206,13 @@ class TrtllmAttentionImpl(AttentionImpl): self.skip = SkipSoftmaxConfig.from_backend_kwargs(backend_kwargs) self._warned_missing_timestep = False - self.quant = QuantConfig.from_backend_kwargs(backend_kwargs) - # Resolve the SAGE quantize fn once at init so the compiled forward path never calls the - # lru_cache-wrapped getter (which triggers a Dynamo graph break every step). + # Override: SageAttention does not support causal attention + if causal: + self.quant = QuantConfig(dtype_qk=None, q_block_size=0, k_block_size=0) + else: + self.quant = QuantConfig.from_backend_kwargs(backend_kwargs) + # Resolve the SAGE quantize fn once at init so the compiled forward path + # does not import it every step. self._sage_quantize_fn = None if self.quant.enabled: if self.quant.dtype_qk not in _QK_QUANT_DTYPES: @@ -226,19 +220,16 @@ class TrtllmAttentionImpl(AttentionImpl): f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got " f"{self.quant.dtype_qk!r}. FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE." ) - if not _sage_kernel_available(): + try: + flashinfer_version = Version(flashinfer.__version__) + except (AttributeError, InvalidVersion): + flashinfer_version = None + if flashinfer_version is not None and flashinfer_version < Version("0.6.18rc10"): raise RuntimeError( - "TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build does not " - "expose the trtllm-gen sage_attn_sfs kernel path. Install a FlashInfer build " - "that provides it, or remove the quant config." - ) - self._sage_quantize_fn = _sage_quantize_fn() - if self._sage_quantize_fn is None: - raise RuntimeError( - "TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build lacks " - "trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade " - "FlashInfer, or remove the quant config." + f"FlashInfer {flashinfer_version} is too old for TRTLLM_ATTN quant (SAGE); " + "install flashinfer >= 0.6.18rc10." ) + self._sage_quantize_fn = flashinfer.trtllm_sage_attention_quantize def set_layer_calibration(self, a: float, b: float) -> None: self.skip = replace(self.skip, a=a, b=b) @@ -330,7 +321,6 @@ class TrtllmAttentionImpl(AttentionImpl): # Canonical packed-padding metadata is [0, valid_kv_tokens], so this slice # is the one-element sequence-length view expected by the kernel. seq_lens=cu_seq_lens_kv[1:], - known_min_kv_length=valid_kv_tokens, ) @staticmethod @@ -407,7 +397,6 @@ class TrtllmAttentionImpl(AttentionImpl): v = value.reshape(physical_batch * kv_len, num_kv_heads, head_dim).contiguous() output_tokens = q.shape[0] - known_min_kv_length: int | None = kv_len if has_packed_metadata: cu_seq_lens_q = extra["cu_seqlens_q"] cu_seq_lens_kv = extra["cu_seqlens_k"] @@ -444,7 +433,6 @@ class TrtllmAttentionImpl(AttentionImpl): cu_seq_lens_kv = packed_layout.cu_seqlens_kv batch = packed_layout.batch_size seq_lens = packed_layout.seq_lens - known_min_kv_length = packed_layout.known_min_kv_length max_q_len = int(extra["max_seqlen_q"]) max_kv_len = int(extra["max_seqlen_k"]) else: @@ -465,24 +453,15 @@ class TrtllmAttentionImpl(AttentionImpl): # SAGE quant is active (which already requires the kernel, checked at init) so the dense # path stays compatible with older builds that lack these parameters. sage_kwargs: dict = {} - # The SAGE kernel requires every KV sequence to contain at least one full - # quantization block. Small auxiliary attention sites use the dense kernel. - use_sage = False if self.quant.enabled: - if known_min_kv_length is not None: - sage_lengths_supported = known_min_kv_length >= self.quant.k_block_size - else: - sage_lengths_supported = bool(torch.all(seq_lens >= self.quant.k_block_size).item()) - use_sage = sage_lengths_supported - if self.quant.enabled and not use_sage: - message = ( - f"TRTLLM_ATTN SAGE quantization is configured for attention role {self.role!r}, but at least one " - f"KV sequence is shorter than k_block_size={self.quant.k_block_size}. Falling back to dense " - "attention for this input." + q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize( + q, + k, + v, + self._sage_quantize_fn, + cu_seq_lens_q, + cu_seq_lens_kv, ) - logger.warning_once(message) - if use_sage: - q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn) sage_kwargs["sage_attn_sfs"] = sage_attn_sfs sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes @@ -505,6 +484,7 @@ class TrtllmAttentionImpl(AttentionImpl): is_causal=self.causal, return_lse=False, skip_softmax_threshold_scale_factor=_skip_factor, + skip_all_rows_active_check=True, **sage_kwargs, ) if out.shape[0] != output_tokens: diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 9bf32a6bc..5dc39adb8 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -2261,7 +2261,7 @@ def build_attention_config( Called exactly once in ``OmniDiffusionConfig.__post_init__``. Handles type-conversion **and** env-var fallback - (``DIFFUSION_ATTENTION_BACKEND``). + (``DIFFUSION_ATTENTION_BACKEND`` and ``DIFFUSION_ATTENTION_QUANT``). """ normalized = parse_attention_config(attention_config) @@ -2275,7 +2275,30 @@ def build_attention_config( if env_attention_backend.lower() == "auto": return normalized - normalized.default = AttentionSpec(backend=env_attention_backend) + # If config file doesn't explicitly setup quantization specs, optionally set + # from the env-var. + quant = None + env_attention_quant = os.environ.get("DIFFUSION_ATTENTION_QUANT") + if env_attention_quant is not None: + fields = [field.strip() for field in env_attention_quant.split(":")] + if len(fields) not in (2, 4) or not all(fields): + raise ValueError( + "DIFFUSION_ATTENTION_QUANT must have the format :[::]." + ) + quant_kwargs: dict[str, Any] = { + "dtype_qk": fields[0], + "dtype_vo": fields[1], + } + if len(fields) == 4: + q_block_size, k_block_size = int(fields[2]), int(fields[3]) + if (q_block_size, k_block_size) != (0, 0): + quant_kwargs.update( + q_block_size=q_block_size, + k_block_size=k_block_size, + ) + quant = AttnQuantSpec(**quant_kwargs) + + normalized.default = AttentionSpec(backend=env_attention_backend, quant=quant) logger.info( "Parsed attention config from DIFFUSION_ATTENTION_BACKEND '%s': default=%s, per_role=%s", env_attention_backend,