[diffusion] use fa3 by default when device supports it (#783)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
1 parent
f266b04534
commit
0df8e80574
4 files changed
+22
-14
No files matched your search
@@ -5,12 +5,14 @@ import importlib
|
||||
import os
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_rocm
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -63,7 +65,22 @@ def get_attn_backend(head_size: int) -> type[AttentionBackend]:
|
||||
The selected attention backend class
|
||||
"""
|
||||
# Check environment variable
|
||||
backend_name: str | None = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
|
||||
|
||||
backend_name = os.environ.get("DIFFUSION_ATTENTION_BACKEND", None)
|
||||
|
||||
if detect_device_type() == "cuda" and not is_rocm():
|
||||
compute_capability = torch.cuda.get_device_capability()
|
||||
major, minor = compute_capability
|
||||
if 80 <= major * 10 + minor < 100:
|
||||
if backend_name is None:
|
||||
backend_name = "FLASH_ATTN"
|
||||
else:
|
||||
if backend_name == "FLASH_ATTN":
|
||||
logger.warning(
|
||||
"""Flash Attention requires GPU with compute capability >= 8.0 or < 10.0. "
|
||||
"Falling back to TORCH_SDPA backend."""
|
||||
)
|
||||
backend_name = "TORCH_SDPA"
|
||||
|
||||
if backend_name is not None:
|
||||
backend_name_upper = backend_name.upper()
|
||||
|
||||
Reference in new issue
Block a user