[Diffusion][Attention] sage attention backend (#243)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
1 parent
c32f52fcc0
commit
c52270cfee
3 files changed
+104
-9
No files matched your search
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
import os
|
||||
from functools import cache
|
||||
|
||||
@@ -9,18 +10,40 @@ from vllm.logger import init_logger
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# environment variable value -> backend class
|
||||
SUPPORTED_BACKENDS = {
|
||||
"FLASH_ATTN": FlashAttentionBackend,
|
||||
"TORCH_SDPA": SDPABackend,
|
||||
# environment variable value -> backend module and class
|
||||
_BACKEND_CONFIG = {
|
||||
"FLASH_ATTN": {
|
||||
"module": "vllm_omni.diffusion.attention.backends.flash_attn",
|
||||
"class": "FlashAttentionBackend",
|
||||
},
|
||||
"TORCH_SDPA": {
|
||||
"module": "vllm_omni.diffusion.attention.backends.sdpa",
|
||||
"class": "SDPABackend",
|
||||
},
|
||||
"SAGE_ATTN": {
|
||||
"module": "vllm_omni.diffusion.attention.backends.sage_attn",
|
||||
"class": "SageAttentionBackend",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_backend(backend_name: str) -> type[AttentionBackend]:
|
||||
config = _BACKEND_CONFIG[backend_name]
|
||||
|
||||
try:
|
||||
module = importlib.import_module(config["module"])
|
||||
backend_class = getattr(module, config["class"])
|
||||
return backend_class
|
||||
except ImportError as e:
|
||||
raise ImportError(f"Failed to import module {config['module']}: {e}")
|
||||
except AttributeError as e:
|
||||
raise AttributeError(f"Class {config['class']} not found in module: {e}")
|
||||
|
||||
|
||||
@cache
|
||||
def get_attn_backend(head_size: int) -> type[AttentionBackend]:
|
||||
"""
|
||||
@@ -41,13 +64,13 @@ def get_attn_backend(head_size: int) -> type[AttentionBackend]:
|
||||
|
||||
if backend_name is not None:
|
||||
backend_name_upper = backend_name.upper()
|
||||
if backend_name_upper not in SUPPORTED_BACKENDS:
|
||||
valid_backends = list(SUPPORTED_BACKENDS.keys())
|
||||
if backend_name_upper not in _BACKEND_CONFIG:
|
||||
valid_backends = list(_BACKEND_CONFIG.keys())
|
||||
raise ValueError(
|
||||
f"Invalid attention backend for diffusion: '{backend_name}'. Valid backends are: {valid_backends}"
|
||||
)
|
||||
logger.info(f"Using attention backend '{backend_name_upper}' for diffusion")
|
||||
return SUPPORTED_BACKENDS[backend_name_upper]
|
||||
return load_backend(backend_name_upper)
|
||||
|
||||
# Default to SDPA
|
||||
return SDPABackend
|
||||
Reference in new issue
Block a user