[Diffusion] Init Attention Backends and Selector for Diffusion (#115)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: samithuang <285365963@qq.com> Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
1 parent
6079b14040
commit
98fe15762d
9 files changed
+338
-35
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class AttentionBackend(ABC):
|
||||
"""Abstract class for diffusion attention backends."""
|
||||
|
||||
accept_output_buffer: bool = False
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_name() -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_impl_cls() -> type["AttentionImpl"]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_metadata_cls() -> type["AttentionMetadata"]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_builder_cls(): # -> Type["AttentionMetadataBuilder"]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
"""Get the list of supported head sizes for this backend."""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def supports_head_size(cls, head_size: int) -> bool:
|
||||
supported_head_sizes = cls.get_supported_head_sizes()
|
||||
return (not supported_head_sizes) or head_size in supported_head_sizes
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttentionMetadata:
|
||||
attn_mask: torch.Tensor | None = None
|
||||
|
||||
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
|
||||
class AttentionImpl(ABC, Generic[T]):
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
softmax_scale: float,
|
||||
causal: bool = False,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: T,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,74 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
AttentionImpl,
|
||||
AttentionMetadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
try:
|
||||
# only tested with flash_attn v3
|
||||
# from flash_attn_interface import flash_attn_func as flash_attn_3_func # not available in flash-attn 2.8.1
|
||||
from flash_attn import flash_attn_func # can be FA2 or FA3
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"FlashAttentionBackend is not available. You may install flash-attn "
|
||||
"by running `uv pip install flash-attn==2.8.1 --no-build-isolation`"
|
||||
" or install pre-built flash-attn from https://github.com/Dao-AILab/flash-attention/releases"
|
||||
)
|
||||
flash_attn_3_func = None
|
||||
|
||||
|
||||
class FlashAttentionBackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
|
||||
@staticmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
return [64, 96, 128, 192, 256]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASH_ATTN"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["FlashAttentionImpl"]:
|
||||
return FlashAttentionImpl
|
||||
|
||||
|
||||
class FlashAttentionImpl(AttentionImpl):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
softmax_scale: float,
|
||||
causal: bool = False,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
self.num_heads = num_heads
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
# TODO: flash_attn_func does not support attn_mask.
|
||||
out: torch.Tensor = flash_attn_func(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
causal=self.causal,
|
||||
softmax_scale=self.softmax_scale,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
AttentionImpl,
|
||||
AttentionMetadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SDPABackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
|
||||
@staticmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
return [x for x in range(1024)] # todo
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "SDPA"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["SDPAImpl"]:
|
||||
return SDPAImpl
|
||||
|
||||
|
||||
class SDPAImpl(AttentionImpl):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
softmax_scale: float,
|
||||
causal: bool = False,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
|
||||
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
||||
|
||||
output = torch.nn.functional.scaled_dot_product_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=attention_mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=self.causal,
|
||||
scale=self.softmax_scale,
|
||||
)
|
||||
out = output.permute(0, 2, 1, 3)
|
||||
return out
|
||||
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.selector import get_attn_backend
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
causal: bool,
|
||||
softmax_scale: float,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.attn_backend = get_attn_backend(-1)
|
||||
self.attn_impl_cls = self.attn_backend.get_impl_cls()
|
||||
self.attention = self.attn_impl_cls(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
num_kv_heads=num_kv_heads,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
# shape: (batch_size, seq_len, num_heads, head_size)
|
||||
attn_output = self.attention.forward(query, key, value, attn_metadata)
|
||||
return attn_output
|
||||
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from functools import cache
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# environment variable value -> backend class
|
||||
SUPPORTED_BACKENDS = {
|
||||
"FLASH_ATTN": FlashAttentionBackend,
|
||||
"TORCH_SDPA": SDPABackend,
|
||||
}
|
||||
|
||||
|
||||
@cache
|
||||
def get_attn_backend(head_size: int) -> type[AttentionBackend]:
|
||||
"""
|
||||
Get attention backend for diffusion models.
|
||||
|
||||
The backend is selected based on the following priority:
|
||||
1. DIFFUSION_ATTENTION_BACKEND environment variable (if set, e.g. export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN)
|
||||
2. Default backend (SDPA)
|
||||
|
||||
Args:
|
||||
head_size: Head size (currently not used for selection, but kept for API compatibility)
|
||||
|
||||
Returns:
|
||||
The selected attention backend class
|
||||
"""
|
||||
# Check environment variable
|
||||
backend_name: str | None = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
|
||||
|
||||
if backend_name is not None:
|
||||
backend_name_upper = backend_name.upper()
|
||||
if backend_name_upper not in SUPPORTED_BACKENDS:
|
||||
valid_backends = list(SUPPORTED_BACKENDS.keys())
|
||||
raise ValueError(
|
||||
f"Invalid attention backend for diffusion: '{backend_name}'. Valid backends are: {valid_backends}"
|
||||
)
|
||||
logger.info(f"Using attention backend '{backend_name_upper}' for diffusion")
|
||||
return SUPPORTED_BACKENDS[backend_name_upper]
|
||||
|
||||
# Default to SDPA
|
||||
return SDPABackend
|
||||
@@ -8,7 +8,6 @@ from typing import Any, Optional, Union
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from diffusers.models.attention import FeedForward
|
||||
from diffusers.models.attention_dispatch import dispatch_attention_fn
|
||||
|
||||
# TODO replace this with vLLM implementation
|
||||
from diffusers.models.embeddings import TimestepEmbedding, Timesteps
|
||||
@@ -18,6 +17,8 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -262,12 +263,12 @@ class QwenImageCrossAttention(nn.Module):
|
||||
self.norm_added_q = RMSNorm(head_dim, eps=eps)
|
||||
self.norm_added_k = RMSNorm(head_dim, eps=eps)
|
||||
|
||||
# self.attn = Attention(
|
||||
# num_heads=num_heads,
|
||||
# head_size=self.head_dim,
|
||||
# scale=1.0, # softmax_scale
|
||||
# attn_type=AttentionType.ENCODER,
|
||||
# )
|
||||
self.attn = Attention(
|
||||
num_heads=num_heads,
|
||||
head_size=self.head_dim,
|
||||
softmax_scale=1.0 / (self.head_dim**0.5),
|
||||
causal=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -316,20 +317,11 @@ class QwenImageCrossAttention(nn.Module):
|
||||
joint_value = torch.cat([txt_value, img_value], dim=1)
|
||||
|
||||
# Compute joint attention
|
||||
# joint_hidden_states = self.attn(
|
||||
# joint_query,
|
||||
# joint_key,
|
||||
# joint_value,
|
||||
# )
|
||||
joint_hidden_states = dispatch_attention_fn(
|
||||
joint_hidden_states = self.attn(
|
||||
joint_query,
|
||||
joint_key,
|
||||
joint_value,
|
||||
dropout_p=0.0,
|
||||
is_causal=False,
|
||||
)
|
||||
|
||||
# Reshape back
|
||||
joint_hidden_states = joint_hidden_states.flatten(2, 3)
|
||||
joint_hidden_states = joint_hidden_states.to(joint_query.dtype)
|
||||
|
||||
@@ -368,19 +360,6 @@ class QwenImageTransformerBlock(nn.Module):
|
||||
nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2
|
||||
)
|
||||
self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
|
||||
# self.attn = Attention(
|
||||
# query_dim=dim,
|
||||
# cross_attention_dim=None, # Enable cross attention for joint computation
|
||||
# added_kv_proj_dim=dim, # Enable added KV projections for text stream
|
||||
# dim_head=attention_head_dim,
|
||||
# heads=num_attention_heads,
|
||||
# out_dim=dim,
|
||||
# context_pre_only=False,
|
||||
# bias=True,
|
||||
# processor=QwenDoubleStreamAttnProcessor2_0(),
|
||||
# qk_norm=qk_norm,
|
||||
# eps=eps,
|
||||
# )
|
||||
self.attn = QwenImageCrossAttention(
|
||||
dim=dim,
|
||||
num_heads=num_attention_heads,
|
||||
|
||||
@@ -22,10 +22,11 @@ from typing import Optional
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from diffusers.models.attention_dispatch import dispatch_attention_fn
|
||||
from torch.nn.utils.rnn import pad_sequence
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
|
||||
ADALN_EMBED_DIM = 256
|
||||
SEQ_MULTI_OF = 32
|
||||
|
||||
@@ -105,6 +106,13 @@ class ZImageAttention(nn.Module):
|
||||
|
||||
self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias=False)])
|
||||
|
||||
self.attn = Attention(
|
||||
num_heads=num_heads,
|
||||
head_size=self.head_dim,
|
||||
softmax_scale=1.0 / (self.head_dim**0.5),
|
||||
causal=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -143,13 +151,11 @@ class ZImageAttention(nn.Module):
|
||||
attention_mask = attention_mask[:, None, None, :]
|
||||
|
||||
# Compute joint attention
|
||||
hidden_states = dispatch_attention_fn(
|
||||
hidden_states = self.attn(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=attention_mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=False,
|
||||
# attn_mask=attention_mask, # we don't support multi prompts now.
|
||||
)
|
||||
|
||||
# Reshape back
|
||||
|
||||
Reference in new issue
Block a user