[Diffusion] Init Attention Backends and Selector for Diffusion (#115)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: samithuang <285365963@qq.com>
Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
Jiangyun ZhuandSamit authored and GitHub committed 2025-12-05 10:46:20 +00:00
1 parent 6079b14040
commit 98fe15762d
9 files changed
+338 -35

No files matched your search

@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Generic, TypeVar
import torch
class AttentionBackend(ABC):
"""Abstract class for diffusion attention backends."""
accept_output_buffer: bool = False
@staticmethod
@abstractmethod
def get_name() -> str:
raise NotImplementedError
@staticmethod
@abstractmethod
def get_impl_cls() -> type["AttentionImpl"]:
raise NotImplementedError
@staticmethod
@abstractmethod
def get_metadata_cls() -> type["AttentionMetadata"]:
raise NotImplementedError
@staticmethod
@abstractmethod
def get_builder_cls(): # -> Type["AttentionMetadataBuilder"]:
raise NotImplementedError
@staticmethod
@abstractmethod
def get_supported_head_sizes() -> list[int]:
"""Get the list of supported head sizes for this backend."""
raise NotImplementedError
@classmethod
def supports_head_size(cls, head_size: int) -> bool:
supported_head_sizes = cls.get_supported_head_sizes()
return (not supported_head_sizes) or head_size in supported_head_sizes
@dataclass
class AttentionMetadata:
attn_mask: torch.Tensor | None = None
T = TypeVar("T", bound=AttentionMetadata)
class AttentionImpl(ABC, Generic[T]):
@abstractmethod
def __init__(
self,
num_heads: int,
head_size: int,
softmax_scale: float,
causal: bool = False,
num_kv_heads: int | None = None,
prefix: str = "",
**extra_impl_args,
) -> None:
raise NotImplementedError
@abstractmethod
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: T,
) -> torch.Tensor:
raise NotImplementedError
@@ -0,0 +1,74 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionBackend,
AttentionImpl,
AttentionMetadata,
)
logger = init_logger(__name__)
try:
# only tested with flash_attn v3
# from flash_attn_interface import flash_attn_func as flash_attn_3_func # not available in flash-attn 2.8.1
from flash_attn import flash_attn_func # can be FA2 or FA3
except ImportError:
logger.warning(
"FlashAttentionBackend is not available. You may install flash-attn "
"by running `uv pip install flash-attn==2.8.1 --no-build-isolation`"
" or install pre-built flash-attn from https://github.com/Dao-AILab/flash-attention/releases"
)
flash_attn_3_func = None
class FlashAttentionBackend(AttentionBackend):
accept_output_buffer: bool = True
@staticmethod
def get_supported_head_sizes() -> list[int]:
return [64, 96, 128, 192, 256]
@staticmethod
def get_name() -> str:
return "FLASH_ATTN"
@staticmethod
def get_impl_cls() -> type["FlashAttentionImpl"]:
return FlashAttentionImpl
class FlashAttentionImpl(AttentionImpl):
def __init__(
self,
num_heads: int,
head_size: int,
softmax_scale: float,
causal: bool = False,
num_kv_heads: int | None = None,
prefix: str = "",
**extra_impl_args,
) -> None:
self.num_heads = num_heads
self.causal = causal
self.softmax_scale = softmax_scale
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata = None,
) -> torch.Tensor:
# TODO: flash_attn_func does not support attn_mask.
out: torch.Tensor = flash_attn_func(
query,
key,
value,
causal=self.causal,
softmax_scale=self.softmax_scale,
)
return out
@@ -0,0 +1,66 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionBackend,
AttentionImpl,
AttentionMetadata,
)
logger = init_logger(__name__)
class SDPABackend(AttentionBackend):
accept_output_buffer: bool = True
@staticmethod
def get_supported_head_sizes() -> list[int]:
return [x for x in range(1024)] # todo
@staticmethod
def get_name() -> str:
return "SDPA"
@staticmethod
def get_impl_cls() -> type["SDPAImpl"]:
return SDPAImpl
class SDPAImpl(AttentionImpl):
def __init__(
self,
num_heads: int,
head_size: int,
softmax_scale: float,
causal: bool = False,
num_kv_heads: int | None = None,
prefix: str = "",
**extra_impl_args,
) -> None:
self.causal = causal
self.softmax_scale = softmax_scale
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata = None,
) -> torch.Tensor:
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
attention_mask = attn_metadata.attn_mask if attn_metadata else None
output = torch.nn.functional.scaled_dot_product_attention(
query,
key,
value,
attn_mask=attention_mask,
dropout_p=0.0,
is_causal=self.causal,
scale=self.softmax_scale,
)
out = output.permute(0, 2, 1, 3)
return out
+43
View File
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionMetadata,
)
from vllm_omni.diffusion.attention.selector import get_attn_backend
class Attention(nn.Module):
def __init__(
self,
num_heads: int,
head_size: int,
causal: bool,
softmax_scale: float,
num_kv_heads: int | None = None,
prefix: str = "",
):
super().__init__()
self.attn_backend = get_attn_backend(-1)
self.attn_impl_cls = self.attn_backend.get_impl_cls()
self.attention = self.attn_impl_cls(
num_heads=num_heads,
head_size=head_size,
softmax_scale=softmax_scale,
causal=causal,
num_kv_heads=num_kv_heads,
)
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata = None,
) -> torch.Tensor:
# shape: (batch_size, seq_len, num_heads, head_size)
attn_output = self.attention.forward(query, key, value, attn_metadata)
return attn_output
+53
View File
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from functools import cache
from vllm.logger import init_logger
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionBackend,
)
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionBackend
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
logger = init_logger(__name__)
# environment variable value -> backend class
SUPPORTED_BACKENDS = {
"FLASH_ATTN": FlashAttentionBackend,
"TORCH_SDPA": SDPABackend,
}
@cache
def get_attn_backend(head_size: int) -> type[AttentionBackend]:
"""
Get attention backend for diffusion models.
The backend is selected based on the following priority:
1. DIFFUSION_ATTENTION_BACKEND environment variable (if set, e.g. export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN)
2. Default backend (SDPA)
Args:
head_size: Head size (currently not used for selection, but kept for API compatibility)
Returns:
The selected attention backend class
"""
# Check environment variable
backend_name: str | None = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
if backend_name is not None:
backend_name_upper = backend_name.upper()
if backend_name_upper not in SUPPORTED_BACKENDS:
valid_backends = list(SUPPORTED_BACKENDS.keys())
raise ValueError(
f"Invalid attention backend for diffusion: '{backend_name}'. Valid backends are: {valid_backends}"
)
logger.info(f"Using attention backend '{backend_name_upper}' for diffusion")
return SUPPORTED_BACKENDS[backend_name_upper]
# Default to SDPA
return SDPABackend
@@ -8,7 +8,6 @@ from typing import Any, Optional, Union
import torch
import torch.nn as nn
from diffusers.models.attention import FeedForward
from diffusers.models.attention_dispatch import dispatch_attention_fn
# TODO replace this with vLLM implementation
from diffusers.models.embeddings import TimestepEmbedding, Timesteps
@@ -18,6 +17,8 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear
from vllm_omni.diffusion.attention.layer import Attention
logger = init_logger(__name__)
@@ -262,12 +263,12 @@ class QwenImageCrossAttention(nn.Module):
self.norm_added_q = RMSNorm(head_dim, eps=eps)
self.norm_added_k = RMSNorm(head_dim, eps=eps)
# self.attn = Attention(
# num_heads=num_heads,
# head_size=self.head_dim,
# scale=1.0, # softmax_scale
# attn_type=AttentionType.ENCODER,
# )
self.attn = Attention(
num_heads=num_heads,
head_size=self.head_dim,
softmax_scale=1.0 / (self.head_dim**0.5),
causal=False,
)
def forward(
self,
@@ -316,20 +317,11 @@ class QwenImageCrossAttention(nn.Module):
joint_value = torch.cat([txt_value, img_value], dim=1)
# Compute joint attention
# joint_hidden_states = self.attn(
# joint_query,
# joint_key,
# joint_value,
# )
joint_hidden_states = dispatch_attention_fn(
joint_hidden_states = self.attn(
joint_query,
joint_key,
joint_value,
dropout_p=0.0,
is_causal=False,
)
# Reshape back
joint_hidden_states = joint_hidden_states.flatten(2, 3)
joint_hidden_states = joint_hidden_states.to(joint_query.dtype)
@@ -368,19 +360,6 @@ class QwenImageTransformerBlock(nn.Module):
nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2
)
self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
# self.attn = Attention(
# query_dim=dim,
# cross_attention_dim=None, # Enable cross attention for joint computation
# added_kv_proj_dim=dim, # Enable added KV projections for text stream
# dim_head=attention_head_dim,
# heads=num_attention_heads,
# out_dim=dim,
# context_pre_only=False,
# bias=True,
# processor=QwenDoubleStreamAttnProcessor2_0(),
# qk_norm=qk_norm,
# eps=eps,
# )
self.attn = QwenImageCrossAttention(
dim=dim,
num_heads=num_attention_heads,
@@ -22,10 +22,11 @@ from typing import Optional
import torch
import torch.nn as nn
import torch.nn.functional as F
from diffusers.models.attention_dispatch import dispatch_attention_fn
from torch.nn.utils.rnn import pad_sequence
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm_omni.diffusion.attention.layer import Attention
ADALN_EMBED_DIM = 256
SEQ_MULTI_OF = 32
@@ -105,6 +106,13 @@ class ZImageAttention(nn.Module):
self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias=False)])
self.attn = Attention(
num_heads=num_heads,
head_size=self.head_dim,
softmax_scale=1.0 / (self.head_dim**0.5),
causal=False,
)
def forward(
self,
hidden_states: torch.Tensor,
@@ -143,13 +151,11 @@ class ZImageAttention(nn.Module):
attention_mask = attention_mask[:, None, None, :]
# Compute joint attention
hidden_states = dispatch_attention_fn(
hidden_states = self.attn(
query,
key,
value,
attn_mask=attention_mask,
dropout_p=0.0,
is_causal=False,
# attn_mask=attention_mask, # we don't support multi prompts now.
)
# Reshape back