diff --git a/vllm_omni/diffusion/attention/__init__.py b/vllm_omni/diffusion/attention/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/vllm_omni/diffusion/attention/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm_omni/diffusion/attention/backends/__init__.py b/vllm_omni/diffusion/attention/backends/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/vllm_omni/diffusion/attention/backends/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm_omni/diffusion/attention/backends/abstract.py b/vllm_omni/diffusion/attention/backends/abstract.py new file mode 100644 index 000000000..5c9c78774 --- /dev/null +++ b/vllm_omni/diffusion/attention/backends/abstract.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Generic, TypeVar + +import torch + + +class AttentionBackend(ABC): + """Abstract class for diffusion attention backends.""" + + accept_output_buffer: bool = False + + @staticmethod + @abstractmethod + def get_name() -> str: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_impl_cls() -> type["AttentionImpl"]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_metadata_cls() -> type["AttentionMetadata"]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_builder_cls(): # -> Type["AttentionMetadataBuilder"]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_supported_head_sizes() -> list[int]: + """Get the list of supported head sizes for this backend.""" + raise NotImplementedError + + @classmethod + def supports_head_size(cls, head_size: int) -> bool: + supported_head_sizes = cls.get_supported_head_sizes() + return (not supported_head_sizes) or head_size in supported_head_sizes + + +@dataclass +class AttentionMetadata: + attn_mask: torch.Tensor | None = None + + +T = TypeVar("T", bound=AttentionMetadata) + + +class AttentionImpl(ABC, Generic[T]): + @abstractmethod + def __init__( + self, + num_heads: int, + head_size: int, + softmax_scale: float, + causal: bool = False, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + raise NotImplementedError + + @abstractmethod + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: T, + ) -> torch.Tensor: + raise NotImplementedError diff --git a/vllm_omni/diffusion/attention/backends/flash_attn.py b/vllm_omni/diffusion/attention/backends/flash_attn.py new file mode 100644 index 000000000..28eca96b3 --- /dev/null +++ b/vllm_omni/diffusion/attention/backends/flash_attn.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from vllm.logger import init_logger + +from vllm_omni.diffusion.attention.backends.abstract import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, +) + +logger = init_logger(__name__) + +try: + # only tested with flash_attn v3 + # from flash_attn_interface import flash_attn_func as flash_attn_3_func # not available in flash-attn 2.8.1 + from flash_attn import flash_attn_func # can be FA2 or FA3 +except ImportError: + logger.warning( + "FlashAttentionBackend is not available. You may install flash-attn " + "by running `uv pip install flash-attn==2.8.1 --no-build-isolation`" + " or install pre-built flash-attn from https://github.com/Dao-AILab/flash-attention/releases" + ) + flash_attn_3_func = None + + +class FlashAttentionBackend(AttentionBackend): + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [64, 96, 128, 192, 256] + + @staticmethod + def get_name() -> str: + return "FLASH_ATTN" + + @staticmethod + def get_impl_cls() -> type["FlashAttentionImpl"]: + return FlashAttentionImpl + + +class FlashAttentionImpl(AttentionImpl): + def __init__( + self, + num_heads: int, + head_size: int, + softmax_scale: float, + causal: bool = False, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.num_heads = num_heads + self.causal = causal + self.softmax_scale = softmax_scale + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata = None, + ) -> torch.Tensor: + # TODO: flash_attn_func does not support attn_mask. + out: torch.Tensor = flash_attn_func( + query, + key, + value, + causal=self.causal, + softmax_scale=self.softmax_scale, + ) + return out diff --git a/vllm_omni/diffusion/attention/backends/sdpa.py b/vllm_omni/diffusion/attention/backends/sdpa.py new file mode 100644 index 000000000..89b3e84a7 --- /dev/null +++ b/vllm_omni/diffusion/attention/backends/sdpa.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from vllm.logger import init_logger + +from vllm_omni.diffusion.attention.backends.abstract import ( + AttentionBackend, + AttentionImpl, + AttentionMetadata, +) + +logger = init_logger(__name__) + + +class SDPABackend(AttentionBackend): + accept_output_buffer: bool = True + + @staticmethod + def get_supported_head_sizes() -> list[int]: + return [x for x in range(1024)] # todo + + @staticmethod + def get_name() -> str: + return "SDPA" + + @staticmethod + def get_impl_cls() -> type["SDPAImpl"]: + return SDPAImpl + + +class SDPAImpl(AttentionImpl): + def __init__( + self, + num_heads: int, + head_size: int, + softmax_scale: float, + causal: bool = False, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.causal = causal + self.softmax_scale = softmax_scale + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata = None, + ) -> torch.Tensor: + query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value)) + attention_mask = attn_metadata.attn_mask if attn_metadata else None + + output = torch.nn.functional.scaled_dot_product_attention( + query, + key, + value, + attn_mask=attention_mask, + dropout_p=0.0, + is_causal=self.causal, + scale=self.softmax_scale, + ) + out = output.permute(0, 2, 1, 3) + return out diff --git a/vllm_omni/diffusion/attention/layer.py b/vllm_omni/diffusion/attention/layer.py new file mode 100644 index 000000000..85d3da802 --- /dev/null +++ b/vllm_omni/diffusion/attention/layer.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +import torch.nn as nn + +from vllm_omni.diffusion.attention.backends.abstract import ( + AttentionMetadata, +) +from vllm_omni.diffusion.attention.selector import get_attn_backend + + +class Attention(nn.Module): + def __init__( + self, + num_heads: int, + head_size: int, + causal: bool, + softmax_scale: float, + num_kv_heads: int | None = None, + prefix: str = "", + ): + super().__init__() + self.attn_backend = get_attn_backend(-1) + self.attn_impl_cls = self.attn_backend.get_impl_cls() + self.attention = self.attn_impl_cls( + num_heads=num_heads, + head_size=head_size, + softmax_scale=softmax_scale, + causal=causal, + num_kv_heads=num_kv_heads, + ) + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata = None, + ) -> torch.Tensor: + # shape: (batch_size, seq_len, num_heads, head_size) + attn_output = self.attention.forward(query, key, value, attn_metadata) + return attn_output diff --git a/vllm_omni/diffusion/attention/selector.py b/vllm_omni/diffusion/attention/selector.py new file mode 100644 index 000000000..20ee4eca9 --- /dev/null +++ b/vllm_omni/diffusion/attention/selector.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import os +from functools import cache + +from vllm.logger import init_logger + +from vllm_omni.diffusion.attention.backends.abstract import ( + AttentionBackend, +) +from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionBackend +from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend + +logger = init_logger(__name__) + +# environment variable value -> backend class +SUPPORTED_BACKENDS = { + "FLASH_ATTN": FlashAttentionBackend, + "TORCH_SDPA": SDPABackend, +} + + +@cache +def get_attn_backend(head_size: int) -> type[AttentionBackend]: + """ + Get attention backend for diffusion models. + + The backend is selected based on the following priority: + 1. DIFFUSION_ATTENTION_BACKEND environment variable (if set, e.g. export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN) + 2. Default backend (SDPA) + + Args: + head_size: Head size (currently not used for selection, but kept for API compatibility) + + Returns: + The selected attention backend class + """ + # Check environment variable + backend_name: str | None = os.environ.get("DIFFUSION_ATTENTION_BACKEND") + + if backend_name is not None: + backend_name_upper = backend_name.upper() + if backend_name_upper not in SUPPORTED_BACKENDS: + valid_backends = list(SUPPORTED_BACKENDS.keys()) + raise ValueError( + f"Invalid attention backend for diffusion: '{backend_name}'. Valid backends are: {valid_backends}" + ) + logger.info(f"Using attention backend '{backend_name_upper}' for diffusion") + return SUPPORTED_BACKENDS[backend_name_upper] + + # Default to SDPA + return SDPABackend diff --git a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py index b2f4e35bd..33df8265c 100644 --- a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py +++ b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py @@ -8,7 +8,6 @@ from typing import Any, Optional, Union import torch import torch.nn as nn from diffusers.models.attention import FeedForward -from diffusers.models.attention_dispatch import dispatch_attention_fn # TODO replace this with vLLM implementation from diffusers.models.embeddings import TimestepEmbedding, Timesteps @@ -18,6 +17,8 @@ from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear +from vllm_omni.diffusion.attention.layer import Attention + logger = init_logger(__name__) @@ -262,12 +263,12 @@ class QwenImageCrossAttention(nn.Module): self.norm_added_q = RMSNorm(head_dim, eps=eps) self.norm_added_k = RMSNorm(head_dim, eps=eps) - # self.attn = Attention( - # num_heads=num_heads, - # head_size=self.head_dim, - # scale=1.0, # softmax_scale - # attn_type=AttentionType.ENCODER, - # ) + self.attn = Attention( + num_heads=num_heads, + head_size=self.head_dim, + softmax_scale=1.0 / (self.head_dim**0.5), + causal=False, + ) def forward( self, @@ -316,20 +317,11 @@ class QwenImageCrossAttention(nn.Module): joint_value = torch.cat([txt_value, img_value], dim=1) # Compute joint attention - # joint_hidden_states = self.attn( - # joint_query, - # joint_key, - # joint_value, - # ) - joint_hidden_states = dispatch_attention_fn( + joint_hidden_states = self.attn( joint_query, joint_key, joint_value, - dropout_p=0.0, - is_causal=False, ) - - # Reshape back joint_hidden_states = joint_hidden_states.flatten(2, 3) joint_hidden_states = joint_hidden_states.to(joint_query.dtype) @@ -368,19 +360,6 @@ class QwenImageTransformerBlock(nn.Module): nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2 ) self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) - # self.attn = Attention( - # query_dim=dim, - # cross_attention_dim=None, # Enable cross attention for joint computation - # added_kv_proj_dim=dim, # Enable added KV projections for text stream - # dim_head=attention_head_dim, - # heads=num_attention_heads, - # out_dim=dim, - # context_pre_only=False, - # bias=True, - # processor=QwenDoubleStreamAttnProcessor2_0(), - # qk_norm=qk_norm, - # eps=eps, - # ) self.attn = QwenImageCrossAttention( dim=dim, num_heads=num_attention_heads, diff --git a/vllm_omni/diffusion/models/z_image/z_image_transformer.py b/vllm_omni/diffusion/models/z_image/z_image_transformer.py index a4f0ffe94..e5e300a5e 100644 --- a/vllm_omni/diffusion/models/z_image/z_image_transformer.py +++ b/vllm_omni/diffusion/models/z_image/z_image_transformer.py @@ -22,10 +22,11 @@ from typing import Optional import torch import torch.nn as nn import torch.nn.functional as F -from diffusers.models.attention_dispatch import dispatch_attention_fn from torch.nn.utils.rnn import pad_sequence from vllm.model_executor.layers.layernorm import RMSNorm +from vllm_omni.diffusion.attention.layer import Attention + ADALN_EMBED_DIM = 256 SEQ_MULTI_OF = 32 @@ -105,6 +106,13 @@ class ZImageAttention(nn.Module): self.to_out = nn.ModuleList([nn.Linear(dim, dim, bias=False)]) + self.attn = Attention( + num_heads=num_heads, + head_size=self.head_dim, + softmax_scale=1.0 / (self.head_dim**0.5), + causal=False, + ) + def forward( self, hidden_states: torch.Tensor, @@ -143,13 +151,11 @@ class ZImageAttention(nn.Module): attention_mask = attention_mask[:, None, None, :] # Compute joint attention - hidden_states = dispatch_attention_fn( + hidden_states = self.attn( query, key, value, - attn_mask=attention_mask, - dropout_p=0.0, - is_causal=False, + # attn_mask=attention_mask, # we don't support multi prompts now. ) # Reshape back