[Diffusion][Bug-Fix]: Fix Ulysses SP Accurary Error (#377)
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
1 parent
b06a5148f6
commit
ee2b78e826
12 files changed
+747
-199
No files matched your search
@@ -48,6 +48,14 @@ class AttentionBackend(ABC):
|
||||
@dataclass
|
||||
class AttentionMetadata:
|
||||
attn_mask: torch.Tensor | None = None
|
||||
joint_query: torch.Tensor | None = None
|
||||
# a replicated tensor among processes appended to the front or rear of query, depends the joint_strategy
|
||||
joint_key: torch.Tensor | None = None
|
||||
# a replicated tensor among processes appended to the front or rear of key, depends the joint_strategy
|
||||
joint_value: torch.Tensor | None = None
|
||||
# a replicated tensor among processes appended to the front or rear of value, depends the joint_strategy
|
||||
joint_strategy: str = "front"
|
||||
# the strategy to joint the query, key, and value, can be "front" or "rear"
|
||||
|
||||
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
@@ -7,17 +7,11 @@
|
||||
# https://github.com/feifeibear/long-context-attention/blob/main/yunchang/attention/layer.py
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.parallel import build_parallel_attention_strategy
|
||||
from vllm_omni.diffusion.attention.selector import get_attn_backend
|
||||
from vllm_omni.diffusion.data import get_current_omni_diffusion_config
|
||||
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_sequence_parallel_world_size, get_sp_group
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
@@ -49,25 +43,13 @@ class Attention(nn.Module):
|
||||
self.scatter_idx = scatter_idx
|
||||
self.gather_idx = gather_idx
|
||||
self.use_sync = use_sync
|
||||
self.ring_pg: dist.ProcessGroup | None = None
|
||||
self.ulysses_pg: dist.ProcessGroup | None = None
|
||||
self.use_ulysses = False
|
||||
|
||||
try:
|
||||
config = get_current_omni_diffusion_config()
|
||||
if config.parallel_config.ulysses_degree > 1:
|
||||
self.use_ulysses = True
|
||||
# Get sequence parallel process group
|
||||
try:
|
||||
sp_group = get_sp_group()
|
||||
self.ring_pg = sp_group.ring_group
|
||||
self.ulysses_pg = sp_group.ulysses_group
|
||||
assert get_sequence_parallel_world_size() > 1, "Sequence parallel world size must be > 1"
|
||||
except (AssertionError, RuntimeError):
|
||||
# If sequence parallel group is not initialized, disable Ulysses
|
||||
self.use_ulysses = False
|
||||
except Exception:
|
||||
self.use_ulysses = False
|
||||
# Parallel attention (communication / resharding) is a pluggable strategy.
|
||||
# This keeps the attention kernel backend selection orthogonal.
|
||||
self.parallel = build_parallel_attention_strategy(
|
||||
scatter_idx=scatter_idx,
|
||||
gather_idx=gather_idx,
|
||||
use_sync=use_sync,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -76,43 +58,13 @@ class Attention(nn.Module):
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
if self.use_ulysses:
|
||||
return self._forward_ulysses(query, key, value, attn_metadata)
|
||||
else:
|
||||
# shape: (batch_size, seq_len, num_heads, head_size)
|
||||
attn_output = self.attention.forward(query, key, value, attn_metadata)
|
||||
return attn_output
|
||||
# Parallel strategy may reshard/communicate QKV before the attention kernel.
|
||||
query, key, value, attn_metadata, ctx = self.parallel.pre_attention(query, key, value, attn_metadata)
|
||||
|
||||
def _forward_ulysses(
|
||||
self,
|
||||
query: Tensor,
|
||||
key: Tensor,
|
||||
value: Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> Tensor:
|
||||
"""Ulysses attention forward pass with sequence parallelism."""
|
||||
# scatter 2, gather 1
|
||||
# (bs, seq_len/N, head_cnt, head_size) -> (bs, seq_len, head_cnt/N, head_size)
|
||||
q = SeqAllToAll4D.apply(self.ulysses_pg, query, self.scatter_idx, self.gather_idx, self.use_sync)
|
||||
k = SeqAllToAll4D.apply(self.ulysses_pg, key, self.scatter_idx, self.gather_idx, self.use_sync)
|
||||
v = SeqAllToAll4D.apply(self.ulysses_pg, value, self.scatter_idx, self.gather_idx, self.use_sync)
|
||||
# shape: (batch_size, seq_len, num_heads, head_size)
|
||||
attn_output = self.attention.forward(query, key, value, attn_metadata)
|
||||
if isinstance(attn_output, tuple):
|
||||
attn_output = attn_output[0]
|
||||
|
||||
softmax_scale = self.softmax_scale
|
||||
if softmax_scale is None:
|
||||
softmax_scale = q.shape[-1] ** -0.5
|
||||
|
||||
context_layer = self.attention.forward(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
|
||||
if isinstance(context_layer, tuple):
|
||||
context_layer = context_layer[0]
|
||||
|
||||
# (bs, seq_len, head_cnt/N, head_size) -> (bs, seq_len/N, head_cnt, head_size)
|
||||
# scatter 1, gather 2
|
||||
output = SeqAllToAll4D.apply(self.ulysses_pg, context_layer, self.gather_idx, self.scatter_idx, self.use_sync)
|
||||
|
||||
return output
|
||||
# Parallel strategy may need to reverse resharding after the kernel.
|
||||
return self.parallel.post_attention(attn_output, ctx)
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Parallel attention strategies.
|
||||
|
||||
This package provides **communication / resharding strategies** for attention,
|
||||
orthogonal to the **attention kernel backend** (SDPA/Flash/Sage).
|
||||
|
||||
The goal is to keep `vllm_omni.diffusion.attention.layer.Attention` small and
|
||||
extensible: adding a new parallelism method should not require editing the core
|
||||
Attention module, only adding a new strategy and selecting it in the factory.
|
||||
"""
|
||||
|
||||
from .base import NoParallelAttention, ParallelAttentionContext, ParallelAttentionStrategy
|
||||
from .factory import build_parallel_attention_strategy
|
||||
|
||||
__all__ = [
|
||||
"ParallelAttentionStrategy",
|
||||
"ParallelAttentionContext",
|
||||
"NoParallelAttention",
|
||||
"build_parallel_attention_strategy",
|
||||
]
|
||||
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ParallelAttentionContext:
|
||||
"""Opaque per-forward context returned by a parallel strategy.
|
||||
|
||||
Strategies may stash whatever they need here to finish post-processing after
|
||||
the attention kernel runs (e.g. reverse resharding, slicing metadata, etc.).
|
||||
"""
|
||||
|
||||
name: str
|
||||
|
||||
|
||||
class ParallelAttentionStrategy(Protocol):
|
||||
"""Pluggable strategy for parallel attention communication/resharding.
|
||||
|
||||
This is intentionally orthogonal to the attention *kernel* backend.
|
||||
The kernel backend implements `AttentionImpl.forward()` for a given device,
|
||||
while the parallel strategy implements how Q/K/V and outputs are sharded /
|
||||
communicated across ranks.
|
||||
"""
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool: ...
|
||||
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
|
||||
def pre_attention(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, AttentionMetadata | None, ParallelAttentionContext | None]:
|
||||
"""Runs before the attention kernel.
|
||||
|
||||
Returns possibly transformed Q/K/V and metadata, and an optional context
|
||||
for `post_attention`.
|
||||
"""
|
||||
|
||||
def post_attention(
|
||||
self,
|
||||
attn_output: torch.Tensor,
|
||||
ctx: ParallelAttentionContext | None,
|
||||
) -> torch.Tensor:
|
||||
"""Runs after the attention kernel."""
|
||||
|
||||
|
||||
class NoParallelAttention:
|
||||
"""Default strategy: do nothing (single device / no SP)."""
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "none"
|
||||
|
||||
def pre_attention(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata | None,
|
||||
):
|
||||
return query, key, value, attn_metadata, None
|
||||
|
||||
def post_attention(self, attn_output: torch.Tensor, ctx: ParallelAttentionContext | None) -> torch.Tensor:
|
||||
return attn_output
|
||||
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from vllm_omni.diffusion.attention.parallel.base import NoParallelAttention, ParallelAttentionStrategy
|
||||
from vllm_omni.diffusion.attention.parallel.ulysses import UlyssesParallelAttention
|
||||
from vllm_omni.diffusion.data import get_current_omni_diffusion_config
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
|
||||
|
||||
|
||||
def build_parallel_attention_strategy(
|
||||
*,
|
||||
scatter_idx: int,
|
||||
gather_idx: int,
|
||||
use_sync: bool,
|
||||
) -> ParallelAttentionStrategy:
|
||||
"""Select a parallel attention strategy based on current diffusion config.
|
||||
|
||||
Design principle:
|
||||
- Attention kernel backend selection remains in `attention/selector.py`.
|
||||
- Parallel attention selection is handled here, based on distributed config
|
||||
and initialized process groups.
|
||||
"""
|
||||
try:
|
||||
cfg = get_current_omni_diffusion_config()
|
||||
p = cfg.parallel_config
|
||||
except Exception:
|
||||
return NoParallelAttention()
|
||||
|
||||
# Current implementation supports Ulysses sequence-parallel attention.
|
||||
# We intentionally do NOT infer ring attention from ring_degree here yet.
|
||||
if getattr(p, "ulysses_degree", 1) > 1:
|
||||
try:
|
||||
sp_group = get_sp_group()
|
||||
except Exception:
|
||||
return NoParallelAttention()
|
||||
return UlyssesParallelAttention(
|
||||
sp_group=sp_group,
|
||||
scatter_idx=scatter_idx,
|
||||
gather_idx=gather_idx,
|
||||
use_sync=use_sync,
|
||||
)
|
||||
|
||||
return NoParallelAttention()
|
||||
@@ -0,0 +1,135 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.parallel.base import ParallelAttentionContext
|
||||
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
|
||||
from vllm_omni.diffusion.distributed.group_coordinator import SequenceParallelGroupCoordinator
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _UlyssesCtx(ParallelAttentionContext):
|
||||
"""Per-forward context for Ulysses sequence-parallel attention."""
|
||||
|
||||
ulysses_pg: dist.ProcessGroup
|
||||
scatter_idx: int
|
||||
gather_idx: int
|
||||
use_sync: bool
|
||||
|
||||
|
||||
class UlyssesParallelAttention:
|
||||
"""Ulysses sequence-parallel strategy (all-to-all over seq/head dims).
|
||||
|
||||
This preserves the semantics previously implemented in
|
||||
`Attention._forward_ulysses`:
|
||||
- If `AttentionMetadata.joint_*` is provided, joint_query is concatenated to
|
||||
query *before* all-to-all; joint_key/value are concatenated *after* all-to-all.
|
||||
- joint_key/value are assumed to be replicated across SP ranks and are sliced
|
||||
by ulysses head rank before concatenation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sp_group: SequenceParallelGroupCoordinator,
|
||||
scatter_idx: int,
|
||||
gather_idx: int,
|
||||
use_sync: bool,
|
||||
) -> None:
|
||||
self._sp_group = sp_group
|
||||
self._ulysses_pg = sp_group.ulysses_group
|
||||
self._scatter_idx = scatter_idx
|
||||
self._gather_idx = gather_idx
|
||||
self._use_sync = use_sync
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "ulysses"
|
||||
|
||||
def pre_attention(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata | None,
|
||||
):
|
||||
joint_tensor_query = joint_tensor_key = joint_tensor_value = None
|
||||
joint_strategy = None
|
||||
|
||||
if attn_metadata is not None:
|
||||
joint_tensor_query = attn_metadata.joint_query
|
||||
joint_tensor_key = attn_metadata.joint_key
|
||||
joint_tensor_value = attn_metadata.joint_value
|
||||
joint_strategy = attn_metadata.joint_strategy
|
||||
|
||||
is_joint = False
|
||||
if joint_tensor_query is not None and joint_tensor_key is not None and joint_tensor_value is not None:
|
||||
supported_joint_strategy = ["front", "rear"]
|
||||
if joint_strategy not in supported_joint_strategy:
|
||||
raise ValueError(
|
||||
f"joint_strategy: {joint_strategy} not supported."
|
||||
f" supported joint strategy: {supported_joint_strategy}"
|
||||
)
|
||||
if joint_strategy == "rear":
|
||||
query = torch.cat([query, joint_tensor_query], dim=1)
|
||||
else:
|
||||
query = torch.cat([joint_tensor_query, query], dim=1)
|
||||
is_joint = True
|
||||
elif joint_tensor_query is None and joint_tensor_key is None and joint_tensor_value is None:
|
||||
pass
|
||||
else:
|
||||
raise ValueError("joint_query, joint_key, and joint_value should be None or not None simultaneously.")
|
||||
|
||||
if is_joint:
|
||||
# Slice joint key/value heads for this ulysses rank.
|
||||
ulysses_world_size = self._sp_group.ulysses_world_size
|
||||
ulysses_rank = self._sp_group.ulysses_rank
|
||||
attn_heads_per_ulysses_rank = joint_tensor_key.shape[-2] // ulysses_world_size
|
||||
joint_tensor_key = joint_tensor_key[
|
||||
...,
|
||||
attn_heads_per_ulysses_rank * ulysses_rank : attn_heads_per_ulysses_rank * (ulysses_rank + 1),
|
||||
:,
|
||||
]
|
||||
joint_tensor_value = joint_tensor_value[
|
||||
...,
|
||||
attn_heads_per_ulysses_rank * ulysses_rank : attn_heads_per_ulysses_rank * (ulysses_rank + 1),
|
||||
:,
|
||||
]
|
||||
|
||||
# (bs, seq_len/P, head_cnt, head_size) -> (bs, seq_len, head_cnt/P, head_size)
|
||||
query = SeqAllToAll4D.apply(self._ulysses_pg, query, self._scatter_idx, self._gather_idx, self._use_sync)
|
||||
key = SeqAllToAll4D.apply(self._ulysses_pg, key, self._scatter_idx, self._gather_idx, self._use_sync)
|
||||
value = SeqAllToAll4D.apply(self._ulysses_pg, value, self._scatter_idx, self._gather_idx, self._use_sync)
|
||||
|
||||
if is_joint:
|
||||
# Concatenate joint key/value after all-to-all (matches previous implementation).
|
||||
if joint_strategy == "front":
|
||||
key = torch.cat([joint_tensor_key, key], dim=1)
|
||||
value = torch.cat([joint_tensor_value, value], dim=1)
|
||||
else: # "rear"
|
||||
key = torch.cat([key, joint_tensor_key], dim=1)
|
||||
value = torch.cat([value, joint_tensor_value], dim=1)
|
||||
|
||||
ctx = _UlyssesCtx(
|
||||
name=self.name,
|
||||
ulysses_pg=self._ulysses_pg,
|
||||
scatter_idx=self._scatter_idx,
|
||||
gather_idx=self._gather_idx,
|
||||
use_sync=self._use_sync,
|
||||
)
|
||||
return query, key, value, attn_metadata, ctx
|
||||
|
||||
def post_attention(self, attn_output: torch.Tensor, ctx: ParallelAttentionContext | None) -> torch.Tensor:
|
||||
assert isinstance(ctx, _UlyssesCtx), f"Unexpected ctx type: {type(ctx)!r}"
|
||||
# Reverse: (bs, seq_len, head_cnt/P, head_size) -> (bs, seq_len/P, head_cnt, head_size)
|
||||
return SeqAllToAll4D.apply(ctx.ulysses_pg, attn_output, ctx.gather_idx, ctx.scatter_idx, ctx.use_sync)
|
||||
@@ -0,0 +1,90 @@
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class ForwardContext:
|
||||
"""
|
||||
set forward context for diffusion models
|
||||
"""
|
||||
|
||||
vllm_config: VllmConfig | None = None
|
||||
omni_diffusion_config: OmniDiffusionConfig | None = None
|
||||
attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None = None
|
||||
split_text_embed_in_sp: bool = False
|
||||
# whether to split the text embed in sequence parallel, if True, the text embed will be split in sequence parallel
|
||||
|
||||
def __post_init__(self):
|
||||
pass
|
||||
|
||||
|
||||
_forward_context: ForwardContext | None = None
|
||||
|
||||
|
||||
def get_forward_context() -> ForwardContext:
|
||||
"""Get the current forward context."""
|
||||
assert _forward_context is not None, (
|
||||
"Forward context is not set. Please use `set_forward_context` to set the forward context."
|
||||
)
|
||||
return _forward_context
|
||||
|
||||
|
||||
def is_forward_context_available() -> bool:
|
||||
return _forward_context is not None
|
||||
|
||||
|
||||
def create_forward_context(
|
||||
vllm_config: VllmConfig | None = None,
|
||||
omni_diffusion_config: OmniDiffusionConfig | None = None,
|
||||
attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None = None,
|
||||
split_text_embed_in_sp: bool = False,
|
||||
):
|
||||
return ForwardContext(
|
||||
vllm_config=vllm_config,
|
||||
omni_diffusion_config=omni_diffusion_config,
|
||||
attn_metadata=attn_metadata,
|
||||
split_text_embed_in_sp=split_text_embed_in_sp,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def override_forward_context(forward_context: ForwardContext | None):
|
||||
"""A context manager that overrides the current forward context.
|
||||
This is used to override the forward context for a specific
|
||||
forward pass.
|
||||
"""
|
||||
global _forward_context
|
||||
prev_context = _forward_context
|
||||
_forward_context = forward_context
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_forward_context = prev_context
|
||||
|
||||
|
||||
@contextmanager
|
||||
def set_forward_context(
|
||||
vllm_config: VllmConfig | None = None,
|
||||
omni_diffusion_config: OmniDiffusionConfig | None = None,
|
||||
attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None = None,
|
||||
split_text_embed_in_sp: bool = False,
|
||||
):
|
||||
"""A context manager that stores the current forward context,
|
||||
can be attention metadata, split_text_embed_in_sp, etc.
|
||||
Here we can inject common logic for every model forward pass.
|
||||
"""
|
||||
forward_context = create_forward_context(
|
||||
vllm_config=vllm_config,
|
||||
omni_diffusion_config=omni_diffusion_config,
|
||||
attn_metadata=attn_metadata,
|
||||
split_text_embed_in_sp=split_text_embed_in_sp,
|
||||
)
|
||||
with override_forward_context(forward_context):
|
||||
yield
|
||||
@@ -19,14 +19,18 @@ from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.base import CachedTransformer
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, get_current_omni_diffusion_config
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_sequence_parallel_rank,
|
||||
get_sequence_parallel_world_size,
|
||||
get_sp_group,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context
|
||||
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -356,6 +360,12 @@ class QwenImageCrossAttention(nn.Module):
|
||||
)
|
||||
self.rope = RotaryEmbedding(is_neox_style=False)
|
||||
|
||||
try:
|
||||
config = get_current_omni_diffusion_config()
|
||||
self.parallel_config = config.parallel_config
|
||||
except Exception:
|
||||
self.parallel_config = None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -405,11 +415,31 @@ class QwenImageCrossAttention(nn.Module):
|
||||
joint_value = torch.cat([txt_value, img_value], dim=1)
|
||||
|
||||
# Compute joint attention
|
||||
joint_hidden_states = self.attn(
|
||||
joint_query,
|
||||
joint_key,
|
||||
joint_value,
|
||||
)
|
||||
|
||||
if (
|
||||
self.parallel_config is not None
|
||||
and self.parallel_config.sequence_parallel_size > 1
|
||||
and not get_forward_context().split_text_embed_in_sp
|
||||
):
|
||||
# if using sequence parallel, but not splitting text embed,
|
||||
# we need to pass text embedding to attention layer as joint qkv
|
||||
joint_hidden_states = self.attn(
|
||||
img_query,
|
||||
img_key,
|
||||
img_value,
|
||||
AttentionMetadata(
|
||||
joint_query=txt_query,
|
||||
joint_key=txt_key,
|
||||
joint_value=txt_value,
|
||||
joint_strategy="front",
|
||||
),
|
||||
)
|
||||
else:
|
||||
joint_hidden_states = self.attn(
|
||||
joint_query,
|
||||
joint_key,
|
||||
joint_value,
|
||||
)
|
||||
joint_hidden_states = joint_hidden_states.flatten(2, 3)
|
||||
joint_hidden_states = joint_hidden_states.to(joint_query.dtype)
|
||||
|
||||
@@ -709,7 +739,10 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
hidden_states = torch.chunk(hidden_states, get_sequence_parallel_world_size(), dim=-2)[
|
||||
get_sequence_parallel_rank()
|
||||
]
|
||||
|
||||
# NOTE:
|
||||
# QwenImage uses *dual-stream* (text + image) and runs a *joint attention*.
|
||||
# text embeddings to be replicated across SP ranks for correctness.
|
||||
get_forward_context().split_text_embed_in_sp = False
|
||||
hidden_states = self.img_in(hidden_states)
|
||||
|
||||
# Ensure timestep tensor is on the same device and dtype as hidden_states
|
||||
@@ -746,6 +779,8 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
if self.parallel_config.sequence_parallel_size > 1:
|
||||
img_freqs, txt_freqs = image_rotary_emb
|
||||
img_freqs = get_rotary_emb_chunk(img_freqs)
|
||||
if get_forward_context().split_text_embed_in_sp:
|
||||
txt_freqs = get_rotary_emb_chunk(txt_freqs)
|
||||
image_rotary_emb = (img_freqs, txt_freqs)
|
||||
|
||||
for index_block, block in enumerate(self.transformer_blocks):
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
|
||||
@@ -65,31 +66,33 @@ class GPUWorker:
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = self.od_config.parallel_config.tensor_parallel_size
|
||||
vllm_config.parallel_config.data_parallel_size = self.od_config.parallel_config.data_parallel_size
|
||||
self.vllm_config = vllm_config
|
||||
with (
|
||||
set_current_omni_diffusion_config(self.od_config),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
init_distributed_environment(world_size=world_size, rank=rank)
|
||||
logger.info(f"Worker {self.rank}: Initialized device and distributed environment.")
|
||||
parallel_config = self.od_config.parallel_config
|
||||
initialize_model_parallel(
|
||||
data_parallel_size=parallel_config.data_parallel_size,
|
||||
cfg_parallel_size=parallel_config.cfg_parallel_size,
|
||||
sequence_parallel_size=parallel_config.sequence_parallel_size,
|
||||
ulysses_degree=parallel_config.ulysses_degree,
|
||||
ring_degree=parallel_config.ring_degree,
|
||||
tensor_parallel_size=parallel_config.tensor_parallel_size,
|
||||
pipeline_parallel_size=parallel_config.pipeline_parallel_size,
|
||||
)
|
||||
|
||||
with set_current_omni_diffusion_config(self.od_config):
|
||||
with set_current_vllm_config(vllm_config):
|
||||
init_distributed_environment(world_size=world_size, rank=rank)
|
||||
logger.info(f"Worker {self.rank}: Initialized device and distributed environment.")
|
||||
parallel_config = self.od_config.parallel_config
|
||||
initialize_model_parallel(
|
||||
data_parallel_size=parallel_config.data_parallel_size,
|
||||
cfg_parallel_size=parallel_config.cfg_parallel_size,
|
||||
sequence_parallel_size=parallel_config.sequence_parallel_size,
|
||||
ulysses_degree=parallel_config.ulysses_degree,
|
||||
ring_degree=parallel_config.ring_degree,
|
||||
tensor_parallel_size=parallel_config.tensor_parallel_size,
|
||||
pipeline_parallel_size=parallel_config.pipeline_parallel_size,
|
||||
load_config = LoadConfig()
|
||||
model_loader = DiffusersPipelineLoader(load_config)
|
||||
time_before_load = time.perf_counter()
|
||||
with DeviceMemoryProfiler() as m:
|
||||
self.pipeline = model_loader.load_model(
|
||||
od_config=self.od_config,
|
||||
load_device=f"cuda:{rank}",
|
||||
)
|
||||
|
||||
load_config = LoadConfig()
|
||||
model_loader = DiffusersPipelineLoader(load_config)
|
||||
time_before_load = time.perf_counter()
|
||||
with DeviceMemoryProfiler() as m:
|
||||
self.pipeline = model_loader.load_model(
|
||||
od_config=self.od_config,
|
||||
load_device=f"cuda:{rank}",
|
||||
)
|
||||
time_after_load = time.perf_counter()
|
||||
time_after_load = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"Model loading took %.4f GiB and %.6f seconds",
|
||||
@@ -116,8 +119,8 @@ class GPUWorker:
|
||||
# Refresh cache context if needed
|
||||
if self.cache_backend is not None and self.cache_backend.is_enabled():
|
||||
self.cache_backend.refresh(self.pipeline, req.num_inference_steps)
|
||||
|
||||
output = self.pipeline.forward(req)
|
||||
with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config):
|
||||
output = self.pipeline.forward(req)
|
||||
return output
|
||||
|
||||
def shutdown(self) -> None:
|
||||
|
||||
Reference in new issue
Block a user