[Feature] Add AllGather-KV sequence-parallel attention str… (#4968)

Signed-off-by: zhuhh97 <zhuhonghan@huawei.com>
Signed-off-by: dph97 <nenbaying@163.com>
Signed-off-by: zhuhh97 <zhh@unknown>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: dengyunyang <584797741@qq.com>
This commit is contained in:
authored and GitHub committed 2026-07-24 14:45:26 +00:00
1 parent 8927ec057c
commit 84bbd57e4d
33 files changed
+1181 -135

No files matched your search

@@ -62,6 +62,7 @@ from vllm_omni.diffusion.attention.backends.abstract import (
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.distributed.parallel_state import (
get_allgather_parallel_world_size,
get_cfg_group,
get_classifier_free_guidance_rank,
get_classifier_free_guidance_world_size,
@@ -78,6 +79,7 @@ from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step
from vllm_omni.diffusion.layers.norm import RMSNorm
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
from vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe import HunyuanFusedMoE
from vllm_omni.diffusion.utils.kv_utils import repeat_kv
from vllm_omni.model_executor.layers.timestep_embedding import timestep_embedding
logger = logging.getLogger(__name__)
@@ -426,22 +428,6 @@ def apply_rotary_pos_emb(
return q_embed, k_embed
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
"""
Equivalent to torch.repeat_interleave(x, dim=2, repeats=n_rep).
Input: (batch, seqlen, num_key_value_heads, head_dim)
Output: (batch, seqlen, num_attention_heads, head_dim)
"""
batch, slen, num_key_value_heads, head_dim = hidden_states.shape
if n_rep == 1:
return hidden_states
hidden_states = hidden_states[:, :, :, None, :].expand(batch, slen, num_key_value_heads, n_rep, head_dim)
return hidden_states.reshape(batch, slen, num_key_value_heads * n_rep, head_dim)
def default(value, default_value):
return value if value is not None else default_value
@@ -957,6 +943,7 @@ class ImageKVCacheManager:
self._injected_ar_kv: list[tuple[torch.Tensor, torch.Tensor]] | None = None
self.sp_size = get_sequence_parallel_world_size()
self.allgather_size = get_allgather_parallel_world_size()
self.sp_rank = get_sequence_parallel_rank()
self.attn = Attention(
num_heads=self.num_heads,
@@ -1130,6 +1117,7 @@ class ImageKVCacheManager:
head_num_per_rank = query.shape[1]
kv_head_num_per_rank = key.shape[1]
repeat_num = head_num_per_rank // kv_head_num_per_rank
keep_kv_compressed = self.allgather_size > 1
head_dim = query.shape[2]
query = query.reshape(bs, q_len, head_num_per_rank, head_dim)
@@ -1171,11 +1159,12 @@ class ImageKVCacheManager:
joint_text_query = query[:, :0, :, :]
joint_text_key, joint_text_value = self._reuse_prompt_kv(key, value, seq_len, bs, shard_image_size)
key = repeat_kv(key, repeat_num)
value = repeat_kv(value, repeat_num)
if self.sp_size > 1:
joint_text_key = repeat_kv(joint_text_key, repeat_num)
joint_text_value = repeat_kv(joint_text_value, repeat_num)
if not keep_kv_compressed:
key = repeat_kv(key, repeat_num)
value = repeat_kv(value, repeat_num)
if self.sp_size > 1:
joint_text_key = repeat_kv(joint_text_key, repeat_num)
joint_text_value = repeat_kv(joint_text_value, repeat_num)
attention_mask = attention_mask.contiguous()