[Diffusion] Add paged KV cache support for HunyuanImage3 DiT on NPU and GPU (#6563)

Signed-off-by: Acerak01-fy <wfy2003324@163.com>
Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
Signed-off-by: zwhzzz0821 <2831474076@qq.com>
Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
Co-authored-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
authored and GitHub committed 2026-08-31 10:15:46 +00:00
1 parent 6f16f9546d
commit fe4a2a7477
28 files changed
+1789 -205

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import inspect
import logging
@@ -1069,18 +1069,98 @@ class ImageKVCacheManager(nn.Module):
)
return key, value
def forward(
def clear_legacy_prompt_kv_cache(self) -> None:
"""Drop model-owned dense KV state before Scheduler-paged execution."""
self.image_kv_cache_map = None
self.image_kv_cache_lens = None
def _forward_paged(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor | None = None,
*,
first_step: bool,
query_lens: list[int],
seq_lens: list[int],
shard_image_size: int | None,
full_attn_spans: list[list[tuple[int, int]]],
uncond_cfg_prefill: bool,
) -> torch.Tensor:
"""Run Hunyuan attention against Worker-managed Scheduler pages.
This model boundary is shared by GPU and NPU: it describes Hunyuan's
Q/K/V layout and mixed-attention spans, while the common Attention
layer resolves the actual CUDA or Ascend paged kernel.
"""
if uncond_cfg_prefill:
raise RuntimeError("Hunyuan negative-CFG prefill must run before paged row activation")
if self._injected_ar_kv is not None:
raise NotImplementedError("Hunyuan Scheduler-paged KV does not support imported AR KV")
if not query_lens or len(seq_lens) != len(query_lens):
raise ValueError("Hunyuan Scheduler-paged KV requires matching query and sequence lengths")
if len(full_attn_spans) != len(query_lens):
raise ValueError("Hunyuan Scheduler-paged KV requires one full-attention span row per sequence")
self.clear_legacy_prompt_kv_cache()
bs = len(query_lens)
q_len = query_lens[0]
seq_len = seq_lens[0]
assert query.shape[0] == bs * q_len, f"{query.shape[0]} != {bs * q_len}"
head_num_per_rank = query.shape[1]
kv_head_num_per_rank = key.shape[1]
head_dim = query.shape[2]
query = query.reshape(bs, q_len, head_num_per_rank, head_dim)
key = key.reshape(bs, q_len, kv_head_num_per_rank, head_dim)
value = value.reshape(bs, q_len, kv_head_num_per_rank, head_dim)
joint_text_query = joint_text_key = joint_text_value = None
if self.sp_size > 1 and first_step:
if shard_image_size is None or shard_image_size <= 0:
raise ValueError("Hunyuan paged Ulysses requires a positive local image shard size")
local_prompt_len = seq_len - shard_image_size
joint_query_len = query.shape[1] - shard_image_size
if local_prompt_len != joint_query_len:
raise ValueError(
"Hunyuan paged Ulysses prompt layout mismatch: "
f"key_prompt={local_prompt_len}, query_prompt={joint_query_len}"
)
joint_text_query = query[:, :joint_query_len]
joint_text_key = key[:, :local_prompt_len]
joint_text_value = value[:, :local_prompt_len]
query = query[:, joint_query_len:]
key = key[:, local_prompt_len:]
value = value[:, local_prompt_len:]
if joint_text_query is None:
attn_metadata = AttentionMetadata(full_attn_spans=full_attn_spans)
else:
attn_metadata = AttentionMetadata(
joint_query=joint_text_query,
joint_key=joint_text_key,
joint_value=joint_text_value,
joint_strategy="front",
full_attn_spans=full_attn_spans,
)
attn_output = self.attn(query, key, value, attn_metadata)
return attn_output.reshape(bs * q_len, head_num_per_rank, head_dim)
def _forward_dense_legacy(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor,
**kwargs,
) -> torch.Tensor:
self.image_token_len = kwargs.get("num_image_tokens")
"""Run the device-neutral, model-owned dense prompt-KV compatibility path."""
first_step = kwargs.get("first_step")
uncond_cfg_prefill = kwargs.get("uncond_cfg_prefill", False)
query_lens = kwargs.get("query_lens")
seq_lens = kwargs.get("seq_lens")
shard_image_size = kwargs.get("shard_image_size") if self.sp_size > 1 else None
@@ -1109,8 +1189,7 @@ class ImageKVCacheManager(nn.Module):
key = key[:, :0, :, :]
value = value[:, :0, :, :]
elif first_step:
self.image_kv_cache_map = None # reset first
self.image_kv_cache_lens = None
self.clear_legacy_prompt_kv_cache()
key, value = self._cache_prompt_kv(
key,
value,
@@ -1134,7 +1213,7 @@ class ImageKVCacheManager(nn.Module):
joint_text_query = query[:, :0, :, :]
joint_text_key, joint_text_value = self._reuse_prompt_kv(key, value, seq_len, bs, shard_image_size)
if not keep_kv_compressed and not self.attn.is_paged_kv_active():
if not keep_kv_compressed:
key = repeat_kv(key, repeat_num)
value = repeat_kv(value, repeat_num)
if self.sp_size > 1:
@@ -1163,6 +1242,34 @@ class ImageKVCacheManager(nn.Module):
attn_output = attn_output.reshape(bs * q_len, head_num_per_rank, head_dim)
return attn_output
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
self.image_token_len = kwargs.get("num_image_tokens")
if self.attn.is_paged_kv_active():
full_attn_spans = kwargs.get("full_attn_spans")
if full_attn_spans is None:
raise ValueError("Hunyuan Scheduler-paged KV requires full_attn_spans metadata")
return self._forward_paged(
query,
key,
value,
first_step=bool(kwargs.get("first_step")),
query_lens=kwargs.get("query_lens"),
seq_lens=kwargs.get("seq_lens"),
shard_image_size=kwargs.get("shard_image_size") if self.sp_size > 1 else None,
full_attn_spans=full_attn_spans,
uncond_cfg_prefill=kwargs.get("uncond_cfg_prefill", False),
)
if attention_mask is None:
raise ValueError("Hunyuan dense attention requires an attention mask")
return self._forward_dense_legacy(query, key, value, attention_mask, **kwargs)
@dataclass
class CausalMMOutputWithPast(CausalLMOutputWithPast):
@@ -2706,7 +2813,6 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
# List[List[...]] per-sample metadata indexed along the CFG batch dim
if isinstance(model_kwargs.get("full_attn_spans"), list):
model_kwargs["full_attn_spans"] = model_kwargs["full_attn_spans"][s.start : s.stop]
# custom_pos_emb: tuple of (cos, sin)
if "custom_pos_emb" in model_kwargs and model_kwargs["custom_pos_emb"] is not None:
cos, sin = model_kwargs["custom_pos_emb"]
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import copy
import logging
@@ -72,7 +72,6 @@ _STEP_OUTPUT_SIZE = "hunyuan_output_size"
_STEP_COT_TEXT_LIST = "hunyuan_cot_text_list"
_STEP_AR_KV = "hunyuan_ar_kv"
_STEP_PROMPT_KV = "hunyuan_prompt_kv"
_HUNYUAN_DEFAULT_OUTPUT_TYPE = "pil"
@@ -466,6 +465,11 @@ class HunyuanImage3Pipeline(
return self._pipeline
def _validate_step_request(self, state: "StepRequestState") -> None:
if self._uses_scheduler_paged_kv():
raise ValueError(
"HunyuanImage3 paged_scheduler currently supports request-level execution only; "
"disable step execution or use dense_legacy."
)
prompt = state.prompt
sampling = state.sampling
if prompt is None:
@@ -663,6 +667,12 @@ class HunyuanImage3Pipeline(
prefix_lens.append(int(prompt_kv[0]["lens"][branch].item()))
return prefix_lens
def _uses_scheduler_paged_kv(self) -> bool:
return (
getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)
is DiffusionKVCacheMode.PAGED_SCHEDULER
)
def _merge_step_model_inputs(
self,
states: list["StepRequestState"],
@@ -693,6 +703,7 @@ class HunyuanImage3Pipeline(
values,
row_branches,
prefix_lens or [None] * len(row_branches),
strict=True,
)
]
continue
@@ -1871,8 +1882,7 @@ class HunyuanImage3Pipeline(
row_branches = [branch for branch in range(cfg_factor) for _ in states]
return row_state_indexes, row_branches
@staticmethod
def _validate_step_group_states(states: list["StepRequestState"]) -> tuple[bool, int]:
def _validate_step_group_states(self, states: list["StepRequestState"]) -> tuple[bool, int]:
if not states:
raise ValueError("HunyuanImage3 denoise_step received an empty group.")
@@ -2035,6 +2045,11 @@ class HunyuanImage3Pipeline(
states = list(input_batch.states)
if not states:
raise ValueError("HunyuanImage3 denoise_step received an empty batch.")
if self._uses_scheduler_paged_kv():
raise ValueError(
"HunyuanImage3 paged_scheduler currently supports request-level execution only; "
"disable step execution or use dense_legacy."
)
self._ensure_grouped_attention_backend_supported(len(states))
outputs: dict[str, torch.Tensor] = {}
for group in self._split_step_groups(states):