[Diffusion] Add paged KV cache support for HunyuanImage3 DiT on NPU and GPU (#6563)

Signed-off-by: Acerak01-fy <wfy2003324@163.com>
Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
Signed-off-by: zwhzzz0821 <2831474076@qq.com>
Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
Co-authored-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
authored and GitHub committed 2026-08-31 10:15:46 +00:00
1 parent 6f16f9546d
commit fe4a2a7477
28 files changed
+1789 -205

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import os
from functools import partial
@@ -7,11 +7,7 @@ from functools import partial
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionBackend,
AttentionImpl,
AttentionMetadata,
)
from vllm_omni.diffusion.attention.backends.abstract import AttentionBackend, AttentionImpl, AttentionMetadata
from vllm_omni.diffusion.attention.backends.sdpa import _maybe_reshape_attn_mask
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import (
piecewise_attn,
@@ -285,7 +281,9 @@ class FlashAttentionImpl(AttentionImpl):
rank-local native cache context. The adapter no longer performs the
attention call itself; this method is the backend-owned execution
boundary. Its native layer wrapper keeps vLLM version-specific cache
and kernel details out of Omni's common ``Attention`` layer.
and kernel details out of Omni's common ``Attention`` layer. CUDA uses
the native vLLM FlashAttention writer/kernel contract. Ascend writes
the complete K/V span once before its piecewise FIA calls.
"""
layer = getattr(paged_kv_context, "layer", None)
@@ -297,8 +295,18 @@ class FlashAttentionImpl(AttentionImpl):
native_impl = getattr(layer, "impl", None)
if native_impl is None:
raise RuntimeError(f"Native attention implementation is not bound for diffusion layer {layer.layer_name!r}")
if not layer.attn_backend.forward_includes_kv_cache_update:
native_impl.do_kv_cache_update(
prewrite_kv = current_omni_platform.requires_diffusion_paged_kv_prewrite()
read_kv_from_cache = prewrite_kv and layer.attn_backend.forward_includes_kv_cache_update
# The GPU/default path preserves native cache-update ownership. Ascend
# prewrites through vLLM-Ascend's normal-layout writer so piecewise FIA
# segments do not repeatedly scatter the same layer K/V.
if not layer.attn_backend.forward_includes_kv_cache_update or prewrite_kv:
cache_update = getattr(native_impl, "do_kv_cache_update", None)
if not callable(cache_update):
raise RuntimeError(
f"Native attention implementation for {layer.layer_name!r} cannot update the KV cache"
)
cache_update(
layer,
paged_kv_context.key_write,
paged_kv_context.value_write,
@@ -308,33 +316,52 @@ class FlashAttentionImpl(AttentionImpl):
def run_native_attention(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
key: torch.Tensor | None,
value: torch.Tensor | None,
native_metadata,
output: torch.Tensor | None,
) -> torch.Tensor:
output = torch.empty(
(query.shape[0], layer.num_heads, layer.head_size_v),
dtype=query.dtype,
device=query.device,
)
if output is None:
output = torch.empty(
(query.shape[0], layer.num_heads, layer.head_size_v),
dtype=query.dtype,
device=query.device,
)
return native_impl.forward(
layer,
query,
key,
value,
None if read_kv_from_cache else key,
None if read_kv_from_cache else value,
kv_cache,
native_metadata,
output,
)
if paged_kv_context.piecewise_plan is not None:
# Ascend FIA can execute identical CFG rows as one batched call
# per piece. Keep that batch layout through the piecewise runner
# so it can concatenate row-major results instead of emitting an
# indexed ScatterUpdate for every segment. The CUDA path keeps
# its existing output-buffer contract (including graph capture).
use_homogeneous_batch = (
current_omni_platform.is_npu() and paged_kv_context.piecewise_plan.homogeneous_batch_shape is not None
)
output = None
if not use_homogeneous_batch:
output = torch.empty(
(paged_kv_context.query.shape[0], layer.num_heads, layer.head_size_v),
dtype=paged_kv_context.query.dtype,
device=paged_kv_context.query.device,
)
output = run_paged_piecewise_plan(
paged_kv_context.query,
paged_kv_context.key_write,
paged_kv_context.value_write,
None if read_kv_from_cache else paged_kv_context.key_write,
None if read_kv_from_cache else paged_kv_context.value_write,
paged_kv_context.piecewise_plan,
paged_kv_context.piecewise_native_metadata,
run_native_attention,
output_buffer=output,
use_homogeneous_batch=use_homogeneous_batch,
)
else:
output = run_native_attention(
@@ -342,6 +369,7 @@ class FlashAttentionImpl(AttentionImpl):
paged_kv_context.key_write,
paged_kv_context.value_write,
paged_kv_context.native_metadata,
None,
)
return paged_kv_context.restore_output(output)
@@ -516,15 +544,8 @@ class FlashAttentionImpl(AttentionImpl):
value: torch.Tensor,
attn_metadata: AttentionMetadata = None,
) -> torch.Tensor:
try:
from mindiesd import attention_forward
except ImportError:
raise ImportError(
"FlashAttentionBackend NPU implementation requires MindIE-SD. "
"Please install MindIE-SD to enable NPU attention support. "
"For installation details, see https://gitcode.com/Ascend/MindIE-SD"
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
)
from mindiesd import attention_forward
# Opt-in mask-free paths (mirror the CUDA cu_seqlens behavior): the
# model marks extra["npu_attn_varlen"] and carries packed metadata, so
# the padding document is excluded without reading or materializing
@@ -633,15 +654,8 @@ class FlashAttentionImpl(AttentionImpl):
return None
seq_q, seq_k = resolved
try:
from mindiesd import attention_forward_varlen
except ImportError:
raise ImportError(
"FlashAttentionBackend NPU implementation requires MindIE-SD. "
"Please install MindIE-SD to enable NPU attention support. "
"For installation details, see https://gitcode.com/Ascend/MindIE-SD"
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
)
from mindiesd import attention_forward_varlen
q = query.squeeze(0) # [T, N, D] == TND
k = key.squeeze(0)
v = value.squeeze(0)
@@ -689,15 +703,8 @@ class FlashAttentionImpl(AttentionImpl):
_, seq_k = resolved
used_k = seq_k[0] # real document length (first cumulative end)
try:
from mindiesd import attention_forward
except ImportError:
raise ImportError(
"FlashAttentionBackend NPU implementation requires MindIE-SD. "
"Please install MindIE-SD to enable NPU attention support. "
"For installation details, see https://gitcode.com/Ascend/MindIE-SD"
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
)
from mindiesd import attention_forward
# mindiesd always takes BSND input regardless of `layout`; the arg
# selects the op-internal layout and laser only supports BNSD, so do
# NOT forward the model's qkv_layout ("BSND" for MiniMax-H3) here.
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Piecewise attention for mixed causal / full (bidirectional) masks.
@@ -92,6 +92,7 @@ def piecewise_attn(
):
_check_homogeneous(full_attn_spans)
spans = full_attn_spans[0]
ranges: tuple[tuple[int, int, int], ...]
if query_ranges is None:
query_len = query.shape[1]
ranges = ((0, query_len, key.shape[1] - query_len),)
@@ -133,6 +134,11 @@ class PagedPiecewiseSegment:
row_segments: tuple[Segment, ...]
query_indices: torch.Tensor
query_range: tuple[int, int] | None
# Local [start, end) range shared by every row when the batch can retain
# its original [B, S] layout. The packed query_indices remain available
# for metadata construction and the heterogeneous fallback.
local_query_range: tuple[int, int] | None = None
@dataclass(frozen=True, slots=True)
@@ -142,10 +148,16 @@ class PagedPiecewisePlan:
spans: tuple[tuple[tuple[int, int], ...], ...]
segments: tuple[PagedPiecewiseSegment, ...]
num_query_tokens: int
segments_cover_query_contiguously: bool
# (batch_size, query_len) when all rows have the same local segment
# layout. Native FIA still receives flattened row-major tokens, but the
# runner can restore outputs with one batch concat instead of one indexed
# scatter per segment.
homogeneous_batch_shape: tuple[int, int] | None = None
PagedPiecewiseRunner = Callable[
[torch.Tensor, torch.Tensor, torch.Tensor, object],
[torch.Tensor, torch.Tensor | None, torch.Tensor | None, object, torch.Tensor | None],
torch.Tensor,
]
@@ -164,7 +176,9 @@ def build_paged_piecewise_plan(
if row_count == 0 or not (row_count == len(query_offsets) == len(query_lens) == len(seq_lens)):
raise ValueError("Paged piecewise inputs must have one entry per row")
spans = tuple(tuple(tuple(span) for span in row_spans) for row_spans in full_attn_spans)
spans: tuple[tuple[tuple[int, int], ...], ...] = tuple(
tuple((span_start, span_end) for span_start, span_end in row_spans) for row_spans in full_attn_spans
)
packed_offsets = [0]
for row_spans, query_offset, query_len, seq_len in zip(spans, query_offsets, query_lens, seq_lens, strict=True):
@@ -188,46 +202,206 @@ def build_paged_piecewise_plan(
for row_segments in zip(*segments_by_row, strict=True):
if len({segment.mode for segment in row_segments}) != 1:
raise ValueError("Paged piecewise rows must use the same attention mode per segment")
query_indices = []
query_indices: list[int] = []
local_ranges: list[tuple[int, int]] = []
for row_index, (segment, query_offset) in enumerate(zip(row_segments, query_offsets, strict=True)):
local_start = segment.q_start - query_offset
local_end = segment.q_end - query_offset
local_ranges.append((local_start, local_end))
query_indices.extend(range(packed_offsets[row_index] + local_start, packed_offsets[row_index] + local_end))
query_range = None
if query_indices and query_indices == list(range(query_indices[0], query_indices[-1] + 1)):
query_range = (query_indices[0], query_indices[-1] + 1)
local_query_range = local_ranges[0] if len(set(local_ranges)) == 1 else None
packed_segments.append(
PagedPiecewiseSegment(
row_segments=tuple(row_segments),
query_indices=torch.tensor(query_indices, dtype=torch.long, device=device),
query_range=query_range,
local_query_range=local_query_range,
)
)
return PagedPiecewisePlan(spans, tuple(packed_segments), packed_offsets[-1])
covered = 0
for segment in packed_segments:
if segment.query_range is None or segment.query_range[0] != covered:
break
covered = segment.query_range[1]
segments_cover_query_contiguously = covered == packed_offsets[-1]
homogeneous_batch_shape: tuple[int, int] | None = None
if (
row_count > 1
and len(set(query_lens)) == 1
and all(segment.local_query_range is not None for segment in packed_segments)
):
# A common local range is sufficient even when rows have different
# global offsets or KV lengths; those values remain row-specific in
# native metadata.
homogeneous_batch_shape = (row_count, int(query_lens[0]))
return PagedPiecewisePlan(
spans=spans,
segments=tuple(packed_segments),
num_query_tokens=packed_offsets[-1],
segments_cover_query_contiguously=segments_cover_query_contiguously,
homogeneous_batch_shape=homogeneous_batch_shape,
)
def _run_homogeneous_paged_piecewise_plan(
query: torch.Tensor,
key: torch.Tensor | None,
value: torch.Tensor | None,
plan: PagedPiecewisePlan,
segment_metadata: Sequence[object],
segment_runner: PagedPiecewiseRunner,
output_buffer: torch.Tensor | None,
) -> torch.Tensor:
"""Run a homogeneous batch while retaining the old [B, S] layout.
The native FIA contract is flattened TND, so each segment is flattened
only for the kernel call. Results are reshaped back to [B, L] and
concatenated along the sequence dimension, which avoids the per-segment
``index_copy_`` used by heterogeneous packed rows.
"""
batch_shape = plan.homogeneous_batch_shape
assert batch_shape is not None
batch_size, query_len = batch_shape
if query.shape[0] != batch_size * query_len:
raise ValueError(
f"Homogeneous paged piecewise plan expects {batch_size * query_len} query tokens, got {query.shape[0]}"
)
query_rows = query.reshape(batch_size, query_len, *query.shape[1:])
key_rows = None if key is None else key.reshape(batch_size, query_len, *key.shape[1:])
value_rows = None if value is None else value.reshape(batch_size, query_len, *value.shape[1:])
segment_outputs: list[torch.Tensor] = []
for segment, metadata in zip(plan.segments, segment_metadata, strict=True):
local_range = segment.local_query_range
if local_range is None:
raise ValueError("Homogeneous paged piecewise execution requires a shared local segment range")
start, end = local_range
segment_query = query_rows[:, start:end].contiguous().reshape(-1, *query.shape[1:])
segment_key = None if key_rows is None else key_rows[:, start:end].contiguous().reshape(-1, *key.shape[1:])
segment_value = (
None if value_rows is None else value_rows[:, start:end].contiguous().reshape(-1, *value.shape[1:])
)
segment_output = segment_runner(segment_query, segment_key, segment_value, metadata, None)
expected_tokens = batch_size * (end - start)
if segment_output.shape[0] != expected_tokens:
raise ValueError(
f"Paged piecewise runner returned {segment_output.shape[0]} tokens "
f"for a {expected_tokens}-token homogeneous segment"
)
segment_outputs.append(segment_output.reshape(batch_size, end - start, *segment_output.shape[1:]))
if not segment_outputs:
result = torch.empty_like(query)
elif len(segment_outputs) == 1:
result = segment_outputs[0].reshape(plan.num_query_tokens, *segment_outputs[0].shape[2:])
else:
result_rows = torch.cat(segment_outputs, dim=1)
result = result_rows.reshape(plan.num_query_tokens, *result_rows.shape[2:])
if output_buffer is not None:
if output_buffer.shape != result.shape:
raise ValueError(
f"Paged piecewise output buffer shape {tuple(output_buffer.shape)} does not match "
f"homogeneous result shape {tuple(result.shape)}"
)
output_buffer.copy_(result)
return output_buffer
return result
def run_paged_piecewise_plan(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
key: torch.Tensor | None,
value: torch.Tensor | None,
plan: PagedPiecewisePlan,
segment_metadata: Sequence[object],
segment_runner: PagedPiecewiseRunner,
output_buffer: torch.Tensor | None = None,
*,
use_homogeneous_batch: bool = False,
) -> torch.Tensor:
"""Run and scatter piecewise attention over packed Q/K/V write tensors."""
"""Run piecewise attention with view fast paths for contiguous segments.
``use_homogeneous_batch`` is an opt-in for native backends that can keep
identical rows in the legacy batch layout. The default remains the
indexed packed path so existing GPU/heterogeneous contracts are unchanged.
"""
if query.shape[0] != plan.num_query_tokens:
raise ValueError(f"Paged piecewise plan has {plan.num_query_tokens} query tokens, got {query.shape[0]}")
if output_buffer is not None and output_buffer.shape[0] != plan.num_query_tokens:
raise ValueError(
f"Paged piecewise output buffer has {output_buffer.shape[0]} tokens; expected {plan.num_query_tokens}"
)
output = None
if use_homogeneous_batch and plan.homogeneous_batch_shape is not None:
return _run_homogeneous_paged_piecewise_plan(
query,
key,
value,
plan,
segment_metadata,
segment_runner,
output_buffer,
)
output = output_buffer
contiguous_outputs = []
for segment, metadata in zip(plan.segments, segment_metadata, strict=True):
indices = segment.query_indices
segment_output_buffer = None
if segment.query_range is None:
segment_query = query.index_select(0, indices)
segment_key = None if key is None else key.index_select(0, indices)
segment_value = None if value is None else value.index_select(0, indices)
else:
start, end = segment.query_range
segment_query = query[start:end]
segment_key = None if key is None else key[start:end]
segment_value = None if value is None else value[start:end]
if output is not None:
segment_output_buffer = output[start:end]
segment_output = segment_runner(
query.index_select(0, indices),
key.index_select(0, indices),
value.index_select(0, indices),
segment_query,
segment_key,
segment_value,
metadata,
segment_output_buffer,
)
if segment_output.shape[0] != indices.shape[0]:
raise ValueError(
f"Paged piecewise runner returned {segment_output.shape[0]} tokens "
f"for a {indices.shape[0]}-token segment"
)
if segment_output_buffer is not None and segment_output is not segment_output_buffer:
if segment_output.shape != segment_output_buffer.shape:
raise ValueError(
f"Paged piecewise runner returned shape {tuple(segment_output.shape)} "
f"for output buffer shape {tuple(segment_output_buffer.shape)}"
)
segment_output_buffer.copy_(segment_output)
segment_output = segment_output_buffer
if plan.segments_cover_query_contiguously:
if output_buffer is None:
contiguous_outputs.append(segment_output)
continue
if output is None:
output = segment_output.new_empty((query.shape[0], *segment_output.shape[1:]))
output.index_copy_(0, indices, segment_output)
if segment.query_range is None:
output.index_copy_(0, indices, segment_output)
else:
start, end = segment.query_range
output[start:end].copy_(segment_output)
if output_buffer is not None and plan.segments_cover_query_contiguously:
return output_buffer
if contiguous_outputs:
if len(contiguous_outputs) == 1:
return contiguous_outputs[0]
return torch.cat(contiguous_outputs, dim=0)
assert output is not None
return output
+111 -6
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Copyright (c) Microsoft Corporation and Jiarui Fang
# SPDX-License-Identifier: Apache-2.0
# DeepSpeed Team & Jiarui Fang
@@ -23,6 +23,7 @@ from vllm_omni.diffusion.attention.parallel.base import NoParallelAttention
from vllm_omni.diffusion.attention.parallel.ring import RingParallelAttention
from vllm_omni.diffusion.attention.selector import get_attn_backend_for_role
from vllm_omni.diffusion.config import get_current_diffusion_config_or_none
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
from vllm_omni.diffusion.forward_context import (
get_forward_context,
@@ -44,6 +45,8 @@ def _try_extract_layer_index(prefix: str) -> int | None:
class Attention(nn.Module):
_scheduler_paged_kv = False
def __init__(
self,
num_heads: int,
@@ -111,6 +114,44 @@ class Attention(nn.Module):
role_category=role_category,
allow_trtllm_default=allow_trtllm_default,
)
scheduler_paged_kv = (
config is not None
and getattr(config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)
is DiffusionKVCacheMode.PAGED_SCHEDULER
)
self._scheduler_paged_kv = scheduler_paged_kv
if (
scheduler_paged_kv
and paged_kv_cache_role is not None
and spec is None
and not attn_backend_cls.supports_paged_kv
):
# FLASH_ATTN is an Omni selector, not a device-specific kernel.
# vLLM resolves it to CUDA FlashAttention on GPU and Ascend native
# attention on NPU. A platform may legitimately default dense
# attention to SDPA (for example when an optional diffusion FA
# package is absent), but a paged layer must advertise the
# capability so the Worker can register its native cache view.
# Resolve the selector only for this marked layer; unmarked dense
# attention keeps the platform default unchanged.
from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec
dense_backend_name = attn_backend_cls.get_name()
paged_attention_config = AttentionConfig(default=AttentionSpec(backend="FLASH_ATTN"))
attn_backend_cls, spec = get_attn_backend_for_role(
role=role,
head_size=head_size,
attention_config=paged_attention_config,
role_category=role_category,
allow_trtllm_default=allow_trtllm_default,
)
logger.info(
"Resolved marked paged diffusion attention role=%r to %r because the platform default %r "
"does not support Scheduler-owned KV",
role,
attn_backend_cls.get_name(),
dense_backend_name,
)
parallel_config = getattr(config, "parallel_config", None)
allgather_degree = getattr(parallel_config, "allgather_degree", 1)
# TODO: Move AllGather-KV compatibility into an AttentionBackend capability
@@ -321,6 +362,17 @@ class Attention(nn.Module):
# Get the appropriate parallel strategy based on SP active state
strategy = self._get_active_parallel_strategy()
paged_adapter = self._active_paged_kv_adapter()
in_kv_memory_profile = is_forward_context_available() and get_forward_context().in_diffusion_kv_memory_profile
if (
self._scheduler_paged_kv
and self.paged_kv_cache_role is not None
and paged_adapter is None
and not in_kv_memory_profile
):
raise RuntimeError(
"Scheduler-paged diffusion attention reached model forward without an active Worker adapter. "
"Only the startup KV memory profile may execute before paged KV initialization."
)
use_paged_attention = paged_adapter is not None and self.paged_kv_cache_role is not None
if use_paged_attention and not getattr(self.attn_backend, "supports_paged_kv", False):
raise NotImplementedError(
@@ -343,9 +395,39 @@ class Attention(nn.Module):
# For Ring: Concat joint_q
query, key, value, attn_metadata, ctx = strategy.pre_attention(query, key, value, attn_metadata)
# 2. Kernel execution stays inside the selected Omni backend. The
# Worker adapter only prepares the native page-table context after
# SP has produced rank-local Q/K/V tensors.
# Scheduler rows describe the logical sequence, while strict Ulysses
# may append synthetic tokens solely to make the image shard divisible.
# Remove those tokens after the all-to-all and put zero placeholders
# back before the reverse all-to-all.
paged_sp_padding = 0
paged_sp_padding_offset = query.shape[1]
if use_paged_attention and strategy is not self._no_parallel_strategy and strategy.name == "ulysses":
forward_ctx = get_forward_context() if is_forward_context_available() else None
paged_sp_padding = int(getattr(forward_ctx, "sp_padding_size", 0))
if paged_sp_padding:
joint_len = int(getattr(ctx, "joint_len", 0))
joint_strategy = str(getattr(ctx, "joint_strategy", "front"))
paged_sp_padding_offset = query.shape[1] - (joint_len if joint_strategy == "rear" else 0)
padding_start = paged_sp_padding_offset - paged_sp_padding
if padding_start < 0:
raise ValueError(
"Paged Ulysses padding exceeds the post-all-to-all sequence: "
f"padding={paged_sp_padding}, sequence={query.shape[1]}"
)
def _remove_paged_sp_padding(tensor: torch.Tensor) -> torch.Tensor:
return torch.cat(
(tensor[:, :padding_start], tensor[:, paged_sp_padding_offset:]),
dim=1,
).contiguous()
query = _remove_paged_sp_padding(query)
key = _remove_paged_sp_padding(key)
value = _remove_paged_sp_padding(value)
# 2. This is the shared GPU/NPU boundary. The Worker adapter prepares
# the native page-table context after SP has produced rank-local Q/K/V;
# backend resolution below selects CUDA or Ascend execution.
if use_paged_attention:
assert paged_adapter is not None
paged_kv_context = paged_adapter.prepare_layer_context(
@@ -363,6 +445,11 @@ class Attention(nn.Module):
else:
out = self._run_local_attention(query, key, value, attn_metadata)
if paged_sp_padding:
padding_start = paged_sp_padding_offset - paged_sp_padding
output_padding = out.new_zeros((out.shape[0], paged_sp_padding, *out.shape[2:]))
out = torch.cat((out[:, :padding_start], output_padding, out[:, padding_start:]), dim=1)
# 3. Post-processing (Reverse Communication)
# For Ulysses: AllToAll Output, and AllGather Joint Output
out = strategy.post_attention(out, ctx)
@@ -371,7 +458,7 @@ class Attention(nn.Module):
@staticmethod
def _active_paged_kv_adapter():
"""Return the opaque Worker adapter installed for this forward."""
"""Return the Worker adapter selected by Runner-owned metadata."""
if not is_forward_context_available():
return None
@@ -392,7 +479,25 @@ class Attention(nn.Module):
)
return self.sdpa_fallback.forward(query, key, value, attn_metadata)
# Fallback to standard attention
in_kv_memory_profile = is_forward_context_available() and get_forward_context().in_diffusion_kv_memory_profile
# NPU only: the startup KV-capacity profile needs tensor shapes, not a
# paged attention result. If dense MindIE-SD is absent, SDPA provides
# that profile forward. Formal GPU/NPU paged requests never use this
# branch because their Worker adapter is active.
if (
self._scheduler_paged_kv
and self.paged_kv_cache_role is not None
and in_kv_memory_profile
and current_omni_platform.is_npu()
and self.attn_backend.get_name() == "FLASH_ATTN"
and not current_omni_platform.supports_diffusion_dense_flash_attention()
):
logger.warning_once(
"The startup KV memory profile is using SDPA because MindIE-SD is unavailable. "
"Formal paged requests still use the platform-native paged attention backend."
)
return self.sdpa_fallback.forward(query, key, value, attn_metadata)
return self.attention.forward(query, key, value, attn_metadata)
def _assert_piecewise_compatible(self, attn_metadata: AttentionMetadata | None) -> None:
+2 -3
View File
@@ -1086,10 +1086,9 @@ class OmniDiffusionConfig:
"need_recv_cache", False
):
raise ValueError(
"paged_scheduler Diffusion KV does not support imported AR KV in Phase 1; "
"disable need_recv_cache until connector-aware admission is implemented"
"paged_scheduler Diffusion KV does not support imported AR KV; "
"disable need_recv_cache until connector-aware import is implemented"
)
self.master_port = self._resolve_master_port()
self.request_batch_max_wait_ms = float(self.request_batch_max_wait_ms or 0.0)
if not math.isfinite(self.request_batch_max_wait_ms) or self.request_batch_max_wait_ms < 0:
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import copy
from collections.abc import Mapping, Sequence
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
@@ -22,9 +22,9 @@ from vllm_omni.diffusion.diffusion_kv.metadata import DiffusionKVMetadata
from vllm_omni.diffusion.diffusion_kv.paged_attention_adapter import (
DiffusionPagedAttentionAdapter,
DiffusionPagedAttentionLayerAdapter,
DiffusionPagedAttentionRow,
DiffusionPagedAttentionMetadata,
DiffusionPagedAttentionRowBinding,
PreparedDiffusionPagedAttentionBatch,
DiffusionPagedAttentionRuntime,
)
from vllm_omni.platforms import current_omni_platform
@@ -32,13 +32,6 @@ DiffusionKVIdentity = tuple[str, int | None, str | None]
DiffusionKVSnapshot = tuple[object, ...]
@dataclass(frozen=True)
class _DiffusionKVRequestState:
generation: int
snapshot: DiffusionKVSnapshot
row_token_lens: tuple[tuple[DiffusionKVIdentity, int], ...]
@dataclass(frozen=True)
class _DiffusionKVRowInstall:
identity: DiffusionKVIdentity
@@ -46,6 +39,13 @@ class _DiffusionKVRowInstall:
block_ids: tuple[tuple[int, ...], ...]
@dataclass(frozen=True)
class _DiffusionKVRequestState:
generation: int
snapshot: DiffusionKVSnapshot
row_installs: tuple[_DiffusionKVRowInstall, ...]
class DiffusionKVModelRunnerBackend:
"""Native paged-KV state and BlockTable operations for a model runner."""
@@ -503,7 +503,7 @@ class DiffusionKVModelRunnerBackend:
self._diffusion_kv_request_states[metadata.request_id] = _DiffusionKVRequestState(
generation=metadata.allocation_generation,
snapshot=snapshot,
row_token_lens=current_state.row_token_lens,
row_installs=current_state.row_installs,
)
return False
@@ -530,7 +530,7 @@ class DiffusionKVModelRunnerBackend:
self._diffusion_kv_request_states[metadata.request_id] = _DiffusionKVRequestState(
generation=metadata.allocation_generation,
snapshot=snapshot,
row_token_lens=tuple((install.identity, install.token_len) for install in installs),
row_installs=tuple(installs),
)
return True
@@ -559,24 +559,11 @@ class DiffusionKVModelRunnerBackend:
raise RuntimeError("paged_scheduler native attention adapter is not initialized")
return self.paged_attention_adapter
def prepare_paged_attention_batch(
self,
rows: Sequence[DiffusionPagedAttentionRow],
) -> PreparedDiffusionPagedAttentionBatch:
"""Build native page-table metadata for one Diffusion forward.
def activate_paged_attention_metadata(self, metadata: DiffusionPagedAttentionMetadata):
"""Activate Runner-owned request metadata for an internal denoise loop."""
The scheduler allocation payload describes capacity, while the model
integration supplies the current ``query_len``/``kv_start_pos`` span
for each row. Keeping this boundary explicit prevents the Worker from
guessing model-specific text/image layout.
"""
return self.get_paged_attention_adapter().prepare_batch(rows)
def activate_paged_attention(self, batch: PreparedDiffusionPagedAttentionBatch):
"""Return the context manager that exposes a prepared batch to Omni Attention."""
return self.get_paged_attention_adapter().activate(batch)
runtime = DiffusionPagedAttentionRuntime(self.get_paged_attention_adapter(), metadata)
return runtime, runtime.activate()
def _resolve_paged_attention_row(
self,
@@ -588,11 +575,12 @@ class DiffusionKVModelRunnerBackend:
identity = (request_id, None, context_id) if context_id is not None else (request_id, sequence_id, None)
request_state = self._diffusion_kv_request_states.get(request_id)
if request_state is not None:
for installed_identity, token_len in request_state.row_token_lens:
if installed_identity == identity:
for install in request_state.row_installs:
if install.identity == identity:
return DiffusionPagedAttentionRowBinding(
row_index=row_index,
max_seq_len=token_len,
max_seq_len=install.token_len,
block_ids=install.block_ids,
)
raise RuntimeError(f"Diffusion KV request state is missing logical length for {identity!r}")
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -34,6 +34,7 @@ class DiffusionPagedAttentionRowBinding:
row_index: int
max_seq_len: int
block_ids: tuple[tuple[int, ...], ...] = ()
DiffusionKVRowResolver = Callable[
@@ -100,6 +101,10 @@ class DiffusionPagedAttentionLayerAdapter(AttentionLayerBase):
finally:
attention_config.backend = previous_backend
attention_config.backend_per_kind = previous_backend_per_kind
attn_backend = current_omni_platform.get_diffusion_paged_kv_attn_backend(
attn_backend,
ulysses_degree=ulysses_degree,
)
canonical_spec = replace(
spec,
num_kv_heads=num_kv_heads,
@@ -152,9 +157,11 @@ class DiffusionPagedAttentionLayerAdapter(AttentionLayerBase):
raise RuntimeError(
f"Native paged-attention backend {self.attn_backend.get_name()!r} has no forward implementation"
)
if not self.attn_backend.forward_includes_kv_cache_update and not callable(
getattr(impl, "do_kv_cache_update", None)
):
needs_explicit_cache_update = (
not self.attn_backend.forward_includes_kv_cache_update
or current_omni_platform.requires_diffusion_paged_kv_prewrite()
)
if needs_explicit_cache_update and not callable(getattr(impl, "do_kv_cache_update", None)):
raise RuntimeError(
f"Native paged-attention backend {self.attn_backend.get_name()!r} cannot update the KV cache"
)
@@ -204,6 +211,30 @@ class DiffusionPagedAttentionRow:
return (self.request_id, self.sequence_id, self.context_id)
@dataclass(frozen=True)
class DiffusionPagedAttentionMetadata:
"""Runner-owned row layouts for one request-level denoise loop."""
prefill_rows: tuple[DiffusionPagedAttentionRow, ...]
denoise_rows: tuple[DiffusionPagedAttentionRow, ...]
def __post_init__(self) -> None:
if not self.prefill_rows or not self.denoise_rows:
raise ValueError("Paged attention metadata requires both prefill and denoise rows")
prefill_identities = tuple(row.identity for row in self.prefill_rows)
denoise_identities = tuple(row.identity for row in self.denoise_rows)
if prefill_identities != denoise_identities:
raise ValueError(
"Paged attention prefill and denoise rows must use the same ordered identities: "
f"prefill={prefill_identities!r}, denoise={denoise_identities!r}"
)
def rows_for_step(self, step_idx: int | None) -> tuple[DiffusionPagedAttentionRow, ...]:
if type(step_idx) is not int or step_idx < 0:
raise ValueError(f"Paged attention requires a non-negative denoise step index, got {step_idx!r}")
return self.prefill_rows if step_idx == 0 else self.denoise_rows
@dataclass(frozen=True)
class PreparedDiffusionPagedAttentionBatch:
"""Native metadata shared by all paged attention layers in one forward."""
@@ -236,13 +267,81 @@ class DiffusionPagedAttentionContext:
piecewise_native_metadata: tuple[Any, ...]
query_token_shape: tuple[int, ...]
query_has_head_dims: bool
output_scatter_indices: torch.Tensor | None = None
def restore_output(self, output: torch.Tensor) -> torch.Tensor:
if self.output_scatter_indices is not None:
flat_output = output.reshape(-1, self.layer.num_heads, self.layer.head_size_v)
restored = output.new_zeros(
(*self.query_token_shape, self.layer.num_heads, self.layer.head_size_v)
).reshape(-1, self.layer.num_heads, self.layer.head_size_v)
restored.index_copy_(0, self.output_scatter_indices, flat_output)
if self.query_has_head_dims:
return restored.reshape(
*self.query_token_shape,
self.layer.num_heads,
self.layer.head_size_v,
)
return restored.reshape(
*self.query_token_shape,
self.layer.num_heads * self.layer.head_size_v,
)
if self.query_has_head_dims:
return output.reshape(*self.query_token_shape, self.layer.num_heads, self.layer.head_size_v)
return output.reshape(*self.query_token_shape, self.layer.num_heads * self.layer.head_size_v)
class DiffusionPagedAttentionRuntime:
"""Activate Runner-prepared rows lazily as the denoise loop advances."""
def __init__(
self,
adapter: DiffusionPagedAttentionAdapter,
metadata: DiffusionPagedAttentionMetadata,
) -> None:
self.adapter = adapter
self.metadata = metadata
self._active_phase: bool | None = None
self._activation: Any | None = None
self._closed = False
def ensure_active(self, step_idx: int | None) -> DiffusionPagedAttentionAdapter:
if self._closed:
raise RuntimeError("Paged attention runtime is already closed")
is_prefill = step_idx == 0
rows = self.metadata.rows_for_step(step_idx)
if self._active_phase == is_prefill:
return self.adapter
self._deactivate()
batch = self.adapter.prepare_batch(rows)
activation = self.adapter.activate(batch)
activation.__enter__()
self._activation = activation
self._active_phase = is_prefill
return self.adapter
def _deactivate(self) -> None:
activation = self._activation
self._activation = None
self._active_phase = None
if activation is not None:
activation.__exit__(None, None, None)
def close(self) -> None:
if self._closed:
return
self._deactivate()
self._closed = True
@contextmanager
def activate(self) -> Iterator[DiffusionPagedAttentionRuntime]:
try:
yield self
finally:
self.close()
class DiffusionPagedAttentionAdapter:
"""Translate diffusion rows into vLLM metadata for an Omni paged backend."""
@@ -545,15 +644,16 @@ class DiffusionPagedAttentionAdapter:
)
@staticmethod
def _validate_token_layout(
def _token_packing_indices(
token_shape: tuple[int, ...],
batch: PreparedDiffusionPagedAttentionBatch,
*,
name: str,
) -> None:
device: torch.device,
) -> torch.Tensor | None:
if len(token_shape) == 1:
if token_shape[0] == batch.num_tokens:
return
return None
raise ValueError(
f"Paged attention {name} token count must match the prepared write batch: "
f"tokens={token_shape[0]}, prepared={batch.num_tokens}"
@@ -561,11 +661,23 @@ class DiffusionPagedAttentionAdapter:
if len(token_shape) == 2:
batch_size, tokens_per_row = token_shape
query_lens = tuple(row.query_len for row in batch.rows)
if batch_size == len(batch.rows) and all(query_len == tokens_per_row for query_len in query_lens):
return
raise ValueError(
f"Paged attention batched {name} layout must match prepared rows for the current write: "
f"shape={token_shape}, row_query_lens={query_lens}"
if batch_size != len(batch.rows) or any(query_len > tokens_per_row for query_len in query_lens):
raise ValueError(
f"Paged attention batched {name} layout must match prepared rows for the current write: "
f"shape={token_shape}, row_query_lens={query_lens}"
)
if all(query_len == tokens_per_row for query_len in query_lens):
return None
return torch.cat(
[
torch.arange(
row_index * tokens_per_row,
row_index * tokens_per_row + query_len,
dtype=torch.long,
device=device,
)
for row_index, query_len in enumerate(query_lens)
]
)
raise ValueError(
f"Paged attention {name} supports packed [T, ...] or uniform batched [B, T, ...] token layouts; "
@@ -626,19 +738,39 @@ class DiffusionPagedAttentionAdapter:
query_offsets.append(query_offsets[-1] + query_len)
query_start_loc_cpu = torch.tensor(query_offsets, dtype=torch.int32)
seq_lens_cpu = torch.tensor(seq_lens_values, dtype=torch.int32)
metadata_by_segment.append(
self._build_native_metadata(
query_lens=query_lens,
query_start_loc=query_start_loc_cpu.to(self.device),
query_start_loc_cpu=query_start_loc_cpu,
seq_lens=seq_lens_cpu.to(self.device),
seq_lens_cpu=seq_lens_cpu,
positions=batch.positions.index_select(0, packed_segment.query_indices),
block_tables=batch.block_tables,
slot_mappings=batch.slot_mappings.index_select(-1, packed_segment.query_indices),
causal=(row_segments[0].mode == "causal"),
)
if packed_segment.query_range is None:
positions = batch.positions.index_select(0, packed_segment.query_indices)
slot_mappings = batch.slot_mappings.index_select(-1, packed_segment.query_indices)
else:
start, end = packed_segment.query_range
positions = batch.positions[start:end]
slot_mappings = batch.slot_mappings[..., start:end]
segment_metadata = self._build_native_metadata(
query_lens=query_lens,
query_start_loc=query_start_loc_cpu.to(self.device),
query_start_loc_cpu=query_start_loc_cpu,
seq_lens=seq_lens_cpu.to(self.device),
seq_lens_cpu=seq_lens_cpu,
positions=positions,
block_tables=batch.block_tables,
slot_mappings=slot_mappings,
causal=(row_segments[0].mode == "causal"),
)
# FA3's full-CUDA-graph metadata builder reuses one persistent
# scheduler buffer across builds. Piecewise attention prepares all
# segments before executing any of them, so each segment needs its
# own snapshot rather than a view overwritten by the next build.
snapshotted = set()
for native_metadata in segment_metadata.values():
metadata_id = id(native_metadata)
if metadata_id in snapshotted:
continue
snapshotted.add(metadata_id)
for field_name in ("scheduler_metadata", "prefix_scheduler_metadata"):
scheduler_metadata = getattr(native_metadata, field_name, None)
if isinstance(scheduler_metadata, torch.Tensor):
setattr(native_metadata, field_name, scheduler_metadata.clone())
metadata_by_segment.append(segment_metadata)
self._active_piecewise_native_metadata = tuple(metadata_by_segment)
return self._active_piecewise_native_metadata
@@ -678,9 +810,36 @@ class DiffusionPagedAttentionAdapter:
head_size=layer.head_size_v,
name="value",
)
self._validate_token_layout(query_token_shape, batch, name="query")
self._validate_token_layout(key_token_shape, batch, name="key")
self._validate_token_layout(value_token_shape, batch, name="value")
if key_token_shape != query_token_shape:
if len(key_token_shape) == 1:
raise ValueError(
"Paged attention key token count must match the query layout: "
f"query={query_token_shape}, key={key_token_shape}"
)
raise ValueError(
"Paged attention batched key layout must match prepared rows and the query layout: "
f"query={query_token_shape}, key={key_token_shape}"
)
if value_token_shape != query_token_shape:
if len(value_token_shape) == 1:
raise ValueError(
"Paged attention value token count must match the query layout: "
f"query={query_token_shape}, value={value_token_shape}"
)
raise ValueError(
"Paged attention batched value layout must match prepared rows and the query layout: "
f"query={query_token_shape}, value={value_token_shape}"
)
output_scatter_indices = self._token_packing_indices(
query_token_shape,
batch,
name="Q/K/V",
device=query.device,
)
if output_scatter_indices is not None:
query_flat = query_flat.index_select(0, output_scatter_indices)
key_flat = key_flat.index_select(0, output_scatter_indices)
value_flat = value_flat.index_select(0, output_scatter_indices)
if query.device != self.device or key.device != self.device or value.device != self.device:
raise ValueError(
f"Paged attention Q/K/V must be on {self.device}; "
@@ -711,10 +870,18 @@ class DiffusionPagedAttentionAdapter:
f"expected {batch.num_tokens}"
)
# vLLM's writer consumes slot_mapping directly, so a span that starts
# vLLM's CUDA writer consumes the source token strides directly, so
# preserve strided QKV-projection views instead of materializing an
# extra K/V copy here. Platform writers with stricter contracts (for
# example, Ascend PA scatter) make their inputs contiguous locally.
# The native CUDA writer still assumes heads and head dimensions are
# packed within each token; materialize unusual inner layouts only.
if key_flat.stride(-1) != 1 or (key_flat.shape[-2] > 1 and key_flat.stride(-2) != key_flat.shape[-1]):
key_flat = key_flat.contiguous()
if value_flat.stride(-1) != 1 or (value_flat.shape[-2] > 1 and value_flat.stride(-2) != value_flat.shape[-1]):
value_flat = value_flat.contiguous()
# The writer also consumes slot_mapping directly, so a span that starts
# inside a physical block naturally writes only the requested suffix.
key_flat = key_flat.contiguous()
value_flat = value_flat.contiguous()
try:
native_metadata = batch.attn_metadata[layer_name]
except KeyError as exc:
@@ -743,6 +910,7 @@ class DiffusionPagedAttentionAdapter:
piecewise_native_metadata=piecewise_native_metadata,
query_token_shape=query_token_shape,
query_has_head_dims=query_has_head_dims,
output_scatter_indices=output_scatter_indices,
)
@staticmethod
+21
View File
@@ -29,11 +29,19 @@ class ForwardContext:
vllm_config: VllmConfig | None = None
omni_diffusion_config: OmniDiffusionConfig | None = None
attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None = None
# Runner-owned paged execution metadata/runtime. Attention resolves the
# active Worker adapter from it; model code must not construct BlockTable
# rows or activate the runtime directly.
paged_kv_runtime: object | None = None
# Active Worker-side paged KV adapter. The adapter is installed only for
# the duration of a paged forward; dense forwards leave this as ``None``.
# Keep the field opaque here to avoid coupling the common context module to
# the diffusion_kv implementation.
paged_kv_adapter: Any | None = None
# Startup-only memory profiling runs before Scheduler-owned pages exist.
# Attention layers use this explicit scope to distinguish that probe from
# a malformed paged request whose Worker adapter was not activated.
in_diffusion_kv_memory_profile: bool = False
split_text_embed_in_sp: bool = False
denoise_step_idx: int | None = None
denoise_timestep: float | None = None
@@ -154,6 +162,8 @@ def create_forward_context(
vllm_config: VllmConfig | None = None,
omni_diffusion_config: OmniDiffusionConfig | None = None,
attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None = None,
paged_kv_runtime: object | None = None,
in_diffusion_kv_memory_profile: bool = False,
split_text_embed_in_sp: bool = False,
denoise_step_idx: int | None = None,
):
@@ -161,6 +171,8 @@ def create_forward_context(
vllm_config=vllm_config,
omni_diffusion_config=omni_diffusion_config,
attn_metadata=attn_metadata,
paged_kv_runtime=paged_kv_runtime,
in_diffusion_kv_memory_profile=in_diffusion_kv_memory_profile,
split_text_embed_in_sp=split_text_embed_in_sp,
denoise_step_idx=denoise_step_idx,
)
@@ -186,6 +198,8 @@ def set_forward_context(
vllm_config: VllmConfig | None = None,
omni_diffusion_config: OmniDiffusionConfig | None = None,
attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None = None,
paged_kv_runtime: object | None = None,
in_diffusion_kv_memory_profile: bool = False,
split_text_embed_in_sp: bool = False,
denoise_step_idx: int | None = None,
):
@@ -197,6 +211,8 @@ def set_forward_context(
vllm_config=vllm_config,
omni_diffusion_config=omni_diffusion_config,
attn_metadata=attn_metadata,
paged_kv_runtime=paged_kv_runtime,
in_diffusion_kv_memory_profile=in_diffusion_kv_memory_profile,
split_text_embed_in_sp=split_text_embed_in_sp,
denoise_step_idx=denoise_step_idx,
)
@@ -246,6 +262,11 @@ def set_forward_context_denoise_step_idx(step_idx: int | None) -> None:
"""Set the current diffusion denoise step on the active ForwardContext."""
if _forward_context is not None:
_forward_context.denoise_step_idx = step_idx
if step_idx is not None:
paged_kv_runtime = getattr(_forward_context, "paged_kv_runtime", None)
ensure_active = getattr(paged_kv_runtime, "ensure_active", None)
if callable(ensure_active):
ensure_active(step_idx)
def set_forward_context_denoise_timestep(timestep: float | None) -> None:
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import inspect
import logging
@@ -1069,18 +1069,98 @@ class ImageKVCacheManager(nn.Module):
)
return key, value
def forward(
def clear_legacy_prompt_kv_cache(self) -> None:
"""Drop model-owned dense KV state before Scheduler-paged execution."""
self.image_kv_cache_map = None
self.image_kv_cache_lens = None
def _forward_paged(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor | None = None,
*,
first_step: bool,
query_lens: list[int],
seq_lens: list[int],
shard_image_size: int | None,
full_attn_spans: list[list[tuple[int, int]]],
uncond_cfg_prefill: bool,
) -> torch.Tensor:
"""Run Hunyuan attention against Worker-managed Scheduler pages.
This model boundary is shared by GPU and NPU: it describes Hunyuan's
Q/K/V layout and mixed-attention spans, while the common Attention
layer resolves the actual CUDA or Ascend paged kernel.
"""
if uncond_cfg_prefill:
raise RuntimeError("Hunyuan negative-CFG prefill must run before paged row activation")
if self._injected_ar_kv is not None:
raise NotImplementedError("Hunyuan Scheduler-paged KV does not support imported AR KV")
if not query_lens or len(seq_lens) != len(query_lens):
raise ValueError("Hunyuan Scheduler-paged KV requires matching query and sequence lengths")
if len(full_attn_spans) != len(query_lens):
raise ValueError("Hunyuan Scheduler-paged KV requires one full-attention span row per sequence")
self.clear_legacy_prompt_kv_cache()
bs = len(query_lens)
q_len = query_lens[0]
seq_len = seq_lens[0]
assert query.shape[0] == bs * q_len, f"{query.shape[0]} != {bs * q_len}"
head_num_per_rank = query.shape[1]
kv_head_num_per_rank = key.shape[1]
head_dim = query.shape[2]
query = query.reshape(bs, q_len, head_num_per_rank, head_dim)
key = key.reshape(bs, q_len, kv_head_num_per_rank, head_dim)
value = value.reshape(bs, q_len, kv_head_num_per_rank, head_dim)
joint_text_query = joint_text_key = joint_text_value = None
if self.sp_size > 1 and first_step:
if shard_image_size is None or shard_image_size <= 0:
raise ValueError("Hunyuan paged Ulysses requires a positive local image shard size")
local_prompt_len = seq_len - shard_image_size
joint_query_len = query.shape[1] - shard_image_size
if local_prompt_len != joint_query_len:
raise ValueError(
"Hunyuan paged Ulysses prompt layout mismatch: "
f"key_prompt={local_prompt_len}, query_prompt={joint_query_len}"
)
joint_text_query = query[:, :joint_query_len]
joint_text_key = key[:, :local_prompt_len]
joint_text_value = value[:, :local_prompt_len]
query = query[:, joint_query_len:]
key = key[:, local_prompt_len:]
value = value[:, local_prompt_len:]
if joint_text_query is None:
attn_metadata = AttentionMetadata(full_attn_spans=full_attn_spans)
else:
attn_metadata = AttentionMetadata(
joint_query=joint_text_query,
joint_key=joint_text_key,
joint_value=joint_text_value,
joint_strategy="front",
full_attn_spans=full_attn_spans,
)
attn_output = self.attn(query, key, value, attn_metadata)
return attn_output.reshape(bs * q_len, head_num_per_rank, head_dim)
def _forward_dense_legacy(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor,
**kwargs,
) -> torch.Tensor:
self.image_token_len = kwargs.get("num_image_tokens")
"""Run the device-neutral, model-owned dense prompt-KV compatibility path."""
first_step = kwargs.get("first_step")
uncond_cfg_prefill = kwargs.get("uncond_cfg_prefill", False)
query_lens = kwargs.get("query_lens")
seq_lens = kwargs.get("seq_lens")
shard_image_size = kwargs.get("shard_image_size") if self.sp_size > 1 else None
@@ -1109,8 +1189,7 @@ class ImageKVCacheManager(nn.Module):
key = key[:, :0, :, :]
value = value[:, :0, :, :]
elif first_step:
self.image_kv_cache_map = None # reset first
self.image_kv_cache_lens = None
self.clear_legacy_prompt_kv_cache()
key, value = self._cache_prompt_kv(
key,
value,
@@ -1134,7 +1213,7 @@ class ImageKVCacheManager(nn.Module):
joint_text_query = query[:, :0, :, :]
joint_text_key, joint_text_value = self._reuse_prompt_kv(key, value, seq_len, bs, shard_image_size)
if not keep_kv_compressed and not self.attn.is_paged_kv_active():
if not keep_kv_compressed:
key = repeat_kv(key, repeat_num)
value = repeat_kv(value, repeat_num)
if self.sp_size > 1:
@@ -1163,6 +1242,34 @@ class ImageKVCacheManager(nn.Module):
attn_output = attn_output.reshape(bs * q_len, head_num_per_rank, head_dim)
return attn_output
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
self.image_token_len = kwargs.get("num_image_tokens")
if self.attn.is_paged_kv_active():
full_attn_spans = kwargs.get("full_attn_spans")
if full_attn_spans is None:
raise ValueError("Hunyuan Scheduler-paged KV requires full_attn_spans metadata")
return self._forward_paged(
query,
key,
value,
first_step=bool(kwargs.get("first_step")),
query_lens=kwargs.get("query_lens"),
seq_lens=kwargs.get("seq_lens"),
shard_image_size=kwargs.get("shard_image_size") if self.sp_size > 1 else None,
full_attn_spans=full_attn_spans,
uncond_cfg_prefill=kwargs.get("uncond_cfg_prefill", False),
)
if attention_mask is None:
raise ValueError("Hunyuan dense attention requires an attention mask")
return self._forward_dense_legacy(query, key, value, attention_mask, **kwargs)
@dataclass
class CausalMMOutputWithPast(CausalLMOutputWithPast):
@@ -2706,7 +2813,6 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
# List[List[...]] per-sample metadata indexed along the CFG batch dim
if isinstance(model_kwargs.get("full_attn_spans"), list):
model_kwargs["full_attn_spans"] = model_kwargs["full_attn_spans"][s.start : s.stop]
# custom_pos_emb: tuple of (cos, sin)
if "custom_pos_emb" in model_kwargs and model_kwargs["custom_pos_emb"] is not None:
cos, sin = model_kwargs["custom_pos_emb"]
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import copy
import logging
@@ -72,7 +72,6 @@ _STEP_OUTPUT_SIZE = "hunyuan_output_size"
_STEP_COT_TEXT_LIST = "hunyuan_cot_text_list"
_STEP_AR_KV = "hunyuan_ar_kv"
_STEP_PROMPT_KV = "hunyuan_prompt_kv"
_HUNYUAN_DEFAULT_OUTPUT_TYPE = "pil"
@@ -466,6 +465,11 @@ class HunyuanImage3Pipeline(
return self._pipeline
def _validate_step_request(self, state: "StepRequestState") -> None:
if self._uses_scheduler_paged_kv():
raise ValueError(
"HunyuanImage3 paged_scheduler currently supports request-level execution only; "
"disable step execution or use dense_legacy."
)
prompt = state.prompt
sampling = state.sampling
if prompt is None:
@@ -663,6 +667,12 @@ class HunyuanImage3Pipeline(
prefix_lens.append(int(prompt_kv[0]["lens"][branch].item()))
return prefix_lens
def _uses_scheduler_paged_kv(self) -> bool:
return (
getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)
is DiffusionKVCacheMode.PAGED_SCHEDULER
)
def _merge_step_model_inputs(
self,
states: list["StepRequestState"],
@@ -693,6 +703,7 @@ class HunyuanImage3Pipeline(
values,
row_branches,
prefix_lens or [None] * len(row_branches),
strict=True,
)
]
continue
@@ -1871,8 +1882,7 @@ class HunyuanImage3Pipeline(
row_branches = [branch for branch in range(cfg_factor) for _ in states]
return row_state_indexes, row_branches
@staticmethod
def _validate_step_group_states(states: list["StepRequestState"]) -> tuple[bool, int]:
def _validate_step_group_states(self, states: list["StepRequestState"]) -> tuple[bool, int]:
if not states:
raise ValueError("HunyuanImage3 denoise_step received an empty group.")
@@ -2035,6 +2045,11 @@ class HunyuanImage3Pipeline(
states = list(input_batch.states)
if not states:
raise ValueError("HunyuanImage3 denoise_step received an empty batch.")
if self._uses_scheduler_paged_kv():
raise ValueError(
"HunyuanImage3 paged_scheduler currently supports request-level execution only; "
"disable step execution or use dense_legacy."
)
self._ensure_grouped_attention_backend_supported(len(states))
outputs: dict[str, torch.Tensor] = {}
for group in self._split_step_groups(states):
+3 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Build native vLLM configuration views for the Omni diffusion runtime."""
@@ -14,6 +14,7 @@ from vllm.transformers_utils.config import get_hf_text_config
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
from vllm_omni.platforms import current_omni_platform
def resolve_diffusion_max_model_len(od_config: OmniDiffusionConfig) -> int:
@@ -227,6 +228,7 @@ def configure_diffusion_vllm_config(vllm_config: VllmConfig, od_config: OmniDiff
max_num_batched_tokens = getattr(od_config, "max_num_batched_tokens", None)
if max_num_batched_tokens is not None:
vllm_config.scheduler_config.max_num_batched_tokens = int(max_num_batched_tokens)
current_omni_platform.configure_diffusion_vllm_config(vllm_config, od_config)
return vllm_config
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Diffusion Model Runner for vLLM-Omni.
@@ -37,9 +37,10 @@ from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
from vllm_omni.diffusion.diffusion_kv.metadata import DiffusionKVMetadata
from vllm_omni.diffusion.diffusion_kv.model_runner_backend import DiffusionKVModelRunnerBackend
from vllm_omni.diffusion.diffusion_kv.paged_attention_adapter import (
DiffusionPagedAttentionMetadata,
DiffusionPagedAttentionRow,
PreparedDiffusionPagedAttentionBatch,
)
from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_rank
from vllm_omni.diffusion.forward_context import set_forward_context
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import (
@@ -420,11 +421,19 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
raise RuntimeError("Model must be loaded before collecting Diffusion KV cache specs")
cache_layers: dict[str, tuple[Attention, KVCacheSpec]] = {}
for layer_name, module in self.pipeline.named_modules():
for module_path, module in self.pipeline.named_modules():
if not isinstance(module, Attention):
continue
spec = module.get_kv_cache_spec(self.vllm_config)
if spec is not None:
layer_name = module.prefix
if not isinstance(layer_name, str) or not layer_name:
raise RuntimeError(
"Paged Diffusion Attention must expose a non-empty canonical prefix; "
f"module_path={module_path!r}"
)
if layer_name in cache_layers:
raise RuntimeError(f"Duplicate canonical paged Diffusion Attention prefix {layer_name!r}")
cache_layers[layer_name] = (module, spec)
if not cache_layers:
raise RuntimeError(
@@ -456,18 +465,51 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
def refresh_diffusion_kv_block_table_layout(self) -> None:
self.diffusion_kv_backend.refresh_block_table_layout()
def prepare_paged_attention_batch(
def _build_paged_attention_metadata(
self,
rows: list[DiffusionPagedAttentionRow] | tuple[DiffusionPagedAttentionRow, ...],
) -> PreparedDiffusionPagedAttentionBatch:
"""Prepare model-specific query/write spans against installed Worker rows."""
metadata: list[DiffusionKVMetadata],
) -> DiffusionPagedAttentionMetadata:
"""Translate Scheduler sequence state into ordered request-level rows."""
return self.diffusion_kv_backend.prepare_paged_attention_batch(rows)
def activate_paged_attention(self, batch: PreparedDiffusionPagedAttentionBatch):
"""Expose a prepared paged batch to Omni Attention for one forward."""
return self.diffusion_kv_backend.activate_paged_attention(batch)
cfg_size = int(getattr(self.od_config.parallel_config, "cfg_parallel_size", 1) or 1)
cfg_rank = get_classifier_free_guidance_rank() if cfg_size > 1 else None
prefill_rows: list[DiffusionPagedAttentionRow] = []
denoise_rows: list[DiffusionPagedAttentionRow] = []
for request_metadata in metadata:
sequences = request_metadata.sequences
if cfg_rank is not None and len(sequences) > 1:
if len(sequences) != cfg_size:
raise ValueError(
"Paged CFG parallel execution requires one Scheduler sequence per CFG rank: "
f"request={request_metadata.request_id!r}, cfg_size={cfg_size}, rows={len(sequences)}"
)
sequences = (sequences[cfg_rank],)
for sequence in sequences:
active_seq_len = sequence.prefix_len + sequence.target_len
if active_seq_len > sequence.seq_len:
raise ValueError(
"Paged denoise span exceeds its Scheduler allocation: "
f"request={request_metadata.request_id!r}, sequence={sequence.sequence_id}, "
f"active={active_seq_len}, allocated={sequence.seq_len}"
)
prefill_rows.append(
DiffusionPagedAttentionRow(
request_id=request_metadata.request_id,
sequence_id=sequence.sequence_id,
query_len=sequence.seq_len,
seq_len=sequence.seq_len,
)
)
denoise_rows.append(
DiffusionPagedAttentionRow(
request_id=request_metadata.request_id,
sequence_id=sequence.sequence_id,
query_len=sequence.target_len,
seq_len=active_seq_len,
kv_start_pos=sequence.prefix_len,
)
)
return DiffusionPagedAttentionMetadata(tuple(prefill_rows), tuple(denoise_rows))
def clear_prompt_embed_cache(self) -> None:
"""Evict all cached text-encoder outputs (e.g. between training epochs).
@@ -623,6 +665,8 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
kv_prefetch_job: KVPrefetchJob | None = None,
record_name: str,
record_output_peak_memory: bool = True,
in_diffusion_kv_memory_profile: bool = False,
diffusion_kv_metadata: list[DiffusionKVMetadata] | None = None,
) -> BatchRunnerOutput:
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
if not reqs:
@@ -655,7 +699,27 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
if is_primary and record_output_peak_memory:
current_omni_platform.reset_peak_memory_stats()
with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=od_config):
paged_kv_runtime = None
paged_kv_context: AbstractContextManager[Any] = nullcontext()
if diffusion_kv_metadata is not None:
if len(diffusion_kv_metadata) != len(reqs):
raise ValueError(
"Diffusion KV metadata count must match the request batch: "
f"metadata={len(diffusion_kv_metadata)}, requests={len(reqs)}"
)
paged_metadata = self._build_paged_attention_metadata(diffusion_kv_metadata)
paged_kv_runtime, paged_kv_context = self.diffusion_kv_backend.activate_paged_attention_metadata(
paged_metadata
)
with (
set_forward_context(
vllm_config=self.vllm_config,
omni_diffusion_config=od_config,
paged_kv_runtime=paged_kv_runtime,
in_diffusion_kv_memory_profile=in_diffusion_kv_memory_profile,
),
paged_kv_context,
):
with record_function(record_name):
raw_outputs = self.pipeline.forward(batch)
outputs = _normalize_pipeline_outputs(
@@ -733,6 +797,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
require_request_batch_support=False,
kv_prefetch_job=kv_prefetch_job,
record_name="pipeline_forward",
diffusion_kv_metadata=[diffusion_kv_metadata] if diffusion_kv_metadata is not None else None,
)
output = runner_output.runner_outputs[0].result
assert output is not None
@@ -773,6 +838,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
scheduler_output,
validate_kv_metadata=False,
record_output_peak_memory=False,
in_diffusion_kv_memory_profile=True,
)
else:
runner_output = self._execute_request_list(
@@ -785,6 +851,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
# peak counters. Resetting them here would discard request
# preparation allocations and understate the budget.
record_output_peak_memory=False,
in_diffusion_kv_memory_profile=True,
)
current_omni_platform.synchronize()
finally:
@@ -824,6 +891,12 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
allow_single_output=False,
require_request_batch_support=True,
record_name="pipeline_forward_batch",
diffusion_kv_metadata=[
new_req.diffusion_kv_metadata
for new_req in scheduler_output.scheduled_new_reqs
if new_req.diffusion_kv_metadata is not None
]
or None,
)
finally:
if installed_request_ids:
@@ -984,6 +1057,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
*,
validate_kv_metadata: bool,
record_output_peak_memory: bool,
in_diffusion_kv_memory_profile: bool = False,
) -> BatchRunnerOutput:
"""Execute one step with explicit validation and profiling policy."""
@@ -1023,6 +1097,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
return self._execute_stepwise_core(
scheduler_output,
record_output_peak_memory=record_output_peak_memory,
in_diffusion_kv_memory_profile=in_diffusion_kv_memory_profile,
)
except Exception:
if installed_request_ids:
@@ -1034,6 +1109,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
scheduler_output: DiffusionSchedulerOutput,
*,
record_output_peak_memory: bool,
in_diffusion_kv_memory_profile: bool = False,
) -> BatchRunnerOutput:
"""Run the denoise step after metadata admission and row installation."""
@@ -1056,10 +1132,14 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
return BatchRunnerOutput.from_list(runner_output_list)
attn_metadata = {}
kv_backend = getattr(self, "diffusion_kv_backend", None)
paged_kv_runtime = kv_backend if getattr(kv_backend, "paged_attention_adapter", None) is not None else None
with set_forward_context(
vllm_config=self.vllm_config,
omni_diffusion_config=self.od_config,
attn_metadata=attn_metadata,
paged_kv_runtime=paged_kv_runtime,
in_diffusion_kv_memory_profile=in_diffusion_kv_memory_profile,
):
clear_pipeline_stage_durations(self.pipeline)
noise_pred = self.pipeline.denoise_step(input_batch, states=states)
@@ -249,11 +249,6 @@ class DiffusionWorker:
# requests, which only carry their request_id in subsequent ticks.
self._step_lora_state: dict[str, tuple[LoRARequest | None, float]] = {}
self.stage_id = getattr(od_config, "stage_id", 0)
if self.od_config.diffusion_kv_mode is DiffusionKVCacheMode.PAGED_SCHEDULER:
logger.warning_once(
"paged_scheduler initializes native paged KV storage, but no production diffusion model uses the "
"paged-attention adapter yet; model attention remains on the dense path."
)
self.init_device()
# Create model runner — one decision chain, in precedence order:
# 1. explicit od_config.diffusion_model_runner_cls (user override),