[Diffusion] Add paged KV cache support for HunyuanImage3 DiT on NPU and GPU (#6563)
Signed-off-by: Acerak01-fy <wfy2003324@163.com> Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com> Signed-off-by: zwhzzz0821 <2831474076@qq.com> Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com> Co-authored-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
28 files changed
+1789
-205
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""
|
||||
Piecewise attention for mixed causal / full (bidirectional) masks.
|
||||
|
||||
@@ -92,6 +92,7 @@ def piecewise_attn(
|
||||
):
|
||||
_check_homogeneous(full_attn_spans)
|
||||
spans = full_attn_spans[0]
|
||||
ranges: tuple[tuple[int, int, int], ...]
|
||||
if query_ranges is None:
|
||||
query_len = query.shape[1]
|
||||
ranges = ((0, query_len, key.shape[1] - query_len),)
|
||||
@@ -133,6 +134,11 @@ class PagedPiecewiseSegment:
|
||||
|
||||
row_segments: tuple[Segment, ...]
|
||||
query_indices: torch.Tensor
|
||||
query_range: tuple[int, int] | None
|
||||
# Local [start, end) range shared by every row when the batch can retain
|
||||
# its original [B, S] layout. The packed query_indices remain available
|
||||
# for metadata construction and the heterogeneous fallback.
|
||||
local_query_range: tuple[int, int] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -142,10 +148,16 @@ class PagedPiecewisePlan:
|
||||
spans: tuple[tuple[tuple[int, int], ...], ...]
|
||||
segments: tuple[PagedPiecewiseSegment, ...]
|
||||
num_query_tokens: int
|
||||
segments_cover_query_contiguously: bool
|
||||
# (batch_size, query_len) when all rows have the same local segment
|
||||
# layout. Native FIA still receives flattened row-major tokens, but the
|
||||
# runner can restore outputs with one batch concat instead of one indexed
|
||||
# scatter per segment.
|
||||
homogeneous_batch_shape: tuple[int, int] | None = None
|
||||
|
||||
|
||||
PagedPiecewiseRunner = Callable[
|
||||
[torch.Tensor, torch.Tensor, torch.Tensor, object],
|
||||
[torch.Tensor, torch.Tensor | None, torch.Tensor | None, object, torch.Tensor | None],
|
||||
torch.Tensor,
|
||||
]
|
||||
|
||||
@@ -164,7 +176,9 @@ def build_paged_piecewise_plan(
|
||||
if row_count == 0 or not (row_count == len(query_offsets) == len(query_lens) == len(seq_lens)):
|
||||
raise ValueError("Paged piecewise inputs must have one entry per row")
|
||||
|
||||
spans = tuple(tuple(tuple(span) for span in row_spans) for row_spans in full_attn_spans)
|
||||
spans: tuple[tuple[tuple[int, int], ...], ...] = tuple(
|
||||
tuple((span_start, span_end) for span_start, span_end in row_spans) for row_spans in full_attn_spans
|
||||
)
|
||||
|
||||
packed_offsets = [0]
|
||||
for row_spans, query_offset, query_len, seq_len in zip(spans, query_offsets, query_lens, seq_lens, strict=True):
|
||||
@@ -188,46 +202,206 @@ def build_paged_piecewise_plan(
|
||||
for row_segments in zip(*segments_by_row, strict=True):
|
||||
if len({segment.mode for segment in row_segments}) != 1:
|
||||
raise ValueError("Paged piecewise rows must use the same attention mode per segment")
|
||||
query_indices = []
|
||||
query_indices: list[int] = []
|
||||
local_ranges: list[tuple[int, int]] = []
|
||||
for row_index, (segment, query_offset) in enumerate(zip(row_segments, query_offsets, strict=True)):
|
||||
local_start = segment.q_start - query_offset
|
||||
local_end = segment.q_end - query_offset
|
||||
local_ranges.append((local_start, local_end))
|
||||
query_indices.extend(range(packed_offsets[row_index] + local_start, packed_offsets[row_index] + local_end))
|
||||
query_range = None
|
||||
if query_indices and query_indices == list(range(query_indices[0], query_indices[-1] + 1)):
|
||||
query_range = (query_indices[0], query_indices[-1] + 1)
|
||||
local_query_range = local_ranges[0] if len(set(local_ranges)) == 1 else None
|
||||
packed_segments.append(
|
||||
PagedPiecewiseSegment(
|
||||
row_segments=tuple(row_segments),
|
||||
query_indices=torch.tensor(query_indices, dtype=torch.long, device=device),
|
||||
query_range=query_range,
|
||||
local_query_range=local_query_range,
|
||||
)
|
||||
)
|
||||
|
||||
return PagedPiecewisePlan(spans, tuple(packed_segments), packed_offsets[-1])
|
||||
covered = 0
|
||||
for segment in packed_segments:
|
||||
if segment.query_range is None or segment.query_range[0] != covered:
|
||||
break
|
||||
covered = segment.query_range[1]
|
||||
segments_cover_query_contiguously = covered == packed_offsets[-1]
|
||||
homogeneous_batch_shape: tuple[int, int] | None = None
|
||||
if (
|
||||
row_count > 1
|
||||
and len(set(query_lens)) == 1
|
||||
and all(segment.local_query_range is not None for segment in packed_segments)
|
||||
):
|
||||
# A common local range is sufficient even when rows have different
|
||||
# global offsets or KV lengths; those values remain row-specific in
|
||||
# native metadata.
|
||||
homogeneous_batch_shape = (row_count, int(query_lens[0]))
|
||||
return PagedPiecewisePlan(
|
||||
spans=spans,
|
||||
segments=tuple(packed_segments),
|
||||
num_query_tokens=packed_offsets[-1],
|
||||
segments_cover_query_contiguously=segments_cover_query_contiguously,
|
||||
homogeneous_batch_shape=homogeneous_batch_shape,
|
||||
)
|
||||
|
||||
|
||||
def _run_homogeneous_paged_piecewise_plan(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None,
|
||||
value: torch.Tensor | None,
|
||||
plan: PagedPiecewisePlan,
|
||||
segment_metadata: Sequence[object],
|
||||
segment_runner: PagedPiecewiseRunner,
|
||||
output_buffer: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Run a homogeneous batch while retaining the old [B, S] layout.
|
||||
|
||||
The native FIA contract is flattened TND, so each segment is flattened
|
||||
only for the kernel call. Results are reshaped back to [B, L] and
|
||||
concatenated along the sequence dimension, which avoids the per-segment
|
||||
``index_copy_`` used by heterogeneous packed rows.
|
||||
"""
|
||||
|
||||
batch_shape = plan.homogeneous_batch_shape
|
||||
assert batch_shape is not None
|
||||
batch_size, query_len = batch_shape
|
||||
if query.shape[0] != batch_size * query_len:
|
||||
raise ValueError(
|
||||
f"Homogeneous paged piecewise plan expects {batch_size * query_len} query tokens, got {query.shape[0]}"
|
||||
)
|
||||
query_rows = query.reshape(batch_size, query_len, *query.shape[1:])
|
||||
key_rows = None if key is None else key.reshape(batch_size, query_len, *key.shape[1:])
|
||||
value_rows = None if value is None else value.reshape(batch_size, query_len, *value.shape[1:])
|
||||
|
||||
segment_outputs: list[torch.Tensor] = []
|
||||
for segment, metadata in zip(plan.segments, segment_metadata, strict=True):
|
||||
local_range = segment.local_query_range
|
||||
if local_range is None:
|
||||
raise ValueError("Homogeneous paged piecewise execution requires a shared local segment range")
|
||||
start, end = local_range
|
||||
segment_query = query_rows[:, start:end].contiguous().reshape(-1, *query.shape[1:])
|
||||
segment_key = None if key_rows is None else key_rows[:, start:end].contiguous().reshape(-1, *key.shape[1:])
|
||||
segment_value = (
|
||||
None if value_rows is None else value_rows[:, start:end].contiguous().reshape(-1, *value.shape[1:])
|
||||
)
|
||||
segment_output = segment_runner(segment_query, segment_key, segment_value, metadata, None)
|
||||
expected_tokens = batch_size * (end - start)
|
||||
if segment_output.shape[0] != expected_tokens:
|
||||
raise ValueError(
|
||||
f"Paged piecewise runner returned {segment_output.shape[0]} tokens "
|
||||
f"for a {expected_tokens}-token homogeneous segment"
|
||||
)
|
||||
segment_outputs.append(segment_output.reshape(batch_size, end - start, *segment_output.shape[1:]))
|
||||
|
||||
if not segment_outputs:
|
||||
result = torch.empty_like(query)
|
||||
elif len(segment_outputs) == 1:
|
||||
result = segment_outputs[0].reshape(plan.num_query_tokens, *segment_outputs[0].shape[2:])
|
||||
else:
|
||||
result_rows = torch.cat(segment_outputs, dim=1)
|
||||
result = result_rows.reshape(plan.num_query_tokens, *result_rows.shape[2:])
|
||||
if output_buffer is not None:
|
||||
if output_buffer.shape != result.shape:
|
||||
raise ValueError(
|
||||
f"Paged piecewise output buffer shape {tuple(output_buffer.shape)} does not match "
|
||||
f"homogeneous result shape {tuple(result.shape)}"
|
||||
)
|
||||
output_buffer.copy_(result)
|
||||
return output_buffer
|
||||
return result
|
||||
|
||||
|
||||
def run_paged_piecewise_plan(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
key: torch.Tensor | None,
|
||||
value: torch.Tensor | None,
|
||||
plan: PagedPiecewisePlan,
|
||||
segment_metadata: Sequence[object],
|
||||
segment_runner: PagedPiecewiseRunner,
|
||||
output_buffer: torch.Tensor | None = None,
|
||||
*,
|
||||
use_homogeneous_batch: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Run and scatter piecewise attention over packed Q/K/V write tensors."""
|
||||
"""Run piecewise attention with view fast paths for contiguous segments.
|
||||
|
||||
``use_homogeneous_batch`` is an opt-in for native backends that can keep
|
||||
identical rows in the legacy batch layout. The default remains the
|
||||
indexed packed path so existing GPU/heterogeneous contracts are unchanged.
|
||||
"""
|
||||
|
||||
if query.shape[0] != plan.num_query_tokens:
|
||||
raise ValueError(f"Paged piecewise plan has {plan.num_query_tokens} query tokens, got {query.shape[0]}")
|
||||
if output_buffer is not None and output_buffer.shape[0] != plan.num_query_tokens:
|
||||
raise ValueError(
|
||||
f"Paged piecewise output buffer has {output_buffer.shape[0]} tokens; expected {plan.num_query_tokens}"
|
||||
)
|
||||
|
||||
output = None
|
||||
if use_homogeneous_batch and plan.homogeneous_batch_shape is not None:
|
||||
return _run_homogeneous_paged_piecewise_plan(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
plan,
|
||||
segment_metadata,
|
||||
segment_runner,
|
||||
output_buffer,
|
||||
)
|
||||
|
||||
output = output_buffer
|
||||
contiguous_outputs = []
|
||||
for segment, metadata in zip(plan.segments, segment_metadata, strict=True):
|
||||
indices = segment.query_indices
|
||||
segment_output_buffer = None
|
||||
if segment.query_range is None:
|
||||
segment_query = query.index_select(0, indices)
|
||||
segment_key = None if key is None else key.index_select(0, indices)
|
||||
segment_value = None if value is None else value.index_select(0, indices)
|
||||
else:
|
||||
start, end = segment.query_range
|
||||
segment_query = query[start:end]
|
||||
segment_key = None if key is None else key[start:end]
|
||||
segment_value = None if value is None else value[start:end]
|
||||
if output is not None:
|
||||
segment_output_buffer = output[start:end]
|
||||
segment_output = segment_runner(
|
||||
query.index_select(0, indices),
|
||||
key.index_select(0, indices),
|
||||
value.index_select(0, indices),
|
||||
segment_query,
|
||||
segment_key,
|
||||
segment_value,
|
||||
metadata,
|
||||
segment_output_buffer,
|
||||
)
|
||||
if segment_output.shape[0] != indices.shape[0]:
|
||||
raise ValueError(
|
||||
f"Paged piecewise runner returned {segment_output.shape[0]} tokens "
|
||||
f"for a {indices.shape[0]}-token segment"
|
||||
)
|
||||
if segment_output_buffer is not None and segment_output is not segment_output_buffer:
|
||||
if segment_output.shape != segment_output_buffer.shape:
|
||||
raise ValueError(
|
||||
f"Paged piecewise runner returned shape {tuple(segment_output.shape)} "
|
||||
f"for output buffer shape {tuple(segment_output_buffer.shape)}"
|
||||
)
|
||||
segment_output_buffer.copy_(segment_output)
|
||||
segment_output = segment_output_buffer
|
||||
if plan.segments_cover_query_contiguously:
|
||||
if output_buffer is None:
|
||||
contiguous_outputs.append(segment_output)
|
||||
continue
|
||||
if output is None:
|
||||
output = segment_output.new_empty((query.shape[0], *segment_output.shape[1:]))
|
||||
output.index_copy_(0, indices, segment_output)
|
||||
if segment.query_range is None:
|
||||
output.index_copy_(0, indices, segment_output)
|
||||
else:
|
||||
start, end = segment.query_range
|
||||
output[start:end].copy_(segment_output)
|
||||
|
||||
if output_buffer is not None and plan.segments_cover_query_contiguously:
|
||||
return output_buffer
|
||||
if contiguous_outputs:
|
||||
if len(contiguous_outputs) == 1:
|
||||
return contiguous_outputs[0]
|
||||
return torch.cat(contiguous_outputs, dim=0)
|
||||
assert output is not None
|
||||
return output
|
||||
Reference in new issue
Block a user