[Diffusion] Add paged KV cache support for HunyuanImage3 DiT on NPU and GPU (#6563)

Signed-off-by: Acerak01-fy <wfy2003324@163.com>
Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
Signed-off-by: zwhzzz0821 <2831474076@qq.com>
Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
Co-authored-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
authored and GitHub committed 2026-08-31 10:15:46 +00:00
1 parent 6f16f9546d
commit fe4a2a7477
28 files changed
+1789 -205

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""End-to-end test for ``piecewise_attn`` (CPU).
Verify that running attention in segments (causal outside full-attn spans,
@@ -98,9 +98,12 @@ def test_paged_piecewise_runner_uses_packed_indices_and_kv_endpoints():
query = torch.arange(12 * 2 * 4, dtype=torch.float32).reshape(12, 2, 4)
key = query + 100
value = query + 200
calls = []
calls: list[object] = []
def run_segment(segment_query, segment_key, segment_value, segment_batch):
output_buffer = torch.empty_like(query)
def run_segment(segment_query, segment_key, segment_value, segment_batch, segment_output_buffer):
assert segment_output_buffer is None
calls.append(
(
[segment.mode for segment in segment_batch.row_segments],
@@ -119,9 +122,13 @@ def test_paged_piecewise_runner_uses_packed_indices_and_kv_endpoints():
plan,
plan.segments,
run_segment,
output_buffer=output_buffer,
)
assert output is output_buffer
torch.testing.assert_close(output, query)
assert all(segment.query_range is None for segment in plan.segments)
assert not plan.segments_cover_query_contiguously
assert calls == [
(["causal", "causal"], [2, 4], [0, 1, 6, 7]),
(["full", "full"], [5, 7], [2, 3, 4, 8, 9, 10]),
@@ -129,6 +136,161 @@ def test_paged_piecewise_runner_uses_packed_indices_and_kv_endpoints():
]
def test_paged_piecewise_runner_preserves_homogeneous_batch_without_scatter(monkeypatch: pytest.MonkeyPatch):
plan = build_paged_piecewise_plan(
[[(2, 5)], [(2, 5)]],
query_offsets=[0, 0],
query_lens=[6, 6],
seq_lens=[6, 6],
device=DEVICE,
)
assert plan.homogeneous_batch_shape == (2, 6)
query = torch.arange(12 * 2 * 4, dtype=torch.float32).reshape(12, 2, 4)
calls: list[torch.Tensor] = []
def fail_index_copy(*_args, **_kwargs):
raise AssertionError("homogeneous piecewise path must not use index_copy_")
monkeypatch.setattr(torch.Tensor, "index_copy_", fail_index_copy)
def run_segment(segment_query, segment_key, segment_value, _metadata, segment_output_buffer):
assert segment_key is None
assert segment_value is None
assert segment_output_buffer is None
calls.append(segment_query.clone())
return segment_query + 1
output_buffer = torch.empty_like(query)
output = run_paged_piecewise_plan(
query,
None,
None,
plan,
plan.segments,
run_segment,
output_buffer=output_buffer,
use_homogeneous_batch=True,
)
assert output is output_buffer
torch.testing.assert_close(output, query + 1)
assert [call.shape[0] for call in calls] == [4, 6, 2]
torch.testing.assert_close(calls[0], query[[0, 1, 6, 7]])
torch.testing.assert_close(calls[1], query[[2, 3, 4, 8, 9, 10]])
torch.testing.assert_close(calls[2], query[[5, 11]])
def test_paged_piecewise_runner_keeps_indexed_fallback_for_heterogeneous_layout():
plan = build_paged_piecewise_plan(
[[(2, 5)], [(1, 4)]],
query_offsets=[0, 0],
query_lens=[6, 6],
seq_lens=[6, 6],
device=DEVICE,
)
assert plan.homogeneous_batch_shape is None
query = torch.arange(12 * 2 * 4, dtype=torch.float32).reshape(12, 2, 4)
output = run_paged_piecewise_plan(
query,
None,
None,
plan,
plan.segments,
lambda segment_query, *_args: segment_query,
use_homogeneous_batch=True,
)
torch.testing.assert_close(output, query)
def test_paged_piecewise_runner_slices_contiguous_segments():
plan = build_paged_piecewise_plan(
[[(2, 5)]],
query_offsets=[0],
query_lens=[6],
seq_lens=[6],
device=DEVICE,
)
query = torch.arange(6 * 2 * 4, dtype=torch.float32).reshape(6, 2, 4)
key = query + 100
value = query + 200
expected_ranges = [(0, 2), (2, 5), (5, 6)]
calls: list[tuple[int, int]] = []
def run_segment(segment_query, segment_key, segment_value, _segment_batch, segment_output_buffer):
assert segment_output_buffer is None
expected_range = expected_ranges[len(calls)]
start, end = expected_range
calls.append(expected_range)
assert segment_query.untyped_storage().data_ptr() == query.untyped_storage().data_ptr()
assert segment_key.untyped_storage().data_ptr() == key.untyped_storage().data_ptr()
assert segment_value.untyped_storage().data_ptr() == value.untyped_storage().data_ptr()
torch.testing.assert_close(segment_query, query[start:end])
return segment_query
output = run_paged_piecewise_plan(query, key, value, plan, plan.segments, run_segment)
assert calls == expected_ranges
assert [segment.query_range for segment in plan.segments] == expected_ranges
assert plan.segments_cover_query_contiguously
torch.testing.assert_close(output, query)
def test_paged_piecewise_runner_writes_contiguous_segments_into_output_buffer():
plan = build_paged_piecewise_plan(
[[(2, 5)]],
query_offsets=[0],
query_lens=[6],
seq_lens=[6],
device=DEVICE,
)
query = torch.arange(6 * 2 * 4, dtype=torch.float32).reshape(6, 2, 4)
output_buffer = torch.empty(6, 2, 3)
targets = []
def run_segment(segment_query, _key, _value, _metadata, segment_output_buffer):
assert segment_output_buffer is not None
assert segment_output_buffer.untyped_storage().data_ptr() == output_buffer.untyped_storage().data_ptr()
targets.append(segment_output_buffer)
return segment_output_buffer.copy_(segment_query[..., :3])
output = run_paged_piecewise_plan(
query,
query,
query,
plan,
plan.segments,
run_segment,
output_buffer=output_buffer,
)
assert output is output_buffer
assert [target.shape[0] for target in targets] == [2, 3, 1]
torch.testing.assert_close(output, query[..., :3])
def test_paged_piecewise_runner_returns_single_full_segment_directly():
plan = build_paged_piecewise_plan(
[[]],
query_offsets=[0],
query_lens=[6],
seq_lens=[6],
device=DEVICE,
)
query = torch.randn(6, 2, 4)
runner_output = query + 1
output = run_paged_piecewise_plan(
query,
query,
query,
plan,
plan.segments,
lambda *_args: runner_output,
)
assert output is runner_output
def test_paged_piecewise_runner_uses_runner_output_shape():
plan = build_paged_piecewise_plan(
[[(2, 5)]],