[Diffusion] Add paged KV cache support for HunyuanImage3 DiT on NPU and GPU (#6563)
Signed-off-by: Acerak01-fy <wfy2003324@163.com> Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com> Signed-off-by: zwhzzz0821 <2831474076@qq.com> Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com> Co-authored-by: zwhzzz0821 <2831474076@qq.com>
This commit is contained in:
28 files changed
+1789
-205
No files matched your search
@@ -11,6 +11,7 @@ Tests cover:
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -466,6 +467,120 @@ class TestCurrentDiffusionConfig:
|
||||
|
||||
|
||||
class TestAttentionInitUsesCurrentDiffusionConfig:
|
||||
@pytest.mark.parametrize("mindiesd_available", [False, True])
|
||||
def test_paged_npu_memory_profile_uses_sdpa_only_without_mindiesd(self, monkeypatch, mindiesd_available):
|
||||
attention = Attention.__new__(Attention)
|
||||
attention._scheduler_paged_kv = True
|
||||
attention.paged_kv_cache_role = "primary"
|
||||
attention.backend_pref = "FLASH_ATTN"
|
||||
attention.attn_backend = SimpleNamespace(supports_piecewise_spans=True, get_name=lambda: "FLASH_ATTN")
|
||||
flash_output = torch.ones(1)
|
||||
sdpa_output = torch.zeros(1)
|
||||
dense_flash = Mock(return_value=flash_output)
|
||||
sdpa = Mock(return_value=sdpa_output)
|
||||
attention.attention = SimpleNamespace(forward=dense_flash)
|
||||
attention.sdpa_fallback = SimpleNamespace(forward=sdpa)
|
||||
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
layer_mod,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(in_diffusion_kv_memory_profile=True),
|
||||
)
|
||||
monkeypatch.setattr(layer_mod.current_omni_platform, "is_npu", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
layer_mod.current_omni_platform,
|
||||
"supports_diffusion_dense_flash_attention",
|
||||
lambda: mindiesd_available,
|
||||
)
|
||||
|
||||
result = attention._run_local_attention(
|
||||
torch.zeros(1, dtype=torch.bfloat16),
|
||||
torch.zeros(1, dtype=torch.bfloat16),
|
||||
torch.zeros(1, dtype=torch.bfloat16),
|
||||
AttentionMetadata(),
|
||||
)
|
||||
|
||||
if mindiesd_available:
|
||||
assert result is flash_output
|
||||
dense_flash.assert_called_once()
|
||||
sdpa.assert_not_called()
|
||||
else:
|
||||
assert result is sdpa_output
|
||||
dense_flash.assert_not_called()
|
||||
sdpa.assert_called_once()
|
||||
|
||||
def test_dense_flash_import_error_is_not_reclassified(self, monkeypatch):
|
||||
attention = Attention.__new__(Attention)
|
||||
attention._scheduler_paged_kv = False
|
||||
attention.paged_kv_cache_role = "primary"
|
||||
attention.backend_pref = "FLASH_ATTN"
|
||||
attention.attn_backend = SimpleNamespace(supports_piecewise_spans=True, get_name=lambda: "FLASH_ATTN")
|
||||
attention.attention = SimpleNamespace(
|
||||
forward=Mock(side_effect=ModuleNotFoundError("No module named 'mindiesd'", name="mindiesd"))
|
||||
)
|
||||
attention.sdpa_fallback = SimpleNamespace(forward=Mock(side_effect=AssertionError("must not fall back")))
|
||||
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: False)
|
||||
|
||||
with pytest.raises(ModuleNotFoundError, match="mindiesd"):
|
||||
attention._run_local_attention(
|
||||
torch.zeros(1, dtype=torch.bfloat16),
|
||||
torch.zeros(1, dtype=torch.bfloat16),
|
||||
torch.zeros(1, dtype=torch.bfloat16),
|
||||
AttentionMetadata(),
|
||||
)
|
||||
|
||||
def test_paged_marker_does_not_change_dense_backend(self, monkeypatch):
|
||||
class _FakeAttentionImpl:
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
|
||||
def forward(self, query, key, value, attn_metadata=None):
|
||||
return query
|
||||
|
||||
class _DenseBackend:
|
||||
supports_paged_kv = False
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TORCH_SDPA"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls():
|
||||
return _FakeAttentionImpl
|
||||
|
||||
calls = []
|
||||
|
||||
def _fake_get_attn_backend_for_role(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
return _DenseBackend, None
|
||||
|
||||
monkeypatch.setattr(layer_mod, "get_attn_backend_for_role", _fake_get_attn_backend_for_role)
|
||||
monkeypatch.setattr(layer_mod.SDPABackend, "get_impl_cls", staticmethod(lambda: _FakeAttentionImpl))
|
||||
monkeypatch.setattr(layer_mod, "build_parallel_attention_strategy", lambda **kwargs: object())
|
||||
monkeypatch.setattr(layer_mod, "is_forward_context_available", lambda: False)
|
||||
|
||||
od_config = SimpleNamespace(
|
||||
diffusion_attention_config=AttentionConfig(),
|
||||
diffusion_kv_mode=layer_mod.DiffusionKVCacheMode.DENSE_LEGACY,
|
||||
parallel_config=SimpleNamespace(ring_degree=1),
|
||||
diffusion_kv_cache_dtype=None,
|
||||
diffusion_kv_cache_skip_step_indices=None,
|
||||
diffusion_kv_cache_skip_layer_indices=None,
|
||||
)
|
||||
|
||||
with set_current_diffusion_config(od_config):
|
||||
attention = Attention(
|
||||
num_heads=4,
|
||||
head_size=64,
|
||||
causal=False,
|
||||
softmax_scale=1.0,
|
||||
paged_kv_cache_role="primary",
|
||||
)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert attention.attn_backend is _DenseBackend
|
||||
assert attention.backend_pref is None
|
||||
|
||||
def test_attention_init_uses_current_diffusion_config_without_forward_context(self, monkeypatch):
|
||||
class _FakeAttentionImpl:
|
||||
def __init__(self, **kwargs):
|
||||
|
||||
Reference in new issue
Block a user