[Refactor][1/N]diffusion/cache: cache-dit (#5226)

Signed-off-by: yangjianjuan <66548561+yangjianjuan@users.noreply.github.com>
This commit is contained in:
Yang_wdrs authored and GitHub committed 2026-07-24 22:42:45 +08:00
1 parent a5d9d57adc
commit 8927ec057c
34 files changed
+635 -436

No files matched your search

+2 -1
View File
@@ -418,7 +418,8 @@ def enable_cache_for_your_model(pipeline, cache_config):
#### 9d. Register the custom enabler
Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
Add your enabler to `CUSTOM_DIT_ENABLERS` in
`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
```python
CUSTOM_DIT_ENABLERS = {
@@ -29,8 +29,13 @@ vLLM-Omni integrates cache-dit through `CacheDiTBackend`:
| `cache_dit.refresh_context()` | Updates cache context when `num_inference_steps` changes |
**Source files**:
- `vllm_omni/diffusion/cache/cache_dit_backend.py` — `CacheDiTBackend`, enablers, `CUSTOM_DIT_ENABLERS`
- `vllm_omni/diffusion/cache/` — cache backend implementations
- `vllm_omni/diffusion/cache/cachedit/__init__.py` — public package API
- `vllm_omni/diffusion/cache/cachedit/backend.py` — generic backend lifecycle and integration
- `vllm_omni/diffusion/cache/cachedit/config.py` — Cache-DiT configuration types
- `vllm_omni/diffusion/cache/cachedit/model_specific.py` — model-specific adapters, enablers, and registry
Outside `vllm_omni/diffusion/cache`, import Cache-DiT symbols only from
`vllm_omni.diffusion.cache.cachedit`.
## Standard Models: Automatic Support
@@ -175,7 +180,8 @@ Inspect your block's `forward()` return type and residual connection pattern to
## Registering Custom Enablers
Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
Add your enabler to `CUSTOM_DIT_ENABLERS` in
`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
```python
CUSTOM_DIT_ENABLERS = {
@@ -248,7 +254,7 @@ Models listed in `_NO_CACHE_ACCELERATION` in `vllm_omni/diffusion/registry.py` d
| Model | Path | Notes |
|-------|------|-------|
| Standard DiT | `cache_dit_backend.py::enable_cache_for_dit` | Default enabler, automatic |
| Wan2.2 | `cache_dit_backend.py::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
| LongCat | `cache_dit_backend.py::enable_cache_for_longcat_image` | Multi-block-list |
| BAGEL | `cache_dit_backend.py::enable_cache_for_bagel` | Complex omni model |
| Standard DiT | `cachedit.backend::enable_cache_for_dit` | Default enabler, automatic |
| Wan2.2 | `cachedit.model_specific::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
| LongCat | `cachedit.config::CacheDiTAdapterConfig` | Declarative multi-block-list adapter |
| BAGEL | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter |
+12 -8
View File
@@ -32,10 +32,13 @@ The library supports three main caching strategies:
vLLM-omni integrates cache-dit through the `CacheDiTBackend` class, which provides a unified interface for managing cache-dit acceleration on diffusion models.
Code outside `vllm_omni/diffusion/cache` must import Cache-DiT symbols from the
package-level API: `vllm_omni.diffusion.cache.cachedit`.
| Method/Class | Purpose | Behavior |
|--------------|---------|----------|
| [`CacheDiTBackend`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/#vllm_omni.diffusion.cache.CacheBackend) | Unified backend interface | Automatically handles enabler selection and cache refresh |
| [`enable_cache_for_dit()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_dit) | Apply caching to transformer | Configures DBCache on transformer blocks |
| `CacheDiTBackend` | Unified backend interface | Automatically handles enabler selection and cache refresh |
| `enable_cache_for_dit()` | Apply caching to transformer | Configures DBCache on transformer blocks |
**Key APIs from Cache-DiT:**
@@ -47,7 +50,7 @@ vLLM-omni integrates cache-dit through the `CacheDiTBackend` class, which provid
| `ForwardPattern` | Defines block forward signature patterns: `Pattern_0`, `Pattern_1`, `Pattern_2` |
| `ParamsModifier` | Per-transformer or per-block-list cache configuration customization |
| `DBCacheConfig` | Configuration for DBCache parameters (warmup steps, cached steps, thresholds) |
| `refresh_context()` | Update cache context | Called when `num_inference_steps` changes |
| `refresh_context()` | Update cache context | Called at every generation boundary |
---
@@ -179,7 +182,8 @@ cache_dit.enable_cache(
### Registering Custom Implementations
After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in
`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
```python
CUSTOM_DIT_ENABLERS = {
@@ -266,10 +270,10 @@ Complete examples in the codebase:
| Model | Path | Pattern | Notes |
|-------|------|---------|-------|
| **Standard DiT** | [`cache_dit_backend.py::enable_cache_for_dit`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_dit) | Default enabler | Single transformer, automatic |
| **Wan2.2** | [`cache_dit_backend.py::enable_cache_for_wan22`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_wan22) | Single or dual-transformer | Auto-detects mode based on transformer_2 presence |
| **LongCat** | [`cache_dit_backend.py::enable_cache_for_longcat_image`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_longcat_image) | Multi-block-list | Two block lists in one transformer |
| **BAGEL** | [`cache_dit_backend.py::enable_cache_for_bagel`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_bagel) | Omni model | Complex architecture |
| **Standard DiT** | `cachedit.backend::enable_cache_for_dit` | Default enabler | Single transformer, automatic |
| **Wan2.2** | `cachedit.model_specific::enable_cache_for_wan22` | Single or dual-transformer | Auto-detects mode based on transformer_2 presence |
| **LongCat** | `cachedit.config::CacheDiTAdapterConfig` | Declarative block adapter | Two block lists in one transformer |
| **BAGEL** | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter | Complex architecture |
---
+1 -1
View File
@@ -769,7 +769,7 @@ class TeaCacheBackend(CacheBackend):
**2. Cache-DiT Backend**
**Location**: `vllm_omni/diffusion/cache/cache_dit_backend.py`
**Location**: `vllm_omni/diffusion/cache/cachedit/`
```python
class CacheDiTBackend(CacheBackend):
@@ -6,13 +6,13 @@ import logging
import torch
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit import QwenImageEditPipeline
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit_plus import QwenImageEditPlusPipeline
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
logger = logging.getLogger(__name__)
class CustomPipeline(QwenImageEditPipeline):
class CustomPipeline(QwenImageEditPlusPipeline):
def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
super().__init__(od_config=od_config, prefix=prefix)
+98 -19
View File
@@ -17,13 +17,14 @@ import cache_dit
import pytest
from cache_dit import ForwardPattern
from vllm_omni.diffusion.cache.cache_dit_backend import (
from vllm_omni.diffusion.cache.cachedit import (
CacheDiTAdapterConfig,
CacheDiTBackend,
CacheDiTConfig,
)
from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend
from vllm_omni.diffusion.cache.magcache import MagCacheBackend
from vllm_omni.diffusion.cache.selector import get_cache_backend
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
from vllm_omni.diffusion.cache.teacache import TeaCacheBackend
from vllm_omni.diffusion.data import DiffusionCacheConfig
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -32,6 +33,19 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class TestCacheDiTBackend:
"""Test CacheDiTBackend implementation."""
def test_config_projection_contains_only_cache_dit_fields(self):
shared_config = DiffusionCacheConfig(
Fn_compute_blocks=4,
max_warmup_steps=8,
rel_l1_thresh=0.3,
)
config = CacheDiTConfig.from_diffusion_config(shared_config)
assert config.Fn_compute_blocks == 4
assert config.max_warmup_steps == 8
assert not hasattr(config, "rel_l1_thresh")
def test_init_with_dict(self):
"""Test initialization with dictionary config."""
config_dict = {"Fn_compute_blocks": 4, "max_warmup_steps": 8}
@@ -47,8 +61,8 @@ class TestCacheDiTBackend:
assert backend.config.Fn_compute_blocks == 4
assert backend.enabled is False
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_enable_single_transformer(self, mock_cache_dit, mock_block_adapter):
"""Test enabling cache-dit on single-transformer pipeline."""
# Mock pipeline
@@ -75,10 +89,61 @@ class TestCacheDiTBackend:
assert backend._refresh_func is not None
mock_cache_dit.enable_cache.assert_called_once()
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
@patch("vllm_omni.diffusion.cache.cachedit.backend.logger")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_enable_without_model_adapter_uses_cache_dit_registry(self, mock_cache_dit, mock_logger):
"""A missing model-declared adapter falls back to Cache-DiT's registry."""
class BuiltinTransformer:
pass
class BuiltinPipeline:
transformer = BuiltinTransformer()
pipeline = BuiltinPipeline()
backend = CacheDiTBackend({"Fn_compute_blocks": 2})
backend.enable(pipeline)
assert backend.enabled is True
assert backend._refresh_func is not None
assert mock_cache_dit.enable_cache.call_args.args[0] is pipeline.transformer
mock_logger.info.assert_any_call(
"Transformer %s does not declare _cache_dit_adapter_config; "
"falling back to Cache-DiT's built-in adapter registry.",
"BuiltinTransformer",
)
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_enable_without_compatible_adapter_has_contextual_error(self, mock_cache_dit):
"""An unsupported fallback reports both pipeline and transformer names."""
class UnsupportedTransformer:
pass
class UnsupportedPipeline:
transformer = UnsupportedTransformer()
pipeline = UnsupportedPipeline()
backend = CacheDiTBackend({"Fn_compute_blocks": 2})
mock_cache_dit.enable_cache.side_effect = ValueError("unsupported")
with pytest.raises(
ValueError,
match=(
"Failed to enable Cache-DiT for pipeline UnsupportedPipeline with transformer UnsupportedTransformer"
),
) as exc_info:
backend.enable(pipeline)
assert isinstance(exc_info.value.__cause__, ValueError)
assert backend.enabled is False
assert backend._refresh_func is None
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_refresh(self, mock_cache_dit, mock_block_adapter):
"""Test refreshing cache context with SCM mask policy updates when num_inference_steps changes."""
"""Test refreshing cache context with SCM mask policy updates."""
# Mock pipeline
mock_pipeline = Mock()
mock_pipeline.__class__.__name__ = "DiTPipeline"
@@ -107,8 +172,6 @@ class TestCacheDiTBackend:
# First refresh with 50 steps
backend.refresh(mock_pipeline, num_inference_steps=50)
assert backend._last_num_inference_steps == 50
# Verify steps_mask was called with mask policy (not direct steps mask)
mock_cache_dit.steps_mask.assert_called_with(mask_policy="fast", total_steps=50)
assert mock_cache_dit.steps_mask.call_count == 1
@@ -137,10 +200,28 @@ class TestCacheDiTBackend:
call_args = mock_cache_dit.refresh_context.call_args
assert call_args[0][0] == mock_transformer
assert "cache_config" in call_args[1]
assert backend._last_num_inference_steps == 100
assert mock_cache_dit.refresh_context.call_count == 1
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_refresh_resets_same_step_count(self, mock_cache_dit, mock_block_adapter):
"""Every generation must reset context, even when the step count is unchanged."""
mock_pipeline = Mock()
mock_pipeline.__class__.__name__ = "DiTPipeline"
mock_pipeline.transformer = Mock()
mock_pipeline.transformer._cache_dit_adapter_config = CacheDiTAdapterConfig(
block_forward_patterns={"layers": ForwardPattern.Pattern_0}
)
backend = CacheDiTBackend(DiffusionCacheConfig())
backend.enable(mock_pipeline)
backend.refresh(mock_pipeline, num_inference_steps=20)
backend.refresh(mock_pipeline, num_inference_steps=20)
assert mock_cache_dit.refresh_context.call_count == 2
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_enable_hunyuan_pipeline_uses_model_transformer(self, mock_cache_dit, mock_block_adapter):
"""Test HunyuanImage3 uses pipeline.transformer for cache enable/refresh.
@@ -179,8 +260,8 @@ class TestCacheDiTBackend:
assert call_args[0][0] is mock_pipeline.model
assert call_args[1]["num_inference_steps"] == 12
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_enable_dreamid_pipeline_uses_fused_blocks(self, mock_cache_dit, mock_block_adapter):
"""Test DreamID uses pipeline.transformer for cache enable/refresh.
@@ -221,8 +302,8 @@ class TestCacheDiTBackend:
assert call_args[1]["num_inference_steps"] == 12
@pytest.mark.parametrize("num_inference_steps", [1, 7])
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_refresh_scm_bypassed_for_unsupported_step_counts(
self, mock_cache_dit, mock_block_adapter, num_inference_steps
):
@@ -362,8 +443,6 @@ class TestCacheSelector:
class TestMagCacheBackend:
"""Test MagCacheBackend implementation."""
from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend
def test_init(self):
"""Test initialization."""
config = DiffusionCacheConfig(mag_threshold=0.1, mag_max_skip_steps=2, mag_calibrate=True)
+80 -12
View File
@@ -5,15 +5,18 @@
Model specific tests for CacheDiT enablement.
"""
import ast
import sys
from pathlib import Path
from unittest.mock import Mock, patch
import pytest
import torch
from cache_dit.caching.cache_blocks.pattern_0_1_2 import CachedBlocks_Pattern_0_1_2
import vllm_omni.diffusion.cache.cache_dit_backend as cd_backend
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig, CacheDiTBackend, cache_summary
import vllm_omni.diffusion.cache.cachedit as cd_backend
import vllm_omni.diffusion.cache.cachedit.model_specific as cd_model_specific
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig, CacheDiTBackend, cache_summary
from vllm_omni.diffusion.data import DiffusionCacheConfig
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
from vllm_omni.diffusion.models.helios.helios_transformer import HeliosTransformer3DModel
@@ -42,8 +45,73 @@ SEPARATE_CFG_TRANSFORMERS = [
SAMPLE_CACHE_CONFIG = DiffusionCacheConfig()
def test_custom_cache_dit_enablers_are_registered_explicitly():
expected_enablers = {
"Wan22Pipeline": cd_model_specific.enable_cache_for_wan22,
"Wan22I2VPipeline": cd_model_specific.enable_cache_for_wan22,
"Wan22TI2VPipeline": cd_model_specific.enable_cache_for_wan22,
"Wan22VACEPipeline": cd_model_specific.enable_cache_for_wan22,
"Wan22S2VPipeline": cd_model_specific.enable_cache_for_wan22_s2v,
"Cosmos3OmniDiffusersPipeline": cd_model_specific.enable_cache_for_cosmos3,
"Cosmos3OmniPipeline": cd_model_specific.enable_cache_for_cosmos3,
"Krea2Pipeline": cd_model_specific.enable_cache_for_krea2,
}
with patch.dict(cd_backend.CUSTOM_DIT_ENABLERS, {}, clear=True):
cd_model_specific.register_custom_dit_enablers()
assert cd_backend.CUSTOM_DIT_ENABLERS == expected_enablers
def test_wan22_vace_uses_wan22_custom_cache_dit_enabler():
assert cd_backend.CUSTOM_DIT_ENABLERS["Wan22VACEPipeline"] is cd_backend.enable_cache_for_wan22
assert cd_backend.CUSTOM_DIT_ENABLERS["Wan22VACEPipeline"] is cd_model_specific.enable_cache_for_wan22
@pytest.mark.parametrize(
"pipeline_name",
["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"],
)
def test_cosmos3_aliases_use_cosmos3_custom_cache_dit_enabler(pipeline_name: str):
assert cd_backend.CUSTOM_DIT_ENABLERS[pipeline_name] is cd_model_specific.enable_cache_for_cosmos3
def test_cachedit_public_api_is_explicit():
assert set(cd_backend.__all__) == {
"BagelCachedAdapter",
"CUSTOM_DIT_ENABLERS",
"CacheDiTAdapterConfig",
"CacheDiTBackend",
"CacheDiTConfig",
"SensenovaCachedAdapter",
"cache_summary",
"enable_cache_for_dit",
}
assert not hasattr(cd_backend, "enable_cache_for_wan22")
assert not hasattr(cd_backend, "enable_cache_for_wan22_s2v")
def test_cachedit_consumers_use_package_api():
cache_dir = Path(cd_backend.__file__).resolve().parents[1]
package_root = cache_dir.parents[1]
legacy_module = "vllm_omni.diffusion.cache.cache_dit_backend"
internal_prefix = "vllm_omni.diffusion.cache.cachedit."
invalid_imports = []
for source_path in package_root.rglob("*.py"):
if source_path.is_relative_to(cache_dir):
continue
tree = ast.parse(source_path.read_text(encoding="utf-8"))
for node in ast.walk(tree):
modules = []
if isinstance(node, ast.ImportFrom) and node.module is not None:
modules.append(node.module)
elif isinstance(node, ast.Import):
modules.extend(alias.name for alias in node.names)
for module in modules:
if module == legacy_module or module.startswith(internal_prefix):
invalid_imports.append(f"{source_path.relative_to(package_root)}:{node.lineno}: {module}")
assert not invalid_imports, "Cache-DiT consumers must use the package API:\n" + "\n".join(invalid_imports)
@pytest.mark.parametrize("transformer_model", SEPARATE_CFG_TRANSFORMERS)
@@ -54,12 +122,12 @@ def test_cache_dit_configs_have_separate_cfg(transformer_model):
assert transformer_model._cache_dit_adapter_config.has_separate_cfg is True
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.cache_dit")
def test_separate_wan22_custom_enabler_has_separate_cfg(mock_cache_dit, mock_block_adapter):
"""Ensure that Wan22, which has a custom enabler, setts custom CFG correctly."""
mock_pipeline = Mock()
cd_backend.enable_cache_for_wan22(mock_pipeline, SAMPLE_CACHE_CONFIG)
cd_model_specific.enable_cache_for_wan22(mock_pipeline, SAMPLE_CACHE_CONFIG)
mock_cache_dit.enable_cache.assert_called_once()
adapter_kwargs = mock_block_adapter.call_args.kwargs
@@ -67,8 +135,8 @@ def test_separate_wan22_custom_enabler_has_separate_cfg(mock_cache_dit, mock_blo
@pytest.mark.parametrize("has_transformer_2", [False, True])
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.cache_dit")
def test_wan22_custom_enabler_passes_taylorseer_calibrator(
mock_cache_dit,
mock_block_adapter,
@@ -82,7 +150,7 @@ def test_wan22_custom_enabler_passes_taylorseer_calibrator(
mock_pipeline.transformer_2 = None
cache_config = DiffusionCacheConfig(enable_taylorseer=True, taylorseer_order=1)
cd_backend.enable_cache_for_wan22(mock_pipeline, cache_config)
cd_model_specific.enable_cache_for_wan22(mock_pipeline, cache_config)
enable_cache_kwargs = mock_cache_dit.enable_cache.call_args.kwargs
calibrator_config = enable_cache_kwargs["calibrator_config"]
@@ -94,8 +162,8 @@ def test_wan22_custom_enabler_passes_taylorseer_calibrator(
assert modifier._context_kwargs["calibrator_config"] is calibrator_config
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
def test_cosmos3_cache_dit_wraps_gen_layers(mock_cache_dit, mock_block_adapter):
"""Cosmos3 should cache only the repeated GEN pathway blocks."""
mock_pipeline = Mock()
@@ -103,7 +171,7 @@ def test_cosmos3_cache_dit_wraps_gen_layers(mock_cache_dit, mock_block_adapter):
mock_pipeline.transformer.gen_layers = gen_layers
mock_pipeline.transformer._cache_dit_adapter_config = Cosmos3VFMTransformer._cache_dit_adapter_config
cd_backend.enable_cache_for_cosmos3(mock_pipeline, SAMPLE_CACHE_CONFIG)
cd_model_specific.enable_cache_for_cosmos3(mock_pipeline, SAMPLE_CACHE_CONFIG)
mock_cache_dit.enable_cache.assert_called_once()
adapter_kwargs = mock_block_adapter.call_args.kwargs
@@ -465,7 +465,7 @@ def test_edge_forward_rejects_unsupported_modes(
def test_pipeline_registered_and_exported() -> None:
from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS
from vllm_omni.diffusion.cache.cachedit import CUSTOM_DIT_ENABLERS
from vllm_omni.diffusion.models import cosmos3
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
@@ -731,7 +731,7 @@ class TestRegistryIntegration:
def test_cache_dit_for_ltx2_does_not_have_custom_enablers_registered(self):
"""Pipeline variants are *not* registered in CUSTOM_DIT_ENABLERS."""
from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS
from vllm_omni.diffusion.cache.cachedit import CUSTOM_DIT_ENABLERS
# NOTE: We used to have custom enablers for this model, but refactored to handle
# it more generically. Now we only need to ensure it has git cache adapter config.
@@ -4,7 +4,7 @@
import pytest
from torch import nn
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel, _make_rms_norm
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+35
View File
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Public API for the Cache-DiT diffusion cache backend."""
from vllm_omni.diffusion.cache.cachedit.backend import (
CUSTOM_DIT_ENABLERS,
CacheDiTBackend,
cache_summary,
enable_cache_for_dit,
)
from vllm_omni.diffusion.cache.cachedit.config import (
CacheDiTAdapterConfig,
CacheDiTConfig,
)
from vllm_omni.diffusion.cache.cachedit.model_specific import (
BagelCachedAdapter,
SensenovaCachedAdapter,
)
from vllm_omni.diffusion.cache.cachedit.model_specific import (
register_custom_dit_enablers as _register_custom_dit_enablers,
)
_register_custom_dit_enablers()
__all__ = [
"BagelCachedAdapter",
"CUSTOM_DIT_ENABLERS",
"CacheDiTAdapterConfig",
"CacheDiTBackend",
"CacheDiTConfig",
"SensenovaCachedAdapter",
"cache_summary",
"enable_cache_for_dit",
]
+222
View File
@@ -0,0 +1,222 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Generic Cache-DiT backend lifecycle and integration."""
from collections.abc import Callable
from typing import Any, TypeAlias
import cache_dit
from cache_dit import BlockAdapter, DBCacheConfig
from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter
from vllm.logger import init_logger
from vllm_omni.diffusion.cache.base import CacheBackend
from vllm_omni.diffusion.cache.cachedit.config import (
CacheDiTAdapterConfig,
CacheDiTConfig,
)
from vllm_omni.diffusion.data import DiffusionCacheConfig
logger = init_logger(__name__)
RefreshCacheContextFunc: TypeAlias = Callable[[Any, int, bool], None]
CacheDiTEnabler: TypeAlias = Callable[[Any, DiffusionCacheConfig], RefreshCacheContextFunc]
# Model-specific implementations register themselves when the package loads.
CUSTOM_DIT_ENABLERS: dict[str, CacheDiTEnabler] = {}
def cache_summary(pipeline: Any, details: bool = True) -> None:
"""Log Cache-DiT statistics for every transformer on the pipeline."""
transformers = [
transformer
for attribute in ("transformer", "transformer_2")
if (transformer := getattr(pipeline, attribute, None)) is not None
]
for transformer in transformers:
cache_dit.summary(transformer, details=details)
if not transformers:
logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute")
def _default_get_pipeline_transformer(pipeline: Any) -> Any:
return pipeline.transformer
def _build_cache_context_refresh(
cache_config: DiffusionCacheConfig,
get_pipeline_transformer: Callable[[Any], Any] = _default_get_pipeline_transformer,
) -> RefreshCacheContextFunc:
"""Build the cache context refresh callback for one transformer."""
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
transformer = get_pipeline_transformer(pipeline)
# Cache-DiT has no predefined SCM mask for these small step counts.
scm_supported_steps = num_inference_steps >= 8 or num_inference_steps in (4, 6)
if projected_config.scm_steps_mask_policy is None or not scm_supported_steps:
cache_dit.refresh_context(transformer, num_inference_steps=num_inference_steps, verbose=verbose)
return
cache_dit.refresh_context(
transformer,
cache_config=DBCacheConfig().reset(
num_inference_steps=num_inference_steps,
steps_computation_mask=cache_dit.steps_mask(
mask_policy=projected_config.scm_steps_mask_policy,
total_steps=num_inference_steps,
),
steps_computation_policy=projected_config.scm_steps_policy,
),
verbose=verbose,
)
return refresh_cache_context
def enable_cache_for_dit(
pipeline: Any,
cache_config: DiffusionCacheConfig,
block_adapter: BlockAdapter | None = None,
adapter_cls: type[CachedAdapter] | None = None,
) -> RefreshCacheContextFunc:
"""Enable Cache-DiT for a standard single-transformer DiT pipeline."""
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
db_cache_config = projected_config.to_db_cache_config()
calibrator_config = projected_config.to_calibrator_config()
logger.info(
"Enabling cache-dit on transformer: Fn=%s, Bn=%s, W=%s",
db_cache_config.Fn_compute_blocks,
db_cache_config.Bn_compute_blocks,
db_cache_config.max_warmup_steps,
)
transformer = _default_get_pipeline_transformer(pipeline)
cache_target = transformer if block_adapter is None else block_adapter
if adapter_cls is not None:
adapter_cls.apply(
cache_target,
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
elif block_adapter is None:
try:
cache_dit.enable_cache(
cache_target,
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
except ValueError as exc:
raise ValueError(
"Failed to enable Cache-DiT for pipeline "
f"{type(pipeline).__name__} with transformer "
f"{type(transformer).__name__}: no model-declared "
"_cache_dit_adapter_config or compatible Cache-DiT built-in "
"adapter was found."
) from exc
else:
cache_dit.enable_cache(
cache_target,
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
return _build_cache_context_refresh(cache_config)
def _maybe_build_block_adapter(pipeline: Any) -> BlockAdapter | None:
"""Build the model-declared block adapter, when one is configured."""
transformer = _default_get_pipeline_transformer(pipeline)
adapter_config: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
if adapter_config is None:
logger.info(
"Transformer %s does not declare _cache_dit_adapter_config; "
"falling back to Cache-DiT's built-in adapter registry.",
type(transformer).__name__,
)
return None
block_attributes, forward_patterns = zip(*adapter_config.block_forward_patterns.items())
missing_attributes = [
block_attribute for block_attribute in block_attributes if not hasattr(transformer, block_attribute)
]
if missing_attributes:
raise AttributeError(f"Missing Cache-DiT block attributes: {missing_attributes}")
return BlockAdapter(
transformer=transformer,
blocks=[getattr(transformer, block_attribute) for block_attribute in block_attributes],
forward_pattern=list(forward_patterns),
has_separate_cfg=adapter_config.has_separate_cfg,
check_forward_pattern=adapter_config.check_forward_pattern,
)
def _maybe_get_cached_adapter_cls(pipeline: Any) -> type[CachedAdapter] | None:
"""Return the custom cached adapter declared by the transformer."""
transformer = _default_get_pipeline_transformer(pipeline)
adapter_config: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
return None if adapter_config is None else adapter_config.cached_adapter_cls
class CacheDiTBackend(CacheBackend):
"""Manage Cache-DiT through the common diffusion cache lifecycle."""
def __init__(self, cache_config: Any = None):
if cache_config is None:
config = DiffusionCacheConfig()
elif isinstance(cache_config, dict):
config = DiffusionCacheConfig.from_dict(cache_config)
else:
config = cache_config
super().__init__(config)
self._refresh_func: RefreshCacheContextFunc | None = None
def enable(self, pipeline: Any) -> None:
pipeline_name = type(pipeline).__name__
custom_enabler = CUSTOM_DIT_ENABLERS.get(pipeline_name)
if custom_enabler is not None:
logger.info("Using custom cache-dit enabler for model: %s", pipeline_name)
self._refresh_func = custom_enabler(pipeline, self.config)
else:
block_adapter = _maybe_build_block_adapter(pipeline)
adapter_cls = _maybe_get_cached_adapter_cls(pipeline)
self._refresh_func = enable_cache_for_dit(
pipeline,
self.config,
block_adapter,
adapter_cls,
)
self.enabled = True
logger.info("Cache-dit enabled successfully on %s", pipeline_name)
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
if not self.enabled or self._refresh_func is None:
logger.warning("Cache-dit is not enabled. Cannot refresh cache context.")
return
if verbose:
logger.info(
"Refreshing cache context for transformer with num_inference_steps: %s",
num_inference_steps,
)
self._refresh_func(pipeline, num_inference_steps, verbose)
__all__ = [
"CUSTOM_DIT_ENABLERS",
"CacheDiTBackend",
"cache_summary",
"enable_cache_for_dit",
]
+96
View File
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration types and conversion helpers for Cache-DiT."""
from __future__ import annotations
from dataclasses import dataclass
from cache_dit import (
CalibratorConfig,
DBCacheConfig,
ForwardPattern,
TaylorSeerCalibratorConfig,
)
from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter
from vllm.logger import init_logger
from vllm_omni.diffusion.data import DiffusionCacheConfig
logger = init_logger(__name__)
@dataclass
class CacheDiTAdapterConfig:
"""Describe how Cache-DiT wraps a model's transformer blocks."""
block_forward_patterns: dict[str, ForwardPattern]
has_separate_cfg: bool = False
cached_adapter_cls: type[CachedAdapter] | None = None
check_forward_pattern: bool = True
@dataclass(frozen=True)
class CacheDiTConfig:
"""Cache-DiT-specific projection of the shared diffusion cache config."""
Fn_compute_blocks: int = 1
Bn_compute_blocks: int = 0
max_warmup_steps: int = 4
max_cached_steps: int = -1
residual_diff_threshold: float = 0.24
max_continuous_cached_steps: int = 3
enable_taylorseer: bool = False
taylorseer_order: int = 1
scm_steps_mask_policy: str | None = None
scm_steps_policy: str = "dynamic"
force_refresh_step_hint: int | None = None
force_refresh_step_policy: str = "once"
@classmethod
def from_diffusion_config(cls, config: DiffusionCacheConfig) -> CacheDiTConfig:
"""Project the shared config onto fields consumed by Cache-DiT."""
return cls(
Fn_compute_blocks=config.Fn_compute_blocks,
Bn_compute_blocks=config.Bn_compute_blocks,
max_warmup_steps=config.max_warmup_steps,
max_cached_steps=config.max_cached_steps,
residual_diff_threshold=config.residual_diff_threshold,
max_continuous_cached_steps=config.max_continuous_cached_steps,
enable_taylorseer=config.enable_taylorseer,
taylorseer_order=config.taylorseer_order,
scm_steps_mask_policy=config.scm_steps_mask_policy,
scm_steps_policy=config.scm_steps_policy,
force_refresh_step_hint=config.force_refresh_step_hint,
force_refresh_step_policy=config.force_refresh_step_policy,
)
def to_db_cache_config(self) -> DBCacheConfig:
"""Build the third-party cache configuration used during installation."""
return DBCacheConfig(
# The request step count is supplied by ``CacheDiTBackend.refresh``.
num_inference_steps=None,
Fn_compute_blocks=self.Fn_compute_blocks,
Bn_compute_blocks=self.Bn_compute_blocks,
max_warmup_steps=self.max_warmup_steps,
max_cached_steps=self.max_cached_steps,
max_continuous_cached_steps=self.max_continuous_cached_steps,
residual_diff_threshold=self.residual_diff_threshold,
force_refresh_step_hint=self.force_refresh_step_hint,
force_refresh_step_policy=self.force_refresh_step_policy,
)
def to_calibrator_config(self) -> CalibratorConfig | None:
"""Build the optional TaylorSeer calibrator configuration."""
if not self.enable_taylorseer:
return None
logger.info("TaylorSeer enabled with order=%s", self.taylorseer_order)
return TaylorSeerCalibratorConfig(taylorseer_order=self.taylorseer_order)
__all__ = ["CacheDiTAdapterConfig", "CacheDiTConfig"]
@@ -1,27 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
cache-dit integration backend for vllm-omni.
This module provides a CacheDiTBackend class to enable cache-dit acceleration on diffusion
pipelines in vllm-omni, supporting both single and dual-transformer architectures.
"""
"""Model-specific Cache-DiT adapters and enablers."""
import functools
from collections.abc import Callable
from contextlib import ExitStack
from dataclasses import dataclass
from typing import Any, Optional, TypeAlias
from typing import Any
import cache_dit
import torch
from cache_dit import (
BlockAdapter,
CalibratorConfig,
DBCacheConfig,
ForwardPattern,
ParamsModifier,
TaylorSeerCalibratorConfig,
)
from cache_dit.caching.block_adapters import FakeDiffusionPipeline
from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter
@@ -31,178 +23,18 @@ from cache_dit.caching.cache_contexts import BasicCacheConfig
from cache_dit.caching.cache_contexts.cache_manager import CachedContextManager
from vllm.logger import init_logger
from vllm_omni.diffusion.cache.base import CacheBackend
from vllm_omni.diffusion.data import DiffusionCacheConfig, OmniDiffusionConfig
from vllm_omni.diffusion.cache.cachedit.backend import (
CUSTOM_DIT_ENABLERS,
RefreshCacheContextFunc,
_build_cache_context_refresh,
_default_get_pipeline_transformer,
_maybe_build_block_adapter,
enable_cache_for_dit,
)
from vllm_omni.diffusion.cache.cachedit.config import CacheDiTConfig
logger = init_logger(__name__)
RefreshCacheContextFunc: TypeAlias = Callable[[Any, int, bool], None]
@dataclass
class CacheDiTAdapterConfig:
"""Config for creating a Cache DiT's block adapter; to enable CacheDiT,
most models just need to define an instance of this class as a class
var in the DiT.
"""
block_forward_patterns: dict[str, ForwardPattern]
has_separate_cfg: bool = False
cached_adapter_cls: type[CachedAdapter] | None = None
check_forward_pattern: bool = True
# Registry of custom cache-dit enablers for specific models
# Maps pipeline names to their cache-dit enablement functions
CUSTOM_DIT_ENABLERS: dict[str, Callable] = {}
# Small helper to centralize cache-dit summaries.
def cache_summary(pipeline: Any, details: bool = True) -> None:
if hasattr(pipeline, "transformer"):
cache_dit.summary(pipeline.transformer, details=details)
if hasattr(pipeline, "transformer_2"):
cache_dit.summary(pipeline.transformer_2, details=details)
if not hasattr(pipeline, "transformer") and not hasattr(pipeline, "transformer_2"):
logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute")
def default_get_pipeline_transformer(pipeline: Any) -> Any:
return pipeline.transformer
def build_cache_context_refresh(
cache_config: DiffusionCacheConfig,
get_pipeline_transformer: Callable[[Any], Any] = default_get_pipeline_transformer,
) -> RefreshCacheContextFunc:
"""Build the cache context refresh func for a single Transformer."""
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""Refresh cache context for the transformer with new num_inference_steps.
Args:
pipeline: The pipeline instance.
num_inference_steps: New number of inference steps.
"""
transformer = get_pipeline_transformer(pipeline)
# Bypass SCM for step counts that don't support predefined masks (e.g., vLLM's 1-step dummy run)
scm_supported_steps = num_inference_steps >= 8 or num_inference_steps in (4, 6)
if cache_config.scm_steps_mask_policy is None or not scm_supported_steps:
cache_dit.refresh_context(transformer, num_inference_steps=num_inference_steps, verbose=verbose)
else:
cache_dit.refresh_context(
transformer,
cache_config=DBCacheConfig().reset(
num_inference_steps=num_inference_steps,
steps_computation_mask=cache_dit.steps_mask(
mask_policy=cache_config.scm_steps_mask_policy,
total_steps=num_inference_steps,
),
steps_computation_policy=cache_config.scm_steps_policy,
),
verbose=verbose,
)
return refresh_cache_context
def _resolve_calibrator_config(cache_config: DiffusionCacheConfig) -> CalibratorConfig | None:
"""Resolves the Calibrator subconfig For DiT cache."""
calibrator = None
if cache_config.enable_taylorseer:
taylorseer_order = cache_config.taylorseer_order
calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
# In the future, more calibrators will likely be added to DiT Cache,
# e.g., focal; handle them generically here.
return calibrator
def _build_db_cache_config(cache_config: DiffusionCacheConfig) -> DBCacheConfig:
"""Build DBCacheConfig with optional SCM (Step Computation Masking) support.
Args:
cache_config: DiffusionCacheConfig instance.
Returns:
DBCacheConfig instance with SCM support if configured.
"""
return DBCacheConfig(
# we will refresh the context when we get num_inference_steps in the first inference request
num_inference_steps=None,
Fn_compute_blocks=cache_config.Fn_compute_blocks,
Bn_compute_blocks=cache_config.Bn_compute_blocks,
max_warmup_steps=cache_config.max_warmup_steps,
max_cached_steps=cache_config.max_cached_steps,
max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
residual_diff_threshold=cache_config.residual_diff_threshold,
force_refresh_step_hint=cache_config.force_refresh_step_hint,
force_refresh_step_policy=cache_config.force_refresh_step_policy,
)
def enable_cache_for_dit(
pipeline: Any,
cache_config: Any,
block_adapter: BlockAdapter | None = None,
adapter_cls: type[CachedAdapter] | None = None,
) -> RefreshCacheContextFunc:
"""Enable cache-dit for regular single-transformer DiT models.
Args:
pipeline: The diffusion pipeline instance.
cache_config: DiffusionCacheConfig instance with cache configuration.
block_adapter: Custom block adapters for specific model architectures.
adapter_cls: Custom cached adapter class for specific model architectures.
Returns:
A refresh function that can be called to update cache context with new num_inference_steps.
"""
# Build DBCacheConfig with optional SCM support
db_cache_config = _build_db_cache_config(cache_config)
# Build calibrator config if TaylorSeer is enabled
calibrator_config = _resolve_calibrator_config(cache_config)
logger.info(
f"Enabling cache-dit on transformer: "
f"Fn={db_cache_config.Fn_compute_blocks}, "
f"Bn={db_cache_config.Bn_compute_blocks}, "
f"W={db_cache_config.max_warmup_steps}, "
)
# Enable cache-dit on the transformer
transformer = default_get_pipeline_transformer(pipeline)
# If we have a custom cached adapter subclass, call apply directly
if adapter_cls is not None:
adapter_cls.apply(
transformer if block_adapter is None else block_adapter,
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
else:
# Otherwise, call enable cache, which will call CachedAdapter.apply for us
cache_dit.enable_cache(
transformer if block_adapter is None else block_adapter,
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
return build_cache_context_refresh(cache_config)
### Complex / custom enablers for DiT cache
# NOTE (Alex): This case is rare; you should only really need to do this if you have a dual transformer
# architecture, since it hasn't been handled generically yet, or if the model class has unique attributes
# that it sets during Cache DiT enablement.
#
# For the vast majority of models, you should only need to add a _cache_dit_adapter_config attribute
# to the Transformer class, which controls the forward pattern, whether or not we have separate CFG,
# and so on.
# from https://github.com/vipshop/cache-dit/pull/542
def _split_wan22_inference_steps(pipeline, num_inference_steps: int) -> tuple[int, int]:
@@ -249,9 +81,9 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont
Returns:
A refresh function that can be called to update cache context with new num_inference_steps.
"""
# Build DBCacheConfig with optional SCM support
db_cache_config = _build_db_cache_config(cache_config)
calibrator_config = _resolve_calibrator_config(cache_config)
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
db_cache_config = projected_config.to_db_cache_config()
calibrator_config = projected_config.to_calibrator_config()
if getattr(pipeline, "transformer_2", None) is None:
logger.info("transformer_2 not found, enabling cache-dit for single transformer mode")
@@ -273,7 +105,7 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
return build_cache_context_refresh(cache_config)
return _build_cache_context_refresh(cache_config)
cache_dit.enable_cache(
BlockAdapter(
@@ -296,8 +128,8 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont
# high-noise transformer only have 30% steps
ParamsModifier(
cache_config=DBCacheConfig().reset(
max_warmup_steps=cache_config.max_warmup_steps,
max_cached_steps=cache_config.max_cached_steps,
max_warmup_steps=projected_config.max_warmup_steps,
max_cached_steps=projected_config.max_cached_steps,
),
calibrator_config=calibrator_config,
),
@@ -315,8 +147,8 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont
calibrator_config=calibrator_config,
)
refresh_trans_one = build_cache_context_refresh(cache_config)
refresh_trans_two = build_cache_context_refresh(cache_config, lambda pipeline: pipeline.transformer_2)
refresh_trans_one = _build_cache_context_refresh(cache_config)
refresh_trans_two = _build_cache_context_refresh(cache_config, lambda pipeline: pipeline.transformer_2)
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""Refresh cache context for both transformers with new num_inference_steps.
@@ -347,12 +179,9 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache
permanently replace it with a no-op on the transformer to prevent double
injection from the main forward loop.
"""
db_cache_config = _build_db_cache_config(cache_config)
calibrator_config = None
if cache_config.enable_taylorseer:
taylorseer_order = cache_config.taylorseer_order
calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
db_cache_config = projected_config.to_db_cache_config()
calibrator_config = projected_config.to_calibrator_config()
# Save the original after_transformer_block before cache-dit wrapping
transformer = pipeline.transformer
@@ -383,7 +212,7 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""Refresh cache context for the S2V transformer."""
if cache_config.scm_steps_mask_policy is None:
if projected_config.scm_steps_mask_policy is None:
cache_dit.refresh_context(
pipeline.transformer,
num_inference_steps=num_inference_steps,
@@ -395,10 +224,10 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache
cache_config=DBCacheConfig().reset(
num_inference_steps=num_inference_steps,
steps_computation_mask=cache_dit.steps_mask(
mask_policy=cache_config.scm_steps_mask_policy,
mask_policy=projected_config.scm_steps_mask_policy,
total_steps=num_inference_steps,
),
steps_computation_policy=cache_config.scm_steps_policy,
steps_computation_policy=projected_config.scm_steps_policy,
),
verbose=verbose,
)
@@ -971,7 +800,7 @@ def enable_cache_for_cosmos3(pipeline: Any, cache_config: Any) -> RefreshCacheCo
# step accounting. Still do both cond/uncond CFG steps when cache-dit is active.
# CFG is instead neutralized via scale=1.0 outside the interval.
pipeline._cache_dit_requires_paired_cfg = True
block_adapter = CacheDiTBackend.maybe_build_block_adapter(pipeline)
block_adapter = _maybe_build_block_adapter(pipeline)
return enable_cache_for_dit(pipeline, cache_config, block_adapter)
@@ -996,7 +825,7 @@ def enable_cache_for_krea2(pipeline: Any, cache_config: Any) -> RefreshCacheCont
Returns:
A refresh function that can be called to update cache context with new num_inference_steps.
"""
transformer = default_get_pipeline_transformer(pipeline)
transformer = _default_get_pipeline_transformer(pipeline)
block_adapter = BlockAdapter(
transformer=transformer,
blocks=[transformer.transformer_blocks],
@@ -1007,171 +836,28 @@ def enable_cache_for_krea2(pipeline: Any, cache_config: Any) -> RefreshCacheCont
return enable_cache_for_dit(pipeline, cache_config, block_adapter)
# Register custom cache-dit enablers after function definitions
CUSTOM_DIT_ENABLERS.update(
{
"Wan22Pipeline": enable_cache_for_wan22,
"Wan22I2VPipeline": enable_cache_for_wan22,
"Wan22TI2VPipeline": enable_cache_for_wan22,
"Wan22VACEPipeline": enable_cache_for_wan22,
"Wan22S2VPipeline": enable_cache_for_wan22_s2v,
"Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,
"Cosmos3OmniPipeline": enable_cache_for_cosmos3,
"Krea2Pipeline": enable_cache_for_krea2,
}
)
def register_custom_dit_enablers() -> None:
"""Register model-specific Cache-DiT enablers.
class CacheDiTBackend(CacheBackend):
"""Backend class for cache-dit acceleration on diffusion pipelines.
This class implements cache-dit acceleration (DBCache, SCM, TaylorSeer) using
the cache-dit library. It inherits from CacheBackend and provides a unified
interface for managing cache-dit acceleration on diffusion models.
Attributes:
config: Cache configuration (DiffusionCacheConfig instance), inherited from CacheBackend.
enabled: Whether cache-dit is enabled on this pipeline, inherited from CacheBackend.
_refresh_func: Internal refresh function for updating cache context.
_last_num_inference_steps: Last num_inference_steps used for refresh optimization.
This is called explicitly by the package initializer so registration does
not depend on unrelated model-specific symbols being imported for their
side effects.
"""
def __init__(self, cache_config: Any = None):
"""Initialize the cache-dit backend.
Args:
cache_config: Cache configuration (DiffusionCacheConfig instance, dict, or None).
If None or empty, uses default DiffusionCacheConfig().
"""
# Use default config if cache_config is not provided or is empty
if cache_config is None:
config = DiffusionCacheConfig()
elif isinstance(cache_config, dict):
# Convert dict to DiffusionCacheConfig, using defaults for missing keys
config = DiffusionCacheConfig.from_dict(cache_config)
else:
config = cache_config
# Initialize base class with normalized config
super().__init__(config)
# Cache-dit specific attributes
self._refresh_func: Callable[[Any, int, bool], None] | None = None
self._last_num_inference_steps: int | None = None
@staticmethod
def maybe_build_block_adapter(pipeline) -> BlockAdapter | None:
"""If a module defines `_cache_dit_adapter_config`, build the corresponding
block adapter.
"""
transformer = default_get_pipeline_transformer(pipeline)
adapter_cfg: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
if adapter_cfg is None:
return None
block_attrs, forward_pattern = zip(*(adapter_cfg.block_forward_patterns).items())
missing_attrs = [block_attr for block_attr in block_attrs if not hasattr(transformer, block_attr)]
if missing_attrs:
logger.warning("Missing Cache DiT block attributes: %s", missing_attrs)
block_adapter = BlockAdapter(
transformer=transformer,
blocks=[getattr(transformer, block_attr) for block_attr in block_attrs],
forward_pattern=list(forward_pattern),
has_separate_cfg=adapter_cfg.has_separate_cfg,
check_forward_pattern=adapter_cfg.check_forward_pattern,
)
return block_adapter
@staticmethod
def maybe_get_cached_adapter_cls(pipeline) -> type[CachedAdapter] | None:
"""If a module has a custom cached adapter type registered, e.g., SenseNova, retrieve it
from the transformer's CacheDiTAdapterConfig."""
transformer = default_get_pipeline_transformer(pipeline)
adapter_cfg: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
if adapter_cfg is None:
return None
return adapter_cfg.cached_adapter_cls
def enable(self, pipeline: Any) -> None:
"""Enable cache-dit on the pipeline if configured.
This method applies cache-dit acceleration to the appropriate transformer(s)
in the pipeline. It handles both single-transformer and dual-transformer
architectures (e.g., Wan2.2).
Args:
pipeline: The diffusion pipeline instance.
"""
# Extract pipeline name from pipeline
pipeline_name = pipeline.__class__.__name__
# Check if this model has a custom cache-dit enabler
if pipeline_name in CUSTOM_DIT_ENABLERS:
logger.info(f"Using custom cache-dit enabler for model: {pipeline_name}")
self._refresh_func = CUSTOM_DIT_ENABLERS[pipeline_name](pipeline, self.config)
else:
# Common case; either the model doesn't explicitly support dit cache yet,
# Or it defines its _cache_dit_adapter_config, which describes how we should
# create its block adapter.
block_adapter = self.maybe_build_block_adapter(pipeline)
adapter_cls = self.maybe_get_cached_adapter_cls(pipeline)
self._refresh_func = enable_cache_for_dit(pipeline, self.config, block_adapter, adapter_cls)
self.enabled = True
logger.info(f"Cache-dit enabled successfully on {pipeline_name}")
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""Refresh cache context with new num_inference_steps.
This method updates the cache context when num_inference_steps changes
during inference. For dual-transformer models (e.g., Wan2.2), it automatically
splits the steps based on boundary_ratio.
Args:
pipeline: The diffusion pipeline instance.
num_inference_steps: New number of inference steps.
verbose: Whether to log refresh operations.
"""
if not self.enabled or self._refresh_func is None:
logger.warning("Cache-dit is not enabled. Cannot refresh cache context.")
return
# Only refresh if num_inference_steps has changed
if self._last_num_inference_steps is None or num_inference_steps != self._last_num_inference_steps:
if verbose:
logger.info(f"Refreshing cache context for transformer with num_inference_steps: {num_inference_steps}")
self._refresh_func(pipeline, num_inference_steps, verbose)
self._last_num_inference_steps = num_inference_steps
def is_enabled(self) -> bool:
"""Check if cache-dit is enabled on this pipeline.
Returns:
True if cache-dit is enabled, False otherwise.
"""
return self.enabled
CUSTOM_DIT_ENABLERS.update(
{
"Wan22Pipeline": enable_cache_for_wan22,
"Wan22I2VPipeline": enable_cache_for_wan22,
"Wan22TI2VPipeline": enable_cache_for_wan22,
"Wan22VACEPipeline": enable_cache_for_wan22,
"Wan22S2VPipeline": enable_cache_for_wan22_s2v,
"Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,
"Cosmos3OmniPipeline": enable_cache_for_cosmos3,
"Krea2Pipeline": enable_cache_for_krea2,
}
)
def may_enable_cache_dit(pipeline: Any, od_config: OmniDiffusionConfig) -> Optional["CacheDiTBackend"]:
"""Enable cache-dit on the pipeline if configured (convenience function).
This is a convenience function that creates and enables a CacheDiTBackend.
For new code, consider using CacheDiTBackend directly.
Args:
pipeline: The diffusion pipeline instance.
od_config: OmniDiffusionConfig with cache configuration.
Returns:
A CacheDiTBackend instance if cache-dit is enabled, None otherwise.
"""
if od_config.cache_backend != "cache-dit" or not od_config.cache_config:
return None
backend = CacheDiTBackend(od_config.cache_config)
backend.enable(pipeline)
return backend if backend.is_enabled() else None
__all__ = [
"BagelCachedAdapter",
"SensenovaCachedAdapter",
]
+2
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend
from vllm_omni.diffusion.cache.magcache.config import MagCacheConfig
from vllm_omni.diffusion.cache.magcache.hook import (
MagCacheBlockHook,
@@ -21,6 +22,7 @@ __all__ = [
"Flux2MagCacheStrategy",
"FluxMagCacheStrategy",
"MagCacheBlockHook",
"MagCacheBackend",
"MagCacheConfig",
"MagCacheHeadHook",
"MagCacheState",
+3 -3
View File
@@ -1,10 +1,10 @@
from typing import Any
from vllm_omni.diffusion.cache.base import CacheBackend
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTBackend
from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend
from vllm_omni.diffusion.cache.cachedit import CacheDiTBackend
from vllm_omni.diffusion.cache.magcache import MagCacheBackend
from vllm_omni.diffusion.cache.stepcache import StepCacheBackend
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
from vllm_omni.diffusion.cache.teacache import TeaCacheBackend
from vllm_omni.diffusion.data import DiffusionCacheConfig
@@ -37,7 +37,7 @@ from vllm.transformers_utils.configs.bagel import BagelConfig
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata as DiffusionAttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention as DiffusionAttention
from vllm_omni.diffusion.cache.cache_dit_backend import BagelCachedAdapter, CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import BagelCachedAdapter, CacheDiTAdapterConfig
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.parallel_state import (
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.quantization.base_config import (
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention as FrameworkAttention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
@@ -9,7 +9,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
@@ -22,7 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.quantization.component_config import safe_quant_config
if TYPE_CHECKING:
@@ -26,7 +26,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
@@ -41,7 +41,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear, QKVParallelL
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
SequenceParallelOutput,
@@ -60,7 +60,7 @@ from vllm_omni.diffusion.attention.backends.abstract import (
AttentionMetadata,
)
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.distributed.parallel_state import (
get_cfg_group,
get_classifier_free_guidance_rank,
@@ -23,7 +23,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
from vllm_omni.diffusion.distributed.parallel_state import get_sequence_parallel_world_size
@@ -19,7 +19,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
@@ -45,7 +45,7 @@ if TYPE_CHECKING:
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.data import OmniDiffusionConfig
logger = init_logger(__name__)
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig, SensenovaCachedAdapter
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig, SensenovaCachedAdapter
logger = init_logger(__name__)
@@ -7,7 +7,7 @@ from transformers import LlamaConfig, LlamaModel
from transformers.modeling_outputs import BaseModelOutputWithPast
from transformers.models.llama.modeling_llama import LlamaDecoderLayer
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
from vllm_omni.diffusion.layers.fourier import GaussianFourierProjection
@@ -22,7 +22,7 @@ from vllm.config import LoadConfig, VllmConfig
from vllm.logger import init_logger
from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes
from vllm_omni.diffusion.cache.cache_dit_backend import cache_summary
from vllm_omni.diffusion.cache.cachedit import cache_summary
from vllm_omni.diffusion.cache.prompt_embed_cache import (
install_prompt_embed_cache,
resolve_prompt_embed_cache_config,