[Refactor][1/N]diffusion/cache: cache-dit (#5226)
Signed-off-by: yangjianjuan <66548561+yangjianjuan@users.noreply.github.com>
This commit is contained in:
1 parent
a5d9d57adc
commit
8927ec057c
34 files changed
+635
-436
No files matched your search
@@ -418,7 +418,8 @@ def enable_cache_for_your_model(pipeline, cache_config):
|
||||
|
||||
#### 9d. Register the custom enabler
|
||||
|
||||
Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
|
||||
Add your enabler to `CUSTOM_DIT_ENABLERS` in
|
||||
`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
|
||||
|
||||
```python
|
||||
CUSTOM_DIT_ENABLERS = {
|
||||
|
||||
@@ -29,8 +29,13 @@ vLLM-Omni integrates cache-dit through `CacheDiTBackend`:
|
||||
| `cache_dit.refresh_context()` | Updates cache context when `num_inference_steps` changes |
|
||||
|
||||
**Source files**:
|
||||
- `vllm_omni/diffusion/cache/cache_dit_backend.py` — `CacheDiTBackend`, enablers, `CUSTOM_DIT_ENABLERS`
|
||||
- `vllm_omni/diffusion/cache/` — cache backend implementations
|
||||
- `vllm_omni/diffusion/cache/cachedit/__init__.py` — public package API
|
||||
- `vllm_omni/diffusion/cache/cachedit/backend.py` — generic backend lifecycle and integration
|
||||
- `vllm_omni/diffusion/cache/cachedit/config.py` — Cache-DiT configuration types
|
||||
- `vllm_omni/diffusion/cache/cachedit/model_specific.py` — model-specific adapters, enablers, and registry
|
||||
|
||||
Outside `vllm_omni/diffusion/cache`, import Cache-DiT symbols only from
|
||||
`vllm_omni.diffusion.cache.cachedit`.
|
||||
|
||||
## Standard Models: Automatic Support
|
||||
|
||||
@@ -175,7 +180,8 @@ Inspect your block's `forward()` return type and residual connection pattern to
|
||||
|
||||
## Registering Custom Enablers
|
||||
|
||||
Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
|
||||
Add your enabler to `CUSTOM_DIT_ENABLERS` in
|
||||
`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
|
||||
|
||||
```python
|
||||
CUSTOM_DIT_ENABLERS = {
|
||||
@@ -248,7 +254,7 @@ Models listed in `_NO_CACHE_ACCELERATION` in `vllm_omni/diffusion/registry.py` d
|
||||
|
||||
| Model | Path | Notes |
|
||||
|-------|------|-------|
|
||||
| Standard DiT | `cache_dit_backend.py::enable_cache_for_dit` | Default enabler, automatic |
|
||||
| Wan2.2 | `cache_dit_backend.py::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
|
||||
| LongCat | `cache_dit_backend.py::enable_cache_for_longcat_image` | Multi-block-list |
|
||||
| BAGEL | `cache_dit_backend.py::enable_cache_for_bagel` | Complex omni model |
|
||||
| Standard DiT | `cachedit.backend::enable_cache_for_dit` | Default enabler, automatic |
|
||||
| Wan2.2 | `cachedit.model_specific::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
|
||||
| LongCat | `cachedit.config::CacheDiTAdapterConfig` | Declarative multi-block-list adapter |
|
||||
| BAGEL | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter |
|
||||
@@ -32,10 +32,13 @@ The library supports three main caching strategies:
|
||||
|
||||
vLLM-omni integrates cache-dit through the `CacheDiTBackend` class, which provides a unified interface for managing cache-dit acceleration on diffusion models.
|
||||
|
||||
Code outside `vllm_omni/diffusion/cache` must import Cache-DiT symbols from the
|
||||
package-level API: `vllm_omni.diffusion.cache.cachedit`.
|
||||
|
||||
| Method/Class | Purpose | Behavior |
|
||||
|--------------|---------|----------|
|
||||
| [`CacheDiTBackend`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/#vllm_omni.diffusion.cache.CacheBackend) | Unified backend interface | Automatically handles enabler selection and cache refresh |
|
||||
| [`enable_cache_for_dit()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_dit) | Apply caching to transformer | Configures DBCache on transformer blocks |
|
||||
| `CacheDiTBackend` | Unified backend interface | Automatically handles enabler selection and cache refresh |
|
||||
| `enable_cache_for_dit()` | Apply caching to transformer | Configures DBCache on transformer blocks |
|
||||
|
||||
**Key APIs from Cache-DiT:**
|
||||
|
||||
@@ -47,7 +50,7 @@ vLLM-omni integrates cache-dit through the `CacheDiTBackend` class, which provid
|
||||
| `ForwardPattern` | Defines block forward signature patterns: `Pattern_0`, `Pattern_1`, `Pattern_2` |
|
||||
| `ParamsModifier` | Per-transformer or per-block-list cache configuration customization |
|
||||
| `DBCacheConfig` | Configuration for DBCache parameters (warmup steps, cached steps, thresholds) |
|
||||
| `refresh_context()` | Update cache context | Called when `num_inference_steps` changes |
|
||||
| `refresh_context()` | Update cache context | Called at every generation boundary |
|
||||
|
||||
---
|
||||
|
||||
@@ -179,7 +182,8 @@ cache_dit.enable_cache(
|
||||
|
||||
### Registering Custom Implementations
|
||||
|
||||
After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
|
||||
After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in
|
||||
`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
|
||||
|
||||
```python
|
||||
CUSTOM_DIT_ENABLERS = {
|
||||
@@ -266,10 +270,10 @@ Complete examples in the codebase:
|
||||
|
||||
| Model | Path | Pattern | Notes |
|
||||
|-------|------|---------|-------|
|
||||
| **Standard DiT** | [`cache_dit_backend.py::enable_cache_for_dit`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_dit) | Default enabler | Single transformer, automatic |
|
||||
| **Wan2.2** | [`cache_dit_backend.py::enable_cache_for_wan22`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_wan22) | Single or dual-transformer | Auto-detects mode based on transformer_2 presence |
|
||||
| **LongCat** | [`cache_dit_backend.py::enable_cache_for_longcat_image`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_longcat_image) | Multi-block-list | Two block lists in one transformer |
|
||||
| **BAGEL** | [`cache_dit_backend.py::enable_cache_for_bagel`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_bagel) | Omni model | Complex architecture |
|
||||
| **Standard DiT** | `cachedit.backend::enable_cache_for_dit` | Default enabler | Single transformer, automatic |
|
||||
| **Wan2.2** | `cachedit.model_specific::enable_cache_for_wan22` | Single or dual-transformer | Auto-detects mode based on transformer_2 presence |
|
||||
| **LongCat** | `cachedit.config::CacheDiTAdapterConfig` | Declarative block adapter | Two block lists in one transformer |
|
||||
| **BAGEL** | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter | Complex architecture |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -769,7 +769,7 @@ class TeaCacheBackend(CacheBackend):
|
||||
|
||||
**2. Cache-DiT Backend**
|
||||
|
||||
**Location**: `vllm_omni/diffusion/cache/cache_dit_backend.py`
|
||||
**Location**: `vllm_omni/diffusion/cache/cachedit/`
|
||||
|
||||
```python
|
||||
class CacheDiTBackend(CacheBackend):
|
||||
|
||||
@@ -6,13 +6,13 @@ import logging
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit import QwenImageEditPipeline
|
||||
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit_plus import QwenImageEditPlusPipeline
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CustomPipeline(QwenImageEditPipeline):
|
||||
class CustomPipeline(QwenImageEditPlusPipeline):
|
||||
def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
|
||||
super().__init__(od_config=od_config, prefix=prefix)
|
||||
|
||||
|
||||
+98
-19
@@ -17,13 +17,14 @@ import cache_dit
|
||||
import pytest
|
||||
from cache_dit import ForwardPattern
|
||||
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import (
|
||||
from vllm_omni.diffusion.cache.cachedit import (
|
||||
CacheDiTAdapterConfig,
|
||||
CacheDiTBackend,
|
||||
CacheDiTConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend
|
||||
from vllm_omni.diffusion.cache.magcache import MagCacheBackend
|
||||
from vllm_omni.diffusion.cache.selector import get_cache_backend
|
||||
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
|
||||
from vllm_omni.diffusion.cache.teacache import TeaCacheBackend
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -32,6 +33,19 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
class TestCacheDiTBackend:
|
||||
"""Test CacheDiTBackend implementation."""
|
||||
|
||||
def test_config_projection_contains_only_cache_dit_fields(self):
|
||||
shared_config = DiffusionCacheConfig(
|
||||
Fn_compute_blocks=4,
|
||||
max_warmup_steps=8,
|
||||
rel_l1_thresh=0.3,
|
||||
)
|
||||
|
||||
config = CacheDiTConfig.from_diffusion_config(shared_config)
|
||||
|
||||
assert config.Fn_compute_blocks == 4
|
||||
assert config.max_warmup_steps == 8
|
||||
assert not hasattr(config, "rel_l1_thresh")
|
||||
|
||||
def test_init_with_dict(self):
|
||||
"""Test initialization with dictionary config."""
|
||||
config_dict = {"Fn_compute_blocks": 4, "max_warmup_steps": 8}
|
||||
@@ -47,8 +61,8 @@ class TestCacheDiTBackend:
|
||||
assert backend.config.Fn_compute_blocks == 4
|
||||
assert backend.enabled is False
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_enable_single_transformer(self, mock_cache_dit, mock_block_adapter):
|
||||
"""Test enabling cache-dit on single-transformer pipeline."""
|
||||
# Mock pipeline
|
||||
@@ -75,10 +89,61 @@ class TestCacheDiTBackend:
|
||||
assert backend._refresh_func is not None
|
||||
mock_cache_dit.enable_cache.assert_called_once()
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.logger")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_enable_without_model_adapter_uses_cache_dit_registry(self, mock_cache_dit, mock_logger):
|
||||
"""A missing model-declared adapter falls back to Cache-DiT's registry."""
|
||||
|
||||
class BuiltinTransformer:
|
||||
pass
|
||||
|
||||
class BuiltinPipeline:
|
||||
transformer = BuiltinTransformer()
|
||||
|
||||
pipeline = BuiltinPipeline()
|
||||
backend = CacheDiTBackend({"Fn_compute_blocks": 2})
|
||||
|
||||
backend.enable(pipeline)
|
||||
|
||||
assert backend.enabled is True
|
||||
assert backend._refresh_func is not None
|
||||
assert mock_cache_dit.enable_cache.call_args.args[0] is pipeline.transformer
|
||||
mock_logger.info.assert_any_call(
|
||||
"Transformer %s does not declare _cache_dit_adapter_config; "
|
||||
"falling back to Cache-DiT's built-in adapter registry.",
|
||||
"BuiltinTransformer",
|
||||
)
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_enable_without_compatible_adapter_has_contextual_error(self, mock_cache_dit):
|
||||
"""An unsupported fallback reports both pipeline and transformer names."""
|
||||
|
||||
class UnsupportedTransformer:
|
||||
pass
|
||||
|
||||
class UnsupportedPipeline:
|
||||
transformer = UnsupportedTransformer()
|
||||
|
||||
pipeline = UnsupportedPipeline()
|
||||
backend = CacheDiTBackend({"Fn_compute_blocks": 2})
|
||||
mock_cache_dit.enable_cache.side_effect = ValueError("unsupported")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"Failed to enable Cache-DiT for pipeline UnsupportedPipeline with transformer UnsupportedTransformer"
|
||||
),
|
||||
) as exc_info:
|
||||
backend.enable(pipeline)
|
||||
|
||||
assert isinstance(exc_info.value.__cause__, ValueError)
|
||||
assert backend.enabled is False
|
||||
assert backend._refresh_func is None
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_refresh(self, mock_cache_dit, mock_block_adapter):
|
||||
"""Test refreshing cache context with SCM mask policy updates when num_inference_steps changes."""
|
||||
"""Test refreshing cache context with SCM mask policy updates."""
|
||||
# Mock pipeline
|
||||
mock_pipeline = Mock()
|
||||
mock_pipeline.__class__.__name__ = "DiTPipeline"
|
||||
@@ -107,8 +172,6 @@ class TestCacheDiTBackend:
|
||||
|
||||
# First refresh with 50 steps
|
||||
backend.refresh(mock_pipeline, num_inference_steps=50)
|
||||
assert backend._last_num_inference_steps == 50
|
||||
|
||||
# Verify steps_mask was called with mask policy (not direct steps mask)
|
||||
mock_cache_dit.steps_mask.assert_called_with(mask_policy="fast", total_steps=50)
|
||||
assert mock_cache_dit.steps_mask.call_count == 1
|
||||
@@ -137,10 +200,28 @@ class TestCacheDiTBackend:
|
||||
call_args = mock_cache_dit.refresh_context.call_args
|
||||
assert call_args[0][0] == mock_transformer
|
||||
assert "cache_config" in call_args[1]
|
||||
assert backend._last_num_inference_steps == 100
|
||||
assert mock_cache_dit.refresh_context.call_count == 1
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_refresh_resets_same_step_count(self, mock_cache_dit, mock_block_adapter):
|
||||
"""Every generation must reset context, even when the step count is unchanged."""
|
||||
mock_pipeline = Mock()
|
||||
mock_pipeline.__class__.__name__ = "DiTPipeline"
|
||||
mock_pipeline.transformer = Mock()
|
||||
mock_pipeline.transformer._cache_dit_adapter_config = CacheDiTAdapterConfig(
|
||||
block_forward_patterns={"layers": ForwardPattern.Pattern_0}
|
||||
)
|
||||
|
||||
backend = CacheDiTBackend(DiffusionCacheConfig())
|
||||
backend.enable(mock_pipeline)
|
||||
backend.refresh(mock_pipeline, num_inference_steps=20)
|
||||
backend.refresh(mock_pipeline, num_inference_steps=20)
|
||||
|
||||
assert mock_cache_dit.refresh_context.call_count == 2
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_enable_hunyuan_pipeline_uses_model_transformer(self, mock_cache_dit, mock_block_adapter):
|
||||
"""Test HunyuanImage3 uses pipeline.transformer for cache enable/refresh.
|
||||
|
||||
@@ -179,8 +260,8 @@ class TestCacheDiTBackend:
|
||||
assert call_args[0][0] is mock_pipeline.model
|
||||
assert call_args[1]["num_inference_steps"] == 12
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_enable_dreamid_pipeline_uses_fused_blocks(self, mock_cache_dit, mock_block_adapter):
|
||||
"""Test DreamID uses pipeline.transformer for cache enable/refresh.
|
||||
|
||||
@@ -221,8 +302,8 @@ class TestCacheDiTBackend:
|
||||
assert call_args[1]["num_inference_steps"] == 12
|
||||
|
||||
@pytest.mark.parametrize("num_inference_steps", [1, 7])
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_refresh_scm_bypassed_for_unsupported_step_counts(
|
||||
self, mock_cache_dit, mock_block_adapter, num_inference_steps
|
||||
):
|
||||
@@ -362,8 +443,6 @@ class TestCacheSelector:
|
||||
class TestMagCacheBackend:
|
||||
"""Test MagCacheBackend implementation."""
|
||||
|
||||
from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend
|
||||
|
||||
def test_init(self):
|
||||
"""Test initialization."""
|
||||
config = DiffusionCacheConfig(mag_threshold=0.1, mag_max_skip_steps=2, mag_calibrate=True)
|
||||
|
||||
+80
-12
@@ -5,15 +5,18 @@
|
||||
Model specific tests for CacheDiT enablement.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from cache_dit.caching.cache_blocks.pattern_0_1_2 import CachedBlocks_Pattern_0_1_2
|
||||
|
||||
import vllm_omni.diffusion.cache.cache_dit_backend as cd_backend
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig, CacheDiTBackend, cache_summary
|
||||
import vllm_omni.diffusion.cache.cachedit as cd_backend
|
||||
import vllm_omni.diffusion.cache.cachedit.model_specific as cd_model_specific
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig, CacheDiTBackend, cache_summary
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
from vllm_omni.diffusion.models.cosmos3.transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
from vllm_omni.diffusion.models.helios.helios_transformer import HeliosTransformer3DModel
|
||||
@@ -42,8 +45,73 @@ SEPARATE_CFG_TRANSFORMERS = [
|
||||
SAMPLE_CACHE_CONFIG = DiffusionCacheConfig()
|
||||
|
||||
|
||||
def test_custom_cache_dit_enablers_are_registered_explicitly():
|
||||
expected_enablers = {
|
||||
"Wan22Pipeline": cd_model_specific.enable_cache_for_wan22,
|
||||
"Wan22I2VPipeline": cd_model_specific.enable_cache_for_wan22,
|
||||
"Wan22TI2VPipeline": cd_model_specific.enable_cache_for_wan22,
|
||||
"Wan22VACEPipeline": cd_model_specific.enable_cache_for_wan22,
|
||||
"Wan22S2VPipeline": cd_model_specific.enable_cache_for_wan22_s2v,
|
||||
"Cosmos3OmniDiffusersPipeline": cd_model_specific.enable_cache_for_cosmos3,
|
||||
"Cosmos3OmniPipeline": cd_model_specific.enable_cache_for_cosmos3,
|
||||
"Krea2Pipeline": cd_model_specific.enable_cache_for_krea2,
|
||||
}
|
||||
|
||||
with patch.dict(cd_backend.CUSTOM_DIT_ENABLERS, {}, clear=True):
|
||||
cd_model_specific.register_custom_dit_enablers()
|
||||
assert cd_backend.CUSTOM_DIT_ENABLERS == expected_enablers
|
||||
|
||||
|
||||
def test_wan22_vace_uses_wan22_custom_cache_dit_enabler():
|
||||
assert cd_backend.CUSTOM_DIT_ENABLERS["Wan22VACEPipeline"] is cd_backend.enable_cache_for_wan22
|
||||
assert cd_backend.CUSTOM_DIT_ENABLERS["Wan22VACEPipeline"] is cd_model_specific.enable_cache_for_wan22
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pipeline_name",
|
||||
["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"],
|
||||
)
|
||||
def test_cosmos3_aliases_use_cosmos3_custom_cache_dit_enabler(pipeline_name: str):
|
||||
assert cd_backend.CUSTOM_DIT_ENABLERS[pipeline_name] is cd_model_specific.enable_cache_for_cosmos3
|
||||
|
||||
|
||||
def test_cachedit_public_api_is_explicit():
|
||||
assert set(cd_backend.__all__) == {
|
||||
"BagelCachedAdapter",
|
||||
"CUSTOM_DIT_ENABLERS",
|
||||
"CacheDiTAdapterConfig",
|
||||
"CacheDiTBackend",
|
||||
"CacheDiTConfig",
|
||||
"SensenovaCachedAdapter",
|
||||
"cache_summary",
|
||||
"enable_cache_for_dit",
|
||||
}
|
||||
assert not hasattr(cd_backend, "enable_cache_for_wan22")
|
||||
assert not hasattr(cd_backend, "enable_cache_for_wan22_s2v")
|
||||
|
||||
|
||||
def test_cachedit_consumers_use_package_api():
|
||||
cache_dir = Path(cd_backend.__file__).resolve().parents[1]
|
||||
package_root = cache_dir.parents[1]
|
||||
legacy_module = "vllm_omni.diffusion.cache.cache_dit_backend"
|
||||
internal_prefix = "vllm_omni.diffusion.cache.cachedit."
|
||||
|
||||
invalid_imports = []
|
||||
for source_path in package_root.rglob("*.py"):
|
||||
if source_path.is_relative_to(cache_dir):
|
||||
continue
|
||||
tree = ast.parse(source_path.read_text(encoding="utf-8"))
|
||||
for node in ast.walk(tree):
|
||||
modules = []
|
||||
if isinstance(node, ast.ImportFrom) and node.module is not None:
|
||||
modules.append(node.module)
|
||||
elif isinstance(node, ast.Import):
|
||||
modules.extend(alias.name for alias in node.names)
|
||||
|
||||
for module in modules:
|
||||
if module == legacy_module or module.startswith(internal_prefix):
|
||||
invalid_imports.append(f"{source_path.relative_to(package_root)}:{node.lineno}: {module}")
|
||||
|
||||
assert not invalid_imports, "Cache-DiT consumers must use the package API:\n" + "\n".join(invalid_imports)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("transformer_model", SEPARATE_CFG_TRANSFORMERS)
|
||||
@@ -54,12 +122,12 @@ def test_cache_dit_configs_have_separate_cfg(transformer_model):
|
||||
assert transformer_model._cache_dit_adapter_config.has_separate_cfg is True
|
||||
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.cache_dit")
|
||||
def test_separate_wan22_custom_enabler_has_separate_cfg(mock_cache_dit, mock_block_adapter):
|
||||
"""Ensure that Wan22, which has a custom enabler, setts custom CFG correctly."""
|
||||
mock_pipeline = Mock()
|
||||
cd_backend.enable_cache_for_wan22(mock_pipeline, SAMPLE_CACHE_CONFIG)
|
||||
cd_model_specific.enable_cache_for_wan22(mock_pipeline, SAMPLE_CACHE_CONFIG)
|
||||
|
||||
mock_cache_dit.enable_cache.assert_called_once()
|
||||
adapter_kwargs = mock_block_adapter.call_args.kwargs
|
||||
@@ -67,8 +135,8 @@ def test_separate_wan22_custom_enabler_has_separate_cfg(mock_cache_dit, mock_blo
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_transformer_2", [False, True])
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.model_specific.cache_dit")
|
||||
def test_wan22_custom_enabler_passes_taylorseer_calibrator(
|
||||
mock_cache_dit,
|
||||
mock_block_adapter,
|
||||
@@ -82,7 +150,7 @@ def test_wan22_custom_enabler_passes_taylorseer_calibrator(
|
||||
mock_pipeline.transformer_2 = None
|
||||
cache_config = DiffusionCacheConfig(enable_taylorseer=True, taylorseer_order=1)
|
||||
|
||||
cd_backend.enable_cache_for_wan22(mock_pipeline, cache_config)
|
||||
cd_model_specific.enable_cache_for_wan22(mock_pipeline, cache_config)
|
||||
|
||||
enable_cache_kwargs = mock_cache_dit.enable_cache.call_args.kwargs
|
||||
calibrator_config = enable_cache_kwargs["calibrator_config"]
|
||||
@@ -94,8 +162,8 @@ def test_wan22_custom_enabler_passes_taylorseer_calibrator(
|
||||
assert modifier._context_kwargs["calibrator_config"] is calibrator_config
|
||||
|
||||
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cache_dit_backend.cache_dit")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")
|
||||
@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")
|
||||
def test_cosmos3_cache_dit_wraps_gen_layers(mock_cache_dit, mock_block_adapter):
|
||||
"""Cosmos3 should cache only the repeated GEN pathway blocks."""
|
||||
mock_pipeline = Mock()
|
||||
@@ -103,7 +171,7 @@ def test_cosmos3_cache_dit_wraps_gen_layers(mock_cache_dit, mock_block_adapter):
|
||||
mock_pipeline.transformer.gen_layers = gen_layers
|
||||
mock_pipeline.transformer._cache_dit_adapter_config = Cosmos3VFMTransformer._cache_dit_adapter_config
|
||||
|
||||
cd_backend.enable_cache_for_cosmos3(mock_pipeline, SAMPLE_CACHE_CONFIG)
|
||||
cd_model_specific.enable_cache_for_cosmos3(mock_pipeline, SAMPLE_CACHE_CONFIG)
|
||||
|
||||
mock_cache_dit.enable_cache.assert_called_once()
|
||||
adapter_kwargs = mock_block_adapter.call_args.kwargs
|
||||
|
||||
@@ -465,7 +465,7 @@ def test_edge_forward_rejects_unsupported_modes(
|
||||
|
||||
|
||||
def test_pipeline_registered_and_exported() -> None:
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS
|
||||
from vllm_omni.diffusion.cache.cachedit import CUSTOM_DIT_ENABLERS
|
||||
from vllm_omni.diffusion.models import cosmos3
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
|
||||
@@ -731,7 +731,7 @@ class TestRegistryIntegration:
|
||||
|
||||
def test_cache_dit_for_ltx2_does_not_have_custom_enablers_registered(self):
|
||||
"""Pipeline variants are *not* registered in CUSTOM_DIT_ENABLERS."""
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CUSTOM_DIT_ENABLERS
|
||||
from vllm_omni.diffusion.cache.cachedit import CUSTOM_DIT_ENABLERS
|
||||
|
||||
# NOTE: We used to have custom enablers for this model, but refactored to handle
|
||||
# it more generically. Now we only need to ensure it has git cache adapter config.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import pytest
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.models.ltx2.ltx2_transformer import LTX2VideoTransformer3DModel, _make_rms_norm
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Public API for the Cache-DiT diffusion cache backend."""
|
||||
|
||||
from vllm_omni.diffusion.cache.cachedit.backend import (
|
||||
CUSTOM_DIT_ENABLERS,
|
||||
CacheDiTBackend,
|
||||
cache_summary,
|
||||
enable_cache_for_dit,
|
||||
)
|
||||
from vllm_omni.diffusion.cache.cachedit.config import (
|
||||
CacheDiTAdapterConfig,
|
||||
CacheDiTConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.cache.cachedit.model_specific import (
|
||||
BagelCachedAdapter,
|
||||
SensenovaCachedAdapter,
|
||||
)
|
||||
from vllm_omni.diffusion.cache.cachedit.model_specific import (
|
||||
register_custom_dit_enablers as _register_custom_dit_enablers,
|
||||
)
|
||||
|
||||
_register_custom_dit_enablers()
|
||||
|
||||
__all__ = [
|
||||
"BagelCachedAdapter",
|
||||
"CUSTOM_DIT_ENABLERS",
|
||||
"CacheDiTAdapterConfig",
|
||||
"CacheDiTBackend",
|
||||
"CacheDiTConfig",
|
||||
"SensenovaCachedAdapter",
|
||||
"cache_summary",
|
||||
"enable_cache_for_dit",
|
||||
]
|
||||
+222
@@ -0,0 +1,222 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Generic Cache-DiT backend lifecycle and integration."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, TypeAlias
|
||||
|
||||
import cache_dit
|
||||
from cache_dit import BlockAdapter, DBCacheConfig
|
||||
from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheBackend
|
||||
from vllm_omni.diffusion.cache.cachedit.config import (
|
||||
CacheDiTAdapterConfig,
|
||||
CacheDiTConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RefreshCacheContextFunc: TypeAlias = Callable[[Any, int, bool], None]
|
||||
CacheDiTEnabler: TypeAlias = Callable[[Any, DiffusionCacheConfig], RefreshCacheContextFunc]
|
||||
|
||||
# Model-specific implementations register themselves when the package loads.
|
||||
CUSTOM_DIT_ENABLERS: dict[str, CacheDiTEnabler] = {}
|
||||
|
||||
|
||||
def cache_summary(pipeline: Any, details: bool = True) -> None:
|
||||
"""Log Cache-DiT statistics for every transformer on the pipeline."""
|
||||
|
||||
transformers = [
|
||||
transformer
|
||||
for attribute in ("transformer", "transformer_2")
|
||||
if (transformer := getattr(pipeline, attribute, None)) is not None
|
||||
]
|
||||
for transformer in transformers:
|
||||
cache_dit.summary(transformer, details=details)
|
||||
|
||||
if not transformers:
|
||||
logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute")
|
||||
|
||||
|
||||
def _default_get_pipeline_transformer(pipeline: Any) -> Any:
|
||||
return pipeline.transformer
|
||||
|
||||
|
||||
def _build_cache_context_refresh(
|
||||
cache_config: DiffusionCacheConfig,
|
||||
get_pipeline_transformer: Callable[[Any], Any] = _default_get_pipeline_transformer,
|
||||
) -> RefreshCacheContextFunc:
|
||||
"""Build the cache context refresh callback for one transformer."""
|
||||
|
||||
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
transformer = get_pipeline_transformer(pipeline)
|
||||
|
||||
# Cache-DiT has no predefined SCM mask for these small step counts.
|
||||
scm_supported_steps = num_inference_steps >= 8 or num_inference_steps in (4, 6)
|
||||
if projected_config.scm_steps_mask_policy is None or not scm_supported_steps:
|
||||
cache_dit.refresh_context(transformer, num_inference_steps=num_inference_steps, verbose=verbose)
|
||||
return
|
||||
|
||||
cache_dit.refresh_context(
|
||||
transformer,
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_inference_steps,
|
||||
steps_computation_mask=cache_dit.steps_mask(
|
||||
mask_policy=projected_config.scm_steps_mask_policy,
|
||||
total_steps=num_inference_steps,
|
||||
),
|
||||
steps_computation_policy=projected_config.scm_steps_policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
return refresh_cache_context
|
||||
|
||||
|
||||
def enable_cache_for_dit(
|
||||
pipeline: Any,
|
||||
cache_config: DiffusionCacheConfig,
|
||||
block_adapter: BlockAdapter | None = None,
|
||||
adapter_cls: type[CachedAdapter] | None = None,
|
||||
) -> RefreshCacheContextFunc:
|
||||
"""Enable Cache-DiT for a standard single-transformer DiT pipeline."""
|
||||
|
||||
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
|
||||
db_cache_config = projected_config.to_db_cache_config()
|
||||
calibrator_config = projected_config.to_calibrator_config()
|
||||
|
||||
logger.info(
|
||||
"Enabling cache-dit on transformer: Fn=%s, Bn=%s, W=%s",
|
||||
db_cache_config.Fn_compute_blocks,
|
||||
db_cache_config.Bn_compute_blocks,
|
||||
db_cache_config.max_warmup_steps,
|
||||
)
|
||||
|
||||
transformer = _default_get_pipeline_transformer(pipeline)
|
||||
cache_target = transformer if block_adapter is None else block_adapter
|
||||
if adapter_cls is not None:
|
||||
adapter_cls.apply(
|
||||
cache_target,
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
elif block_adapter is None:
|
||||
try:
|
||||
cache_dit.enable_cache(
|
||||
cache_target,
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
"Failed to enable Cache-DiT for pipeline "
|
||||
f"{type(pipeline).__name__} with transformer "
|
||||
f"{type(transformer).__name__}: no model-declared "
|
||||
"_cache_dit_adapter_config or compatible Cache-DiT built-in "
|
||||
"adapter was found."
|
||||
) from exc
|
||||
else:
|
||||
cache_dit.enable_cache(
|
||||
cache_target,
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
|
||||
return _build_cache_context_refresh(cache_config)
|
||||
|
||||
|
||||
def _maybe_build_block_adapter(pipeline: Any) -> BlockAdapter | None:
|
||||
"""Build the model-declared block adapter, when one is configured."""
|
||||
|
||||
transformer = _default_get_pipeline_transformer(pipeline)
|
||||
adapter_config: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
|
||||
if adapter_config is None:
|
||||
logger.info(
|
||||
"Transformer %s does not declare _cache_dit_adapter_config; "
|
||||
"falling back to Cache-DiT's built-in adapter registry.",
|
||||
type(transformer).__name__,
|
||||
)
|
||||
return None
|
||||
|
||||
block_attributes, forward_patterns = zip(*adapter_config.block_forward_patterns.items())
|
||||
missing_attributes = [
|
||||
block_attribute for block_attribute in block_attributes if not hasattr(transformer, block_attribute)
|
||||
]
|
||||
if missing_attributes:
|
||||
raise AttributeError(f"Missing Cache-DiT block attributes: {missing_attributes}")
|
||||
|
||||
return BlockAdapter(
|
||||
transformer=transformer,
|
||||
blocks=[getattr(transformer, block_attribute) for block_attribute in block_attributes],
|
||||
forward_pattern=list(forward_patterns),
|
||||
has_separate_cfg=adapter_config.has_separate_cfg,
|
||||
check_forward_pattern=adapter_config.check_forward_pattern,
|
||||
)
|
||||
|
||||
|
||||
def _maybe_get_cached_adapter_cls(pipeline: Any) -> type[CachedAdapter] | None:
|
||||
"""Return the custom cached adapter declared by the transformer."""
|
||||
|
||||
transformer = _default_get_pipeline_transformer(pipeline)
|
||||
adapter_config: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
|
||||
return None if adapter_config is None else adapter_config.cached_adapter_cls
|
||||
|
||||
|
||||
class CacheDiTBackend(CacheBackend):
|
||||
"""Manage Cache-DiT through the common diffusion cache lifecycle."""
|
||||
|
||||
def __init__(self, cache_config: Any = None):
|
||||
if cache_config is None:
|
||||
config = DiffusionCacheConfig()
|
||||
elif isinstance(cache_config, dict):
|
||||
config = DiffusionCacheConfig.from_dict(cache_config)
|
||||
else:
|
||||
config = cache_config
|
||||
|
||||
super().__init__(config)
|
||||
self._refresh_func: RefreshCacheContextFunc | None = None
|
||||
|
||||
def enable(self, pipeline: Any) -> None:
|
||||
pipeline_name = type(pipeline).__name__
|
||||
custom_enabler = CUSTOM_DIT_ENABLERS.get(pipeline_name)
|
||||
if custom_enabler is not None:
|
||||
logger.info("Using custom cache-dit enabler for model: %s", pipeline_name)
|
||||
self._refresh_func = custom_enabler(pipeline, self.config)
|
||||
else:
|
||||
block_adapter = _maybe_build_block_adapter(pipeline)
|
||||
adapter_cls = _maybe_get_cached_adapter_cls(pipeline)
|
||||
self._refresh_func = enable_cache_for_dit(
|
||||
pipeline,
|
||||
self.config,
|
||||
block_adapter,
|
||||
adapter_cls,
|
||||
)
|
||||
|
||||
self.enabled = True
|
||||
logger.info("Cache-dit enabled successfully on %s", pipeline_name)
|
||||
|
||||
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
if not self.enabled or self._refresh_func is None:
|
||||
logger.warning("Cache-dit is not enabled. Cannot refresh cache context.")
|
||||
return
|
||||
|
||||
if verbose:
|
||||
logger.info(
|
||||
"Refreshing cache context for transformer with num_inference_steps: %s",
|
||||
num_inference_steps,
|
||||
)
|
||||
self._refresh_func(pipeline, num_inference_steps, verbose)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CUSTOM_DIT_ENABLERS",
|
||||
"CacheDiTBackend",
|
||||
"cache_summary",
|
||||
"enable_cache_for_dit",
|
||||
]
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Configuration types and conversion helpers for Cache-DiT."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from cache_dit import (
|
||||
CalibratorConfig,
|
||||
DBCacheConfig,
|
||||
ForwardPattern,
|
||||
TaylorSeerCalibratorConfig,
|
||||
)
|
||||
from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CacheDiTAdapterConfig:
|
||||
"""Describe how Cache-DiT wraps a model's transformer blocks."""
|
||||
|
||||
block_forward_patterns: dict[str, ForwardPattern]
|
||||
has_separate_cfg: bool = False
|
||||
cached_adapter_cls: type[CachedAdapter] | None = None
|
||||
check_forward_pattern: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CacheDiTConfig:
|
||||
"""Cache-DiT-specific projection of the shared diffusion cache config."""
|
||||
|
||||
Fn_compute_blocks: int = 1
|
||||
Bn_compute_blocks: int = 0
|
||||
max_warmup_steps: int = 4
|
||||
max_cached_steps: int = -1
|
||||
residual_diff_threshold: float = 0.24
|
||||
max_continuous_cached_steps: int = 3
|
||||
enable_taylorseer: bool = False
|
||||
taylorseer_order: int = 1
|
||||
scm_steps_mask_policy: str | None = None
|
||||
scm_steps_policy: str = "dynamic"
|
||||
force_refresh_step_hint: int | None = None
|
||||
force_refresh_step_policy: str = "once"
|
||||
|
||||
@classmethod
|
||||
def from_diffusion_config(cls, config: DiffusionCacheConfig) -> CacheDiTConfig:
|
||||
"""Project the shared config onto fields consumed by Cache-DiT."""
|
||||
|
||||
return cls(
|
||||
Fn_compute_blocks=config.Fn_compute_blocks,
|
||||
Bn_compute_blocks=config.Bn_compute_blocks,
|
||||
max_warmup_steps=config.max_warmup_steps,
|
||||
max_cached_steps=config.max_cached_steps,
|
||||
residual_diff_threshold=config.residual_diff_threshold,
|
||||
max_continuous_cached_steps=config.max_continuous_cached_steps,
|
||||
enable_taylorseer=config.enable_taylorseer,
|
||||
taylorseer_order=config.taylorseer_order,
|
||||
scm_steps_mask_policy=config.scm_steps_mask_policy,
|
||||
scm_steps_policy=config.scm_steps_policy,
|
||||
force_refresh_step_hint=config.force_refresh_step_hint,
|
||||
force_refresh_step_policy=config.force_refresh_step_policy,
|
||||
)
|
||||
|
||||
def to_db_cache_config(self) -> DBCacheConfig:
|
||||
"""Build the third-party cache configuration used during installation."""
|
||||
|
||||
return DBCacheConfig(
|
||||
# The request step count is supplied by ``CacheDiTBackend.refresh``.
|
||||
num_inference_steps=None,
|
||||
Fn_compute_blocks=self.Fn_compute_blocks,
|
||||
Bn_compute_blocks=self.Bn_compute_blocks,
|
||||
max_warmup_steps=self.max_warmup_steps,
|
||||
max_cached_steps=self.max_cached_steps,
|
||||
max_continuous_cached_steps=self.max_continuous_cached_steps,
|
||||
residual_diff_threshold=self.residual_diff_threshold,
|
||||
force_refresh_step_hint=self.force_refresh_step_hint,
|
||||
force_refresh_step_policy=self.force_refresh_step_policy,
|
||||
)
|
||||
|
||||
def to_calibrator_config(self) -> CalibratorConfig | None:
|
||||
"""Build the optional TaylorSeer calibrator configuration."""
|
||||
|
||||
if not self.enable_taylorseer:
|
||||
return None
|
||||
|
||||
logger.info("TaylorSeer enabled with order=%s", self.taylorseer_order)
|
||||
return TaylorSeerCalibratorConfig(taylorseer_order=self.taylorseer_order)
|
||||
|
||||
|
||||
__all__ = ["CacheDiTAdapterConfig", "CacheDiTConfig"]
|
||||
+48
-362
@@ -1,27 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
cache-dit integration backend for vllm-omni.
|
||||
|
||||
This module provides a CacheDiTBackend class to enable cache-dit acceleration on diffusion
|
||||
pipelines in vllm-omni, supporting both single and dual-transformer architectures.
|
||||
"""
|
||||
"""Model-specific Cache-DiT adapters and enablers."""
|
||||
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional, TypeAlias
|
||||
from typing import Any
|
||||
|
||||
import cache_dit
|
||||
import torch
|
||||
from cache_dit import (
|
||||
BlockAdapter,
|
||||
CalibratorConfig,
|
||||
DBCacheConfig,
|
||||
ForwardPattern,
|
||||
ParamsModifier,
|
||||
TaylorSeerCalibratorConfig,
|
||||
)
|
||||
from cache_dit.caching.block_adapters import FakeDiffusionPipeline
|
||||
from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter
|
||||
@@ -31,178 +23,18 @@ from cache_dit.caching.cache_contexts import BasicCacheConfig
|
||||
from cache_dit.caching.cache_contexts.cache_manager import CachedContextManager
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheBackend
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.cache.cachedit.backend import (
|
||||
CUSTOM_DIT_ENABLERS,
|
||||
RefreshCacheContextFunc,
|
||||
_build_cache_context_refresh,
|
||||
_default_get_pipeline_transformer,
|
||||
_maybe_build_block_adapter,
|
||||
enable_cache_for_dit,
|
||||
)
|
||||
from vllm_omni.diffusion.cache.cachedit.config import CacheDiTConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RefreshCacheContextFunc: TypeAlias = Callable[[Any, int, bool], None]
|
||||
|
||||
|
||||
@dataclass
|
||||
class CacheDiTAdapterConfig:
|
||||
"""Config for creating a Cache DiT's block adapter; to enable CacheDiT,
|
||||
most models just need to define an instance of this class as a class
|
||||
var in the DiT.
|
||||
"""
|
||||
|
||||
block_forward_patterns: dict[str, ForwardPattern]
|
||||
has_separate_cfg: bool = False
|
||||
cached_adapter_cls: type[CachedAdapter] | None = None
|
||||
check_forward_pattern: bool = True
|
||||
|
||||
|
||||
# Registry of custom cache-dit enablers for specific models
|
||||
# Maps pipeline names to their cache-dit enablement functions
|
||||
CUSTOM_DIT_ENABLERS: dict[str, Callable] = {}
|
||||
|
||||
|
||||
# Small helper to centralize cache-dit summaries.
|
||||
def cache_summary(pipeline: Any, details: bool = True) -> None:
|
||||
if hasattr(pipeline, "transformer"):
|
||||
cache_dit.summary(pipeline.transformer, details=details)
|
||||
if hasattr(pipeline, "transformer_2"):
|
||||
cache_dit.summary(pipeline.transformer_2, details=details)
|
||||
if not hasattr(pipeline, "transformer") and not hasattr(pipeline, "transformer_2"):
|
||||
logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute")
|
||||
|
||||
|
||||
def default_get_pipeline_transformer(pipeline: Any) -> Any:
|
||||
return pipeline.transformer
|
||||
|
||||
|
||||
def build_cache_context_refresh(
|
||||
cache_config: DiffusionCacheConfig,
|
||||
get_pipeline_transformer: Callable[[Any], Any] = default_get_pipeline_transformer,
|
||||
) -> RefreshCacheContextFunc:
|
||||
"""Build the cache context refresh func for a single Transformer."""
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""Refresh cache context for the transformer with new num_inference_steps.
|
||||
|
||||
Args:
|
||||
pipeline: The pipeline instance.
|
||||
num_inference_steps: New number of inference steps.
|
||||
"""
|
||||
transformer = get_pipeline_transformer(pipeline)
|
||||
|
||||
# Bypass SCM for step counts that don't support predefined masks (e.g., vLLM's 1-step dummy run)
|
||||
scm_supported_steps = num_inference_steps >= 8 or num_inference_steps in (4, 6)
|
||||
|
||||
if cache_config.scm_steps_mask_policy is None or not scm_supported_steps:
|
||||
cache_dit.refresh_context(transformer, num_inference_steps=num_inference_steps, verbose=verbose)
|
||||
else:
|
||||
cache_dit.refresh_context(
|
||||
transformer,
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_inference_steps,
|
||||
steps_computation_mask=cache_dit.steps_mask(
|
||||
mask_policy=cache_config.scm_steps_mask_policy,
|
||||
total_steps=num_inference_steps,
|
||||
),
|
||||
steps_computation_policy=cache_config.scm_steps_policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
return refresh_cache_context
|
||||
|
||||
|
||||
def _resolve_calibrator_config(cache_config: DiffusionCacheConfig) -> CalibratorConfig | None:
|
||||
"""Resolves the Calibrator subconfig For DiT cache."""
|
||||
calibrator = None
|
||||
if cache_config.enable_taylorseer:
|
||||
taylorseer_order = cache_config.taylorseer_order
|
||||
calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
|
||||
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
|
||||
# In the future, more calibrators will likely be added to DiT Cache,
|
||||
# e.g., focal; handle them generically here.
|
||||
return calibrator
|
||||
|
||||
|
||||
def _build_db_cache_config(cache_config: DiffusionCacheConfig) -> DBCacheConfig:
|
||||
"""Build DBCacheConfig with optional SCM (Step Computation Masking) support.
|
||||
|
||||
Args:
|
||||
cache_config: DiffusionCacheConfig instance.
|
||||
|
||||
Returns:
|
||||
DBCacheConfig instance with SCM support if configured.
|
||||
"""
|
||||
return DBCacheConfig(
|
||||
# we will refresh the context when we get num_inference_steps in the first inference request
|
||||
num_inference_steps=None,
|
||||
Fn_compute_blocks=cache_config.Fn_compute_blocks,
|
||||
Bn_compute_blocks=cache_config.Bn_compute_blocks,
|
||||
max_warmup_steps=cache_config.max_warmup_steps,
|
||||
max_cached_steps=cache_config.max_cached_steps,
|
||||
max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
|
||||
residual_diff_threshold=cache_config.residual_diff_threshold,
|
||||
force_refresh_step_hint=cache_config.force_refresh_step_hint,
|
||||
force_refresh_step_policy=cache_config.force_refresh_step_policy,
|
||||
)
|
||||
|
||||
|
||||
def enable_cache_for_dit(
|
||||
pipeline: Any,
|
||||
cache_config: Any,
|
||||
block_adapter: BlockAdapter | None = None,
|
||||
adapter_cls: type[CachedAdapter] | None = None,
|
||||
) -> RefreshCacheContextFunc:
|
||||
"""Enable cache-dit for regular single-transformer DiT models.
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
cache_config: DiffusionCacheConfig instance with cache configuration.
|
||||
block_adapter: Custom block adapters for specific model architectures.
|
||||
adapter_cls: Custom cached adapter class for specific model architectures.
|
||||
|
||||
Returns:
|
||||
A refresh function that can be called to update cache context with new num_inference_steps.
|
||||
"""
|
||||
# Build DBCacheConfig with optional SCM support
|
||||
db_cache_config = _build_db_cache_config(cache_config)
|
||||
|
||||
# Build calibrator config if TaylorSeer is enabled
|
||||
calibrator_config = _resolve_calibrator_config(cache_config)
|
||||
|
||||
logger.info(
|
||||
f"Enabling cache-dit on transformer: "
|
||||
f"Fn={db_cache_config.Fn_compute_blocks}, "
|
||||
f"Bn={db_cache_config.Bn_compute_blocks}, "
|
||||
f"W={db_cache_config.max_warmup_steps}, "
|
||||
)
|
||||
|
||||
# Enable cache-dit on the transformer
|
||||
transformer = default_get_pipeline_transformer(pipeline)
|
||||
|
||||
# If we have a custom cached adapter subclass, call apply directly
|
||||
if adapter_cls is not None:
|
||||
adapter_cls.apply(
|
||||
transformer if block_adapter is None else block_adapter,
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
else:
|
||||
# Otherwise, call enable cache, which will call CachedAdapter.apply for us
|
||||
cache_dit.enable_cache(
|
||||
transformer if block_adapter is None else block_adapter,
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
|
||||
return build_cache_context_refresh(cache_config)
|
||||
|
||||
|
||||
### Complex / custom enablers for DiT cache
|
||||
# NOTE (Alex): This case is rare; you should only really need to do this if you have a dual transformer
|
||||
# architecture, since it hasn't been handled generically yet, or if the model class has unique attributes
|
||||
# that it sets during Cache DiT enablement.
|
||||
#
|
||||
# For the vast majority of models, you should only need to add a _cache_dit_adapter_config attribute
|
||||
# to the Transformer class, which controls the forward pattern, whether or not we have separate CFG,
|
||||
# and so on.
|
||||
|
||||
|
||||
# from https://github.com/vipshop/cache-dit/pull/542
|
||||
def _split_wan22_inference_steps(pipeline, num_inference_steps: int) -> tuple[int, int]:
|
||||
@@ -249,9 +81,9 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont
|
||||
Returns:
|
||||
A refresh function that can be called to update cache context with new num_inference_steps.
|
||||
"""
|
||||
# Build DBCacheConfig with optional SCM support
|
||||
db_cache_config = _build_db_cache_config(cache_config)
|
||||
calibrator_config = _resolve_calibrator_config(cache_config)
|
||||
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
|
||||
db_cache_config = projected_config.to_db_cache_config()
|
||||
calibrator_config = projected_config.to_calibrator_config()
|
||||
|
||||
if getattr(pipeline, "transformer_2", None) is None:
|
||||
logger.info("transformer_2 not found, enabling cache-dit for single transformer mode")
|
||||
@@ -273,7 +105,7 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
return build_cache_context_refresh(cache_config)
|
||||
return _build_cache_context_refresh(cache_config)
|
||||
|
||||
cache_dit.enable_cache(
|
||||
BlockAdapter(
|
||||
@@ -296,8 +128,8 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont
|
||||
# high-noise transformer only have 30% steps
|
||||
ParamsModifier(
|
||||
cache_config=DBCacheConfig().reset(
|
||||
max_warmup_steps=cache_config.max_warmup_steps,
|
||||
max_cached_steps=cache_config.max_cached_steps,
|
||||
max_warmup_steps=projected_config.max_warmup_steps,
|
||||
max_cached_steps=projected_config.max_cached_steps,
|
||||
),
|
||||
calibrator_config=calibrator_config,
|
||||
),
|
||||
@@ -315,8 +147,8 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheCont
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
|
||||
refresh_trans_one = build_cache_context_refresh(cache_config)
|
||||
refresh_trans_two = build_cache_context_refresh(cache_config, lambda pipeline: pipeline.transformer_2)
|
||||
refresh_trans_one = _build_cache_context_refresh(cache_config)
|
||||
refresh_trans_two = _build_cache_context_refresh(cache_config, lambda pipeline: pipeline.transformer_2)
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""Refresh cache context for both transformers with new num_inference_steps.
|
||||
@@ -347,12 +179,9 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache
|
||||
permanently replace it with a no-op on the transformer to prevent double
|
||||
injection from the main forward loop.
|
||||
"""
|
||||
db_cache_config = _build_db_cache_config(cache_config)
|
||||
calibrator_config = None
|
||||
if cache_config.enable_taylorseer:
|
||||
taylorseer_order = cache_config.taylorseer_order
|
||||
calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
|
||||
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
|
||||
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
|
||||
db_cache_config = projected_config.to_db_cache_config()
|
||||
calibrator_config = projected_config.to_calibrator_config()
|
||||
|
||||
# Save the original after_transformer_block before cache-dit wrapping
|
||||
transformer = pipeline.transformer
|
||||
@@ -383,7 +212,7 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""Refresh cache context for the S2V transformer."""
|
||||
if cache_config.scm_steps_mask_policy is None:
|
||||
if projected_config.scm_steps_mask_policy is None:
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer,
|
||||
num_inference_steps=num_inference_steps,
|
||||
@@ -395,10 +224,10 @@ def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> RefreshCache
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_inference_steps,
|
||||
steps_computation_mask=cache_dit.steps_mask(
|
||||
mask_policy=cache_config.scm_steps_mask_policy,
|
||||
mask_policy=projected_config.scm_steps_mask_policy,
|
||||
total_steps=num_inference_steps,
|
||||
),
|
||||
steps_computation_policy=cache_config.scm_steps_policy,
|
||||
steps_computation_policy=projected_config.scm_steps_policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
@@ -971,7 +800,7 @@ def enable_cache_for_cosmos3(pipeline: Any, cache_config: Any) -> RefreshCacheCo
|
||||
# step accounting. Still do both cond/uncond CFG steps when cache-dit is active.
|
||||
# CFG is instead neutralized via scale=1.0 outside the interval.
|
||||
pipeline._cache_dit_requires_paired_cfg = True
|
||||
block_adapter = CacheDiTBackend.maybe_build_block_adapter(pipeline)
|
||||
block_adapter = _maybe_build_block_adapter(pipeline)
|
||||
return enable_cache_for_dit(pipeline, cache_config, block_adapter)
|
||||
|
||||
|
||||
@@ -996,7 +825,7 @@ def enable_cache_for_krea2(pipeline: Any, cache_config: Any) -> RefreshCacheCont
|
||||
Returns:
|
||||
A refresh function that can be called to update cache context with new num_inference_steps.
|
||||
"""
|
||||
transformer = default_get_pipeline_transformer(pipeline)
|
||||
transformer = _default_get_pipeline_transformer(pipeline)
|
||||
block_adapter = BlockAdapter(
|
||||
transformer=transformer,
|
||||
blocks=[transformer.transformer_blocks],
|
||||
@@ -1007,171 +836,28 @@ def enable_cache_for_krea2(pipeline: Any, cache_config: Any) -> RefreshCacheCont
|
||||
return enable_cache_for_dit(pipeline, cache_config, block_adapter)
|
||||
|
||||
|
||||
# Register custom cache-dit enablers after function definitions
|
||||
CUSTOM_DIT_ENABLERS.update(
|
||||
{
|
||||
"Wan22Pipeline": enable_cache_for_wan22,
|
||||
"Wan22I2VPipeline": enable_cache_for_wan22,
|
||||
"Wan22TI2VPipeline": enable_cache_for_wan22,
|
||||
"Wan22VACEPipeline": enable_cache_for_wan22,
|
||||
"Wan22S2VPipeline": enable_cache_for_wan22_s2v,
|
||||
"Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,
|
||||
"Cosmos3OmniPipeline": enable_cache_for_cosmos3,
|
||||
"Krea2Pipeline": enable_cache_for_krea2,
|
||||
}
|
||||
)
|
||||
def register_custom_dit_enablers() -> None:
|
||||
"""Register model-specific Cache-DiT enablers.
|
||||
|
||||
|
||||
class CacheDiTBackend(CacheBackend):
|
||||
"""Backend class for cache-dit acceleration on diffusion pipelines.
|
||||
|
||||
This class implements cache-dit acceleration (DBCache, SCM, TaylorSeer) using
|
||||
the cache-dit library. It inherits from CacheBackend and provides a unified
|
||||
interface for managing cache-dit acceleration on diffusion models.
|
||||
|
||||
Attributes:
|
||||
config: Cache configuration (DiffusionCacheConfig instance), inherited from CacheBackend.
|
||||
enabled: Whether cache-dit is enabled on this pipeline, inherited from CacheBackend.
|
||||
_refresh_func: Internal refresh function for updating cache context.
|
||||
_last_num_inference_steps: Last num_inference_steps used for refresh optimization.
|
||||
This is called explicitly by the package initializer so registration does
|
||||
not depend on unrelated model-specific symbols being imported for their
|
||||
side effects.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_config: Any = None):
|
||||
"""Initialize the cache-dit backend.
|
||||
|
||||
Args:
|
||||
cache_config: Cache configuration (DiffusionCacheConfig instance, dict, or None).
|
||||
If None or empty, uses default DiffusionCacheConfig().
|
||||
"""
|
||||
# Use default config if cache_config is not provided or is empty
|
||||
if cache_config is None:
|
||||
config = DiffusionCacheConfig()
|
||||
elif isinstance(cache_config, dict):
|
||||
# Convert dict to DiffusionCacheConfig, using defaults for missing keys
|
||||
config = DiffusionCacheConfig.from_dict(cache_config)
|
||||
else:
|
||||
config = cache_config
|
||||
|
||||
# Initialize base class with normalized config
|
||||
super().__init__(config)
|
||||
|
||||
# Cache-dit specific attributes
|
||||
self._refresh_func: Callable[[Any, int, bool], None] | None = None
|
||||
self._last_num_inference_steps: int | None = None
|
||||
|
||||
@staticmethod
|
||||
def maybe_build_block_adapter(pipeline) -> BlockAdapter | None:
|
||||
"""If a module defines `_cache_dit_adapter_config`, build the corresponding
|
||||
block adapter.
|
||||
"""
|
||||
transformer = default_get_pipeline_transformer(pipeline)
|
||||
|
||||
adapter_cfg: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
|
||||
if adapter_cfg is None:
|
||||
return None
|
||||
|
||||
block_attrs, forward_pattern = zip(*(adapter_cfg.block_forward_patterns).items())
|
||||
missing_attrs = [block_attr for block_attr in block_attrs if not hasattr(transformer, block_attr)]
|
||||
|
||||
if missing_attrs:
|
||||
logger.warning("Missing Cache DiT block attributes: %s", missing_attrs)
|
||||
|
||||
block_adapter = BlockAdapter(
|
||||
transformer=transformer,
|
||||
blocks=[getattr(transformer, block_attr) for block_attr in block_attrs],
|
||||
forward_pattern=list(forward_pattern),
|
||||
has_separate_cfg=adapter_cfg.has_separate_cfg,
|
||||
check_forward_pattern=adapter_cfg.check_forward_pattern,
|
||||
)
|
||||
return block_adapter
|
||||
|
||||
@staticmethod
|
||||
def maybe_get_cached_adapter_cls(pipeline) -> type[CachedAdapter] | None:
|
||||
"""If a module has a custom cached adapter type registered, e.g., SenseNova, retrieve it
|
||||
from the transformer's CacheDiTAdapterConfig."""
|
||||
transformer = default_get_pipeline_transformer(pipeline)
|
||||
|
||||
adapter_cfg: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
|
||||
if adapter_cfg is None:
|
||||
return None
|
||||
return adapter_cfg.cached_adapter_cls
|
||||
|
||||
def enable(self, pipeline: Any) -> None:
|
||||
"""Enable cache-dit on the pipeline if configured.
|
||||
|
||||
This method applies cache-dit acceleration to the appropriate transformer(s)
|
||||
in the pipeline. It handles both single-transformer and dual-transformer
|
||||
architectures (e.g., Wan2.2).
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
"""
|
||||
|
||||
# Extract pipeline name from pipeline
|
||||
pipeline_name = pipeline.__class__.__name__
|
||||
# Check if this model has a custom cache-dit enabler
|
||||
if pipeline_name in CUSTOM_DIT_ENABLERS:
|
||||
logger.info(f"Using custom cache-dit enabler for model: {pipeline_name}")
|
||||
self._refresh_func = CUSTOM_DIT_ENABLERS[pipeline_name](pipeline, self.config)
|
||||
else:
|
||||
# Common case; either the model doesn't explicitly support dit cache yet,
|
||||
# Or it defines its _cache_dit_adapter_config, which describes how we should
|
||||
# create its block adapter.
|
||||
block_adapter = self.maybe_build_block_adapter(pipeline)
|
||||
adapter_cls = self.maybe_get_cached_adapter_cls(pipeline)
|
||||
self._refresh_func = enable_cache_for_dit(pipeline, self.config, block_adapter, adapter_cls)
|
||||
|
||||
self.enabled = True
|
||||
logger.info(f"Cache-dit enabled successfully on {pipeline_name}")
|
||||
|
||||
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""Refresh cache context with new num_inference_steps.
|
||||
|
||||
This method updates the cache context when num_inference_steps changes
|
||||
during inference. For dual-transformer models (e.g., Wan2.2), it automatically
|
||||
splits the steps based on boundary_ratio.
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
num_inference_steps: New number of inference steps.
|
||||
verbose: Whether to log refresh operations.
|
||||
"""
|
||||
if not self.enabled or self._refresh_func is None:
|
||||
logger.warning("Cache-dit is not enabled. Cannot refresh cache context.")
|
||||
return
|
||||
|
||||
# Only refresh if num_inference_steps has changed
|
||||
if self._last_num_inference_steps is None or num_inference_steps != self._last_num_inference_steps:
|
||||
if verbose:
|
||||
logger.info(f"Refreshing cache context for transformer with num_inference_steps: {num_inference_steps}")
|
||||
self._refresh_func(pipeline, num_inference_steps, verbose)
|
||||
self._last_num_inference_steps = num_inference_steps
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
"""Check if cache-dit is enabled on this pipeline.
|
||||
|
||||
Returns:
|
||||
True if cache-dit is enabled, False otherwise.
|
||||
"""
|
||||
return self.enabled
|
||||
CUSTOM_DIT_ENABLERS.update(
|
||||
{
|
||||
"Wan22Pipeline": enable_cache_for_wan22,
|
||||
"Wan22I2VPipeline": enable_cache_for_wan22,
|
||||
"Wan22TI2VPipeline": enable_cache_for_wan22,
|
||||
"Wan22VACEPipeline": enable_cache_for_wan22,
|
||||
"Wan22S2VPipeline": enable_cache_for_wan22_s2v,
|
||||
"Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,
|
||||
"Cosmos3OmniPipeline": enable_cache_for_cosmos3,
|
||||
"Krea2Pipeline": enable_cache_for_krea2,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def may_enable_cache_dit(pipeline: Any, od_config: OmniDiffusionConfig) -> Optional["CacheDiTBackend"]:
|
||||
"""Enable cache-dit on the pipeline if configured (convenience function).
|
||||
|
||||
This is a convenience function that creates and enables a CacheDiTBackend.
|
||||
For new code, consider using CacheDiTBackend directly.
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
od_config: OmniDiffusionConfig with cache configuration.
|
||||
|
||||
Returns:
|
||||
A CacheDiTBackend instance if cache-dit is enabled, None otherwise.
|
||||
"""
|
||||
if od_config.cache_backend != "cache-dit" or not od_config.cache_config:
|
||||
return None
|
||||
|
||||
backend = CacheDiTBackend(od_config.cache_config)
|
||||
backend.enable(pipeline)
|
||||
return backend if backend.is_enabled() else None
|
||||
__all__ = [
|
||||
"BagelCachedAdapter",
|
||||
"SensenovaCachedAdapter",
|
||||
]
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend
|
||||
from vllm_omni.diffusion.cache.magcache.config import MagCacheConfig
|
||||
from vllm_omni.diffusion.cache.magcache.hook import (
|
||||
MagCacheBlockHook,
|
||||
@@ -21,6 +22,7 @@ __all__ = [
|
||||
"Flux2MagCacheStrategy",
|
||||
"FluxMagCacheStrategy",
|
||||
"MagCacheBlockHook",
|
||||
"MagCacheBackend",
|
||||
"MagCacheConfig",
|
||||
"MagCacheHeadHook",
|
||||
"MagCacheState",
|
||||
|
||||
+3
-3
@@ -1,10 +1,10 @@
|
||||
from typing import Any
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheBackend
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTBackend
|
||||
from vllm_omni.diffusion.cache.magcache.backend import MagCacheBackend
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTBackend
|
||||
from vllm_omni.diffusion.cache.magcache import MagCacheBackend
|
||||
from vllm_omni.diffusion.cache.stepcache import StepCacheBackend
|
||||
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
|
||||
from vllm_omni.diffusion.cache.teacache import TeaCacheBackend
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.transformers_utils.configs.bagel import BagelConfig
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata as DiffusionAttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention as DiffusionAttention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import BagelCachedAdapter, CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import BagelCachedAdapter, CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
|
||||
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention as FrameworkAttention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
|
||||
|
||||
@@ -9,7 +9,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
@@ -22,7 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.quantization.component_config import safe_quant_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import ColumnParallelLinear, QKVParallelL
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
SequenceParallelOutput,
|
||||
|
||||
@@ -60,7 +60,7 @@ from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_sequence_parallel_world_size
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
|
||||
@@ -45,7 +45,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
|
||||
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
|
||||
|
||||
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig, SensenovaCachedAdapter
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig, SensenovaCachedAdapter
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ from transformers import LlamaConfig, LlamaModel
|
||||
from transformers.modeling_outputs import BaseModelOutputWithPast
|
||||
from transformers.models.llama.modeling_llama import LlamaDecoderLayer
|
||||
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
|
||||
from vllm_omni.diffusion.layers.fourier import GaussianFourierProjection
|
||||
|
||||
@@ -22,7 +22,7 @@ from vllm.config import LoadConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes
|
||||
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import cache_summary
|
||||
from vllm_omni.diffusion.cache.cachedit import cache_summary
|
||||
from vllm_omni.diffusion.cache.prompt_embed_cache import (
|
||||
install_prompt_embed_cache,
|
||||
resolve_prompt_embed_cache_config,
|
||||
|
||||
Reference in new issue
Block a user