[Diffusion] Add cache-dit and unify diffusion cache backend interface (#250)
Signed-off-by: samithuang <285365963@qq.com> Signed-off-by: SamitHuang <285365963@qq.com> Signed-off-by: Samit <285365963@qq.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
1 parent
63f1e347ad
commit
2b4a9fe729
24 files changed
+1644
-367
No files matched your search
+7
-21
@@ -1,41 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Cache module for diffusion model inference acceleration.
|
||||
|
||||
This module provides a unified cache adapter system for different caching strategies:
|
||||
This module provides a unified cache backend system for different caching strategies:
|
||||
- TeaCache: Timestep Embedding Aware Cache for adaptive transformer caching
|
||||
- (Future: DeepCache, DistriFusion, etc.)
|
||||
- cache-dit: DBCache, SCM, and TaylorSeer caching strategies
|
||||
|
||||
The cache adapter system uses a registry pattern where cache types can be
|
||||
dynamically registered and instantiated. Cache is configured via OmniDiffusionConfig
|
||||
or DIFFUSION_CACHE_ADAPTER environment variable.
|
||||
Cache backends are instantiated directly via their constructors and configured via OmniDiffusionConfig.
|
||||
"""
|
||||
|
||||
from vllm_omni.diffusion.cache.apply import setup_cache
|
||||
from vllm_omni.diffusion.cache.base import CacheAdapter
|
||||
from vllm_omni.diffusion.cache.registry import (
|
||||
CacheType,
|
||||
get_cache_adapter,
|
||||
register_cache_adapter,
|
||||
)
|
||||
from vllm_omni.diffusion.cache.base import CacheBackend
|
||||
from vllm_omni.diffusion.cache.teacache import (
|
||||
CacheContext,
|
||||
TeaCacheConfig,
|
||||
apply_teacache_hook,
|
||||
)
|
||||
|
||||
# Import teacache adapter to trigger registration
|
||||
from vllm_omni.diffusion.cache.teacache.adapter import TeaCacheAdapter # noqa: F401
|
||||
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
|
||||
|
||||
__all__ = [
|
||||
"CacheAdapter",
|
||||
"CacheType",
|
||||
"CacheBackend",
|
||||
"TeaCacheConfig",
|
||||
"CacheContext",
|
||||
"setup_cache",
|
||||
"get_cache_adapter",
|
||||
"register_cache_adapter",
|
||||
"TeaCacheBackend",
|
||||
"apply_teacache_hook",
|
||||
]
|
||||
Vendored
-82
@@ -1,82 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Unified cache setup function for diffusion models.
|
||||
|
||||
This module provides the main entry point for setting up cache adapters
|
||||
on transformer models. It handles cache type selection, adapter instantiation,
|
||||
and application to the model.
|
||||
"""
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheAdapter
|
||||
from vllm_omni.diffusion.cache.registry import get_cache_adapter
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def setup_cache(
|
||||
transformer: torch.nn.Module,
|
||||
cache_type: str = "none",
|
||||
cache_config: Optional[dict[str, Any]] = None,
|
||||
) -> Optional[CacheAdapter]:
|
||||
"""
|
||||
Setup cache adapter for transformer (one-time initialization).
|
||||
|
||||
This is the main entry point for applying caching to diffusion transformers.
|
||||
It's called once during pipeline initialization and applies the appropriate
|
||||
cache adapter based on configuration.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to apply cache to
|
||||
cache_type: Type of cache to apply. Options:
|
||||
- "none": No caching (default)
|
||||
- "tea_cache": TeaCache adaptive caching
|
||||
- Additional types can be registered via register_cache_adapter()
|
||||
cache_config: Cache-specific configuration dictionary. Contents depend
|
||||
on cache type:
|
||||
- For "tea_cache": {"rel_l1_thresh": 0.2, "model_type": "QwenImagePipeline"}
|
||||
- For other types: see their respective adapter documentation
|
||||
|
||||
Returns:
|
||||
CacheAdapter instance for state management, or None if cache_type="none"
|
||||
|
||||
Example:
|
||||
>>> # Setup TeaCache during pipeline initialization
|
||||
>>> adapter = setup_cache(
|
||||
... transformer,
|
||||
... cache_type="tea_cache",
|
||||
... cache_config={"rel_l1_thresh": 0.2}
|
||||
... )
|
||||
>>>
|
||||
>>> # Reset cache state before each generation
|
||||
>>> if adapter is not None:
|
||||
... adapter.reset(transformer)
|
||||
"""
|
||||
# No caching requested
|
||||
if cache_type == "none":
|
||||
logger.debug("No cache adapter requested (cache_type='none')")
|
||||
return None
|
||||
|
||||
# Get adapter instance from registry
|
||||
try:
|
||||
adapter = get_cache_adapter(cache_type, cache_config or {})
|
||||
except ValueError as e:
|
||||
logger.error(f"Failed to get cache adapter: {e}")
|
||||
raise
|
||||
|
||||
# Apply cache to transformer
|
||||
logger.info(f"Applying {cache_type} cache to transformer")
|
||||
try:
|
||||
adapter.apply(transformer)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to apply cache adapter: {e}")
|
||||
raise
|
||||
|
||||
logger.info(f"Cache adapter '{cache_type}' successfully applied")
|
||||
return adapter
|
||||
Vendored
+58
-25
@@ -2,66 +2,99 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Base cache adapter interface for diffusion models.
|
||||
Base cache backend interface for diffusion models.
|
||||
|
||||
This module defines the abstract base class that all cache adapters must implement.
|
||||
Cache adapters provide a unified interface for applying different caching strategies
|
||||
(TeaCache, DeepCache, etc.) to transformer models using hooks.
|
||||
This module defines the abstract base class that all cache backends must implement.
|
||||
Cache backends provide a unified interface for applying different caching strategies
|
||||
to transformer models.
|
||||
|
||||
Main cache backend implementations:
|
||||
1. CacheDiTBackend: Implements cache-dit acceleration (DBCache, SCM, TaylorSeer) using
|
||||
the cache-dit library. Inherits from CacheBackend. Used via cache_backend="cache_dit".
|
||||
2. TeaCacheBackend: Hook-based backend for TeaCache acceleration. Inherits from
|
||||
CacheBackend. Used via cache_backend="tea_cache".
|
||||
|
||||
All backends implement the same interface:
|
||||
- enable(pipeline): Enable cache on the pipeline
|
||||
- refresh(pipeline, num_inference_steps, verbose): Refresh cache state
|
||||
- is_enabled(): Check if cache is enabled
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
|
||||
|
||||
class CacheAdapter(ABC):
|
||||
class CacheBackend(ABC):
|
||||
"""
|
||||
Abstract base class for cache adapters.
|
||||
Abstract base class for cache backends.
|
||||
|
||||
Cache adapters apply caching strategies to transformer models to accelerate
|
||||
inference. Each adapter type (TeaCache, DeepCache, etc.) implements the
|
||||
apply() and reset() methods to manage cache lifecycle.
|
||||
All cache backend implementations (CacheDiTBackend, TeaCacheBackend, etc.) inherit
|
||||
from this base class and implement the enable() and refresh() methods to manage
|
||||
cache lifecycle.
|
||||
|
||||
Cache backends apply caching strategies to transformer models to accelerate
|
||||
inference. Different backends use different underlying mechanisms (e.g., cache-dit
|
||||
library for CacheDiTBackend, hooks for TeaCacheBackend), but all share the same
|
||||
unified interface.
|
||||
|
||||
Attributes:
|
||||
config: Dictionary containing cache-specific configuration parameters
|
||||
config: DiffusionCacheConfig instance containing cache-specific configuration parameters
|
||||
enabled: Boolean flag indicating whether cache is enabled (set to True after enable() is called)
|
||||
"""
|
||||
|
||||
def __init__(self, config: dict[str, Any]):
|
||||
def __init__(self, config: DiffusionCacheConfig):
|
||||
"""
|
||||
Initialize cache adapter with configuration.
|
||||
Initialize cache backend with configuration.
|
||||
|
||||
Args:
|
||||
config: Cache-specific configuration dictionary
|
||||
config: DiffusionCacheConfig instance with cache-specific parameters
|
||||
"""
|
||||
self.config = config
|
||||
self.enabled = False
|
||||
|
||||
@abstractmethod
|
||||
def apply(self, transformer: torch.nn.Module) -> None:
|
||||
def enable(self, pipeline: Any) -> None:
|
||||
"""
|
||||
Apply cache to transformer using hooks.
|
||||
Enable cache on the pipeline.
|
||||
|
||||
This method should register the appropriate hooks on the transformer
|
||||
to enable caching during inference. Called once during pipeline
|
||||
initialization.
|
||||
This method applies the caching strategy to the transformer(s) in the pipeline.
|
||||
The specific implementation depends on the backend (e.g., hooks for TeaCacheBackend,
|
||||
cache-dit library for CacheDiTBackend). Called once during pipeline initialization.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to apply cache to
|
||||
pipeline: Diffusion pipeline instance. The backend can extract:
|
||||
- transformer: via pipeline.transformer
|
||||
- model_type: via pipeline.__class__.__name__
|
||||
"""
|
||||
raise NotImplementedError("Subclasses must implement apply()")
|
||||
raise NotImplementedError("Subclasses must implement enable()")
|
||||
|
||||
@abstractmethod
|
||||
def reset(self, transformer: torch.nn.Module) -> None:
|
||||
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""
|
||||
Reset cache state for new generation.
|
||||
Refresh cache state for new generation.
|
||||
|
||||
This method should clear any cached values and reset counters/accumulators.
|
||||
Called at the start of each generation to ensure clean state.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to reset cache on
|
||||
pipeline: Diffusion pipeline instance. The backend can extract:
|
||||
- transformer: via pipeline.transformer
|
||||
num_inference_steps: Number of inference steps for the current generation.
|
||||
May be used for cache context updates.
|
||||
verbose: Whether to log refresh operations (default: True)
|
||||
"""
|
||||
raise NotImplementedError("Subclasses must implement reset()")
|
||||
raise NotImplementedError("Subclasses must implement refresh()")
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
"""
|
||||
Check if cache is enabled on this backend.
|
||||
|
||||
Returns:
|
||||
True if cache is enabled, False otherwise.
|
||||
"""
|
||||
return self.enabled
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(config={self.config})"
|
||||
+394
@@ -0,0 +1,394 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
cache-dit integration backend for vllm-omni.
|
||||
|
||||
This module provides a CacheDiTBackend class to enable cache-dit acceleration on diffusion
|
||||
pipelines in vllm-omni, supporting both single and dual-transformer architectures.
|
||||
"""
|
||||
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheBackend
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig, OmniDiffusionConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
try:
|
||||
import cache_dit
|
||||
from cache_dit import BlockAdapter, DBCacheConfig, ForwardPattern, ParamsModifier, TaylorSeerCalibratorConfig
|
||||
|
||||
CACHE_DIT_AVAILABLE = True
|
||||
except ImportError:
|
||||
CACHE_DIT_AVAILABLE = False
|
||||
logger.warning("cache-dit is not installed. Cache-dit acceleration will not be available.")
|
||||
|
||||
|
||||
# Registry of custom cache-dit enablers for specific models
|
||||
# Maps pipeline names to their cache-dit enablement functions
|
||||
# Models in this registry require custom handling (e.g., dual-transformer architectures)
|
||||
# Will be populated after function definitions
|
||||
CUSTOM_DIT_ENABLERS: dict[str, Callable] = {}
|
||||
|
||||
|
||||
def _build_db_cache_config(cache_config: Any) -> DBCacheConfig:
|
||||
"""Build DBCacheConfig with optional SCM (Step Computation Masking) support.
|
||||
|
||||
Args:
|
||||
cache_config: DiffusionCacheConfig instance.
|
||||
|
||||
Returns:
|
||||
DBCacheConfig instance with SCM support if configured.
|
||||
"""
|
||||
|
||||
return DBCacheConfig(
|
||||
# we will refresh the context when gets num_inference_steps in the first inference request
|
||||
num_inference_steps=None,
|
||||
Fn_compute_blocks=cache_config.Fn_compute_blocks,
|
||||
Bn_compute_blocks=cache_config.Bn_compute_blocks,
|
||||
max_warmup_steps=cache_config.max_warmup_steps,
|
||||
max_cached_steps=cache_config.max_cached_steps,
|
||||
max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
|
||||
residual_diff_threshold=cache_config.residual_diff_threshold,
|
||||
)
|
||||
|
||||
|
||||
def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
||||
"""Enable cache-dit for Wan2.2 dual-transformer architecture.
|
||||
|
||||
Wan2.2 uses two transformers (transformer and transformer_2) that need
|
||||
to be enabled together using BlockAdapter.
|
||||
|
||||
Args:
|
||||
pipeline: The Wan2.2 pipeline instance.
|
||||
cache_config: DiffusionCacheConfig instance with cache configuration.
|
||||
|
||||
Returns:
|
||||
A refresh function that can be called to update cache context with new num_inference_steps.
|
||||
"""
|
||||
|
||||
# Build DBCacheConfig for primary transformer
|
||||
primary_cache_config = _build_db_cache_config(cache_config)
|
||||
|
||||
# FIXME: secondary cache shares the same config with primary cache for now,
|
||||
# but we should support different config for secondary transformer in the future
|
||||
# Build DBCacheConfig for secondary transformer (can use same or different config)
|
||||
secondary_cache_config = _build_db_cache_config(cache_config)
|
||||
|
||||
# Build calibrator configs if TaylorSeer is enabled
|
||||
primary_calibrator = None
|
||||
secondary_calibrator = None
|
||||
if cache_config.enable_taylorseer:
|
||||
taylorseer_order = cache_config.taylorseer_order
|
||||
primary_calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
|
||||
secondary_calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
|
||||
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
|
||||
|
||||
# Build ParamsModifier for each transformer
|
||||
primary_modifier = ParamsModifier(
|
||||
cache_config=primary_cache_config,
|
||||
calibrator_config=primary_calibrator,
|
||||
)
|
||||
secondary_modifier = ParamsModifier(
|
||||
cache_config=secondary_cache_config,
|
||||
calibrator_config=secondary_calibrator,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Enabling cache-dit on Wan2.2 dual transformers with BlockAdapter: "
|
||||
f"Fn={primary_cache_config.Fn_compute_blocks}, "
|
||||
f"Bn={primary_cache_config.Bn_compute_blocks}, "
|
||||
f"W={primary_cache_config.max_warmup_steps}, "
|
||||
)
|
||||
|
||||
transformer = pipeline.transformer
|
||||
transformer_2 = pipeline.transformer_2
|
||||
transformer_blocks = transformer.blocks
|
||||
transformer_2_blocks = transformer_2.blocks
|
||||
# Enable cache-dit using BlockAdapter for both transformers simultaneously
|
||||
cache_dit.enable_cache(
|
||||
BlockAdapter(
|
||||
transformer=[transformer, transformer_2],
|
||||
blocks=[transformer_blocks, transformer_2_blocks],
|
||||
forward_pattern=[ForwardPattern.Pattern_2, ForwardPattern.Pattern_2],
|
||||
params_modifiers=[primary_modifier, secondary_modifier],
|
||||
has_separate_cfg=True,
|
||||
),
|
||||
)
|
||||
|
||||
# from https://github.com/vipshop/cache-dit/pull/542
|
||||
def _split_inference_steps(num_inference_steps: int) -> tuple[int, int]:
|
||||
"""Split inference steps into high-noise and low-noise steps for Wan2.2.
|
||||
|
||||
This is an internal helper function specific to Wan2.2's dual-transformer
|
||||
architecture that uses boundary_ratio to determine the split point.
|
||||
|
||||
Args:
|
||||
num_inference_steps: Total number of inference steps.
|
||||
|
||||
Returns:
|
||||
A tuple of (num_high_noise_steps, num_low_noise_steps).
|
||||
"""
|
||||
if pipeline.config.boundary_ratio is not None:
|
||||
boundary_timestep = pipeline.config.boundary_ratio * pipeline.scheduler.config.num_train_timesteps
|
||||
else:
|
||||
boundary_timestep = None
|
||||
|
||||
# Set timesteps to calculate the split
|
||||
device = next(pipeline.transformer.parameters()).device
|
||||
pipeline.scheduler.set_timesteps(num_inference_steps, device=device)
|
||||
|
||||
timesteps = pipeline.scheduler.timesteps
|
||||
num_high_noise_steps = 0 # high-noise steps for transformer
|
||||
for t in timesteps:
|
||||
if boundary_timestep is None or t >= boundary_timestep:
|
||||
num_high_noise_steps += 1
|
||||
# low-noise steps for transformer_2
|
||||
num_low_noise_steps = num_inference_steps - num_high_noise_steps
|
||||
return num_high_noise_steps, num_low_noise_steps
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""Refresh cache context for both transformers with new num_inference_steps.
|
||||
|
||||
Args:
|
||||
pipeline: The Wan2.2 pipeline instance.
|
||||
num_inference_steps: New number of inference steps.
|
||||
"""
|
||||
num_high_noise_steps, num_low_noise_steps = _split_inference_steps(num_inference_steps)
|
||||
# Refresh context for high-noise transformer
|
||||
if cache_config.scm_steps_mask_policy is None:
|
||||
cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_high_noise_steps, verbose=verbose)
|
||||
else:
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer,
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_high_noise_steps,
|
||||
steps_computation_mask=cache_dit.steps_mask(
|
||||
mask_policy=cache_config.scm_steps_mask_policy, total_steps=num_high_noise_steps
|
||||
),
|
||||
steps_computation_policy=cache_config.scm_steps_policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
# Refresh context for low-noise transformer
|
||||
if cache_config.scm_steps_mask_policy is None:
|
||||
cache_dit.refresh_context(pipeline.transformer_2, num_inference_steps=num_low_noise_steps, verbose=verbose)
|
||||
else:
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer_2,
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_low_noise_steps,
|
||||
steps_computation_mask=cache_dit.steps_mask(
|
||||
mask_policy=cache_config.scm_steps_mask_policy, total_steps=num_low_noise_steps
|
||||
),
|
||||
steps_computation_policy=cache_config.scm_steps_policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
return refresh_cache_context
|
||||
|
||||
|
||||
def enable_cache_for_flux(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
||||
"""Enable cache-dit for Flux dual-transformer architecture.
|
||||
|
||||
Flux uses two transformers (transformer and transformer_2) that need
|
||||
to be enabled together using BlockAdapter.
|
||||
|
||||
Args:
|
||||
pipeline: The Flux pipeline instance.
|
||||
cache_config: DiffusionCacheConfig instance with cache configuration.
|
||||
|
||||
Returns:
|
||||
A refresh function that can be called to update cache context with new num_inference_steps.
|
||||
"""
|
||||
raise NotImplementedError("cache-dit is not implemented for Flux pipeline.")
|
||||
|
||||
|
||||
def enable_cache_for_dit(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
||||
"""Enable cache-dit for regular single-transformer DiT models.
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
cache_config: DiffusionCacheConfig instance with cache configuration.
|
||||
|
||||
Returns:
|
||||
A refresh function that can be called to update cache context with new num_inference_steps.
|
||||
"""
|
||||
# Build DBCacheConfig with optional SCM support
|
||||
db_cache_config = _build_db_cache_config(cache_config)
|
||||
|
||||
# Build calibrator config if TaylorSeer is enabled
|
||||
calibrator_config = None
|
||||
if cache_config.enable_taylorseer:
|
||||
taylorseer_order = cache_config.taylorseer_order
|
||||
calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
|
||||
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
|
||||
|
||||
logger.info(
|
||||
f"Enabling cache-dit on transformer: "
|
||||
f"Fn={db_cache_config.Fn_compute_blocks}, "
|
||||
f"Bn={db_cache_config.Bn_compute_blocks}, "
|
||||
f"W={db_cache_config.max_warmup_steps}, "
|
||||
)
|
||||
|
||||
# Enable cache-dit on the transformer
|
||||
cache_dit.enable_cache(
|
||||
pipeline.transformer,
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""Refresh cache context for the transformer with new num_inference_steps.
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
num_inference_steps: New number of inference steps.
|
||||
"""
|
||||
if cache_config.scm_steps_mask_policy is None:
|
||||
cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose)
|
||||
else:
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer,
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_inference_steps,
|
||||
steps_computation_mask=cache_dit.steps_mask(
|
||||
mask_policy=cache_config.scm_steps_mask_policy,
|
||||
total_steps=num_inference_steps,
|
||||
),
|
||||
steps_computation_policy=cache_config.scm_steps_policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
return refresh_cache_context
|
||||
|
||||
|
||||
# Register custom cache-dit enablers after function definitions
|
||||
CUSTOM_DIT_ENABLERS.update(
|
||||
{
|
||||
"WanPipeline": enable_cache_for_wan22,
|
||||
"FluxPipeline": enable_cache_for_flux,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class CacheDiTBackend(CacheBackend):
|
||||
"""Backend class for cache-dit acceleration on diffusion pipelines.
|
||||
|
||||
This class implements cache-dit acceleration (DBCache, SCM, TaylorSeer) using
|
||||
the cache-dit library. It inherits from CacheBackend and provides a unified
|
||||
interface for managing cache-dit acceleration on diffusion models.
|
||||
|
||||
Attributes:
|
||||
config: Cache configuration (DiffusionCacheConfig instance), inherited from CacheBackend.
|
||||
enabled: Whether cache-dit is enabled on this pipeline, inherited from CacheBackend.
|
||||
_refresh_func: Internal refresh function for updating cache context.
|
||||
_last_num_inference_steps: Last num_inference_steps used for refresh optimization.
|
||||
"""
|
||||
|
||||
def __init__(self, cache_config: Any = None):
|
||||
"""Initialize the cache-dit backend.
|
||||
|
||||
Args:
|
||||
cache_config: Cache configuration (DiffusionCacheConfig instance, dict, or None).
|
||||
If None or empty, uses default DiffusionCacheConfig().
|
||||
"""
|
||||
# Use default config if cache_config is not provided or is empty
|
||||
if cache_config is None:
|
||||
config = DiffusionCacheConfig()
|
||||
elif isinstance(cache_config, dict):
|
||||
# Convert dict to DiffusionCacheConfig, using defaults for missing keys
|
||||
config = DiffusionCacheConfig.from_dict(cache_config)
|
||||
else:
|
||||
config = cache_config
|
||||
|
||||
# Initialize base class with normalized config
|
||||
super().__init__(config)
|
||||
|
||||
# Cache-dit specific attributes
|
||||
self._refresh_func: Optional[Callable[[Any, int, bool], None]] = None
|
||||
self._last_num_inference_steps: Optional[int] = None
|
||||
|
||||
def enable(self, pipeline: Any) -> None:
|
||||
"""Enable cache-dit on the pipeline if configured.
|
||||
|
||||
This method applies cache-dit acceleration to the appropriate transformer(s)
|
||||
in the pipeline. It handles both single-transformer and dual-transformer
|
||||
architectures (e.g., Wan2.2).
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
"""
|
||||
if not CACHE_DIT_AVAILABLE:
|
||||
logger.warning("cache-dit is not available, skipping cache-dit setup.")
|
||||
return
|
||||
|
||||
# Extract pipeline name from pipeline
|
||||
pipeline_name = pipeline.__class__.__name__
|
||||
# Check if this model has a custom cache-dit enabler
|
||||
if pipeline_name in CUSTOM_DIT_ENABLERS:
|
||||
logger.info(f"Using custom cache-dit enabler for model: {pipeline_name}")
|
||||
self._refresh_func = CUSTOM_DIT_ENABLERS[pipeline_name](pipeline, self.config)
|
||||
else:
|
||||
# For regular single-transformer models
|
||||
self._refresh_func = enable_cache_for_dit(pipeline, self.config)
|
||||
|
||||
self.enabled = True
|
||||
logger.info(f"Cache-dit enabled successfully on {pipeline_name}")
|
||||
|
||||
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""Refresh cache context with new num_inference_steps.
|
||||
|
||||
This method updates the cache context when num_inference_steps changes
|
||||
during inference. For dual-transformer models (e.g., Wan2.2), it automatically
|
||||
splits the steps based on boundary_ratio.
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
num_inference_steps: New number of inference steps.
|
||||
verbose: Whether to log refresh operations.
|
||||
"""
|
||||
if not self.enabled or self._refresh_func is None:
|
||||
logger.warning("Cache-dit is not enabled. Cannot refresh cache context.")
|
||||
return
|
||||
|
||||
# Only refresh if num_inference_steps has changed
|
||||
if self._last_num_inference_steps is None or num_inference_steps != self._last_num_inference_steps:
|
||||
if verbose:
|
||||
logger.info(f"Refreshing cache context for transformer with num_inference_steps: {num_inference_steps}")
|
||||
self._refresh_func(pipeline, num_inference_steps, verbose)
|
||||
self._last_num_inference_steps = num_inference_steps
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
"""Check if cache-dit is enabled on this pipeline.
|
||||
|
||||
Returns:
|
||||
True if cache-dit is enabled, False otherwise.
|
||||
"""
|
||||
return self.enabled
|
||||
|
||||
|
||||
def may_enable_cache_dit(pipeline: Any, od_config: OmniDiffusionConfig) -> Optional["CacheDiTBackend"]:
|
||||
"""Enable cache-dit on the pipeline if configured (convenience function).
|
||||
|
||||
This is a convenience function that creates and enables a CacheDiTBackend.
|
||||
For new code, consider using CacheDiTBackend directly.
|
||||
|
||||
Args:
|
||||
pipeline: The diffusion pipeline instance.
|
||||
od_config: OmniDiffusionConfig with cache configuration.
|
||||
|
||||
Returns:
|
||||
A CacheDiTBackend instance if cache-dit is enabled, None otherwise.
|
||||
"""
|
||||
if od_config.cache_backend != "cache-dit" or not od_config.cache_config:
|
||||
return None
|
||||
|
||||
backend = CacheDiTBackend(od_config.cache_config)
|
||||
backend.enable(pipeline)
|
||||
return backend if backend.is_enabled() else None
|
||||
-100
@@ -1,100 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Cache adapter registry for diffusion models.
|
||||
|
||||
This module provides a registry pattern for cache adapters, allowing dynamic
|
||||
registration and instantiation of different cache types (TeaCache, DeepCache, etc.).
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheAdapter
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CacheType(Enum):
|
||||
"""Supported cache adapter types."""
|
||||
|
||||
NONE = "none"
|
||||
TEA_CACHE = "tea_cache"
|
||||
# Future cache types can be added here:
|
||||
# DEEP_CACHE = "deep_cache"
|
||||
# DISTRI_FUSION = "distri_fusion"
|
||||
|
||||
|
||||
# Global registry mapping cache types to adapter classes
|
||||
CACHE_ADAPTER_REGISTRY: dict[CacheType, type[CacheAdapter]] = {}
|
||||
|
||||
|
||||
def register_cache_adapter(cache_type: CacheType, adapter_class: type[CacheAdapter]) -> None:
|
||||
"""
|
||||
Register a cache adapter class for a given cache type.
|
||||
|
||||
This allows extending the system with new cache types without modifying
|
||||
the core cache infrastructure.
|
||||
|
||||
Args:
|
||||
cache_type: CacheType enum value
|
||||
adapter_class: CacheAdapter subclass to register
|
||||
|
||||
Example:
|
||||
>>> register_cache_adapter(CacheType.TEA_CACHE, TeaCacheAdapter)
|
||||
"""
|
||||
if not issubclass(adapter_class, CacheAdapter):
|
||||
raise TypeError(f"{adapter_class} must be a subclass of CacheAdapter")
|
||||
|
||||
CACHE_ADAPTER_REGISTRY[cache_type] = adapter_class
|
||||
logger.debug(f"Registered cache adapter: {cache_type.value} -> {adapter_class.__name__}")
|
||||
|
||||
|
||||
def get_cache_adapter(cache_type: str, config: dict[str, Any]) -> CacheAdapter:
|
||||
"""
|
||||
Factory function to get cache adapter instance.
|
||||
|
||||
Converts string cache type to enum, looks up in registry, and instantiates
|
||||
the appropriate adapter class with the provided configuration.
|
||||
|
||||
Args:
|
||||
cache_type: String name of cache type ("tea_cache", "deep_cache", etc.)
|
||||
config: Configuration dictionary to pass to adapter constructor
|
||||
|
||||
Returns:
|
||||
Instantiated CacheAdapter subclass
|
||||
|
||||
Raises:
|
||||
ValueError: If cache_type is unknown or not registered
|
||||
|
||||
Example:
|
||||
>>> adapter = get_cache_adapter("tea_cache", {"rel_l1_thresh": 0.2})
|
||||
>>> adapter.apply(transformer)
|
||||
"""
|
||||
# Normalize cache type string
|
||||
cache_type_str = cache_type.lower().strip()
|
||||
|
||||
# Convert string to enum
|
||||
try:
|
||||
cache_enum = CacheType(cache_type_str)
|
||||
except ValueError:
|
||||
available = [ct.value for ct in CacheType if ct != CacheType.NONE]
|
||||
raise ValueError(f"Unknown cache type: '{cache_type}'. Available types: {available}")
|
||||
|
||||
# Check if it's the special "none" case
|
||||
if cache_enum == CacheType.NONE:
|
||||
raise ValueError("Cannot instantiate adapter for cache_type='none'. Use setup_cache() which handles this case.")
|
||||
|
||||
# Lookup in registry
|
||||
if cache_enum not in CACHE_ADAPTER_REGISTRY:
|
||||
raise ValueError(
|
||||
f"Cache type '{cache_type}' is not registered. Registered types: {list(CACHE_ADAPTER_REGISTRY.keys())}"
|
||||
)
|
||||
|
||||
adapter_class = CACHE_ADAPTER_REGISTRY[cache_enum]
|
||||
|
||||
# Instantiate and return
|
||||
return adapter_class(config)
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
from typing import Any
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheBackend
|
||||
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTBackend
|
||||
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
|
||||
|
||||
def get_cache_backend(cache_backend: str | None, cache_config: Any) -> CacheBackend | None:
|
||||
"""Get cache backend instance based on cache_backend string.
|
||||
|
||||
This is a selector function that routes to the appropriate backend implementation.
|
||||
- cache_dit: Uses CacheDiTBackend with enable()/refresh() interface
|
||||
- tea_cache: Uses TeaCacheBackend with enable()/refresh() interface
|
||||
|
||||
Args:
|
||||
cache_backend: Cache backend name ("cache_dit", "tea_cache", or None).
|
||||
cache_config: Cache configuration (dict or DiffusionCacheConfig instance).
|
||||
|
||||
Returns:
|
||||
Cache backend instance (CacheDiTBackend or TeaCacheBackend) if cache_backend is set,
|
||||
None otherwise.
|
||||
|
||||
Raises:
|
||||
ValueError: If cache_backend is unsupported.
|
||||
"""
|
||||
if cache_backend is None or cache_backend == "none":
|
||||
return None
|
||||
|
||||
if isinstance(cache_config, dict):
|
||||
cache_config = DiffusionCacheConfig.from_dict(cache_config)
|
||||
|
||||
if cache_backend == "cache_dit":
|
||||
return CacheDiTBackend(cache_config)
|
||||
elif cache_backend == "tea_cache":
|
||||
return TeaCacheBackend(cache_config)
|
||||
else:
|
||||
raise ValueError(f"Unsupported cache backend: {cache_backend}. Supported: 'cache_dit', 'tea_cache'")
|
||||
+4
-4
@@ -16,16 +16,16 @@ Usage:
|
||||
|
||||
omni = Omni(
|
||||
model="Qwen/Qwen-Image",
|
||||
cache_adapter="tea_cache",
|
||||
cache_backend="tea_cache",
|
||||
cache_config={"rel_l1_thresh": 0.2}
|
||||
)
|
||||
images = omni.generate("a cat")
|
||||
|
||||
# Alternative: Using environment variable
|
||||
# export DIFFUSION_CACHE_ADAPTER=tea_cache
|
||||
# export DIFFUSION_CACHE_BACKEND=tea_cache
|
||||
"""
|
||||
|
||||
from vllm_omni.diffusion.cache.teacache.adapter import TeaCacheAdapter
|
||||
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
|
||||
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
|
||||
from vllm_omni.diffusion.cache.teacache.extractors import (
|
||||
CacheContext,
|
||||
@@ -35,7 +35,7 @@ from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache
|
||||
from vllm_omni.diffusion.cache.teacache.state import TeaCacheState
|
||||
|
||||
__all__ = [
|
||||
"TeaCacheAdapter",
|
||||
"TeaCacheBackend",
|
||||
"TeaCacheConfig",
|
||||
"TeaCacheState",
|
||||
"TeaCacheHook",
|
||||
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
TeaCache adapter implementation.
|
||||
|
||||
This module provides the TeaCache adapter that implements the CacheAdapter
|
||||
interface using the hooks-based TeaCache system.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheAdapter
|
||||
from vllm_omni.diffusion.cache.registry import CacheType, register_cache_adapter
|
||||
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
|
||||
from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache_hook
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TeaCacheAdapter(CacheAdapter):
|
||||
"""
|
||||
TeaCache implementation using hooks.
|
||||
|
||||
TeaCache (Timestep Embedding Aware Cache) is an adaptive caching technique
|
||||
that speeds up diffusion inference by reusing transformer block computations
|
||||
when consecutive timestep embeddings are similar.
|
||||
|
||||
The adapter applies TeaCache hooks to the transformer which intercept the
|
||||
forward pass and implement the caching logic transparently.
|
||||
|
||||
Example:
|
||||
>>> adapter = TeaCacheAdapter({"rel_l1_thresh": 0.2})
|
||||
>>> adapter.apply(transformer)
|
||||
>>> # Generate with cache enabled
|
||||
>>> adapter.reset(transformer) # Reset before each generation
|
||||
"""
|
||||
|
||||
def apply(self, transformer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Apply TeaCache to transformer using hooks.
|
||||
|
||||
This creates a TeaCacheConfig from the adapter's config dict and
|
||||
applies the TeaCache hook to the transformer.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to apply TeaCache to
|
||||
"""
|
||||
# Validate that model_type is provided
|
||||
if "model_type" not in self.config:
|
||||
raise ValueError(
|
||||
"model_type must be provided in cache_config. "
|
||||
"Should be set to OmniDiffusionConfig.model_class_name (e.g., 'QwenImagePipeline')"
|
||||
)
|
||||
|
||||
# Create TeaCacheConfig from dict
|
||||
try:
|
||||
teacache_config = TeaCacheConfig(**self.config)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create TeaCacheConfig: {e}")
|
||||
raise ValueError(
|
||||
f"Invalid TeaCache configuration: {e}. "
|
||||
f"Expected keys: rel_l1_thresh, model_type (pipeline class name), coefficients (optional)"
|
||||
)
|
||||
|
||||
# Apply hook to transformer
|
||||
apply_teacache_hook(transformer, teacache_config)
|
||||
logger.info(
|
||||
f"TeaCache applied with rel_l1_thresh={teacache_config.rel_l1_thresh}, "
|
||||
f"model_type={teacache_config.model_type}"
|
||||
)
|
||||
|
||||
def reset(self, transformer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Reset TeaCache state for new generation.
|
||||
|
||||
Clears all cached residuals and resets counters/accumulators.
|
||||
Should be called before each generation to ensure clean state.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to reset cache on
|
||||
"""
|
||||
if hasattr(transformer, "_hook_registry"):
|
||||
hook = transformer._hook_registry.get_hook(TeaCacheHook._HOOK_NAME)
|
||||
if hook is not None:
|
||||
transformer._hook_registry.reset_hook(TeaCacheHook._HOOK_NAME)
|
||||
logger.debug("TeaCache state reset")
|
||||
else:
|
||||
logger.warning("TeaCache hook not found, nothing to reset")
|
||||
else:
|
||||
logger.warning("Transformer has no hook registry, TeaCache may not be applied")
|
||||
|
||||
|
||||
# Register TeaCache adapter in the global registry
|
||||
register_cache_adapter(CacheType.TEA_CACHE, TeaCacheAdapter)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
TeaCache backend implementation.
|
||||
|
||||
This module provides the TeaCache backend that implements the CacheBackend
|
||||
interface using the hooks-based TeaCache system.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheBackend
|
||||
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
|
||||
from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache_hook
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TeaCacheBackend(CacheBackend):
|
||||
"""
|
||||
TeaCache implementation using hooks.
|
||||
|
||||
TeaCache (Timestep Embedding Aware Cache) is an adaptive caching technique
|
||||
that speeds up diffusion inference by reusing transformer block computations
|
||||
when consecutive timestep embeddings are similar.
|
||||
|
||||
The backend applies TeaCache hooks to the transformer which intercept the
|
||||
forward pass and implement the caching logic transparently.
|
||||
|
||||
Example:
|
||||
>>> from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
>>> backend = TeaCacheBackend(DiffusionCacheConfig(rel_l1_thresh=0.2))
|
||||
>>> backend.enable(pipeline)
|
||||
>>> # Generate with cache enabled
|
||||
>>> backend.refresh(pipeline, num_inference_steps=50) # Refresh before each generation
|
||||
>>> # Access config attributes: backend.config.rel_l1_thresh
|
||||
"""
|
||||
|
||||
def enable(self, pipeline: Any) -> None:
|
||||
"""
|
||||
Enable TeaCache on transformer using hooks.
|
||||
|
||||
This creates a TeaCacheConfig from the backend's DiffusionCacheConfig
|
||||
and applies the TeaCache hook to the transformer.
|
||||
|
||||
Args:
|
||||
pipeline: Diffusion pipeline instance. Extracts transformer and model_type:
|
||||
- transformer: pipeline.transformer
|
||||
- model_type: pipeline.__class__.__name__
|
||||
"""
|
||||
# Extract transformer and model_type from pipeline
|
||||
transformer = pipeline.transformer
|
||||
model_type = pipeline.__class__.__name__
|
||||
|
||||
# Create TeaCacheConfig from DiffusionCacheConfig with model_type
|
||||
# Access parameters via attribute access: config.rel_l1_thresh
|
||||
# rel_l1_thresh already has a default value of 0.2 in DiffusionCacheConfig
|
||||
try:
|
||||
teacache_config = TeaCacheConfig(
|
||||
model_type=model_type,
|
||||
rel_l1_thresh=self.config.rel_l1_thresh,
|
||||
coefficients=self.config.coefficients,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create TeaCacheConfig: {e}")
|
||||
raise ValueError(
|
||||
f"Invalid TeaCache configuration: {e}. "
|
||||
f"Expected keys: rel_l1_thresh, coefficients (optional). "
|
||||
f"model_type is automatically extracted from pipeline.__class__.__name__."
|
||||
)
|
||||
|
||||
# Apply hook to transformer
|
||||
apply_teacache_hook(transformer, teacache_config)
|
||||
|
||||
# Store backend reference on pipeline for pipeline's internal use
|
||||
# (e.g., checking if cache is enabled for cache_branch parameter)
|
||||
pipeline._cache_backend = self
|
||||
|
||||
# Mark as enabled
|
||||
self.enabled = True
|
||||
|
||||
logger.info(
|
||||
f"TeaCache applied with rel_l1_thresh={teacache_config.rel_l1_thresh}, "
|
||||
f"model_type={teacache_config.model_type}"
|
||||
)
|
||||
|
||||
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
"""
|
||||
Refresh TeaCache state for new generation.
|
||||
|
||||
Clears all cached residuals and resets counters/accumulators.
|
||||
Should be called before each generation to ensure clean state.
|
||||
|
||||
Args:
|
||||
pipeline: Diffusion pipeline instance. Extracts transformer via pipeline.transformer.
|
||||
num_inference_steps: Number of inference steps for the current generation.
|
||||
Currently not used by TeaCache but accepted for interface consistency.
|
||||
verbose: Whether to log refresh operations (default: True)
|
||||
"""
|
||||
# Extract transformer from pipeline
|
||||
transformer = pipeline.transformer
|
||||
|
||||
if hasattr(transformer, "_hook_registry"):
|
||||
hook = transformer._hook_registry.get_hook(TeaCacheHook._HOOK_NAME)
|
||||
if hook is not None:
|
||||
transformer._hook_registry.reset_hook(TeaCacheHook._HOOK_NAME)
|
||||
if verbose:
|
||||
logger.debug(f"TeaCache state refreshed (num_inference_steps={num_inference_steps})")
|
||||
else:
|
||||
if verbose:
|
||||
logger.warning("TeaCache hook not found, nothing to refresh")
|
||||
else:
|
||||
if verbose:
|
||||
logger.warning("Transformer has no hook registry, TeaCache may not be applied")
|
||||
Reference in new issue
Block a user