[Diffusion] Add cache-dit and unify diffusion cache backend interface (#250)

Signed-off-by: samithuang <285365963@qq.com>
Signed-off-by: SamitHuang <285365963@qq.com>
Signed-off-by: Samit <285365963@qq.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
SamitandJiangyun Zhu authored and GitHub committed 2025-12-15 15:10:45 +08:00
1 parent 63f1e347ad
commit 2b4a9fe729
24 files changed
+1644 -367

No files matched your search

+7 -21
View File
@@ -1,41 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Cache module for diffusion model inference acceleration.
This module provides a unified cache adapter system for different caching strategies:
This module provides a unified cache backend system for different caching strategies:
- TeaCache: Timestep Embedding Aware Cache for adaptive transformer caching
- (Future: DeepCache, DistriFusion, etc.)
- cache-dit: DBCache, SCM, and TaylorSeer caching strategies
The cache adapter system uses a registry pattern where cache types can be
dynamically registered and instantiated. Cache is configured via OmniDiffusionConfig
or DIFFUSION_CACHE_ADAPTER environment variable.
Cache backends are instantiated directly via their constructors and configured via OmniDiffusionConfig.
"""
from vllm_omni.diffusion.cache.apply import setup_cache
from vllm_omni.diffusion.cache.base import CacheAdapter
from vllm_omni.diffusion.cache.registry import (
CacheType,
get_cache_adapter,
register_cache_adapter,
)
from vllm_omni.diffusion.cache.base import CacheBackend
from vllm_omni.diffusion.cache.teacache import (
CacheContext,
TeaCacheConfig,
apply_teacache_hook,
)
# Import teacache adapter to trigger registration
from vllm_omni.diffusion.cache.teacache.adapter import TeaCacheAdapter # noqa: F401
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
__all__ = [
"CacheAdapter",
"CacheType",
"CacheBackend",
"TeaCacheConfig",
"CacheContext",
"setup_cache",
"get_cache_adapter",
"register_cache_adapter",
"TeaCacheBackend",
"apply_teacache_hook",
]
-82
View File
@@ -1,82 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unified cache setup function for diffusion models.
This module provides the main entry point for setting up cache adapters
on transformer models. It handles cache type selection, adapter instantiation,
and application to the model.
"""
from typing import Any, Optional
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.cache.base import CacheAdapter
from vllm_omni.diffusion.cache.registry import get_cache_adapter
logger = init_logger(__name__)
def setup_cache(
transformer: torch.nn.Module,
cache_type: str = "none",
cache_config: Optional[dict[str, Any]] = None,
) -> Optional[CacheAdapter]:
"""
Setup cache adapter for transformer (one-time initialization).
This is the main entry point for applying caching to diffusion transformers.
It's called once during pipeline initialization and applies the appropriate
cache adapter based on configuration.
Args:
transformer: Transformer module to apply cache to
cache_type: Type of cache to apply. Options:
- "none": No caching (default)
- "tea_cache": TeaCache adaptive caching
- Additional types can be registered via register_cache_adapter()
cache_config: Cache-specific configuration dictionary. Contents depend
on cache type:
- For "tea_cache": {"rel_l1_thresh": 0.2, "model_type": "QwenImagePipeline"}
- For other types: see their respective adapter documentation
Returns:
CacheAdapter instance for state management, or None if cache_type="none"
Example:
>>> # Setup TeaCache during pipeline initialization
>>> adapter = setup_cache(
... transformer,
... cache_type="tea_cache",
... cache_config={"rel_l1_thresh": 0.2}
... )
>>>
>>> # Reset cache state before each generation
>>> if adapter is not None:
... adapter.reset(transformer)
"""
# No caching requested
if cache_type == "none":
logger.debug("No cache adapter requested (cache_type='none')")
return None
# Get adapter instance from registry
try:
adapter = get_cache_adapter(cache_type, cache_config or {})
except ValueError as e:
logger.error(f"Failed to get cache adapter: {e}")
raise
# Apply cache to transformer
logger.info(f"Applying {cache_type} cache to transformer")
try:
adapter.apply(transformer)
except Exception as e:
logger.error(f"Failed to apply cache adapter: {e}")
raise
logger.info(f"Cache adapter '{cache_type}' successfully applied")
return adapter
+58 -25
View File
@@ -2,66 +2,99 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Base cache adapter interface for diffusion models.
Base cache backend interface for diffusion models.
This module defines the abstract base class that all cache adapters must implement.
Cache adapters provide a unified interface for applying different caching strategies
(TeaCache, DeepCache, etc.) to transformer models using hooks.
This module defines the abstract base class that all cache backends must implement.
Cache backends provide a unified interface for applying different caching strategies
to transformer models.
Main cache backend implementations:
1. CacheDiTBackend: Implements cache-dit acceleration (DBCache, SCM, TaylorSeer) using
the cache-dit library. Inherits from CacheBackend. Used via cache_backend="cache_dit".
2. TeaCacheBackend: Hook-based backend for TeaCache acceleration. Inherits from
CacheBackend. Used via cache_backend="tea_cache".
All backends implement the same interface:
- enable(pipeline): Enable cache on the pipeline
- refresh(pipeline, num_inference_steps, verbose): Refresh cache state
- is_enabled(): Check if cache is enabled
"""
from abc import ABC, abstractmethod
from typing import Any
import torch
from vllm_omni.diffusion.data import DiffusionCacheConfig
class CacheAdapter(ABC):
class CacheBackend(ABC):
"""
Abstract base class for cache adapters.
Abstract base class for cache backends.
Cache adapters apply caching strategies to transformer models to accelerate
inference. Each adapter type (TeaCache, DeepCache, etc.) implements the
apply() and reset() methods to manage cache lifecycle.
All cache backend implementations (CacheDiTBackend, TeaCacheBackend, etc.) inherit
from this base class and implement the enable() and refresh() methods to manage
cache lifecycle.
Cache backends apply caching strategies to transformer models to accelerate
inference. Different backends use different underlying mechanisms (e.g., cache-dit
library for CacheDiTBackend, hooks for TeaCacheBackend), but all share the same
unified interface.
Attributes:
config: Dictionary containing cache-specific configuration parameters
config: DiffusionCacheConfig instance containing cache-specific configuration parameters
enabled: Boolean flag indicating whether cache is enabled (set to True after enable() is called)
"""
def __init__(self, config: dict[str, Any]):
def __init__(self, config: DiffusionCacheConfig):
"""
Initialize cache adapter with configuration.
Initialize cache backend with configuration.
Args:
config: Cache-specific configuration dictionary
config: DiffusionCacheConfig instance with cache-specific parameters
"""
self.config = config
self.enabled = False
@abstractmethod
def apply(self, transformer: torch.nn.Module) -> None:
def enable(self, pipeline: Any) -> None:
"""
Apply cache to transformer using hooks.
Enable cache on the pipeline.
This method should register the appropriate hooks on the transformer
to enable caching during inference. Called once during pipeline
initialization.
This method applies the caching strategy to the transformer(s) in the pipeline.
The specific implementation depends on the backend (e.g., hooks for TeaCacheBackend,
cache-dit library for CacheDiTBackend). Called once during pipeline initialization.
Args:
transformer: Transformer module to apply cache to
pipeline: Diffusion pipeline instance. The backend can extract:
- transformer: via pipeline.transformer
- model_type: via pipeline.__class__.__name__
"""
raise NotImplementedError("Subclasses must implement apply()")
raise NotImplementedError("Subclasses must implement enable()")
@abstractmethod
def reset(self, transformer: torch.nn.Module) -> None:
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""
Reset cache state for new generation.
Refresh cache state for new generation.
This method should clear any cached values and reset counters/accumulators.
Called at the start of each generation to ensure clean state.
Args:
transformer: Transformer module to reset cache on
pipeline: Diffusion pipeline instance. The backend can extract:
- transformer: via pipeline.transformer
num_inference_steps: Number of inference steps for the current generation.
May be used for cache context updates.
verbose: Whether to log refresh operations (default: True)
"""
raise NotImplementedError("Subclasses must implement reset()")
raise NotImplementedError("Subclasses must implement refresh()")
def is_enabled(self) -> bool:
"""
Check if cache is enabled on this backend.
Returns:
True if cache is enabled, False otherwise.
"""
return self.enabled
def __repr__(self) -> str:
return f"{self.__class__.__name__}(config={self.config})"
+394
View File
@@ -0,0 +1,394 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
cache-dit integration backend for vllm-omni.
This module provides a CacheDiTBackend class to enable cache-dit acceleration on diffusion
pipelines in vllm-omni, supporting both single and dual-transformer architectures.
"""
from typing import Any, Callable, Optional
from vllm.logger import init_logger
from vllm_omni.diffusion.cache.base import CacheBackend
from vllm_omni.diffusion.data import DiffusionCacheConfig, OmniDiffusionConfig
logger = init_logger(__name__)
try:
import cache_dit
from cache_dit import BlockAdapter, DBCacheConfig, ForwardPattern, ParamsModifier, TaylorSeerCalibratorConfig
CACHE_DIT_AVAILABLE = True
except ImportError:
CACHE_DIT_AVAILABLE = False
logger.warning("cache-dit is not installed. Cache-dit acceleration will not be available.")
# Registry of custom cache-dit enablers for specific models
# Maps pipeline names to their cache-dit enablement functions
# Models in this registry require custom handling (e.g., dual-transformer architectures)
# Will be populated after function definitions
CUSTOM_DIT_ENABLERS: dict[str, Callable] = {}
def _build_db_cache_config(cache_config: Any) -> DBCacheConfig:
"""Build DBCacheConfig with optional SCM (Step Computation Masking) support.
Args:
cache_config: DiffusionCacheConfig instance.
Returns:
DBCacheConfig instance with SCM support if configured.
"""
return DBCacheConfig(
# we will refresh the context when gets num_inference_steps in the first inference request
num_inference_steps=None,
Fn_compute_blocks=cache_config.Fn_compute_blocks,
Bn_compute_blocks=cache_config.Bn_compute_blocks,
max_warmup_steps=cache_config.max_warmup_steps,
max_cached_steps=cache_config.max_cached_steps,
max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
residual_diff_threshold=cache_config.residual_diff_threshold,
)
def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
"""Enable cache-dit for Wan2.2 dual-transformer architecture.
Wan2.2 uses two transformers (transformer and transformer_2) that need
to be enabled together using BlockAdapter.
Args:
pipeline: The Wan2.2 pipeline instance.
cache_config: DiffusionCacheConfig instance with cache configuration.
Returns:
A refresh function that can be called to update cache context with new num_inference_steps.
"""
# Build DBCacheConfig for primary transformer
primary_cache_config = _build_db_cache_config(cache_config)
# FIXME: secondary cache shares the same config with primary cache for now,
# but we should support different config for secondary transformer in the future
# Build DBCacheConfig for secondary transformer (can use same or different config)
secondary_cache_config = _build_db_cache_config(cache_config)
# Build calibrator configs if TaylorSeer is enabled
primary_calibrator = None
secondary_calibrator = None
if cache_config.enable_taylorseer:
taylorseer_order = cache_config.taylorseer_order
primary_calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
secondary_calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
# Build ParamsModifier for each transformer
primary_modifier = ParamsModifier(
cache_config=primary_cache_config,
calibrator_config=primary_calibrator,
)
secondary_modifier = ParamsModifier(
cache_config=secondary_cache_config,
calibrator_config=secondary_calibrator,
)
logger.info(
"Enabling cache-dit on Wan2.2 dual transformers with BlockAdapter: "
f"Fn={primary_cache_config.Fn_compute_blocks}, "
f"Bn={primary_cache_config.Bn_compute_blocks}, "
f"W={primary_cache_config.max_warmup_steps}, "
)
transformer = pipeline.transformer
transformer_2 = pipeline.transformer_2
transformer_blocks = transformer.blocks
transformer_2_blocks = transformer_2.blocks
# Enable cache-dit using BlockAdapter for both transformers simultaneously
cache_dit.enable_cache(
BlockAdapter(
transformer=[transformer, transformer_2],
blocks=[transformer_blocks, transformer_2_blocks],
forward_pattern=[ForwardPattern.Pattern_2, ForwardPattern.Pattern_2],
params_modifiers=[primary_modifier, secondary_modifier],
has_separate_cfg=True,
),
)
# from https://github.com/vipshop/cache-dit/pull/542
def _split_inference_steps(num_inference_steps: int) -> tuple[int, int]:
"""Split inference steps into high-noise and low-noise steps for Wan2.2.
This is an internal helper function specific to Wan2.2's dual-transformer
architecture that uses boundary_ratio to determine the split point.
Args:
num_inference_steps: Total number of inference steps.
Returns:
A tuple of (num_high_noise_steps, num_low_noise_steps).
"""
if pipeline.config.boundary_ratio is not None:
boundary_timestep = pipeline.config.boundary_ratio * pipeline.scheduler.config.num_train_timesteps
else:
boundary_timestep = None
# Set timesteps to calculate the split
device = next(pipeline.transformer.parameters()).device
pipeline.scheduler.set_timesteps(num_inference_steps, device=device)
timesteps = pipeline.scheduler.timesteps
num_high_noise_steps = 0 # high-noise steps for transformer
for t in timesteps:
if boundary_timestep is None or t >= boundary_timestep:
num_high_noise_steps += 1
# low-noise steps for transformer_2
num_low_noise_steps = num_inference_steps - num_high_noise_steps
return num_high_noise_steps, num_low_noise_steps
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""Refresh cache context for both transformers with new num_inference_steps.
Args:
pipeline: The Wan2.2 pipeline instance.
num_inference_steps: New number of inference steps.
"""
num_high_noise_steps, num_low_noise_steps = _split_inference_steps(num_inference_steps)
# Refresh context for high-noise transformer
if cache_config.scm_steps_mask_policy is None:
cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_high_noise_steps, verbose=verbose)
else:
cache_dit.refresh_context(
pipeline.transformer,
cache_config=DBCacheConfig().reset(
num_inference_steps=num_high_noise_steps,
steps_computation_mask=cache_dit.steps_mask(
mask_policy=cache_config.scm_steps_mask_policy, total_steps=num_high_noise_steps
),
steps_computation_policy=cache_config.scm_steps_policy,
),
verbose=verbose,
)
# Refresh context for low-noise transformer
if cache_config.scm_steps_mask_policy is None:
cache_dit.refresh_context(pipeline.transformer_2, num_inference_steps=num_low_noise_steps, verbose=verbose)
else:
cache_dit.refresh_context(
pipeline.transformer_2,
cache_config=DBCacheConfig().reset(
num_inference_steps=num_low_noise_steps,
steps_computation_mask=cache_dit.steps_mask(
mask_policy=cache_config.scm_steps_mask_policy, total_steps=num_low_noise_steps
),
steps_computation_policy=cache_config.scm_steps_policy,
),
verbose=verbose,
)
return refresh_cache_context
def enable_cache_for_flux(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
"""Enable cache-dit for Flux dual-transformer architecture.
Flux uses two transformers (transformer and transformer_2) that need
to be enabled together using BlockAdapter.
Args:
pipeline: The Flux pipeline instance.
cache_config: DiffusionCacheConfig instance with cache configuration.
Returns:
A refresh function that can be called to update cache context with new num_inference_steps.
"""
raise NotImplementedError("cache-dit is not implemented for Flux pipeline.")
def enable_cache_for_dit(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
"""Enable cache-dit for regular single-transformer DiT models.
Args:
pipeline: The diffusion pipeline instance.
cache_config: DiffusionCacheConfig instance with cache configuration.
Returns:
A refresh function that can be called to update cache context with new num_inference_steps.
"""
# Build DBCacheConfig with optional SCM support
db_cache_config = _build_db_cache_config(cache_config)
# Build calibrator config if TaylorSeer is enabled
calibrator_config = None
if cache_config.enable_taylorseer:
taylorseer_order = cache_config.taylorseer_order
calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
logger.info(
f"Enabling cache-dit on transformer: "
f"Fn={db_cache_config.Fn_compute_blocks}, "
f"Bn={db_cache_config.Bn_compute_blocks}, "
f"W={db_cache_config.max_warmup_steps}, "
)
# Enable cache-dit on the transformer
cache_dit.enable_cache(
pipeline.transformer,
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""Refresh cache context for the transformer with new num_inference_steps.
Args:
pipeline: The diffusion pipeline instance.
num_inference_steps: New number of inference steps.
"""
if cache_config.scm_steps_mask_policy is None:
cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose)
else:
cache_dit.refresh_context(
pipeline.transformer,
cache_config=DBCacheConfig().reset(
num_inference_steps=num_inference_steps,
steps_computation_mask=cache_dit.steps_mask(
mask_policy=cache_config.scm_steps_mask_policy,
total_steps=num_inference_steps,
),
steps_computation_policy=cache_config.scm_steps_policy,
),
verbose=verbose,
)
return refresh_cache_context
# Register custom cache-dit enablers after function definitions
CUSTOM_DIT_ENABLERS.update(
{
"WanPipeline": enable_cache_for_wan22,
"FluxPipeline": enable_cache_for_flux,
}
)
class CacheDiTBackend(CacheBackend):
"""Backend class for cache-dit acceleration on diffusion pipelines.
This class implements cache-dit acceleration (DBCache, SCM, TaylorSeer) using
the cache-dit library. It inherits from CacheBackend and provides a unified
interface for managing cache-dit acceleration on diffusion models.
Attributes:
config: Cache configuration (DiffusionCacheConfig instance), inherited from CacheBackend.
enabled: Whether cache-dit is enabled on this pipeline, inherited from CacheBackend.
_refresh_func: Internal refresh function for updating cache context.
_last_num_inference_steps: Last num_inference_steps used for refresh optimization.
"""
def __init__(self, cache_config: Any = None):
"""Initialize the cache-dit backend.
Args:
cache_config: Cache configuration (DiffusionCacheConfig instance, dict, or None).
If None or empty, uses default DiffusionCacheConfig().
"""
# Use default config if cache_config is not provided or is empty
if cache_config is None:
config = DiffusionCacheConfig()
elif isinstance(cache_config, dict):
# Convert dict to DiffusionCacheConfig, using defaults for missing keys
config = DiffusionCacheConfig.from_dict(cache_config)
else:
config = cache_config
# Initialize base class with normalized config
super().__init__(config)
# Cache-dit specific attributes
self._refresh_func: Optional[Callable[[Any, int, bool], None]] = None
self._last_num_inference_steps: Optional[int] = None
def enable(self, pipeline: Any) -> None:
"""Enable cache-dit on the pipeline if configured.
This method applies cache-dit acceleration to the appropriate transformer(s)
in the pipeline. It handles both single-transformer and dual-transformer
architectures (e.g., Wan2.2).
Args:
pipeline: The diffusion pipeline instance.
"""
if not CACHE_DIT_AVAILABLE:
logger.warning("cache-dit is not available, skipping cache-dit setup.")
return
# Extract pipeline name from pipeline
pipeline_name = pipeline.__class__.__name__
# Check if this model has a custom cache-dit enabler
if pipeline_name in CUSTOM_DIT_ENABLERS:
logger.info(f"Using custom cache-dit enabler for model: {pipeline_name}")
self._refresh_func = CUSTOM_DIT_ENABLERS[pipeline_name](pipeline, self.config)
else:
# For regular single-transformer models
self._refresh_func = enable_cache_for_dit(pipeline, self.config)
self.enabled = True
logger.info(f"Cache-dit enabled successfully on {pipeline_name}")
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""Refresh cache context with new num_inference_steps.
This method updates the cache context when num_inference_steps changes
during inference. For dual-transformer models (e.g., Wan2.2), it automatically
splits the steps based on boundary_ratio.
Args:
pipeline: The diffusion pipeline instance.
num_inference_steps: New number of inference steps.
verbose: Whether to log refresh operations.
"""
if not self.enabled or self._refresh_func is None:
logger.warning("Cache-dit is not enabled. Cannot refresh cache context.")
return
# Only refresh if num_inference_steps has changed
if self._last_num_inference_steps is None or num_inference_steps != self._last_num_inference_steps:
if verbose:
logger.info(f"Refreshing cache context for transformer with num_inference_steps: {num_inference_steps}")
self._refresh_func(pipeline, num_inference_steps, verbose)
self._last_num_inference_steps = num_inference_steps
def is_enabled(self) -> bool:
"""Check if cache-dit is enabled on this pipeline.
Returns:
True if cache-dit is enabled, False otherwise.
"""
return self.enabled
def may_enable_cache_dit(pipeline: Any, od_config: OmniDiffusionConfig) -> Optional["CacheDiTBackend"]:
"""Enable cache-dit on the pipeline if configured (convenience function).
This is a convenience function that creates and enables a CacheDiTBackend.
For new code, consider using CacheDiTBackend directly.
Args:
pipeline: The diffusion pipeline instance.
od_config: OmniDiffusionConfig with cache configuration.
Returns:
A CacheDiTBackend instance if cache-dit is enabled, None otherwise.
"""
if od_config.cache_backend != "cache-dit" or not od_config.cache_config:
return None
backend = CacheDiTBackend(od_config.cache_config)
backend.enable(pipeline)
return backend if backend.is_enabled() else None
-100
View File
@@ -1,100 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Cache adapter registry for diffusion models.
This module provides a registry pattern for cache adapters, allowing dynamic
registration and instantiation of different cache types (TeaCache, DeepCache, etc.).
"""
from enum import Enum
from typing import Any
from vllm.logger import init_logger
from vllm_omni.diffusion.cache.base import CacheAdapter
logger = init_logger(__name__)
class CacheType(Enum):
"""Supported cache adapter types."""
NONE = "none"
TEA_CACHE = "tea_cache"
# Future cache types can be added here:
# DEEP_CACHE = "deep_cache"
# DISTRI_FUSION = "distri_fusion"
# Global registry mapping cache types to adapter classes
CACHE_ADAPTER_REGISTRY: dict[CacheType, type[CacheAdapter]] = {}
def register_cache_adapter(cache_type: CacheType, adapter_class: type[CacheAdapter]) -> None:
"""
Register a cache adapter class for a given cache type.
This allows extending the system with new cache types without modifying
the core cache infrastructure.
Args:
cache_type: CacheType enum value
adapter_class: CacheAdapter subclass to register
Example:
>>> register_cache_adapter(CacheType.TEA_CACHE, TeaCacheAdapter)
"""
if not issubclass(adapter_class, CacheAdapter):
raise TypeError(f"{adapter_class} must be a subclass of CacheAdapter")
CACHE_ADAPTER_REGISTRY[cache_type] = adapter_class
logger.debug(f"Registered cache adapter: {cache_type.value} -> {adapter_class.__name__}")
def get_cache_adapter(cache_type: str, config: dict[str, Any]) -> CacheAdapter:
"""
Factory function to get cache adapter instance.
Converts string cache type to enum, looks up in registry, and instantiates
the appropriate adapter class with the provided configuration.
Args:
cache_type: String name of cache type ("tea_cache", "deep_cache", etc.)
config: Configuration dictionary to pass to adapter constructor
Returns:
Instantiated CacheAdapter subclass
Raises:
ValueError: If cache_type is unknown or not registered
Example:
>>> adapter = get_cache_adapter("tea_cache", {"rel_l1_thresh": 0.2})
>>> adapter.apply(transformer)
"""
# Normalize cache type string
cache_type_str = cache_type.lower().strip()
# Convert string to enum
try:
cache_enum = CacheType(cache_type_str)
except ValueError:
available = [ct.value for ct in CacheType if ct != CacheType.NONE]
raise ValueError(f"Unknown cache type: '{cache_type}'. Available types: {available}")
# Check if it's the special "none" case
if cache_enum == CacheType.NONE:
raise ValueError("Cannot instantiate adapter for cache_type='none'. Use setup_cache() which handles this case.")
# Lookup in registry
if cache_enum not in CACHE_ADAPTER_REGISTRY:
raise ValueError(
f"Cache type '{cache_type}' is not registered. Registered types: {list(CACHE_ADAPTER_REGISTRY.keys())}"
)
adapter_class = CACHE_ADAPTER_REGISTRY[cache_enum]
# Instantiate and return
return adapter_class(config)
+38
View File
@@ -0,0 +1,38 @@
from typing import Any
from vllm_omni.diffusion.cache.base import CacheBackend
from vllm_omni.diffusion.cache.cache_dit_backend import CacheDiTBackend
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
from vllm_omni.diffusion.data import DiffusionCacheConfig
def get_cache_backend(cache_backend: str | None, cache_config: Any) -> CacheBackend | None:
"""Get cache backend instance based on cache_backend string.
This is a selector function that routes to the appropriate backend implementation.
- cache_dit: Uses CacheDiTBackend with enable()/refresh() interface
- tea_cache: Uses TeaCacheBackend with enable()/refresh() interface
Args:
cache_backend: Cache backend name ("cache_dit", "tea_cache", or None).
cache_config: Cache configuration (dict or DiffusionCacheConfig instance).
Returns:
Cache backend instance (CacheDiTBackend or TeaCacheBackend) if cache_backend is set,
None otherwise.
Raises:
ValueError: If cache_backend is unsupported.
"""
if cache_backend is None or cache_backend == "none":
return None
if isinstance(cache_config, dict):
cache_config = DiffusionCacheConfig.from_dict(cache_config)
if cache_backend == "cache_dit":
return CacheDiTBackend(cache_config)
elif cache_backend == "tea_cache":
return TeaCacheBackend(cache_config)
else:
raise ValueError(f"Unsupported cache backend: {cache_backend}. Supported: 'cache_dit', 'tea_cache'")
+4 -4
View File
@@ -16,16 +16,16 @@ Usage:
omni = Omni(
model="Qwen/Qwen-Image",
cache_adapter="tea_cache",
cache_backend="tea_cache",
cache_config={"rel_l1_thresh": 0.2}
)
images = omni.generate("a cat")
# Alternative: Using environment variable
# export DIFFUSION_CACHE_ADAPTER=tea_cache
# export DIFFUSION_CACHE_BACKEND=tea_cache
"""
from vllm_omni.diffusion.cache.teacache.adapter import TeaCacheAdapter
from vllm_omni.diffusion.cache.teacache.backend import TeaCacheBackend
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
from vllm_omni.diffusion.cache.teacache.extractors import (
CacheContext,
@@ -35,7 +35,7 @@ from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache
from vllm_omni.diffusion.cache.teacache.state import TeaCacheState
__all__ = [
"TeaCacheAdapter",
"TeaCacheBackend",
"TeaCacheConfig",
"TeaCacheState",
"TeaCacheHook",
-96
View File
@@ -1,96 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
TeaCache adapter implementation.
This module provides the TeaCache adapter that implements the CacheAdapter
interface using the hooks-based TeaCache system.
"""
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.cache.base import CacheAdapter
from vllm_omni.diffusion.cache.registry import CacheType, register_cache_adapter
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache_hook
logger = init_logger(__name__)
class TeaCacheAdapter(CacheAdapter):
"""
TeaCache implementation using hooks.
TeaCache (Timestep Embedding Aware Cache) is an adaptive caching technique
that speeds up diffusion inference by reusing transformer block computations
when consecutive timestep embeddings are similar.
The adapter applies TeaCache hooks to the transformer which intercept the
forward pass and implement the caching logic transparently.
Example:
>>> adapter = TeaCacheAdapter({"rel_l1_thresh": 0.2})
>>> adapter.apply(transformer)
>>> # Generate with cache enabled
>>> adapter.reset(transformer) # Reset before each generation
"""
def apply(self, transformer: torch.nn.Module) -> None:
"""
Apply TeaCache to transformer using hooks.
This creates a TeaCacheConfig from the adapter's config dict and
applies the TeaCache hook to the transformer.
Args:
transformer: Transformer module to apply TeaCache to
"""
# Validate that model_type is provided
if "model_type" not in self.config:
raise ValueError(
"model_type must be provided in cache_config. "
"Should be set to OmniDiffusionConfig.model_class_name (e.g., 'QwenImagePipeline')"
)
# Create TeaCacheConfig from dict
try:
teacache_config = TeaCacheConfig(**self.config)
except Exception as e:
logger.error(f"Failed to create TeaCacheConfig: {e}")
raise ValueError(
f"Invalid TeaCache configuration: {e}. "
f"Expected keys: rel_l1_thresh, model_type (pipeline class name), coefficients (optional)"
)
# Apply hook to transformer
apply_teacache_hook(transformer, teacache_config)
logger.info(
f"TeaCache applied with rel_l1_thresh={teacache_config.rel_l1_thresh}, "
f"model_type={teacache_config.model_type}"
)
def reset(self, transformer: torch.nn.Module) -> None:
"""
Reset TeaCache state for new generation.
Clears all cached residuals and resets counters/accumulators.
Should be called before each generation to ensure clean state.
Args:
transformer: Transformer module to reset cache on
"""
if hasattr(transformer, "_hook_registry"):
hook = transformer._hook_registry.get_hook(TeaCacheHook._HOOK_NAME)
if hook is not None:
transformer._hook_registry.reset_hook(TeaCacheHook._HOOK_NAME)
logger.debug("TeaCache state reset")
else:
logger.warning("TeaCache hook not found, nothing to reset")
else:
logger.warning("Transformer has no hook registry, TeaCache may not be applied")
# Register TeaCache adapter in the global registry
register_cache_adapter(CacheType.TEA_CACHE, TeaCacheAdapter)
+117
View File
@@ -0,0 +1,117 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
TeaCache backend implementation.
This module provides the TeaCache backend that implements the CacheBackend
interface using the hooks-based TeaCache system.
"""
from typing import Any
from vllm.logger import init_logger
from vllm_omni.diffusion.cache.base import CacheBackend
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache_hook
logger = init_logger(__name__)
class TeaCacheBackend(CacheBackend):
"""
TeaCache implementation using hooks.
TeaCache (Timestep Embedding Aware Cache) is an adaptive caching technique
that speeds up diffusion inference by reusing transformer block computations
when consecutive timestep embeddings are similar.
The backend applies TeaCache hooks to the transformer which intercept the
forward pass and implement the caching logic transparently.
Example:
>>> from vllm_omni.diffusion.data import DiffusionCacheConfig
>>> backend = TeaCacheBackend(DiffusionCacheConfig(rel_l1_thresh=0.2))
>>> backend.enable(pipeline)
>>> # Generate with cache enabled
>>> backend.refresh(pipeline, num_inference_steps=50) # Refresh before each generation
>>> # Access config attributes: backend.config.rel_l1_thresh
"""
def enable(self, pipeline: Any) -> None:
"""
Enable TeaCache on transformer using hooks.
This creates a TeaCacheConfig from the backend's DiffusionCacheConfig
and applies the TeaCache hook to the transformer.
Args:
pipeline: Diffusion pipeline instance. Extracts transformer and model_type:
- transformer: pipeline.transformer
- model_type: pipeline.__class__.__name__
"""
# Extract transformer and model_type from pipeline
transformer = pipeline.transformer
model_type = pipeline.__class__.__name__
# Create TeaCacheConfig from DiffusionCacheConfig with model_type
# Access parameters via attribute access: config.rel_l1_thresh
# rel_l1_thresh already has a default value of 0.2 in DiffusionCacheConfig
try:
teacache_config = TeaCacheConfig(
model_type=model_type,
rel_l1_thresh=self.config.rel_l1_thresh,
coefficients=self.config.coefficients,
)
except Exception as e:
logger.error(f"Failed to create TeaCacheConfig: {e}")
raise ValueError(
f"Invalid TeaCache configuration: {e}. "
f"Expected keys: rel_l1_thresh, coefficients (optional). "
f"model_type is automatically extracted from pipeline.__class__.__name__."
)
# Apply hook to transformer
apply_teacache_hook(transformer, teacache_config)
# Store backend reference on pipeline for pipeline's internal use
# (e.g., checking if cache is enabled for cache_branch parameter)
pipeline._cache_backend = self
# Mark as enabled
self.enabled = True
logger.info(
f"TeaCache applied with rel_l1_thresh={teacache_config.rel_l1_thresh}, "
f"model_type={teacache_config.model_type}"
)
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""
Refresh TeaCache state for new generation.
Clears all cached residuals and resets counters/accumulators.
Should be called before each generation to ensure clean state.
Args:
pipeline: Diffusion pipeline instance. Extracts transformer via pipeline.transformer.
num_inference_steps: Number of inference steps for the current generation.
Currently not used by TeaCache but accepted for interface consistency.
verbose: Whether to log refresh operations (default: True)
"""
# Extract transformer from pipeline
transformer = pipeline.transformer
if hasattr(transformer, "_hook_registry"):
hook = transformer._hook_registry.get_hook(TeaCacheHook._HOOK_NAME)
if hook is not None:
transformer._hook_registry.reset_hook(TeaCacheHook._HOOK_NAME)
if verbose:
logger.debug(f"TeaCache state refreshed (num_inference_steps={num_inference_steps})")
else:
if verbose:
logger.warning("TeaCache hook not found, nothing to refresh")
else:
if verbose:
logger.warning("Transformer has no hook registry, TeaCache may not be applied")