# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ Model-specific extractors for TeaCache. This module provides a registry of extractor functions that know how to extract modulated inputs from different transformer architectures. Adding support for a new model requires only adding a new extractor function to the registry. With Option B enhancement, extractors now return a CacheContext object containing all model-specific information needed for generic caching, including preprocessing, transformer execution, and postprocessing logic. """ from dataclasses import dataclass from typing import Any, Callable, Union import torch import torch.nn as nn @dataclass class CacheContext: """ Context object containing all model-specific information for caching. This allows the TeaCacheHook to remain completely generic - all model-specific logic is encapsulated in the extractor that returns this context. Attributes: modulated_input: Tensor used for cache decision (similarity comparison). Must be a torch.Tensor extracted from the first transformer block, typically after applying normalization and modulation. hidden_states: Current hidden states (will be modified by caching). Must be a torch.Tensor representing the main image/latent states after preprocessing but before transformer blocks. encoder_hidden_states: Optional encoder states (for dual-stream models). Set to None for single-stream models (e.g., Flux). For dual-stream models (e.g., Qwen), contains text encoder outputs. temb: Timestep embedding tensor. Must be a torch.Tensor containing the timestep conditioning. run_transformer_blocks: Callable that executes model-specific transformer blocks. Signature: () -> tuple[torch.Tensor, ...] Returns: tuple containing: - [0]: processed hidden_states (required) - [1]: processed encoder_hidden_states (optional, only for dual-stream) Example for single-stream: def run_blocks(): h = hidden_states for block in module.transformer_blocks: h = block(h, temb=temb) return (h,) Example for dual-stream: def run_blocks(): h, e = hidden_states, encoder_hidden_states for block in module.transformer_blocks: e, h = block(h, e, temb=temb) return (h, e) postprocess: Callable that does model-specific output postprocessing. Signature: (torch.Tensor) -> Union[torch.Tensor, Transformer2DModelOutput, tuple] Takes the processed hidden_states and applies final transformations (normalization, projection) to produce the model output. Example: def postprocess(h): h = module.norm_out(h, temb) output = module.proj_out(h) return Transformer2DModelOutput(sample=output) extra_states: Optional dict for additional model-specific state. Use this for models that need to pass additional context beyond the standard fields. """ modulated_input: torch.Tensor hidden_states: torch.Tensor encoder_hidden_states: torch.Tensor | None temb: torch.Tensor run_transformer_blocks: Callable[[], tuple[torch.Tensor, ...]] postprocess: Callable[[torch.Tensor], Any] extra_states: dict[str, Any] | None = None def validate(self) -> None: """ Validate that the CacheContext contains valid data. Raises: TypeError: If fields have wrong types ValueError: If tensors have invalid properties RuntimeError: If callables fail basic invocation tests This method should be called after creating a CacheContext to catch common developer errors early with clear error messages. """ # Validate tensor fields if not isinstance(self.modulated_input, torch.Tensor): raise TypeError(f"modulated_input must be torch.Tensor, got {type(self.modulated_input)}") if not isinstance(self.hidden_states, torch.Tensor): raise TypeError(f"hidden_states must be torch.Tensor, got {type(self.hidden_states)}") if self.encoder_hidden_states is not None and not isinstance(self.encoder_hidden_states, torch.Tensor): raise TypeError( f"encoder_hidden_states must be torch.Tensor or None, got {type(self.encoder_hidden_states)}" ) if not isinstance(self.temb, torch.Tensor): raise TypeError(f"temb must be torch.Tensor, got {type(self.temb)}") # Validate callables if not callable(self.run_transformer_blocks): raise TypeError(f"run_transformer_blocks must be callable, got {type(self.run_transformer_blocks)}") if not callable(self.postprocess): raise TypeError(f"postprocess must be callable, got {type(self.postprocess)}") # Validate tensor shapes are compatible if self.modulated_input.shape[0] != self.hidden_states.shape[0]: raise ValueError( f"Batch size mismatch: modulated_input has batch size " f"{self.modulated_input.shape[0]}, but hidden_states has " f"{self.hidden_states.shape[0]}" ) # Validate devices match if self.modulated_input.device != self.hidden_states.device: raise ValueError( f"Device mismatch: modulated_input on {self.modulated_input.device}, " f"hidden_states on {self.hidden_states.device}" ) def extract_qwen_context( module: nn.Module, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, encoder_hidden_states_mask: torch.Tensor, timestep: Union[torch.Tensor, float, int], img_shapes: torch.Tensor, txt_seq_lens: torch.Tensor, guidance: torch.Tensor | None = None, attention_kwargs: dict[str, Any] | None = None, **kwargs: Any, ) -> CacheContext: """ Extract cache context for QwenImageTransformer2DModel. This is the ONLY Qwen-specific code needed for TeaCache support. It encapsulates preprocessing, modulated input extraction, transformer execution, and postprocessing logic. Args: module: QwenImageTransformer2DModel instance hidden_states: Input hidden states tensor encoder_hidden_states: Text encoder outputs encoder_hidden_states_mask: Mask for text encoder timestep: Current diffusion timestep img_shapes: Image shapes for position embedding txt_seq_lens: Text sequence lengths guidance: Optional guidance scale for CFG attention_kwargs: Additional attention arguments **kwargs: Additional keyword arguments ignored by this extractor Returns: CacheContext with all information needed for generic caching """ from diffusers.models.modeling_outputs import Transformer2DModelOutput if not hasattr(module, "transformer_blocks") or len(module.transformer_blocks) == 0: raise ValueError("Module must have transformer_blocks") # ============================================================================ # PREPROCESSING (Qwen-specific) # ============================================================================ hidden_states = module.img_in(hidden_states) timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype) encoder_hidden_states = module.txt_norm(encoder_hidden_states) encoder_hidden_states = module.txt_in(encoder_hidden_states) if guidance is not None: guidance = guidance.to(hidden_states.dtype) * 1000 temb = ( module.time_text_embed(timestep, hidden_states) if guidance is None else module.time_text_embed(timestep, guidance, hidden_states) ) image_rotary_emb = module.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device) # ============================================================================ # EXTRACT MODULATED INPUT (for cache decision) # ============================================================================ block = module.transformer_blocks[0] img_mod_params = block.img_mod(temb) img_mod1, _ = img_mod_params.chunk(2, dim=-1) img_normed = block.img_norm1(hidden_states) img_modulated, _ = block._modulate(img_normed, img_mod1) # ============================================================================ # DEFINE TRANSFORMER EXECUTION (Qwen-specific) # ============================================================================ def run_transformer_blocks(): """Execute all Qwen transformer blocks.""" h = hidden_states e = encoder_hidden_states for block in module.transformer_blocks: e, h = block( hidden_states=h, encoder_hidden_states=e, encoder_hidden_states_mask=encoder_hidden_states_mask, temb=temb, image_rotary_emb=image_rotary_emb, joint_attention_kwargs=attention_kwargs, ) return (h, e) # ============================================================================ # DEFINE POSTPROCESSING (Qwen-specific) # ============================================================================ return_dict = kwargs.get("return_dict", True) def postprocess(h): """Apply Qwen-specific output postprocessing.""" h = module.norm_out(h, temb) output = module.proj_out(h) if not return_dict: return (output,) return Transformer2DModelOutput(sample=output) # ============================================================================ # RETURN CONTEXT # ============================================================================ return CacheContext( modulated_input=img_modulated, hidden_states=hidden_states, encoder_hidden_states=encoder_hidden_states, temb=temb, run_transformer_blocks=run_transformer_blocks, postprocess=postprocess, ) # Registry for model-specific extractors # Key: Pipeline class name (from OmniDiffusionConfig.model_class_name) # Value: extractor function with signature (module, *args, **kwargs) -> CacheContext # # Note: Use the exact pipeline class name as specified in OmniDiffusionConfig.model_class_name # to ensure consistent lookup without substring matching. EXTRACTOR_REGISTRY: dict[str, Callable] = { "QwenImagePipeline": extract_qwen_context, # Future models: # "FluxPipeline": extract_flux_context, # "CogVideoXPipeline": extract_cogvideox_context, } def register_extractor(model_identifier: str, extractor_fn: Callable) -> None: """ Register a new extractor function for a model type. This allows extending TeaCache support to new models without modifying the core TeaCache code. Args: model_identifier: Model type identifier (class name or type string) extractor_fn: Function with signature (module, *args, **kwargs) -> CacheContext Example: >>> def extract_flux_context(module, hidden_states, timestep, guidance=None, **kwargs): ... # Preprocessing ... temb = module.time_text_embed(timestep, guidance) ... # Extract modulated input ... modulated = module.transformer_blocks[0].norm1(hidden_states, emb=temb) ... # Define execution ... def run_blocks(): ... h = hidden_states ... for block in module.transformer_blocks: ... h = block(h, temb=temb) ... return (h,) ... # Define postprocessing ... def postprocess(h): ... return module.proj_out(module.norm_out(h, temb)) ... # Return context ... return CacheContext(modulated, hidden_states, None, temb, run_blocks, postprocess) >>> register_extractor("FluxTransformer2DModel", extract_flux_context) """ EXTRACTOR_REGISTRY[model_identifier] = extractor_fn def get_extractor(model_type: str) -> Callable: """ Get extractor function for given model type. This function looks up the extractor based on the exact model_type string, which should match the pipeline class name from OmniDiffusionConfig.model_class_name. Args: model_type: Model type string (e.g., "QwenImagePipeline", "FluxPipeline") Must exactly match a key in EXTRACTOR_REGISTRY. Returns: Extractor function with signature (module, *args, **kwargs) -> CacheContext Raises: ValueError: If model type not found in registry Example: >>> # Get extractor for Qwen model >>> extractor = get_extractor("QwenImagePipeline") >>> ctx = extractor(transformer, hidden_states, encoder_hidden_states, timestep, ...) """ # Direct lookup - no substring matching if model_type in EXTRACTOR_REGISTRY: return EXTRACTOR_REGISTRY[model_type] # No match found available_types = list(EXTRACTOR_REGISTRY.keys()) raise ValueError( f"Unknown model type: '{model_type}'. " f"Available types: {available_types}\n" f"To add support for a new model, use register_extractor() or add to EXTRACTOR_REGISTRY." )