[RFC] Clean and refactor TeaCache registry (#416)
Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
This commit is contained in:
1 parent
a16476bebf
commit
edbd610060
9 files changed
+89
-87
No files matched your search
+24
-24
@@ -151,6 +151,7 @@ def extract_qwen_context(
|
||||
img_shapes: torch.Tensor,
|
||||
txt_seq_lens: torch.Tensor,
|
||||
guidance: torch.Tensor | None = None,
|
||||
additional_t_cond: torch.Tensor | None = None,
|
||||
attention_kwargs: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> CacheContext:
|
||||
@@ -170,6 +171,7 @@ def extract_qwen_context(
|
||||
img_shapes: Image shapes for position embedding
|
||||
txt_seq_lens: Text sequence lengths
|
||||
guidance: Optional guidance scale for CFG
|
||||
additional_t_cond: Optional additional timestep conditioning
|
||||
attention_kwargs: Additional attention arguments
|
||||
**kwargs: Additional keyword arguments ignored by this extractor
|
||||
|
||||
@@ -193,9 +195,9 @@ def extract_qwen_context(
|
||||
guidance = guidance.to(hidden_states.dtype) * 1000
|
||||
|
||||
temb = (
|
||||
module.time_text_embed(timestep, hidden_states)
|
||||
module.time_text_embed(timestep, hidden_states, additional_t_cond)
|
||||
if guidance is None
|
||||
else module.time_text_embed(timestep, guidance, hidden_states)
|
||||
else module.time_text_embed(timestep, guidance, hidden_states, additional_t_cond)
|
||||
)
|
||||
|
||||
image_rotary_emb = module.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
|
||||
@@ -254,22 +256,20 @@ def extract_qwen_context(
|
||||
|
||||
|
||||
# Registry for model-specific extractors
|
||||
# Key: Pipeline class name (from OmniDiffusionConfig.model_class_name)
|
||||
# Key: Transformer class name
|
||||
# Value: extractor function with signature (module, *args, **kwargs) -> CacheContext
|
||||
#
|
||||
# Note: Use the exact pipeline class name as specified in OmniDiffusionConfig.model_class_name
|
||||
# to ensure consistent lookup without substring matching.
|
||||
# Note: Use the transformer class name as specified in pipelines as TeaCache hooks operate
|
||||
# on the transformer module and multiple pipelines can share the same transformer.
|
||||
EXTRACTOR_REGISTRY: dict[str, Callable] = {
|
||||
"QwenImagePipeline": extract_qwen_context,
|
||||
"QwenImageEditPipeline": extract_qwen_context,
|
||||
"QwenImageEditPlusPipeline": extract_qwen_context,
|
||||
"QwenImageTransformer2DModel": extract_qwen_context,
|
||||
# Future models:
|
||||
# "FluxPipeline": extract_flux_context,
|
||||
# "CogVideoXPipeline": extract_cogvideox_context,
|
||||
# "FluxTransformer2DModel": extract_flux_context,
|
||||
# "CogVideoXTransformer3DModel": extract_cogvideox_context,
|
||||
}
|
||||
|
||||
|
||||
def register_extractor(model_identifier: str, extractor_fn: Callable) -> None:
|
||||
def register_extractor(transformer_cls_name: str, extractor_fn: Callable) -> None:
|
||||
"""
|
||||
Register a new extractor function for a model type.
|
||||
|
||||
@@ -277,7 +277,7 @@ def register_extractor(model_identifier: str, extractor_fn: Callable) -> None:
|
||||
the core TeaCache code.
|
||||
|
||||
Args:
|
||||
model_identifier: Model type identifier (class name or type string)
|
||||
transformer_cls_name: Transformer model type identifier (class name or type string)
|
||||
extractor_fn: Function with signature (module, *args, **kwargs) -> CacheContext
|
||||
|
||||
Example:
|
||||
@@ -299,19 +299,19 @@ def register_extractor(model_identifier: str, extractor_fn: Callable) -> None:
|
||||
... return CacheContext(modulated, hidden_states, None, temb, run_blocks, postprocess)
|
||||
>>> register_extractor("FluxTransformer2DModel", extract_flux_context)
|
||||
"""
|
||||
EXTRACTOR_REGISTRY[model_identifier] = extractor_fn
|
||||
EXTRACTOR_REGISTRY[transformer_cls_name] = extractor_fn
|
||||
|
||||
|
||||
def get_extractor(model_type: str) -> Callable:
|
||||
def get_extractor(transformer_cls_name: str) -> Callable:
|
||||
"""
|
||||
Get extractor function for given model type.
|
||||
Get extractor function for given transformer class.
|
||||
|
||||
This function looks up the extractor based on the exact model_type string,
|
||||
which should match the pipeline class name from OmniDiffusionConfig.model_class_name.
|
||||
This function looks up the extractor based on the exact transformer_cls_name string,
|
||||
which should match the transformer type in the pipeline (i.e., pipeline.transformer.__class__.__name__).
|
||||
|
||||
Args:
|
||||
model_type: Model type string (e.g., "QwenImagePipeline", "FluxPipeline")
|
||||
Must exactly match a key in EXTRACTOR_REGISTRY.
|
||||
transformer_cls_name: Transformer class name (e.g., "QwenImageTransformer2DModel")
|
||||
Must exactly match a key in EXTRACTOR_REGISTRY.
|
||||
|
||||
Returns:
|
||||
Extractor function with signature (module, *args, **kwargs) -> CacheContext
|
||||
@@ -320,18 +320,18 @@ def get_extractor(model_type: str) -> Callable:
|
||||
ValueError: If model type not found in registry
|
||||
|
||||
Example:
|
||||
>>> # Get extractor for Qwen model
|
||||
>>> extractor = get_extractor("QwenImagePipeline")
|
||||
>>> # Get extractor for QwenImageTransformer2DModel
|
||||
>>> extractor = get_extractor("QwenImageTransformer2DModel")
|
||||
>>> ctx = extractor(transformer, hidden_states, encoder_hidden_states, timestep, ...)
|
||||
"""
|
||||
# Direct lookup - no substring matching
|
||||
if model_type in EXTRACTOR_REGISTRY:
|
||||
return EXTRACTOR_REGISTRY[model_type]
|
||||
if transformer_cls_name in EXTRACTOR_REGISTRY:
|
||||
return EXTRACTOR_REGISTRY[transformer_cls_name]
|
||||
|
||||
# No match found
|
||||
available_types = list(EXTRACTOR_REGISTRY.keys())
|
||||
raise ValueError(
|
||||
f"Unknown model type: '{model_type}'. "
|
||||
f"Unknown model type: '{transformer_cls_name}'. "
|
||||
f"Available types: {available_types}\n"
|
||||
f"To add support for a new model, use register_extractor() or add to EXTRACTOR_REGISTRY."
|
||||
)
|
||||
Reference in new issue
Block a user