[Feature]: support Flux.2-dev tea_cache (#1871)

Co-authored-by: wuzhongjian <wuzhongjian@cmss.chinamobile.com>
This commit is contained in:
JohnJanandwuzhongjian authored and GitHub committed 2026-04-13 17:51:48 +08:00
1 parent 0d4e975e1b
commit cd2761e15c
5 files changed
+281 -2

No files matched your search

@@ -13,6 +13,7 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.bagel.pipeline_bagel import BagelPipeline
from vllm_omni.diffusion.models.flux2.pipeline_flux2 import Flux2Pipeline
from vllm_omni.diffusion.models.stable_audio.pipeline_stable_audio import StableAudioPipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -103,6 +104,31 @@ class StableAudioAdapter:
registry.register_hook(hook._HOOK_NAME, hook)
class Flux2Adapter:
"""Adapter for Flux2 model coefficient estimation."""
@staticmethod
def load_pipeline(model_path: str, device: str = "cuda", dtype: torch.dtype = torch.bfloat16) -> Flux2Pipeline:
"""Load Flux2 pipeline for coefficient estimation."""
od_config = OmniDiffusionConfig.from_kwargs(model=model_path, dtype=dtype)
od_config.model_class_name = "Flux2Pipeline"
pipeline = Flux2Pipeline(od_config=od_config)
loader = DiffusersPipelineLoader(LoadConfig())
loader.load_weights(pipeline)
pipeline.to(device)
return pipeline
@staticmethod
def get_transformer(pipeline: Any) -> tuple[Any, str]:
return pipeline.transformer, pipeline.transformer.__class__.__name__
@staticmethod
def install_hook(transformer: Any, hook: DataCollectionHook) -> None:
registry = HookRegistry.get_or_create(transformer)
registry.register_hook(hook._HOOK_NAME, hook)
class DefaultAdapter:
"""Default adapter for standard diffusers pipelines."""
@@ -123,6 +149,7 @@ class DefaultAdapter:
_MODEL_ADAPTERS: dict[str, type] = {
"Bagel": BagelAdapter,
"StableAudio": StableAudioAdapter,
"Flux2": Flux2Adapter,
}
_EPSILON = 1e-6
+9
View File
@@ -64,6 +64,15 @@ _MODEL_COEFFICIENTS = {
-1.04182570e01,
6.78098549e-01,
],
# Flux2 transformer coefficients
# Copied from Qwen-Image, need to be tuned specifically for Flux2 in future
"Flux2Transformer2DModel": [
-4.50000000e02,
2.80000000e02,
-4.50000000e01,
3.20000000e00,
-2.00000000e-02,
],
}
+140
View File
@@ -21,6 +21,7 @@ import torch
import torch.nn as nn
from vllm_omni.diffusion.forward_context import get_forward_context
from vllm_omni.platforms import current_omni_platform
@dataclass
@@ -827,6 +828,144 @@ def extract_stable_audio_context(
)
def extract_flux2_context(
module: nn.Module,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor = None,
timestep: torch.LongTensor = None,
img_ids: torch.Tensor = None,
txt_ids: torch.Tensor = None,
guidance: torch.Tensor | None = None,
joint_attention_kwargs: dict[str, Any] | None = None,
return_dict: bool = True,
**kwargs: Any,
) -> CacheContext:
"""
Extract cache context for Flux2Transformer2DModel.
This is the ONLY Flux2-specific code needed for TeaCache support.
It encapsulates preprocessing, modulated input extraction, transformer execution,
and postprocessing logic.
Args:
module: Flux2Transformer2DModel instance
hidden_states: Input hidden states tensor
encoder_hidden_states: Text encoder outputs
timestep: Current diffusion timestep
img_ids: Image inputs for position embedding
txt_ids: Text inputs for position embedding
guidance: Optional guidance scale for CFG
joint_attention_kwargs: Additional attention arguments
return_dict: Whether to return a Transformer2DModelOutput instead of a plain tensor
**kwargs: Additional keyword arguments ignored by this extractor
Returns:
CacheContext with all information needed for generic caching
"""
from diffusers.models.modeling_outputs import Transformer2DModelOutput
if not hasattr(module, "transformer_blocks") or len(module.transformer_blocks) == 0:
raise ValueError("Module must have transformer_blocks")
# ============================================================================
# PREPROCESSING (Flux2-specific)
# ============================================================================
num_txt_tokens = encoder_hidden_states.shape[1]
timestep = timestep.to(hidden_states.dtype) * 1000
if guidance is not None:
guidance = guidance.to(hidden_states.dtype) * 1000
temb = module.time_guidance_embed(timestep, guidance)
double_stream_mod_img = module.double_stream_modulation_img(temb)
double_stream_mod_txt = module.double_stream_modulation_txt(temb)
single_stream_mod = module.single_stream_modulation(temb)[0]
hidden_states = module.x_embedder(hidden_states)
encoder_hidden_states = module.context_embedder(encoder_hidden_states)
if img_ids.ndim == 3:
img_ids = img_ids[0]
if txt_ids.ndim == 3:
txt_ids = txt_ids[0]
if current_omni_platform.is_npu():
freqs_cos_image, freqs_sin_image = module.pos_embed(img_ids.cpu())
image_rotary_emb = (freqs_cos_image.npu(), freqs_sin_image.npu())
freqs_cos_text, freqs_sin_text = module.pos_embed(txt_ids.cpu())
text_rotary_emb = (freqs_cos_text.npu(), freqs_sin_text.npu())
else:
image_rotary_emb = module.pos_embed(img_ids)
text_rotary_emb = module.pos_embed(txt_ids)
concat_rotary_emb = (
torch.cat([text_rotary_emb[0], image_rotary_emb[0]], dim=0),
torch.cat([text_rotary_emb[1], image_rotary_emb[1]], dim=0),
)
# ============================================================================
# EXTRACT MODULATED INPUT (for cache decision)
# ============================================================================
block = module.transformer_blocks[0]
(shift_msa, scale_msa, gate_msa), _ = double_stream_mod_img
modulated_input = block.norm1(hidden_states)
modulated_input = (1 + scale_msa) * modulated_input + shift_msa
# ============================================================================
# DEFINE TRANSFORMER EXECUTION (Flux2-specific)
# ============================================================================
def run_transformer_blocks():
"""Execute all Flux2 transformer blocks."""
h = hidden_states
e = encoder_hidden_states
for transformer_block in module.transformer_blocks:
e, h = transformer_block(
hidden_states=h,
encoder_hidden_states=e,
temb_mod_params_img=double_stream_mod_img,
temb_mod_params_txt=double_stream_mod_txt,
image_rotary_emb=concat_rotary_emb,
joint_attention_kwargs=joint_attention_kwargs,
)
h = torch.cat([e, h], dim=1)
for single_transformer_block in module.single_transformer_blocks:
h = single_transformer_block(
hidden_states=h,
encoder_hidden_states=None,
temb_mod_params=single_stream_mod,
image_rotary_emb=concat_rotary_emb,
joint_attention_kwargs=joint_attention_kwargs,
)
h = h[:, num_txt_tokens:, ...]
return (h,)
# ============================================================================
# DEFINE POSTPROCESSING
# ============================================================================
def postprocess(h):
h = module.norm_out(h, temb)
output = module.proj_out(h)
if not return_dict:
return (output,)
return Transformer2DModelOutput(sample=output)
# ============================================================================
# RETURN CONTEXT
# ============================================================================
return CacheContext(
modulated_input=modulated_input,
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
temb=temb,
run_transformer_blocks=run_transformer_blocks,
postprocess=postprocess,
)
# Registry for model-specific extractors
# Key: Transformer class name
# Value: extractor function with signature (module, *args, **kwargs) -> CacheContext
@@ -839,6 +978,7 @@ EXTRACTOR_REGISTRY: dict[str, Callable] = {
"ZImageTransformer2DModel": extract_zimage_context,
"Flux2Klein": extract_flux2_klein_context,
"StableAudioDiTModel": extract_stable_audio_context,
"Flux2Transformer2DModel": extract_flux2_context,
# Future models:
# "FluxTransformer2DModel": extract_flux_context,
# "CogVideoXTransformer3DModel": extract_cogvideox_context,