[XPU][S2V] Optimize Wan2.2 S2V: RoPE refactor + cache_dit enabling (#4062)

Signed-off-by: Chendi Xue <chendi.xue@intel.com>
Co-authored-by: Mengkejiergeli Ba <mengkejiergeli.ba@intel.com>
This commit is contained in:
Chendi.XueandMengkejiergeli Ba authored and GitHub committed 2026-06-05 06:20:16 +08:00
1 parent fccd95b12b
commit 31f882935d
7 files changed
+841 -514

No files matched your search

+220
View File
@@ -242,6 +242,80 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> Callable[[int],
return refresh_cache_context
def enable_cache_for_wan22_s2v(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
"""Enable cache-dit for Wan2.2 S2V.
S2V uses a single transformer, but unlike the other Wan2.2 variants its
block loop calls each block as ``block(hidden_states, **kwargs)`` and keeps
the timestep modulation state in ``e`` rather than a second positional
tensor. CacheDiT Pattern_3 matches that contract: cache hidden states only
and pass the remaining conditioning through kwargs unchanged.
The S2V transformer has an ``after_transformer_block`` method that injects
audio embeddings after specific layers. The cached blocks wrapper
(Wan22S2VCachedBlocks._run_block) calls the original internally, so we
permanently replace it with a no-op on the transformer to prevent double
injection from the main forward loop.
"""
db_cache_config = _build_db_cache_config(cache_config)
calibrator_config = None
if cache_config.enable_taylorseer:
taylorseer_order = cache_config.taylorseer_order
calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
# Save the original after_transformer_block before cache-dit wrapping
transformer = pipeline.transformer
if hasattr(transformer, "after_transformer_block"):
transformer._cache_dit_original_after_transformer_block = transformer.after_transformer_block
Wan22S2VCachedAdapter.apply(
BlockAdapter(
transformer=transformer,
blocks=[transformer.blocks],
forward_pattern=[ForwardPattern.Pattern_3],
params_modifiers=[
ParamsModifier(cache_config=db_cache_config, calibrator_config=calibrator_config),
],
has_separate_cfg=True,
),
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
# Permanently replace after_transformer_block with a no-op.
# The cached blocks wrapper (Wan22S2VCachedBlocks._run_block) already calls
# the original via _cache_dit_original_after_transformer_block.
def _noop_after_transformer_block(block_idx: int, hidden_states: torch.Tensor) -> torch.Tensor:
return hidden_states
transformer.after_transformer_block = _noop_after_transformer_block
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
"""Refresh cache context for the S2V transformer."""
if cache_config.scm_steps_mask_policy is None:
cache_dit.refresh_context(
pipeline.transformer,
num_inference_steps=num_inference_steps,
verbose=verbose,
)
else:
cache_dit.refresh_context(
pipeline.transformer,
cache_config=DBCacheConfig().reset(
num_inference_steps=num_inference_steps,
steps_computation_mask=cache_dit.steps_mask(
mask_policy=cache_config.scm_steps_mask_policy,
total_steps=num_inference_steps,
),
steps_computation_policy=cache_config.scm_steps_policy,
),
verbose=verbose,
)
return refresh_cache_context
def enable_cache_for_longcat_image(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
"""Enable cache-dit for LongCatImage pipeline.
@@ -986,6 +1060,151 @@ class BagelCachedBlocks(CachedBlocks_Pattern_0_1_2):
return hidden_states, encoder_hidden_states
class Wan22S2VCachedBlocks(CachedBlocks_Pattern_3_4_5):
"""CacheDiT blocks wrapper that preserves S2V per-layer audio injection."""
def _run_block(self, block_id: int, block: torch.nn.Module, hidden_states: torch.Tensor, *args, **kwargs):
hidden_states = block(hidden_states, *args, **kwargs)
hidden_states, new_encoder_hidden_states = self._process_block_outputs(hidden_states)
original_after_transformer_block = getattr(
self.transformer,
"_cache_dit_original_after_transformer_block",
getattr(self.transformer, "after_transformer_block", None),
)
if original_after_transformer_block is not None:
hidden_states = original_after_transformer_block(block_id, hidden_states)
return hidden_states, new_encoder_hidden_states
def call_blocks(
self,
hidden_states: torch.Tensor,
*args,
**kwargs,
):
new_encoder_hidden_states = None
for block_id, block in enumerate(self.transformer_blocks):
hidden_states, new_encoder_hidden_states = self._run_block(
block_id,
block,
hidden_states,
*args,
**kwargs,
)
return hidden_states, new_encoder_hidden_states
def call_Fn_blocks(
self,
hidden_states: torch.Tensor,
*args,
**kwargs,
):
new_encoder_hidden_states = None
for block_id, block in enumerate(self._Fn_blocks()):
hidden_states, new_encoder_hidden_states = self._run_block(
block_id,
block,
hidden_states,
*args,
**kwargs,
)
return hidden_states, new_encoder_hidden_states
def call_Mn_blocks(
self,
hidden_states: torch.Tensor,
*args,
**kwargs,
):
original_hidden_states = hidden_states
new_encoder_hidden_states = None
start_idx = self.context_manager.Fn_compute_blocks()
for block_id, block in enumerate(self._Mn_blocks(), start=start_idx):
hidden_states, new_encoder_hidden_states = self._run_block(
block_id,
block,
hidden_states,
*args,
**kwargs,
)
hidden_states = hidden_states.contiguous()
hidden_states_residual = hidden_states - original_hidden_states.to(hidden_states.device)
return (
hidden_states,
new_encoder_hidden_states,
hidden_states_residual,
)
def call_Bn_blocks(
self,
hidden_states: torch.Tensor,
*args,
**kwargs,
):
new_encoder_hidden_states = None
if self.context_manager.Bn_compute_blocks() == 0:
return hidden_states, new_encoder_hidden_states
start_idx = len(self.transformer_blocks) - self.context_manager.Bn_compute_blocks()
for block_id, block in enumerate(self._Bn_blocks(), start=start_idx):
hidden_states, new_encoder_hidden_states = self._run_block(
block_id,
block,
hidden_states,
*args,
**kwargs,
)
return hidden_states, new_encoder_hidden_states
class Wan22S2VCachedAdapter(CachedAdapter):
"""CacheDiT adapter that uses Wan22S2VCachedBlocks for S2V audio injection.
Only overrides collect_unified_blocks to use Wan22S2VCachedBlocks (which
calls after_transformer_block per-layer internally). The base class
mock_transformer handles the forward wrapping — after_transformer_block is
permanently replaced with a no-op in enable_cache_for_wan22_s2v() to prevent
double injection.
"""
@classmethod
def collect_unified_blocks(
cls,
block_adapter: BlockAdapter,
contexts_kwargs: list[dict],
) -> list[dict[str, torch.nn.ModuleList]]:
BlockAdapter.assert_normalized(block_adapter)
total_cached_blocks: list[dict[str, torch.nn.ModuleList]] = []
assert hasattr(block_adapter.pipe, "_context_manager")
for i in range(len(block_adapter.transformer)):
unified_blocks_bind_context = {}
for j in range(len(block_adapter.blocks[i])):
cache_config: BasicCacheConfig = contexts_kwargs[i * len(block_adapter.blocks[i]) + j]["cache_config"]
unified_blocks_bind_context[block_adapter.unique_blocks_name[i][j]] = torch.nn.ModuleList(
[
Wan22S2VCachedBlocks(
block_adapter.blocks[i][j],
transformer=block_adapter.transformer[i],
forward_pattern=block_adapter.forward_pattern[i][j],
check_forward_pattern=block_adapter.check_forward_pattern,
check_num_outputs=block_adapter.check_num_outputs,
cache_prefix=block_adapter.blocks_name[i][j],
cache_context=block_adapter.unique_blocks_name[i][j],
context_manager=block_adapter.pipe._context_manager,
cache_type=cache_config.cache_type,
)
]
)
total_cached_blocks.append(unified_blocks_bind_context)
return total_cached_blocks
class BagelCachedAdapter(CachedAdapter):
"""
Custom CachedAdapter for Bagel that uses BagelCachedContextManager and BagelCachedBlocks.
@@ -1808,6 +2027,7 @@ CUSTOM_DIT_ENABLERS.update(
{
"Wan22Pipeline": enable_cache_for_wan22,
"Wan22I2VPipeline": enable_cache_for_wan22,
"Wan22S2VPipeline": enable_cache_for_wan22_s2v,
"HunyuanImage3Pipeline": enable_cache_for_hunyuan_image3,
"FluxPipeline": enable_cache_for_flux,
"Flux2KleinPipeline": enable_cache_for_flux2_klein,