Perf/minimax h3 dit swiglu rope cache npu (#6410)

Signed-off-by: Wenjia Li <wjialish@gmail.com>
Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com>
This commit is contained in:
Wenjia LiandWeiQing Chen authored and GitHub committed 2026-08-29 09:14:52 +00:00
1 parent bb5bf8fd0a
commit d9980c75e8
11 files changed
+920 -33

No files matched your search

+84 -17
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Model-specific extractors for TeaCache.
@@ -13,6 +13,7 @@ all model-specific information needed for generic caching, including preprocessi
transformer execution, and postprocessing logic.
"""
import math
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
@@ -1263,7 +1264,6 @@ def extract_minimax_h3_context(
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
_FORWARD_SUPPORTED_KWARGS,
MINIMAX_H3_ADALN_MODALITY_NUM,
_build_rope_table,
_required_kwarg,
)
@@ -1320,8 +1320,20 @@ def extract_minimax_h3_context(
if inverse_indices.shape[0] != seq_len:
raise ValueError(f"inverse_indices must be [{seq_len}], got {list(inverse_indices.shape)}")
device = x.device
rope_table = _build_rope_table(module.rope(img_position_ids).to(device))
local_span = module._rope_local_span(seq_len)
local_start, local_len = local_span
rope_table = kwargs.get("rope_table")
if rope_table is None:
rope_table = module.prepare_rope_table(
img_position_ids,
seq_len=seq_len,
)
else:
module._validate_prepared_rope_table(
rope_table,
local_len=local_len,
device=device,
)
decoder_input, t_emb = module._embed(
x=x,
@@ -1336,28 +1348,59 @@ def extract_minimax_h3_context(
num_requests=num_requests,
seq_len=seq_len,
device=device,
local_span=(0, seq_len),
local_span=local_span,
)
combined_indices = (inverse_indices * MINIMAX_H3_ADALN_MODALITY_NUM + token_tags.clamp(min=0)).to(device)
inverse_indices = inverse_indices.to(device)
cu_seqlens = cu_seqlens.to(device)
# In strict SP, _embed already returns this rank's packed rows while
# combined_indices still describes the full packed sequence. TeaCache uses
# the first block's modulation as part of its rank-local cache state, so
# the modulation indices must be sliced to the same local layout. Keep the
# full indices below: local_sp_prepare owns the global-to-local conversion
# for transformer-block inputs.
state_combined_indices = combined_indices.narrow(0, local_start, local_len)
if local_len == seq_len:
state_combined_indices = combined_indices
shift_msa, scale_msa, *_ = module.blocks[0].adaln_proj(t_emb)
modulated_input = module.blocks[0].norm1(decoder_input)
modulated_input = indexed_scale_shift_(
modulated_input,
shift_msa,
scale_msa,
combined_indices,
state_combined_indices,
)
def run_transformer_blocks() -> tuple[torch.Tensor, ...]:
hidden, block_rope, block_combined = module.sp_prepare(
decoder_input,
rope_table,
combined_indices,
def synchronize_cache_decision(local_should_compute: bool) -> bool:
if local_len == seq_len:
return local_should_compute
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
decision = torch.tensor(
int(local_should_compute),
dtype=torch.int32,
device=device,
)
get_sp_group().all_reduce(decision, op=torch.distributed.ReduceOp.MAX)
return bool(decision.item())
def run_transformer_blocks() -> tuple[torch.Tensor, ...]:
if local_len == seq_len:
hidden, block_rope, block_combined = module.sp_prepare(
decoder_input,
rope_table,
combined_indices,
)
else:
hidden, block_rope, block_combined = module.local_sp_prepare(
decoder_input,
rope_table,
combined_indices,
)
for block in module.blocks:
hidden = block(
hidden,
@@ -1369,15 +1412,36 @@ def extract_minimax_h3_context(
packed_total=seq_len,
video_layout=video_layout,
)
hidden = module.sp_gather(hidden)
# TeaCache stores residuals as block_output - decoder_input. Strict SP
# must keep both tensors rank-local until postprocess projects local
# rows to compact logits and gathers them exactly once.
if local_len == seq_len:
hidden = module.sp_gather(hidden)
return (hidden,)
def postprocess(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
video_logits, audio_logits = module.final_layer(
hidden,
t_emb=t_emb,
inverse_indices=inverse_indices,
)
if local_len == seq_len:
video_logits, audio_logits = module.final_layer(
hidden,
t_emb=t_emb,
inverse_indices=inverse_indices,
)
else:
local_inverse_indices = inverse_indices.narrow(
0,
local_start,
local_len,
)
video_logits, audio_logits = module.final_layer(
hidden,
t_emb=t_emb,
inverse_indices=local_inverse_indices,
)
compact_logits = torch.cat((video_logits, audio_logits), dim=-1)
compact_logits = module.sp_gather(compact_logits)
video_width = module.arch.latents_dim * math.prod(module.arch.patch_size)
video_logits = compact_logits[..., :video_width]
audio_logits = compact_logits[..., video_width:]
video_logits = video_logits.index_select(0, infer_out_pos.to(device))
audio_logits = audio_logits.index_select(0, audio_pos.to(device))
if not skip_mask_out_condition:
@@ -1396,6 +1460,9 @@ def extract_minimax_h3_context(
temb=t_emb,
run_transformer_blocks=run_transformer_blocks,
postprocess=postprocess,
extra_states={
"synchronize_cache_decision": synchronize_cache_decision,
},
)
+10 -1
View File
@@ -138,7 +138,16 @@ class TeaCacheHook(ModelHook):
state = self.state_manager.get_state()
# Decide whether to compute or cache based on modulated input similarity
should_compute = self._should_compute_full_transformer(state, ctx.modulated_input)
local_should_compute = self._should_compute_full_transformer(state, ctx.modulated_input)
sync_cache_decision = (ctx.extra_states or {}).get("synchronize_cache_decision")
if sync_cache_decision is not None:
should_compute = sync_cache_decision(local_should_compute)
# A rank that locally chose the cache path must reset its counter
# when another SP rank requires a full collective block execution.
if should_compute and not local_should_compute:
state.accumulated_rel_l1_distance = 0.0
else:
should_compute = local_should_compute
if not should_compute and state.previous_residual is not None:
# ============================================================================