Perf/minimax h3 dit swiglu rope cache npu (#6410)
Signed-off-by: Wenjia Li <wjialish@gmail.com> Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com>
This commit is contained in:
1 parent
bb5bf8fd0a
commit
d9980c75e8
11 files changed
+920
-33
No files matched your search
+84
-17
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Model-specific extractors for TeaCache.
|
||||
@@ -13,6 +13,7 @@ all model-specific information needed for generic caching, including preprocessi
|
||||
transformer execution, and postprocessing logic.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
@@ -1263,7 +1264,6 @@ def extract_minimax_h3_context(
|
||||
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
|
||||
_FORWARD_SUPPORTED_KWARGS,
|
||||
MINIMAX_H3_ADALN_MODALITY_NUM,
|
||||
_build_rope_table,
|
||||
_required_kwarg,
|
||||
)
|
||||
|
||||
@@ -1320,8 +1320,20 @@ def extract_minimax_h3_context(
|
||||
if inverse_indices.shape[0] != seq_len:
|
||||
raise ValueError(f"inverse_indices must be [{seq_len}], got {list(inverse_indices.shape)}")
|
||||
device = x.device
|
||||
|
||||
rope_table = _build_rope_table(module.rope(img_position_ids).to(device))
|
||||
local_span = module._rope_local_span(seq_len)
|
||||
local_start, local_len = local_span
|
||||
rope_table = kwargs.get("rope_table")
|
||||
if rope_table is None:
|
||||
rope_table = module.prepare_rope_table(
|
||||
img_position_ids,
|
||||
seq_len=seq_len,
|
||||
)
|
||||
else:
|
||||
module._validate_prepared_rope_table(
|
||||
rope_table,
|
||||
local_len=local_len,
|
||||
device=device,
|
||||
)
|
||||
|
||||
decoder_input, t_emb = module._embed(
|
||||
x=x,
|
||||
@@ -1336,28 +1348,59 @@ def extract_minimax_h3_context(
|
||||
num_requests=num_requests,
|
||||
seq_len=seq_len,
|
||||
device=device,
|
||||
local_span=(0, seq_len),
|
||||
local_span=local_span,
|
||||
)
|
||||
|
||||
combined_indices = (inverse_indices * MINIMAX_H3_ADALN_MODALITY_NUM + token_tags.clamp(min=0)).to(device)
|
||||
inverse_indices = inverse_indices.to(device)
|
||||
cu_seqlens = cu_seqlens.to(device)
|
||||
|
||||
# In strict SP, _embed already returns this rank's packed rows while
|
||||
# combined_indices still describes the full packed sequence. TeaCache uses
|
||||
# the first block's modulation as part of its rank-local cache state, so
|
||||
# the modulation indices must be sliced to the same local layout. Keep the
|
||||
# full indices below: local_sp_prepare owns the global-to-local conversion
|
||||
# for transformer-block inputs.
|
||||
state_combined_indices = combined_indices.narrow(0, local_start, local_len)
|
||||
if local_len == seq_len:
|
||||
state_combined_indices = combined_indices
|
||||
|
||||
shift_msa, scale_msa, *_ = module.blocks[0].adaln_proj(t_emb)
|
||||
modulated_input = module.blocks[0].norm1(decoder_input)
|
||||
modulated_input = indexed_scale_shift_(
|
||||
modulated_input,
|
||||
shift_msa,
|
||||
scale_msa,
|
||||
combined_indices,
|
||||
state_combined_indices,
|
||||
)
|
||||
|
||||
def run_transformer_blocks() -> tuple[torch.Tensor, ...]:
|
||||
hidden, block_rope, block_combined = module.sp_prepare(
|
||||
decoder_input,
|
||||
rope_table,
|
||||
combined_indices,
|
||||
def synchronize_cache_decision(local_should_compute: bool) -> bool:
|
||||
if local_len == seq_len:
|
||||
return local_should_compute
|
||||
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
|
||||
|
||||
decision = torch.tensor(
|
||||
int(local_should_compute),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
get_sp_group().all_reduce(decision, op=torch.distributed.ReduceOp.MAX)
|
||||
return bool(decision.item())
|
||||
|
||||
def run_transformer_blocks() -> tuple[torch.Tensor, ...]:
|
||||
if local_len == seq_len:
|
||||
hidden, block_rope, block_combined = module.sp_prepare(
|
||||
decoder_input,
|
||||
rope_table,
|
||||
combined_indices,
|
||||
)
|
||||
else:
|
||||
hidden, block_rope, block_combined = module.local_sp_prepare(
|
||||
decoder_input,
|
||||
rope_table,
|
||||
combined_indices,
|
||||
)
|
||||
for block in module.blocks:
|
||||
hidden = block(
|
||||
hidden,
|
||||
@@ -1369,15 +1412,36 @@ def extract_minimax_h3_context(
|
||||
packed_total=seq_len,
|
||||
video_layout=video_layout,
|
||||
)
|
||||
hidden = module.sp_gather(hidden)
|
||||
# TeaCache stores residuals as block_output - decoder_input. Strict SP
|
||||
# must keep both tensors rank-local until postprocess projects local
|
||||
# rows to compact logits and gathers them exactly once.
|
||||
if local_len == seq_len:
|
||||
hidden = module.sp_gather(hidden)
|
||||
return (hidden,)
|
||||
|
||||
def postprocess(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
video_logits, audio_logits = module.final_layer(
|
||||
hidden,
|
||||
t_emb=t_emb,
|
||||
inverse_indices=inverse_indices,
|
||||
)
|
||||
if local_len == seq_len:
|
||||
video_logits, audio_logits = module.final_layer(
|
||||
hidden,
|
||||
t_emb=t_emb,
|
||||
inverse_indices=inverse_indices,
|
||||
)
|
||||
else:
|
||||
local_inverse_indices = inverse_indices.narrow(
|
||||
0,
|
||||
local_start,
|
||||
local_len,
|
||||
)
|
||||
video_logits, audio_logits = module.final_layer(
|
||||
hidden,
|
||||
t_emb=t_emb,
|
||||
inverse_indices=local_inverse_indices,
|
||||
)
|
||||
compact_logits = torch.cat((video_logits, audio_logits), dim=-1)
|
||||
compact_logits = module.sp_gather(compact_logits)
|
||||
video_width = module.arch.latents_dim * math.prod(module.arch.patch_size)
|
||||
video_logits = compact_logits[..., :video_width]
|
||||
audio_logits = compact_logits[..., video_width:]
|
||||
video_logits = video_logits.index_select(0, infer_out_pos.to(device))
|
||||
audio_logits = audio_logits.index_select(0, audio_pos.to(device))
|
||||
if not skip_mask_out_condition:
|
||||
@@ -1396,6 +1460,9 @@ def extract_minimax_h3_context(
|
||||
temb=t_emb,
|
||||
run_transformer_blocks=run_transformer_blocks,
|
||||
postprocess=postprocess,
|
||||
extra_states={
|
||||
"synchronize_cache_decision": synchronize_cache_decision,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
|
||||
+10
-1
@@ -138,7 +138,16 @@ class TeaCacheHook(ModelHook):
|
||||
state = self.state_manager.get_state()
|
||||
|
||||
# Decide whether to compute or cache based on modulated input similarity
|
||||
should_compute = self._should_compute_full_transformer(state, ctx.modulated_input)
|
||||
local_should_compute = self._should_compute_full_transformer(state, ctx.modulated_input)
|
||||
sync_cache_decision = (ctx.extra_states or {}).get("synchronize_cache_decision")
|
||||
if sync_cache_decision is not None:
|
||||
should_compute = sync_cache_decision(local_should_compute)
|
||||
# A rank that locally chose the cache path must reset its counter
|
||||
# when another SP rank requires a full collective block execution.
|
||||
if should_compute and not local_should_compute:
|
||||
state.accumulated_rel_l1_distance = 0.0
|
||||
else:
|
||||
should_compute = local_should_compute
|
||||
|
||||
if not should_compute and state.previous_residual is not None:
|
||||
# ============================================================================
|
||||
|
||||
Reference in new issue
Block a user