perf(qwen3-tts): cache decoder masks and compile pre-transformer (#5107)
Signed-off-by: Ziyang Zhang <hafuhafu@qq.com> Co-authored-by: Sy03 <1370724210@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
3 files changed
+249
-40
No files matched your search
+83
-38
@@ -25,12 +25,9 @@ from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import MimiConfig, MimiModel
|
||||
from transformers.activations import ACT2FN
|
||||
from transformers.cache_utils import Cache, DynamicCache
|
||||
from transformers.cache_utils import Cache
|
||||
from transformers.integrations import use_kernel_forward_from_hub
|
||||
from transformers.masking_utils import (
|
||||
create_causal_mask,
|
||||
create_sliding_window_causal_mask,
|
||||
)
|
||||
from transformers.masking_utils import create_sliding_window_causal_mask
|
||||
from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
|
||||
from transformers.modeling_layers import GradientCheckpointingLayer
|
||||
from transformers.modeling_outputs import BaseModelOutputWithPast
|
||||
@@ -523,15 +520,64 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr
|
||||
self.norm = Qwen3TTSTokenizerV2DecoderRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.rotary_emb = Qwen3TTSTokenizerV2DecoderRotatoryEmbedding(config=config)
|
||||
self.gradient_checkpointing = False
|
||||
self.has_sliding_layers = "sliding_attention" in self.config.layer_types
|
||||
self.window_size = config.sliding_window
|
||||
|
||||
self.input_proj = nn.Linear(config.latent_dim, config.hidden_size)
|
||||
self.output_proj = nn.Linear(config.hidden_size, config.latent_dim)
|
||||
|
||||
# Code2Wav is cache-free and every decoder layer uses sliding attention.
|
||||
# Cache an exact, compact mask per warmup shape instead of retaining a
|
||||
# max_position_embeddings square buffer. CUDA Graph warmup populates
|
||||
# these entries before capture, and dtype/device are part of the key.
|
||||
self._sliding_attention_mask_cache: dict[tuple[int, torch.dtype, torch.device, str], torch.Tensor | None] = {}
|
||||
|
||||
# Initialize weights and apply final processing
|
||||
self.post_init()
|
||||
|
||||
def _apply(self, fn, recurse=True):
|
||||
# Cached masks are keyed by dtype/device and should not retain storage
|
||||
# from the model's previous placement.
|
||||
self._sliding_attention_mask_cache.clear()
|
||||
return super()._apply(fn, recurse=recurse)
|
||||
|
||||
def _get_sliding_attention_mask(self, inputs_embeds: torch.Tensor) -> torch.Tensor | None:
|
||||
sequence_length = int(inputs_embeds.shape[1])
|
||||
max_position_embeddings = int(self.config.max_position_embeddings)
|
||||
if sequence_length > max_position_embeddings:
|
||||
raise ValueError(
|
||||
f"Input length {sequence_length} exceeds max_position_embeddings {max_position_embeddings}"
|
||||
)
|
||||
|
||||
attention_implementation = str(self.config._attn_implementation)
|
||||
cache_key = (
|
||||
sequence_length,
|
||||
inputs_embeds.dtype,
|
||||
inputs_embeds.device,
|
||||
attention_implementation,
|
||||
)
|
||||
if cache_key not in self._sliding_attention_mask_cache:
|
||||
# A batch-one mask broadcasts across requests and avoids caching a
|
||||
# duplicate tensor for every batch size.
|
||||
mask_inputs = inputs_embeds[:1]
|
||||
mask_kwargs = {
|
||||
"config": self.config,
|
||||
"attention_mask": None,
|
||||
"past_key_values": None,
|
||||
"position_ids": None,
|
||||
}
|
||||
sig = inspect.signature(create_sliding_window_causal_mask)
|
||||
if "input_embeds" in sig.parameters:
|
||||
mask_kwargs["input_embeds"] = mask_inputs
|
||||
mask_kwargs["cache_position"] = torch.arange(sequence_length, device=inputs_embeds.device)
|
||||
else:
|
||||
mask_kwargs["inputs_embeds"] = mask_inputs
|
||||
sliding_attention_mask = create_sliding_window_causal_mask(**mask_kwargs)
|
||||
if sliding_attention_mask is not None:
|
||||
sliding_attention_mask = sliding_attention_mask.contiguous()
|
||||
self._sliding_attention_mask_cache[cache_key] = sliding_attention_mask
|
||||
|
||||
return self._sliding_attention_mask_cache[cache_key]
|
||||
|
||||
# Note: @check_model_inputs decorator removed for vLLM compatibility
|
||||
# The decorator causes "unexpected keyword argument 'inputs_embeds'" error
|
||||
@auto_docstring
|
||||
@@ -554,57 +600,56 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr
|
||||
raise ValueError("input_ids is not expected")
|
||||
if (input_ids is None) ^ (inputs_embeds is not None):
|
||||
raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
|
||||
if use_cache:
|
||||
raise ValueError("Code2Wav sliding-mask caching does not support use_cache=True")
|
||||
if past_key_values is not None:
|
||||
raise ValueError("Code2Wav sliding-mask caching does not support past_key_values")
|
||||
if attention_mask is not None and not isinstance(attention_mask, dict):
|
||||
raise ValueError("Code2Wav cached masks require attention_mask to be None or a prepared mask mapping")
|
||||
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
|
||||
inputs_embeds = self.input_proj(inputs_embeds)
|
||||
|
||||
if use_cache and past_key_values is None:
|
||||
past_key_values = DynamicCache(config=self.config)
|
||||
sequence_length = inputs_embeds.shape[1]
|
||||
expected_cache_position = torch.arange(sequence_length, device=inputs_embeds.device)
|
||||
|
||||
if cache_position is None:
|
||||
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
|
||||
cache_position = torch.arange(
|
||||
past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
|
||||
)
|
||||
cache_position = expected_cache_position
|
||||
elif attention_mask is None and (
|
||||
cache_position.shape != expected_cache_position.shape
|
||||
or cache_position.device != expected_cache_position.device
|
||||
or not torch.equal(cache_position, expected_cache_position)
|
||||
):
|
||||
raise ValueError("Cached Code2Wav masks require contiguous zero-based cache_position")
|
||||
|
||||
if position_ids is None:
|
||||
position_ids = cache_position.unsqueeze(0)
|
||||
|
||||
# It may already have been prepared by e.g. `generate`
|
||||
if not isinstance(causal_mask_mapping := attention_mask, dict):
|
||||
# Prepare mask arguments
|
||||
mask_kwargs = {
|
||||
"config": self.config,
|
||||
"attention_mask": attention_mask,
|
||||
"past_key_values": past_key_values,
|
||||
"position_ids": position_ids,
|
||||
}
|
||||
# Handle API changes across transformers versions
|
||||
sig = inspect.signature(create_causal_mask)
|
||||
if "input_embeds" in sig.parameters:
|
||||
mask_kwargs["input_embeds"] = inputs_embeds
|
||||
mask_kwargs["cache_position"] = cache_position
|
||||
else:
|
||||
mask_kwargs["inputs_embeds"] = inputs_embeds
|
||||
# Create the masks
|
||||
causal_mask_mapping = {
|
||||
"full_attention": create_causal_mask(**mask_kwargs),
|
||||
}
|
||||
# The sliding window alternating layers are not always activated depending on the config
|
||||
if self.has_sliding_layers:
|
||||
causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(**mask_kwargs)
|
||||
elif attention_mask is None:
|
||||
expected_position_ids = expected_cache_position.unsqueeze(0)
|
||||
if (
|
||||
position_ids.ndim != 2
|
||||
or position_ids.shape[-1] != sequence_length
|
||||
or position_ids.device != expected_position_ids.device
|
||||
or not torch.equal(position_ids, expected_position_ids.expand_as(position_ids))
|
||||
):
|
||||
raise ValueError("Cached Code2Wav masks require contiguous zero-based position_ids")
|
||||
|
||||
hidden_states = inputs_embeds
|
||||
|
||||
if isinstance(attention_mask, dict):
|
||||
sliding_attention_mask = attention_mask["sliding_attention"]
|
||||
else:
|
||||
sliding_attention_mask = self._get_sliding_attention_mask(inputs_embeds)
|
||||
|
||||
# create position embeddings to be shared across the decoder layers
|
||||
position_embeddings = self.rotary_emb(hidden_states, position_ids)
|
||||
|
||||
for decoder_layer in self.layers[: self.config.num_hidden_layers]:
|
||||
hidden_states = decoder_layer(
|
||||
hidden_states,
|
||||
attention_mask=causal_mask_mapping[decoder_layer.attention_type],
|
||||
attention_mask=sliding_attention_mask,
|
||||
position_ids=position_ids,
|
||||
past_key_values=past_key_values,
|
||||
use_cache=use_cache,
|
||||
@@ -617,7 +662,7 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr
|
||||
hidden_states = self.output_proj(hidden_states)
|
||||
return BaseModelOutputWithPast(
|
||||
last_hidden_state=hidden_states,
|
||||
past_key_values=past_key_values if use_cache else None,
|
||||
past_key_values=None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user