[Bugfix] Fix transformers 5.x compat issues in online TTS serving (#1536)

Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-03-04 01:28:32 +08:00
1 parent 742d845b8e
commit a3240fd484
6 files changed
+57 -13

No files matched your search

@@ -397,7 +397,8 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
bsz = int(inputs_embeds.shape[0])
qlen = 2
# Flatten to token-major.
hs = inputs_embeds.to(dtype=torch.bfloat16).reshape(bsz * qlen, -1)
model_dtype = next(self.model.parameters()).dtype
hs = inputs_embeds.to(dtype=model_dtype).reshape(bsz * qlen, -1)
hs = self.small_to_mtp_projection(hs)
query_lens = torch.full((bsz,), qlen, dtype=torch.int32)
@@ -498,11 +499,15 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
for step in range(1, num_groups):
# Sample or argmax from logits.
if do_sample and temperature > 0:
scaled = logits / temperature
scaled = logits.float() / temperature
if top_k > 0:
topk_vals, _ = scaled.topk(top_k, dim=-1)
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
probs = torch.softmax(scaled, dim=-1)
# Guard against NaN/Inf from lower-precision dtypes.
probs = probs.clamp(min=0.0)
row_sums = probs.sum(dim=-1, keepdim=True)
probs = torch.where(row_sums > 0, probs / row_sums, torch.ones_like(probs) / probs.shape[-1])
next_ids = torch.multinomial(probs, num_samples=1) # [B, 1]
else:
next_ids = logits.argmax(dim=-1, keepdim=True) # [B, 1]
@@ -638,12 +638,12 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
def _get_tokenizer(self):
if self._tokenizer is None:
self._tokenizer = AutoTokenizer.from_pretrained(
self.model_path,
trust_remote_code=True,
fix_mistral_regex=True,
use_fast=True,
)
import transformers
kwargs = dict(trust_remote_code=True, use_fast=True)
if transformers.__version__ < "5":
kwargs["fix_mistral_regex"] = True
self._tokenizer = AutoTokenizer.from_pretrained(self.model_path, **kwargs)
self._tokenizer.padding_side = "left"
return self._tokenizer
@@ -48,6 +48,22 @@ from .configuration_qwen3_tts_tokenizer_v2 import (
logger = logging.get_logger(__name__)
def _default_rope_init(config, device=None, seq_len=None, layer_type=None):
"""Vanilla sinusoidal RoPE (no scaling).
transformers>=5.0 removed the 'default' entry from
``ROPE_INIT_FUNCTIONS`` (see ``modeling_rope_utils.py`` in
huggingface/transformers), but the speech tokenizer config still
declares ``rope_type="default"``. This reimplements the original
behaviour.
"""
head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
inv_freq = 1.0 / (
config.rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32, device=device) / head_dim)
)
return inv_freq, 1.0
@dataclass
@auto_docstring
class Qwen3TTSTokenizerV2EncoderOutput(ModelOutput):
@@ -256,7 +272,15 @@ class Qwen3TTSTokenizerV2DecoderRotatoryEmbedding(nn.Module):
self.original_max_seq_len = config.max_position_embeddings
self.config = config
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
if self.rope_type in ROPE_INIT_FUNCTIONS:
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
elif self.rope_type == "default":
self.rope_init_fn = _default_rope_init
else:
raise ValueError(
f"Unsupported rope_type '{self.rope_type}'. Expected one of {list(ROPE_INIT_FUNCTIONS)} or 'default'."
)
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
self.register_buffer("inv_freq", inv_freq, persistent=False)