[Bugfix] Fix transformers 5.x compat issues in online TTS serving (#1536)
Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
1 parent
742d845b8e
commit
a3240fd484
6 files changed
+57
-13
No files matched your search
@@ -397,7 +397,8 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
|
||||
bsz = int(inputs_embeds.shape[0])
|
||||
qlen = 2
|
||||
# Flatten to token-major.
|
||||
hs = inputs_embeds.to(dtype=torch.bfloat16).reshape(bsz * qlen, -1)
|
||||
model_dtype = next(self.model.parameters()).dtype
|
||||
hs = inputs_embeds.to(dtype=model_dtype).reshape(bsz * qlen, -1)
|
||||
hs = self.small_to_mtp_projection(hs)
|
||||
|
||||
query_lens = torch.full((bsz,), qlen, dtype=torch.int32)
|
||||
@@ -498,11 +499,15 @@ class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
|
||||
for step in range(1, num_groups):
|
||||
# Sample or argmax from logits.
|
||||
if do_sample and temperature > 0:
|
||||
scaled = logits / temperature
|
||||
scaled = logits.float() / temperature
|
||||
if top_k > 0:
|
||||
topk_vals, _ = scaled.topk(top_k, dim=-1)
|
||||
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
|
||||
probs = torch.softmax(scaled, dim=-1)
|
||||
# Guard against NaN/Inf from lower-precision dtypes.
|
||||
probs = probs.clamp(min=0.0)
|
||||
row_sums = probs.sum(dim=-1, keepdim=True)
|
||||
probs = torch.where(row_sums > 0, probs / row_sums, torch.ones_like(probs) / probs.shape[-1])
|
||||
next_ids = torch.multinomial(probs, num_samples=1) # [B, 1]
|
||||
else:
|
||||
next_ids = logits.argmax(dim=-1, keepdim=True) # [B, 1]
|
||||
|
||||
@@ -638,12 +638,12 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
def _get_tokenizer(self):
|
||||
if self._tokenizer is None:
|
||||
self._tokenizer = AutoTokenizer.from_pretrained(
|
||||
self.model_path,
|
||||
trust_remote_code=True,
|
||||
fix_mistral_regex=True,
|
||||
use_fast=True,
|
||||
)
|
||||
import transformers
|
||||
|
||||
kwargs = dict(trust_remote_code=True, use_fast=True)
|
||||
if transformers.__version__ < "5":
|
||||
kwargs["fix_mistral_regex"] = True
|
||||
self._tokenizer = AutoTokenizer.from_pretrained(self.model_path, **kwargs)
|
||||
self._tokenizer.padding_side = "left"
|
||||
return self._tokenizer
|
||||
|
||||
|
||||
+25
-1
@@ -48,6 +48,22 @@ from .configuration_qwen3_tts_tokenizer_v2 import (
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
def _default_rope_init(config, device=None, seq_len=None, layer_type=None):
|
||||
"""Vanilla sinusoidal RoPE (no scaling).
|
||||
|
||||
transformers>=5.0 removed the 'default' entry from
|
||||
``ROPE_INIT_FUNCTIONS`` (see ``modeling_rope_utils.py`` in
|
||||
huggingface/transformers), but the speech tokenizer config still
|
||||
declares ``rope_type="default"``. This reimplements the original
|
||||
behaviour.
|
||||
"""
|
||||
head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
|
||||
inv_freq = 1.0 / (
|
||||
config.rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32, device=device) / head_dim)
|
||||
)
|
||||
return inv_freq, 1.0
|
||||
|
||||
|
||||
@dataclass
|
||||
@auto_docstring
|
||||
class Qwen3TTSTokenizerV2EncoderOutput(ModelOutput):
|
||||
@@ -256,7 +272,15 @@ class Qwen3TTSTokenizerV2DecoderRotatoryEmbedding(nn.Module):
|
||||
self.original_max_seq_len = config.max_position_embeddings
|
||||
|
||||
self.config = config
|
||||
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
|
||||
if self.rope_type in ROPE_INIT_FUNCTIONS:
|
||||
self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
elif self.rope_type == "default":
|
||||
self.rope_init_fn = _default_rope_init
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported rope_type '{self.rope_type}'. Expected one of {list(ROPE_INIT_FUNCTIONS)} or 'default'."
|
||||
)
|
||||
|
||||
inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
|
||||
Reference in new issue
Block a user