[Feature] : Support disaggregated inference pipeline for Qwen3_TTS (#1161)

Signed-off-by: Sy03 <1370724210@qq.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Sy03andHongsheng Liu authored and GitHub committed 2026-02-21 07:36:44 +08:00
1 parent 5295f0f81b
commit efbe4116ff
23 files changed
+2970 -2592

No files matched your search

@@ -504,19 +504,39 @@ class Qwen3TTSConfig(PretrainedConfig):
self.tts_bos_token_id = tts_bos_token_id
self.tts_eos_token_id = tts_eos_token_id
# TODO: remove these dummy values after
self.image_token_id = 0 # dummy image token id
self.video_token_id = 0 # dummy video token id
self.vision_start_token_id = 0 # dummy vision start token id
# Dummy vision token IDs that must never collide with real codec tokens.
# mrope scans prompt_token_ids for these; using -1 ensures no false match.
self.image_token_id = -1
self.video_token_id = -1
self.vision_start_token_id = -1
self.vision_config = PretrainedConfig() # dummy vision config
self.vision_config.spatial_merge_size = 1
@property
def codec_frame_rate_hz(self) -> float | None:
pos_per_sec = getattr(self.talker_config, "position_id_per_seconds", None)
if pos_per_sec is None:
return None
try:
fps = float(pos_per_sec)
except (TypeError, ValueError):
return None
return fps if fps > 0 else None
def get_text_config(self, **kwargs):
# vLLM expects text config to expose hidden_size/num_attention_heads.
# For Qwen3 TTS, the talker config is the text model config.
config = self.talker_config
# if hasattr(config, "rope_parameters"):
# delattr(config, "rope_parameters")
# Code2Wav is a pure convolutional waveform decoder; it does NOT use
# rotary position embeddings. When hf_overrides sets architectures
# to [Qwen3TTSCode2Wav], strip rope_parameters so that the model
# runner sees uses_mrope == False and skips mrope position computation
# on codec tokens. Each stage loads its own config instance, so this
# in-place mutation does not affect the Talker stage.
archs = getattr(self, "architectures", []) or []
if any("Code2Wav" in str(a) for a in archs):
if hasattr(config, "rope_parameters"):
delattr(config, "rope_parameters")
return config
File diff suppressed because it is too large. Load diff
@@ -1,102 +0,0 @@
# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from transformers.feature_extraction_utils import BatchFeature
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin
class Qwen3TTSProcessorKwargs(ProcessingKwargs, total=False):
_defaults = {
"text_kwargs": {
"padding": False,
"padding_side": "left",
}
}
class Qwen3TTSProcessor(ProcessorMixin):
r"""
Constructs a Qwen3TTS processor.
Args:
tokenizer ([`Qwen2TokenizerFast`], *optional*):
The text tokenizer.
chat_template (`Optional[str]`, *optional*):
The Jinja template to use for formatting the conversation.
If not provided, the default chat template is used.
"""
attributes = ["tokenizer"]
tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
def __init__(self, tokenizer=None, chat_template=None):
super().__init__(tokenizer, chat_template=chat_template)
def __call__(self, text=None, **kwargs) -> BatchFeature:
"""
Main method to prepare for the model one or several sequences(s) and audio(s).
This method forwards the `text` and `kwargs` arguments to
Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None`
to encode the text.
Args:
text (`str`, `List[str]`, `List[List[str]]`):
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
"""
if text is None:
raise ValueError("You need to specify either a `text` input to process.")
output_kwargs = self._merge_kwargs(
Qwen3TTSProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
if not isinstance(text, list):
text = [text]
texts_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
return BatchFeature(
data={**texts_inputs},
tensor_type=kwargs.get("return_tensors"),
)
def batch_decode(self, *args, **kwargs):
"""
This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
refer to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
the docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
def apply_chat_template(self, conversations, chat_template=None, **kwargs):
if isinstance(conversations[0], dict):
conversations = [conversations]
return super().apply_chat_template(conversations, chat_template, **kwargs)
@property
def model_input_names(self):
tokenizer_input_names = self.tokenizer.model_input_names
return list(dict.fromkeys(tokenizer_input_names))
__all__ = ["Qwen3TTSProcessor"]
@@ -0,0 +1,239 @@
from __future__ import annotations
import os
from collections.abc import Iterable
from typing import Any
import numpy as np
import torch
import torch.nn as nn
from transformers.utils.hub import cached_file
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm_omni.model_executor.models.output_templates import OmniOutput
from .qwen3_tts_tokenizer import Qwen3TTSTokenizer
logger = init_logger(__name__)
class Qwen3TTSCode2Wav(nn.Module):
"""Stage-1 code2wav model for Qwen3-TTS (GenerationModelRunner).
Consumes frame-aligned codec tokens from input_ids and decodes waveform via SpeechTokenizer."""
input_modalities = "audio"
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.vllm_config = vllm_config
self.model_path = vllm_config.model_config.model
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
# Generation-only stage (no logits / sampling).
self.requires_raw_input_tokens = True
self._speech_tokenizer: Qwen3TTSTokenizer | None = None
self._num_quantizers: int | None = None
self._decode_upsample_rate: int | None = None
self._output_sample_rate: int | None = None
self._logged_codec_stats = False
@staticmethod
def _module_device(module: nn.Module) -> torch.device:
try:
return next(module.parameters()).device
except StopIteration:
for _, buf in module.named_buffers(recurse=True):
return buf.device
return torch.device("cpu")
def _ensure_speech_tokenizer_loaded(self) -> Qwen3TTSTokenizer:
if self._speech_tokenizer is not None:
return self._speech_tokenizer
# Locate speech_tokenizer dir from HF cache (or local path).
cfg_path = cached_file(self.model_path, "speech_tokenizer/config.json")
if cfg_path is None:
raise ValueError(f"{self.model_path}/speech_tokenizer/config.json not found")
speech_tokenizer_dir = os.path.dirname(cfg_path)
# Stage-1 only needs decode; skip HF feature extractor to avoid heavy optional deps.
# Still require preprocessor_config.json (use cached_file so online runs can fetch it).
prep_cfg = cached_file(self.model_path, "speech_tokenizer/preprocessor_config.json")
if prep_cfg is None:
raise ValueError(
f"{self.model_path}/speech_tokenizer/preprocessor_config.json not found. "
"Please make sure the checkpoint contains the required HF preprocessing files."
)
tok = Qwen3TTSTokenizer.from_pretrained(
speech_tokenizer_dir,
torch_dtype=torch.bfloat16,
load_feature_extractor=False,
)
# Align device with vLLM worker, then read back from module.
if tok.model is not None:
tok.model.to(device=self.vllm_config.device_config.device)
tok.device = self._module_device(tok.model)
# Derive codec group count and rates from tokenizer config.
dec_cfg = getattr(tok.model.config, "decoder_config", None)
num_q = getattr(dec_cfg, "num_quantizers", None) if dec_cfg is not None else None
if num_q is None:
raise ValueError("speech_tokenizer decoder_config.num_quantizers not found")
num_q = int(num_q)
if num_q <= 0:
raise ValueError(f"Invalid speech_tokenizer num_quantizers={num_q}")
try:
upsample = int(tok.get_decode_upsample_rate())
except Exception as e:
raise ValueError(f"Failed to get decode upsample rate: {e}") from e
if upsample <= 0:
raise ValueError(f"Invalid decode upsample rate: {upsample}")
try:
out_sr = int(tok.get_output_sample_rate())
except Exception as e:
raise ValueError(f"Failed to get output sample rate: {e}") from e
self._speech_tokenizer = tok
self._num_quantizers = num_q
self._decode_upsample_rate = upsample
self._output_sample_rate = out_sr
return tok
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
# This stage ignores token embeddings. Keep a stable dummy embedding for vLLM runner.
if input_ids.numel() == 0:
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> None:
return None
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: Any,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Decode codec codes into audio waveform.
input_ids layout: [codec_context_frames, *flat_codes]
where flat_codes is codebook-major [q*F].
"""
tok = self._ensure_speech_tokenizer_loaded()
assert self._num_quantizers is not None
assert self._output_sample_rate is not None
sr_val = self._output_sample_rate
empty_ret = (
torch.zeros((0,), dtype=torch.float32),
torch.tensor(sr_val, dtype=torch.int32),
)
if input_ids is None:
return empty_ret
q = int(self._num_quantizers)
ids = input_ids.reshape(-1).to(dtype=torch.long)
n_tokens = ids.numel()
if n_tokens == 0:
return empty_ret
# input_ids[0] = codec_context_frames (prepended by stage_input_processor).
ctx_frames = int(ids[0].item())
ids = ids[1:]
n_tokens = ids.numel()
if n_tokens == 0:
return empty_ret
# Warmup / dummy_run: not divisible by num_quantizers.
if n_tokens % q != 0:
logger.warning(
"Code2Wav input_ids length %d not divisible by num_quantizers %d, "
"likely a warmup run; returning empty audio.",
n_tokens,
q,
)
return empty_ret
total_frames = n_tokens // q
# Reshape codebook-major flat [q*F] -> [q, F] -> [F, q] for SpeechTokenizer.
codes_fq = ids.reshape(q, total_frames).transpose(0, 1).contiguous()
if not self._logged_codec_stats and total_frames > 1:
self._logged_codec_stats = True
try:
uniq = int(torch.unique(codes_fq).numel())
cmin = int(codes_fq.min().item())
cmax = int(codes_fq.max().item())
head = codes_fq[: min(2, total_frames), : min(8, q)].cpu().tolist()
logger.info(
"Code2Wav codec: frames=%d q=%d uniq=%d range=[%d,%d] head=%s",
total_frames,
q,
uniq,
cmin,
cmax,
head,
)
except Exception:
pass
wavs, sr = tok.decode({"audio_codes": codes_fq})
if not wavs:
raise ValueError("SpeechTokenizer code2wav produced empty waveform list.")
audio_np = wavs[0].astype(np.float32, copy=False)
# Trim left-context waveform samples (streaming sliding window).
if ctx_frames > 0:
upsample = self._decode_upsample_rate
if upsample is None or upsample <= 0:
raise ValueError(f"Invalid decode upsample rate: {upsample}")
cut = ctx_frames * upsample
if cut < audio_np.shape[0]:
audio_np = audio_np[cut:]
else:
logger.warning(
"Context trim %d >= decoded length %d; returning empty audio.",
cut,
audio_np.shape[0],
)
return empty_ret
audio_tensor = torch.from_numpy(audio_np).to(dtype=torch.float32).reshape(-1)
sr_tensor = torch.tensor(int(sr), dtype=torch.int32)
return audio_tensor, sr_tensor
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
if not (isinstance(model_outputs, tuple) and len(model_outputs) == 2):
raise TypeError(f"Qwen3TTSCode2Wav expected (audio_tensor, sr) outputs, got {type(model_outputs)}")
audio_tensor, sr = model_outputs
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={
"model_outputs": audio_tensor,
"sr": sr,
},
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# SpeechTokenizer weights live under `speech_tokenizer/` and are loaded
# lazily from that directory. Ignore main checkpoint weights.
return set()
@@ -0,0 +1,495 @@
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.vllm import set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.qwen3 import Qwen3DecoderLayer
from vllm.model_executor.models.utils import is_pp_missing_parameter
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec, KVCacheSpec, KVCacheTensor
from vllm.v1.worker.gpu import attn_utils
from .configuration_qwen3_tts import Qwen3TTSTalkerCodePredictorConfig, Qwen3TTSTalkerConfig
class _LocalPredictorKVCache:
"""Minimal local KV cache + attention metadata for running
code_predictor inside one worker (independent of engine KV)."""
def __init__(
self,
*,
vllm_config: VllmConfig,
max_seq_len: int,
max_batch_size: int,
device: torch.device,
) -> None:
self.vllm_config = vllm_config
self.device = device
# Collect attention layers registered in this vllm_config.
kv_cache_spec_by_layer = attn_utils.get_kv_cache_spec(vllm_config)
if not kv_cache_spec_by_layer:
raise RuntimeError("Local predictor KVCache requires vLLM Attention layers to be registered.")
# We only need enough blocks for a tiny per-frame sequence (<= max_seq_len).
any_spec = next(iter(kv_cache_spec_by_layer.values()))
block_size = int(any_spec.block_size)
blocks_per_seq = (int(max_seq_len) + block_size - 1) // block_size
num_blocks = max(1, int(max_batch_size) * int(blocks_per_seq))
# Allocate per-layer KV caches (small, independent).
kv_cache_tensors: list[KVCacheTensor] = []
for layer_name, spec in kv_cache_spec_by_layer.items():
kv_cache_tensors.append(KVCacheTensor(size=int(spec.page_size_bytes) * num_blocks, shared_by=[layer_name]))
merged_spec: KVCacheSpec = KVCacheSpec.merge(list(kv_cache_spec_by_layer.values()))
self.kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=[
KVCacheGroupSpec(layer_names=list(kv_cache_spec_by_layer.keys()), kv_cache_spec=merged_spec)
],
)
# Init backend + bind KV cache tensors to attention modules.
self.attn_backends, self.attn_metadata_builders = attn_utils.init_attn_backend(
self.kv_cache_config, vllm_config, device
)
self.runner_kv_caches: list[torch.Tensor] = []
attn_utils.init_kv_cache(
self.runner_kv_caches,
vllm_config.compilation_config.static_forward_context,
self.kv_cache_config,
self.attn_backends,
device,
)
# Precompute a fixed block table mapping for the maximum batch.
self.block_size = block_size
self.blocks_per_seq = blocks_per_seq
self.max_batch_size = int(max_batch_size)
bt = torch.full((self.max_batch_size, self.blocks_per_seq), -1, dtype=torch.int32, device=device)
for i in range(self.max_batch_size):
for j in range(self.blocks_per_seq):
bt[i, j] = i * self.blocks_per_seq + j
self._block_table = bt
def build_attn_metadata(
self,
*,
num_reqs: int,
query_lens: torch.Tensor, # (num_reqs,) int32 on cpu
seq_lens: torch.Tensor, # (num_reqs,) int32 on cpu
) -> tuple[dict[str, Any], torch.Tensor, dict[str, torch.Tensor]]:
"""Build attention metadata, positions, and slot_mapping dict.
Returns:
(attn_metadata, positions, slot_mappings_by_layer)
- attn_metadata: per-layer attention metadata for attn backends.
- positions: (num_tokens,) position IDs on device.
- slot_mappings_by_layer: {layer_name: slot_mapping_tensor} for
set_forward_context so that unified_kv_cache_update can write
the KV cache correctly.
"""
num_reqs = int(num_reqs)
if num_reqs <= 0:
return {}, torch.empty((0,), dtype=torch.int64, device=self.device), {}
if num_reqs > self.max_batch_size:
raise ValueError(f"num_reqs={num_reqs} exceeds local predictor max_batch_size={self.max_batch_size}")
query_lens_i32 = query_lens.to(dtype=torch.int32, device="cpu")
seq_lens_i32 = seq_lens.to(dtype=torch.int32, device="cpu")
# query_start_loc: prefix sums of query_lens.
qsl = torch.zeros((num_reqs + 1,), dtype=torch.int32, device="cpu")
qsl[1:] = torch.cumsum(query_lens_i32, dim=0)
num_tokens = int(qsl[-1].item())
if num_tokens <= 0:
return {}, torch.empty((0,), dtype=torch.int64, device=self.device), {}
# positions: for each request i, emit positions [seq_len-query_len .. seq_len-1]
pos_list: list[torch.Tensor] = []
for i in range(num_reqs):
ql = int(query_lens_i32[i].item())
sl = int(seq_lens_i32[i].item())
start = sl - ql
pos_list.append(torch.arange(start, sl, dtype=torch.int64))
positions_cpu = torch.cat(pos_list, dim=0)
# slot_mapping: map each query token to a physical slot in the paged KV cache.
# We allocate per-request contiguous blocks; slot = base + position.
slot_mapping = torch.empty((num_tokens,), dtype=torch.int64, device="cpu")
cursor = 0
for i in range(num_reqs):
ql = int(query_lens_i32[i].item())
sl = int(seq_lens_i32[i].item())
start = sl - ql
for p in range(start, sl):
block_idx = p // self.block_size
offset = p % self.block_size
block_id = int(self._block_table[i, block_idx].item())
slot_mapping[cursor] = block_id * self.block_size + offset
cursor += 1
max_seq_len = int(seq_lens_i32[:num_reqs].max().item())
query_start_loc_gpu = qsl.to(device=self.device)
seq_lens_gpu = seq_lens_i32.to(device=self.device)
block_table = self._block_table[:num_reqs].contiguous()
slot_mapping_gpu = slot_mapping.to(device=self.device)
attn_metadata = attn_utils.build_attn_metadata(
self.attn_metadata_builders,
num_reqs=num_reqs,
num_tokens=num_tokens,
query_start_loc_gpu=query_start_loc_gpu,
query_start_loc_cpu=qsl,
seq_lens=seq_lens_gpu,
max_seq_len=max_seq_len,
block_tables=[block_table],
slot_mappings=[slot_mapping_gpu],
kv_cache_config=self.kv_cache_config,
)
# Build slot_mappings_by_layer for set_forward_context.
# Fix for vllm 0.15.0
slot_mappings_by_layer: dict[str, torch.Tensor] = {}
for kv_cache_group in self.kv_cache_config.kv_cache_groups:
for layer_name in kv_cache_group.layer_names:
slot_mappings_by_layer[layer_name] = slot_mapping_gpu
return attn_metadata, positions_cpu.to(device=self.device), slot_mappings_by_layer
class Qwen3TTSTalkerCodePredictorModelVLLM(nn.Module):
def __init__(
self,
config: Qwen3TTSTalkerCodePredictorConfig,
*,
talker_hidden_size: int | None = None,
cache_config=None,
quant_config=None,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.quant_config = quant_config
self.layers = nn.ModuleList(
[
Qwen3DecoderLayer(
config, cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.layers.{i}"
)
for i in range(config.num_hidden_layers)
]
)
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# Official code_predictor uses one embedding table per residual group.
# Some Qwen3-TTS checkpoints store codec embeddings in the talker hidden
# space, even when `code_predictor_config.hidden_size` is smaller.
# We keep the embedding dim aligned with the checkpoint and project down
# via `small_to_mtp_projection` in the wrapper module.
emb_dim = int(talker_hidden_size) if talker_hidden_size is not None else int(config.hidden_size)
self.codec_embedding = nn.ModuleList(
[nn.Embedding(config.vocab_size, emb_dim) for _ in range(config.num_code_groups - 1)]
)
def get_input_embeddings(self) -> nn.ModuleList:
return self.codec_embedding
def forward(self, positions: torch.Tensor, inputs_embeds: torch.Tensor) -> torch.Tensor:
# Token-major: [num_tokens, hidden]
hidden_states = inputs_embeds
residual = None
for layer in self.layers:
hidden_states, residual = layer(positions, hidden_states, residual)
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Match vLLM Qwen2/Qwen3 packing conventions: q_proj/k_proj/v_proj -> qkv_proj,
# gate_proj/up_proj -> gate_up_proj.
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):
param = params_dict[scale_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
weight_loader(param, loaded_weight)
loaded_params.add(scale_name)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
mapped = name.replace(weight_name, param_name)
if mapped.endswith(".bias") and mapped not in params_dict:
continue
if is_pp_missing_parameter(mapped, self):
continue
if mapped.endswith("scale"):
mapped = maybe_remap_kv_scale_name(mapped, params_dict)
if mapped is None:
continue
param = params_dict.get(mapped)
if param is None:
continue
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(mapped)
break
else:
mapped = maybe_remap_kv_scale_name(name, params_dict)
if mapped is None:
continue
if name.endswith(".bias") and mapped not in params_dict:
continue
if is_pp_missing_parameter(mapped, self):
continue
param = params_dict.get(mapped)
if param is None:
continue
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(mapped)
return loaded_params
class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
"""vLLM-native code_predictor used by the AR talker (residual codebooks)."""
def __init__(
self,
*,
vllm_config: VllmConfig,
config: Qwen3TTSTalkerCodePredictorConfig,
talker_config: Qwen3TTSTalkerConfig,
prefix: str = "code_predictor",
) -> None:
super().__init__()
self._vllm_config = vllm_config
self.config = config
self.talker_config = talker_config
# Keep module/weight names aligned with official checkpoint (talker.code_predictor.model.*).
self.model = Qwen3TTSTalkerCodePredictorModelVLLM(
config,
talker_hidden_size=int(talker_config.hidden_size),
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=f"{prefix}.model",
)
# One head per residual group.
self.lm_head = nn.ModuleList(
[nn.Linear(config.hidden_size, config.vocab_size, bias=False) for _ in range(config.num_code_groups - 1)]
)
if config.hidden_size != talker_config.hidden_size:
self.small_to_mtp_projection = nn.Linear(talker_config.hidden_size, config.hidden_size, bias=True)
else:
self.small_to_mtp_projection = nn.Identity()
self._kv_cache: _LocalPredictorKVCache | None = None
def get_input_embeddings(self) -> nn.ModuleList:
return self.model.get_input_embeddings()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Ensure all vLLM custom layers consult the predictor vllm_config
# (esp. for Attention static_forward_context).
with set_current_vllm_config(self._vllm_config):
loaded: set[str] = set()
model_weights: list[tuple[str, torch.Tensor]] = []
other_weights: list[tuple[str, torch.Tensor]] = []
for name, w in weights:
if name.startswith("model."):
model_weights.append((name[len("model.") :], w))
else:
other_weights.append((name, w))
loaded_model = self.model.load_weights(model_weights)
loaded |= {f"model.{n}" for n in loaded_model}
params = dict(self.named_parameters(remove_duplicate=False))
for name, w in other_weights:
if name not in params:
continue
default_weight_loader(params[name], w)
loaded.add(name)
return loaded
def _maybe_init_kv_cache(self, device: torch.device) -> None:
if self._kv_cache is not None:
return
max_seq_len = int(getattr(self.config, "num_code_groups", 16) or 16)
# Upper bound on batch size: vLLM scheduler max_num_seqs (fallback 8).
max_batch = int(getattr(self._vllm_config.scheduler_config, "max_num_seqs", 8) or 8)
max_batch = max(1, max_batch)
self._kv_cache = _LocalPredictorKVCache(
vllm_config=self._vllm_config,
max_seq_len=max_seq_len,
max_batch_size=max_batch,
device=device,
)
@torch.inference_mode()
def reset_cache(self) -> None:
# We reuse a fixed kv cache buffer and overwrite starting at slot 0.
# No action required here (seq_lens controls what is read).
return
@torch.inference_mode()
def prefill_logits(self, inputs_embeds: torch.Tensor) -> torch.Tensor:
"""Prefill with 2 tokens: [past_hidden, layer0_embed]. Returns logits for residual group 0."""
self._maybe_init_kv_cache(inputs_embeds.device)
assert self._kv_cache is not None
bsz = int(inputs_embeds.shape[0])
qlen = 2
# Flatten to token-major.
hs = inputs_embeds.to(dtype=torch.bfloat16).reshape(bsz * qlen, -1)
hs = self.small_to_mtp_projection(hs)
query_lens = torch.full((bsz,), qlen, dtype=torch.int32)
seq_lens = query_lens.clone()
attn_metadata, positions, slot_mappings = self._kv_cache.build_attn_metadata(
num_reqs=bsz, query_lens=query_lens, seq_lens=seq_lens
)
with (
set_current_vllm_config(self._vllm_config),
set_forward_context(
attn_metadata,
self._vllm_config,
num_tokens=int(hs.shape[0]),
slot_mapping=slot_mappings,
),
):
out = self.model(positions=positions, inputs_embeds=hs)
# Gather last token per request.
last_idx = torch.arange(qlen - 1, bsz * qlen, step=qlen, device=out.device, dtype=torch.long)
last_h = out.index_select(0, last_idx)
logits = self.lm_head[0](last_h)
return logits
@torch.inference_mode()
def decode_logits(self, input_ids: torch.Tensor, *, generation_step: int, past_seq_len: int) -> torch.Tensor:
"""Decode one new token for residual group `generation_step` (1..Q-1)."""
self._maybe_init_kv_cache(input_ids.device)
assert self._kv_cache is not None
bsz = int(input_ids.shape[0])
if generation_step <= 0:
raise ValueError("generation_step must be >= 1 for decode_logits")
embed_idx = generation_step - 1
hs = self.model.get_input_embeddings()[embed_idx](input_ids.to(dtype=torch.long).reshape(bsz, 1))
hs = self.small_to_mtp_projection(hs.reshape(bsz, -1))
query_lens = torch.ones((bsz,), dtype=torch.int32)
seq_lens = torch.full((bsz,), int(past_seq_len) + 1, dtype=torch.int32)
attn_metadata, positions, slot_mappings = self._kv_cache.build_attn_metadata(
num_reqs=bsz, query_lens=query_lens, seq_lens=seq_lens
)
with (
set_current_vllm_config(self._vllm_config),
set_forward_context(
attn_metadata,
self._vllm_config,
num_tokens=int(hs.shape[0]),
slot_mapping=slot_mappings,
),
):
out = self.model(positions=positions, inputs_embeds=hs)
logits = self.lm_head[generation_step](out)
return logits
@torch.inference_mode()
def forward(
self,
layer0_code: torch.Tensor,
layer0_embed: torch.Tensor,
last_talker_hidden: torch.Tensor,
do_sample: bool = True,
temperature: float = 1.0,
top_k: int = 50,
top_p: float = 1.0,
) -> torch.Tensor:
"""Full autoregressive prediction of residual codebooks 1..Q-1.
Args:
layer0_code: [B, 1] first-layer codec token ids.
layer0_embed: [B, 1, H] embedding of layer0_code.
last_talker_hidden: [B, 1, H] hidden state from the talker.
do_sample: whether to sample or take argmax.
temperature: sampling temperature.
top_k: top-k filtering.
top_p: top-p (nucleus) filtering.
Returns:
audio_codes: [B, Q] all codebook tokens (layer0 + residuals).
"""
bsz = int(layer0_code.shape[0])
num_groups = int(self.config.num_code_groups)
max_steps = num_groups - 1
# Reset KV cache for a fresh sequence.
self.reset_cache()
# Prefill: feed [last_talker_hidden, layer0_embed] → logits for group 1.
prefill_input = torch.cat([last_talker_hidden, layer0_embed], dim=1) # [B, 2, H]
logits = self.prefill_logits(prefill_input) # [B, vocab]
all_codes = [layer0_code.reshape(bsz, 1)]
past_seq_len = 2
for step in range(1, num_groups):
# Sample or argmax from logits.
if do_sample and temperature > 0:
scaled = logits / temperature
if top_k > 0:
topk_vals, _ = scaled.topk(top_k, dim=-1)
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
probs = torch.softmax(scaled, dim=-1)
next_ids = torch.multinomial(probs, num_samples=1) # [B, 1]
else:
next_ids = logits.argmax(dim=-1, keepdim=True) # [B, 1]
all_codes.append(next_ids)
# If not the last step, decode one more token.
if step < max_steps:
logits = self.decode_logits(
next_ids.reshape(bsz),
generation_step=step,
past_seq_len=past_seq_len,
)
past_seq_len += 1
return torch.cat(all_codes, dim=1) # [B, Q]
File diff suppressed because it is too large. Load diff
@@ -80,16 +80,21 @@ class Qwen3TTSTokenizer:
"""
inst = cls()
AutoConfig.register("qwen3_tts_tokenizer_25hz", Qwen3TTSTokenizerV1Config)
AutoModel.register(Qwen3TTSTokenizerV1Config, Qwen3TTSTokenizerV1Model)
load_feature_extractor = bool(kwargs.pop("load_feature_extractor", True))
AutoConfig.register("qwen3_tts_tokenizer_12hz", Qwen3TTSTokenizerV2Config)
AutoModel.register(Qwen3TTSTokenizerV2Config, Qwen3TTSTokenizerV2Model)
inst.feature_extractor = AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)
AutoConfig.register("qwen3_tts_tokenizer_25hz", Qwen3TTSTokenizerV1Config)
AutoModel.register(Qwen3TTSTokenizerV1Config, Qwen3TTSTokenizerV1Model)
inst.model = AutoModel.from_pretrained(pretrained_model_name_or_path, **kwargs)
inst.config = inst.model.config
inst.feature_extractor = (
AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path) if load_feature_extractor else None
)
inst.device = getattr(inst.model, "device", None)
if inst.device is None:
# fallback: infer from first parameter device
@@ -212,12 +217,7 @@ class Qwen3TTSTokenizer:
audios: AudioInput,
sr: int | None = None,
return_dict: bool = True,
) -> (
Qwen3TTSTokenizerV1EncoderOutput
| Qwen3TTSTokenizerV2EncoderOutput
| tuple[list[torch.Tensor], list[torch.Tensor] | None, list[torch.Tensor] | None]
| tuple[list[torch.Tensor]]
):
) -> Qwen3TTSTokenizerV1EncoderOutput | Qwen3TTSTokenizerV2EncoderOutput | tuple:
"""
Batch-encode audio into discrete codes (and optional conditioning, depending on 25Hz/12Hz).
+11 -1
View File
@@ -50,8 +50,18 @@ _OMNI_MODELS = {
),
"Qwen3TTSForConditionalGeneration": (
"qwen3_tts",
"qwen3_tts_talker",
"Qwen3TTSTalkerForConditionalGeneration",
),
"Qwen3TTSTalkerForConditionalGeneration": (
"qwen3_tts",
"Qwen3TTSModelForGeneration",
"qwen3_tts_talker",
"Qwen3TTSTalkerForConditionalGeneration",
),
"Qwen3TTSCode2Wav": (
"qwen3_tts",
"qwen3_tts_code2wav",
"Qwen3TTSCode2Wav",
),
}