[Feature] : Support disaggregated inference pipeline for Qwen3_TTS (#1161)
Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
5295f0f81b
commit
efbe4116ff
23 files changed
+2970
-2592
No files matched your search
@@ -504,19 +504,39 @@ class Qwen3TTSConfig(PretrainedConfig):
|
||||
self.tts_bos_token_id = tts_bos_token_id
|
||||
self.tts_eos_token_id = tts_eos_token_id
|
||||
|
||||
# TODO: remove these dummy values after
|
||||
self.image_token_id = 0 # dummy image token id
|
||||
self.video_token_id = 0 # dummy video token id
|
||||
self.vision_start_token_id = 0 # dummy vision start token id
|
||||
# Dummy vision token IDs that must never collide with real codec tokens.
|
||||
# mrope scans prompt_token_ids for these; using -1 ensures no false match.
|
||||
self.image_token_id = -1
|
||||
self.video_token_id = -1
|
||||
self.vision_start_token_id = -1
|
||||
self.vision_config = PretrainedConfig() # dummy vision config
|
||||
self.vision_config.spatial_merge_size = 1
|
||||
|
||||
@property
|
||||
def codec_frame_rate_hz(self) -> float | None:
|
||||
pos_per_sec = getattr(self.talker_config, "position_id_per_seconds", None)
|
||||
if pos_per_sec is None:
|
||||
return None
|
||||
try:
|
||||
fps = float(pos_per_sec)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return fps if fps > 0 else None
|
||||
|
||||
def get_text_config(self, **kwargs):
|
||||
# vLLM expects text config to expose hidden_size/num_attention_heads.
|
||||
# For Qwen3 TTS, the talker config is the text model config.
|
||||
config = self.talker_config
|
||||
# if hasattr(config, "rope_parameters"):
|
||||
# delattr(config, "rope_parameters")
|
||||
# Code2Wav is a pure convolutional waveform decoder; it does NOT use
|
||||
# rotary position embeddings. When hf_overrides sets architectures
|
||||
# to [Qwen3TTSCode2Wav], strip rope_parameters so that the model
|
||||
# runner sees uses_mrope == False and skips mrope position computation
|
||||
# on codec tokens. Each stage loads its own config instance, so this
|
||||
# in-place mutation does not affect the Talker stage.
|
||||
archs = getattr(self, "architectures", []) or []
|
||||
if any("Code2Wav" in str(a) for a in archs):
|
||||
if hasattr(config, "rope_parameters"):
|
||||
delattr(config, "rope_parameters")
|
||||
return config
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,102 +0,0 @@
|
||||
# Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin
|
||||
|
||||
|
||||
class Qwen3TTSProcessorKwargs(ProcessingKwargs, total=False):
|
||||
_defaults = {
|
||||
"text_kwargs": {
|
||||
"padding": False,
|
||||
"padding_side": "left",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class Qwen3TTSProcessor(ProcessorMixin):
|
||||
r"""
|
||||
Constructs a Qwen3TTS processor.
|
||||
|
||||
Args:
|
||||
tokenizer ([`Qwen2TokenizerFast`], *optional*):
|
||||
The text tokenizer.
|
||||
chat_template (`Optional[str]`, *optional*):
|
||||
The Jinja template to use for formatting the conversation.
|
||||
If not provided, the default chat template is used.
|
||||
"""
|
||||
|
||||
attributes = ["tokenizer"]
|
||||
tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
|
||||
|
||||
def __init__(self, tokenizer=None, chat_template=None):
|
||||
super().__init__(tokenizer, chat_template=chat_template)
|
||||
|
||||
def __call__(self, text=None, **kwargs) -> BatchFeature:
|
||||
"""
|
||||
Main method to prepare for the model one or several sequences(s) and audio(s).
|
||||
This method forwards the `text` and `kwargs` arguments to
|
||||
Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None`
|
||||
to encode the text.
|
||||
|
||||
Args:
|
||||
text (`str`, `List[str]`, `List[List[str]]`):
|
||||
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
|
||||
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
|
||||
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
|
||||
"""
|
||||
|
||||
if text is None:
|
||||
raise ValueError("You need to specify either a `text` input to process.")
|
||||
|
||||
output_kwargs = self._merge_kwargs(
|
||||
Qwen3TTSProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
texts_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
|
||||
|
||||
return BatchFeature(
|
||||
data={**texts_inputs},
|
||||
tensor_type=kwargs.get("return_tensors"),
|
||||
)
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
|
||||
refer to the docstring of this method for more information.
|
||||
"""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
|
||||
the docstring of this method for more information.
|
||||
"""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
def apply_chat_template(self, conversations, chat_template=None, **kwargs):
|
||||
if isinstance(conversations[0], dict):
|
||||
conversations = [conversations]
|
||||
return super().apply_chat_template(conversations, chat_template, **kwargs)
|
||||
|
||||
@property
|
||||
def model_input_names(self):
|
||||
tokenizer_input_names = self.tokenizer.model_input_names
|
||||
return list(dict.fromkeys(tokenizer_input_names))
|
||||
|
||||
|
||||
__all__ = ["Qwen3TTSProcessor"]
|
||||
@@ -0,0 +1,239 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers.utils.hub import cached_file
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
from .qwen3_tts_tokenizer import Qwen3TTSTokenizer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Qwen3TTSCode2Wav(nn.Module):
|
||||
"""Stage-1 code2wav model for Qwen3-TTS (GenerationModelRunner).
|
||||
Consumes frame-aligned codec tokens from input_ids and decodes waveform via SpeechTokenizer."""
|
||||
|
||||
input_modalities = "audio"
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = vllm_config.model_config.model
|
||||
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
# Generation-only stage (no logits / sampling).
|
||||
self.requires_raw_input_tokens = True
|
||||
|
||||
self._speech_tokenizer: Qwen3TTSTokenizer | None = None
|
||||
self._num_quantizers: int | None = None
|
||||
self._decode_upsample_rate: int | None = None
|
||||
self._output_sample_rate: int | None = None
|
||||
self._logged_codec_stats = False
|
||||
|
||||
@staticmethod
|
||||
def _module_device(module: nn.Module) -> torch.device:
|
||||
try:
|
||||
return next(module.parameters()).device
|
||||
except StopIteration:
|
||||
for _, buf in module.named_buffers(recurse=True):
|
||||
return buf.device
|
||||
return torch.device("cpu")
|
||||
|
||||
def _ensure_speech_tokenizer_loaded(self) -> Qwen3TTSTokenizer:
|
||||
if self._speech_tokenizer is not None:
|
||||
return self._speech_tokenizer
|
||||
|
||||
# Locate speech_tokenizer dir from HF cache (or local path).
|
||||
cfg_path = cached_file(self.model_path, "speech_tokenizer/config.json")
|
||||
if cfg_path is None:
|
||||
raise ValueError(f"{self.model_path}/speech_tokenizer/config.json not found")
|
||||
speech_tokenizer_dir = os.path.dirname(cfg_path)
|
||||
|
||||
# Stage-1 only needs decode; skip HF feature extractor to avoid heavy optional deps.
|
||||
# Still require preprocessor_config.json (use cached_file so online runs can fetch it).
|
||||
prep_cfg = cached_file(self.model_path, "speech_tokenizer/preprocessor_config.json")
|
||||
if prep_cfg is None:
|
||||
raise ValueError(
|
||||
f"{self.model_path}/speech_tokenizer/preprocessor_config.json not found. "
|
||||
"Please make sure the checkpoint contains the required HF preprocessing files."
|
||||
)
|
||||
|
||||
tok = Qwen3TTSTokenizer.from_pretrained(
|
||||
speech_tokenizer_dir,
|
||||
torch_dtype=torch.bfloat16,
|
||||
load_feature_extractor=False,
|
||||
)
|
||||
|
||||
# Align device with vLLM worker, then read back from module.
|
||||
if tok.model is not None:
|
||||
tok.model.to(device=self.vllm_config.device_config.device)
|
||||
tok.device = self._module_device(tok.model)
|
||||
|
||||
# Derive codec group count and rates from tokenizer config.
|
||||
dec_cfg = getattr(tok.model.config, "decoder_config", None)
|
||||
num_q = getattr(dec_cfg, "num_quantizers", None) if dec_cfg is not None else None
|
||||
if num_q is None:
|
||||
raise ValueError("speech_tokenizer decoder_config.num_quantizers not found")
|
||||
num_q = int(num_q)
|
||||
if num_q <= 0:
|
||||
raise ValueError(f"Invalid speech_tokenizer num_quantizers={num_q}")
|
||||
|
||||
try:
|
||||
upsample = int(tok.get_decode_upsample_rate())
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to get decode upsample rate: {e}") from e
|
||||
if upsample <= 0:
|
||||
raise ValueError(f"Invalid decode upsample rate: {upsample}")
|
||||
|
||||
try:
|
||||
out_sr = int(tok.get_output_sample_rate())
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to get output sample rate: {e}") from e
|
||||
|
||||
self._speech_tokenizer = tok
|
||||
self._num_quantizers = num_q
|
||||
self._decode_upsample_rate = upsample
|
||||
self._output_sample_rate = out_sr
|
||||
return tok
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
# This stage ignores token embeddings. Keep a stable dummy embedding for vLLM runner.
|
||||
if input_ids.numel() == 0:
|
||||
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
|
||||
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> None:
|
||||
return None
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: Any,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Decode codec codes into audio waveform.
|
||||
|
||||
input_ids layout: [codec_context_frames, *flat_codes]
|
||||
where flat_codes is codebook-major [q*F].
|
||||
"""
|
||||
tok = self._ensure_speech_tokenizer_loaded()
|
||||
assert self._num_quantizers is not None
|
||||
assert self._output_sample_rate is not None
|
||||
|
||||
sr_val = self._output_sample_rate
|
||||
empty_ret = (
|
||||
torch.zeros((0,), dtype=torch.float32),
|
||||
torch.tensor(sr_val, dtype=torch.int32),
|
||||
)
|
||||
|
||||
if input_ids is None:
|
||||
return empty_ret
|
||||
|
||||
q = int(self._num_quantizers)
|
||||
ids = input_ids.reshape(-1).to(dtype=torch.long)
|
||||
n_tokens = ids.numel()
|
||||
|
||||
if n_tokens == 0:
|
||||
return empty_ret
|
||||
|
||||
# input_ids[0] = codec_context_frames (prepended by stage_input_processor).
|
||||
ctx_frames = int(ids[0].item())
|
||||
ids = ids[1:]
|
||||
n_tokens = ids.numel()
|
||||
|
||||
if n_tokens == 0:
|
||||
return empty_ret
|
||||
|
||||
# Warmup / dummy_run: not divisible by num_quantizers.
|
||||
if n_tokens % q != 0:
|
||||
logger.warning(
|
||||
"Code2Wav input_ids length %d not divisible by num_quantizers %d, "
|
||||
"likely a warmup run; returning empty audio.",
|
||||
n_tokens,
|
||||
q,
|
||||
)
|
||||
return empty_ret
|
||||
|
||||
total_frames = n_tokens // q
|
||||
|
||||
# Reshape codebook-major flat [q*F] -> [q, F] -> [F, q] for SpeechTokenizer.
|
||||
codes_fq = ids.reshape(q, total_frames).transpose(0, 1).contiguous()
|
||||
|
||||
if not self._logged_codec_stats and total_frames > 1:
|
||||
self._logged_codec_stats = True
|
||||
try:
|
||||
uniq = int(torch.unique(codes_fq).numel())
|
||||
cmin = int(codes_fq.min().item())
|
||||
cmax = int(codes_fq.max().item())
|
||||
head = codes_fq[: min(2, total_frames), : min(8, q)].cpu().tolist()
|
||||
logger.info(
|
||||
"Code2Wav codec: frames=%d q=%d uniq=%d range=[%d,%d] head=%s",
|
||||
total_frames,
|
||||
q,
|
||||
uniq,
|
||||
cmin,
|
||||
cmax,
|
||||
head,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
wavs, sr = tok.decode({"audio_codes": codes_fq})
|
||||
if not wavs:
|
||||
raise ValueError("SpeechTokenizer code2wav produced empty waveform list.")
|
||||
audio_np = wavs[0].astype(np.float32, copy=False)
|
||||
|
||||
# Trim left-context waveform samples (streaming sliding window).
|
||||
if ctx_frames > 0:
|
||||
upsample = self._decode_upsample_rate
|
||||
if upsample is None or upsample <= 0:
|
||||
raise ValueError(f"Invalid decode upsample rate: {upsample}")
|
||||
cut = ctx_frames * upsample
|
||||
if cut < audio_np.shape[0]:
|
||||
audio_np = audio_np[cut:]
|
||||
else:
|
||||
logger.warning(
|
||||
"Context trim %d >= decoded length %d; returning empty audio.",
|
||||
cut,
|
||||
audio_np.shape[0],
|
||||
)
|
||||
return empty_ret
|
||||
|
||||
audio_tensor = torch.from_numpy(audio_np).to(dtype=torch.float32).reshape(-1)
|
||||
sr_tensor = torch.tensor(int(sr), dtype=torch.int32)
|
||||
return audio_tensor, sr_tensor
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
|
||||
if not (isinstance(model_outputs, tuple) and len(model_outputs) == 2):
|
||||
raise TypeError(f"Qwen3TTSCode2Wav expected (audio_tensor, sr) outputs, got {type(model_outputs)}")
|
||||
|
||||
audio_tensor, sr = model_outputs
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={
|
||||
"model_outputs": audio_tensor,
|
||||
"sr": sr,
|
||||
},
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# SpeechTokenizer weights live under `speech_tokenizer/` and are loaded
|
||||
# lazily from that directory. Ignore main checkpoint weights.
|
||||
return set()
|
||||
@@ -0,0 +1,495 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3 import Qwen3DecoderLayer
|
||||
from vllm.model_executor.models.utils import is_pp_missing_parameter
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheGroupSpec, KVCacheSpec, KVCacheTensor
|
||||
from vllm.v1.worker.gpu import attn_utils
|
||||
|
||||
from .configuration_qwen3_tts import Qwen3TTSTalkerCodePredictorConfig, Qwen3TTSTalkerConfig
|
||||
|
||||
|
||||
class _LocalPredictorKVCache:
|
||||
"""Minimal local KV cache + attention metadata for running
|
||||
code_predictor inside one worker (independent of engine KV)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
max_seq_len: int,
|
||||
max_batch_size: int,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
|
||||
# Collect attention layers registered in this vllm_config.
|
||||
kv_cache_spec_by_layer = attn_utils.get_kv_cache_spec(vllm_config)
|
||||
if not kv_cache_spec_by_layer:
|
||||
raise RuntimeError("Local predictor KVCache requires vLLM Attention layers to be registered.")
|
||||
|
||||
# We only need enough blocks for a tiny per-frame sequence (<= max_seq_len).
|
||||
any_spec = next(iter(kv_cache_spec_by_layer.values()))
|
||||
block_size = int(any_spec.block_size)
|
||||
blocks_per_seq = (int(max_seq_len) + block_size - 1) // block_size
|
||||
num_blocks = max(1, int(max_batch_size) * int(blocks_per_seq))
|
||||
|
||||
# Allocate per-layer KV caches (small, independent).
|
||||
kv_cache_tensors: list[KVCacheTensor] = []
|
||||
for layer_name, spec in kv_cache_spec_by_layer.items():
|
||||
kv_cache_tensors.append(KVCacheTensor(size=int(spec.page_size_bytes) * num_blocks, shared_by=[layer_name]))
|
||||
|
||||
merged_spec: KVCacheSpec = KVCacheSpec.merge(list(kv_cache_spec_by_layer.values()))
|
||||
self.kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=kv_cache_tensors,
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=list(kv_cache_spec_by_layer.keys()), kv_cache_spec=merged_spec)
|
||||
],
|
||||
)
|
||||
|
||||
# Init backend + bind KV cache tensors to attention modules.
|
||||
self.attn_backends, self.attn_metadata_builders = attn_utils.init_attn_backend(
|
||||
self.kv_cache_config, vllm_config, device
|
||||
)
|
||||
self.runner_kv_caches: list[torch.Tensor] = []
|
||||
attn_utils.init_kv_cache(
|
||||
self.runner_kv_caches,
|
||||
vllm_config.compilation_config.static_forward_context,
|
||||
self.kv_cache_config,
|
||||
self.attn_backends,
|
||||
device,
|
||||
)
|
||||
|
||||
# Precompute a fixed block table mapping for the maximum batch.
|
||||
self.block_size = block_size
|
||||
self.blocks_per_seq = blocks_per_seq
|
||||
self.max_batch_size = int(max_batch_size)
|
||||
|
||||
bt = torch.full((self.max_batch_size, self.blocks_per_seq), -1, dtype=torch.int32, device=device)
|
||||
for i in range(self.max_batch_size):
|
||||
for j in range(self.blocks_per_seq):
|
||||
bt[i, j] = i * self.blocks_per_seq + j
|
||||
self._block_table = bt
|
||||
|
||||
def build_attn_metadata(
|
||||
self,
|
||||
*,
|
||||
num_reqs: int,
|
||||
query_lens: torch.Tensor, # (num_reqs,) int32 on cpu
|
||||
seq_lens: torch.Tensor, # (num_reqs,) int32 on cpu
|
||||
) -> tuple[dict[str, Any], torch.Tensor, dict[str, torch.Tensor]]:
|
||||
"""Build attention metadata, positions, and slot_mapping dict.
|
||||
|
||||
Returns:
|
||||
(attn_metadata, positions, slot_mappings_by_layer)
|
||||
- attn_metadata: per-layer attention metadata for attn backends.
|
||||
- positions: (num_tokens,) position IDs on device.
|
||||
- slot_mappings_by_layer: {layer_name: slot_mapping_tensor} for
|
||||
set_forward_context so that unified_kv_cache_update can write
|
||||
the KV cache correctly.
|
||||
"""
|
||||
num_reqs = int(num_reqs)
|
||||
if num_reqs <= 0:
|
||||
return {}, torch.empty((0,), dtype=torch.int64, device=self.device), {}
|
||||
if num_reqs > self.max_batch_size:
|
||||
raise ValueError(f"num_reqs={num_reqs} exceeds local predictor max_batch_size={self.max_batch_size}")
|
||||
|
||||
query_lens_i32 = query_lens.to(dtype=torch.int32, device="cpu")
|
||||
seq_lens_i32 = seq_lens.to(dtype=torch.int32, device="cpu")
|
||||
|
||||
# query_start_loc: prefix sums of query_lens.
|
||||
qsl = torch.zeros((num_reqs + 1,), dtype=torch.int32, device="cpu")
|
||||
qsl[1:] = torch.cumsum(query_lens_i32, dim=0)
|
||||
num_tokens = int(qsl[-1].item())
|
||||
if num_tokens <= 0:
|
||||
return {}, torch.empty((0,), dtype=torch.int64, device=self.device), {}
|
||||
|
||||
# positions: for each request i, emit positions [seq_len-query_len .. seq_len-1]
|
||||
pos_list: list[torch.Tensor] = []
|
||||
for i in range(num_reqs):
|
||||
ql = int(query_lens_i32[i].item())
|
||||
sl = int(seq_lens_i32[i].item())
|
||||
start = sl - ql
|
||||
pos_list.append(torch.arange(start, sl, dtype=torch.int64))
|
||||
positions_cpu = torch.cat(pos_list, dim=0)
|
||||
|
||||
# slot_mapping: map each query token to a physical slot in the paged KV cache.
|
||||
# We allocate per-request contiguous blocks; slot = base + position.
|
||||
slot_mapping = torch.empty((num_tokens,), dtype=torch.int64, device="cpu")
|
||||
cursor = 0
|
||||
for i in range(num_reqs):
|
||||
ql = int(query_lens_i32[i].item())
|
||||
sl = int(seq_lens_i32[i].item())
|
||||
start = sl - ql
|
||||
for p in range(start, sl):
|
||||
block_idx = p // self.block_size
|
||||
offset = p % self.block_size
|
||||
block_id = int(self._block_table[i, block_idx].item())
|
||||
slot_mapping[cursor] = block_id * self.block_size + offset
|
||||
cursor += 1
|
||||
|
||||
max_seq_len = int(seq_lens_i32[:num_reqs].max().item())
|
||||
query_start_loc_gpu = qsl.to(device=self.device)
|
||||
seq_lens_gpu = seq_lens_i32.to(device=self.device)
|
||||
block_table = self._block_table[:num_reqs].contiguous()
|
||||
slot_mapping_gpu = slot_mapping.to(device=self.device)
|
||||
|
||||
attn_metadata = attn_utils.build_attn_metadata(
|
||||
self.attn_metadata_builders,
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
query_start_loc_gpu=query_start_loc_gpu,
|
||||
query_start_loc_cpu=qsl,
|
||||
seq_lens=seq_lens_gpu,
|
||||
max_seq_len=max_seq_len,
|
||||
block_tables=[block_table],
|
||||
slot_mappings=[slot_mapping_gpu],
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
)
|
||||
|
||||
# Build slot_mappings_by_layer for set_forward_context.
|
||||
# Fix for vllm 0.15.0
|
||||
slot_mappings_by_layer: dict[str, torch.Tensor] = {}
|
||||
for kv_cache_group in self.kv_cache_config.kv_cache_groups:
|
||||
for layer_name in kv_cache_group.layer_names:
|
||||
slot_mappings_by_layer[layer_name] = slot_mapping_gpu
|
||||
|
||||
return attn_metadata, positions_cpu.to(device=self.device), slot_mappings_by_layer
|
||||
|
||||
|
||||
class Qwen3TTSTalkerCodePredictorModelVLLM(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen3TTSTalkerCodePredictorConfig,
|
||||
*,
|
||||
talker_hidden_size: int | None = None,
|
||||
cache_config=None,
|
||||
quant_config=None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
Qwen3DecoderLayer(
|
||||
config, cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.layers.{i}"
|
||||
)
|
||||
for i in range(config.num_hidden_layers)
|
||||
]
|
||||
)
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
# Official code_predictor uses one embedding table per residual group.
|
||||
# Some Qwen3-TTS checkpoints store codec embeddings in the talker hidden
|
||||
# space, even when `code_predictor_config.hidden_size` is smaller.
|
||||
# We keep the embedding dim aligned with the checkpoint and project down
|
||||
# via `small_to_mtp_projection` in the wrapper module.
|
||||
emb_dim = int(talker_hidden_size) if talker_hidden_size is not None else int(config.hidden_size)
|
||||
self.codec_embedding = nn.ModuleList(
|
||||
[nn.Embedding(config.vocab_size, emb_dim) for _ in range(config.num_code_groups - 1)]
|
||||
)
|
||||
|
||||
def get_input_embeddings(self) -> nn.ModuleList:
|
||||
return self.codec_embedding
|
||||
|
||||
def forward(self, positions: torch.Tensor, inputs_embeds: torch.Tensor) -> torch.Tensor:
|
||||
# Token-major: [num_tokens, hidden]
|
||||
hidden_states = inputs_embeds
|
||||
residual = None
|
||||
for layer in self.layers:
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Match vLLM Qwen2/Qwen3 packing conventions: q_proj/k_proj/v_proj -> qkv_proj,
|
||||
# gate_proj/up_proj -> gate_up_proj.
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):
|
||||
param = params_dict[scale_name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(scale_name)
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped = name.replace(weight_name, param_name)
|
||||
if mapped.endswith(".bias") and mapped not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(mapped, self):
|
||||
continue
|
||||
if mapped.endswith("scale"):
|
||||
mapped = maybe_remap_kv_scale_name(mapped, params_dict)
|
||||
if mapped is None:
|
||||
continue
|
||||
param = params_dict.get(mapped)
|
||||
if param is None:
|
||||
continue
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(mapped)
|
||||
break
|
||||
else:
|
||||
mapped = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if mapped is None:
|
||||
continue
|
||||
if name.endswith(".bias") and mapped not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(mapped, self):
|
||||
continue
|
||||
param = params_dict.get(mapped)
|
||||
if param is None:
|
||||
continue
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(mapped)
|
||||
return loaded_params
|
||||
|
||||
|
||||
class Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM(nn.Module):
|
||||
"""vLLM-native code_predictor used by the AR talker (residual codebooks)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
config: Qwen3TTSTalkerCodePredictorConfig,
|
||||
talker_config: Qwen3TTSTalkerConfig,
|
||||
prefix: str = "code_predictor",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._vllm_config = vllm_config
|
||||
self.config = config
|
||||
self.talker_config = talker_config
|
||||
|
||||
# Keep module/weight names aligned with official checkpoint (talker.code_predictor.model.*).
|
||||
self.model = Qwen3TTSTalkerCodePredictorModelVLLM(
|
||||
config,
|
||||
talker_hidden_size=int(talker_config.hidden_size),
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=f"{prefix}.model",
|
||||
)
|
||||
|
||||
# One head per residual group.
|
||||
self.lm_head = nn.ModuleList(
|
||||
[nn.Linear(config.hidden_size, config.vocab_size, bias=False) for _ in range(config.num_code_groups - 1)]
|
||||
)
|
||||
|
||||
if config.hidden_size != talker_config.hidden_size:
|
||||
self.small_to_mtp_projection = nn.Linear(talker_config.hidden_size, config.hidden_size, bias=True)
|
||||
else:
|
||||
self.small_to_mtp_projection = nn.Identity()
|
||||
|
||||
self._kv_cache: _LocalPredictorKVCache | None = None
|
||||
|
||||
def get_input_embeddings(self) -> nn.ModuleList:
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Ensure all vLLM custom layers consult the predictor vllm_config
|
||||
# (esp. for Attention static_forward_context).
|
||||
with set_current_vllm_config(self._vllm_config):
|
||||
loaded: set[str] = set()
|
||||
model_weights: list[tuple[str, torch.Tensor]] = []
|
||||
other_weights: list[tuple[str, torch.Tensor]] = []
|
||||
for name, w in weights:
|
||||
if name.startswith("model."):
|
||||
model_weights.append((name[len("model.") :], w))
|
||||
else:
|
||||
other_weights.append((name, w))
|
||||
|
||||
loaded_model = self.model.load_weights(model_weights)
|
||||
loaded |= {f"model.{n}" for n in loaded_model}
|
||||
|
||||
params = dict(self.named_parameters(remove_duplicate=False))
|
||||
for name, w in other_weights:
|
||||
if name not in params:
|
||||
continue
|
||||
default_weight_loader(params[name], w)
|
||||
loaded.add(name)
|
||||
return loaded
|
||||
|
||||
def _maybe_init_kv_cache(self, device: torch.device) -> None:
|
||||
if self._kv_cache is not None:
|
||||
return
|
||||
max_seq_len = int(getattr(self.config, "num_code_groups", 16) or 16)
|
||||
# Upper bound on batch size: vLLM scheduler max_num_seqs (fallback 8).
|
||||
max_batch = int(getattr(self._vllm_config.scheduler_config, "max_num_seqs", 8) or 8)
|
||||
max_batch = max(1, max_batch)
|
||||
self._kv_cache = _LocalPredictorKVCache(
|
||||
vllm_config=self._vllm_config,
|
||||
max_seq_len=max_seq_len,
|
||||
max_batch_size=max_batch,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def reset_cache(self) -> None:
|
||||
# We reuse a fixed kv cache buffer and overwrite starting at slot 0.
|
||||
# No action required here (seq_lens controls what is read).
|
||||
return
|
||||
|
||||
@torch.inference_mode()
|
||||
def prefill_logits(self, inputs_embeds: torch.Tensor) -> torch.Tensor:
|
||||
"""Prefill with 2 tokens: [past_hidden, layer0_embed]. Returns logits for residual group 0."""
|
||||
self._maybe_init_kv_cache(inputs_embeds.device)
|
||||
assert self._kv_cache is not None
|
||||
|
||||
bsz = int(inputs_embeds.shape[0])
|
||||
qlen = 2
|
||||
# Flatten to token-major.
|
||||
hs = inputs_embeds.to(dtype=torch.bfloat16).reshape(bsz * qlen, -1)
|
||||
hs = self.small_to_mtp_projection(hs)
|
||||
|
||||
query_lens = torch.full((bsz,), qlen, dtype=torch.int32)
|
||||
seq_lens = query_lens.clone()
|
||||
attn_metadata, positions, slot_mappings = self._kv_cache.build_attn_metadata(
|
||||
num_reqs=bsz, query_lens=query_lens, seq_lens=seq_lens
|
||||
)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(self._vllm_config),
|
||||
set_forward_context(
|
||||
attn_metadata,
|
||||
self._vllm_config,
|
||||
num_tokens=int(hs.shape[0]),
|
||||
slot_mapping=slot_mappings,
|
||||
),
|
||||
):
|
||||
out = self.model(positions=positions, inputs_embeds=hs)
|
||||
|
||||
# Gather last token per request.
|
||||
last_idx = torch.arange(qlen - 1, bsz * qlen, step=qlen, device=out.device, dtype=torch.long)
|
||||
last_h = out.index_select(0, last_idx)
|
||||
logits = self.lm_head[0](last_h)
|
||||
return logits
|
||||
|
||||
@torch.inference_mode()
|
||||
def decode_logits(self, input_ids: torch.Tensor, *, generation_step: int, past_seq_len: int) -> torch.Tensor:
|
||||
"""Decode one new token for residual group `generation_step` (1..Q-1)."""
|
||||
self._maybe_init_kv_cache(input_ids.device)
|
||||
assert self._kv_cache is not None
|
||||
bsz = int(input_ids.shape[0])
|
||||
if generation_step <= 0:
|
||||
raise ValueError("generation_step must be >= 1 for decode_logits")
|
||||
|
||||
embed_idx = generation_step - 1
|
||||
hs = self.model.get_input_embeddings()[embed_idx](input_ids.to(dtype=torch.long).reshape(bsz, 1))
|
||||
hs = self.small_to_mtp_projection(hs.reshape(bsz, -1))
|
||||
|
||||
query_lens = torch.ones((bsz,), dtype=torch.int32)
|
||||
seq_lens = torch.full((bsz,), int(past_seq_len) + 1, dtype=torch.int32)
|
||||
attn_metadata, positions, slot_mappings = self._kv_cache.build_attn_metadata(
|
||||
num_reqs=bsz, query_lens=query_lens, seq_lens=seq_lens
|
||||
)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(self._vllm_config),
|
||||
set_forward_context(
|
||||
attn_metadata,
|
||||
self._vllm_config,
|
||||
num_tokens=int(hs.shape[0]),
|
||||
slot_mapping=slot_mappings,
|
||||
),
|
||||
):
|
||||
out = self.model(positions=positions, inputs_embeds=hs)
|
||||
|
||||
logits = self.lm_head[generation_step](out)
|
||||
return logits
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
layer0_code: torch.Tensor,
|
||||
layer0_embed: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
do_sample: bool = True,
|
||||
temperature: float = 1.0,
|
||||
top_k: int = 50,
|
||||
top_p: float = 1.0,
|
||||
) -> torch.Tensor:
|
||||
"""Full autoregressive prediction of residual codebooks 1..Q-1.
|
||||
|
||||
Args:
|
||||
layer0_code: [B, 1] first-layer codec token ids.
|
||||
layer0_embed: [B, 1, H] embedding of layer0_code.
|
||||
last_talker_hidden: [B, 1, H] hidden state from the talker.
|
||||
do_sample: whether to sample or take argmax.
|
||||
temperature: sampling temperature.
|
||||
top_k: top-k filtering.
|
||||
top_p: top-p (nucleus) filtering.
|
||||
|
||||
Returns:
|
||||
audio_codes: [B, Q] all codebook tokens (layer0 + residuals).
|
||||
"""
|
||||
bsz = int(layer0_code.shape[0])
|
||||
num_groups = int(self.config.num_code_groups)
|
||||
max_steps = num_groups - 1
|
||||
|
||||
# Reset KV cache for a fresh sequence.
|
||||
self.reset_cache()
|
||||
|
||||
# Prefill: feed [last_talker_hidden, layer0_embed] → logits for group 1.
|
||||
prefill_input = torch.cat([last_talker_hidden, layer0_embed], dim=1) # [B, 2, H]
|
||||
logits = self.prefill_logits(prefill_input) # [B, vocab]
|
||||
|
||||
all_codes = [layer0_code.reshape(bsz, 1)]
|
||||
past_seq_len = 2
|
||||
|
||||
for step in range(1, num_groups):
|
||||
# Sample or argmax from logits.
|
||||
if do_sample and temperature > 0:
|
||||
scaled = logits / temperature
|
||||
if top_k > 0:
|
||||
topk_vals, _ = scaled.topk(top_k, dim=-1)
|
||||
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
|
||||
probs = torch.softmax(scaled, dim=-1)
|
||||
next_ids = torch.multinomial(probs, num_samples=1) # [B, 1]
|
||||
else:
|
||||
next_ids = logits.argmax(dim=-1, keepdim=True) # [B, 1]
|
||||
all_codes.append(next_ids)
|
||||
|
||||
# If not the last step, decode one more token.
|
||||
if step < max_steps:
|
||||
logits = self.decode_logits(
|
||||
next_ids.reshape(bsz),
|
||||
generation_step=step,
|
||||
past_seq_len=past_seq_len,
|
||||
)
|
||||
past_seq_len += 1
|
||||
|
||||
return torch.cat(all_codes, dim=1) # [B, Q]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -80,16 +80,21 @@ class Qwen3TTSTokenizer:
|
||||
"""
|
||||
inst = cls()
|
||||
|
||||
AutoConfig.register("qwen3_tts_tokenizer_25hz", Qwen3TTSTokenizerV1Config)
|
||||
AutoModel.register(Qwen3TTSTokenizerV1Config, Qwen3TTSTokenizerV1Model)
|
||||
load_feature_extractor = bool(kwargs.pop("load_feature_extractor", True))
|
||||
|
||||
AutoConfig.register("qwen3_tts_tokenizer_12hz", Qwen3TTSTokenizerV2Config)
|
||||
AutoModel.register(Qwen3TTSTokenizerV2Config, Qwen3TTSTokenizerV2Model)
|
||||
|
||||
inst.feature_extractor = AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)
|
||||
AutoConfig.register("qwen3_tts_tokenizer_25hz", Qwen3TTSTokenizerV1Config)
|
||||
AutoModel.register(Qwen3TTSTokenizerV1Config, Qwen3TTSTokenizerV1Model)
|
||||
|
||||
inst.model = AutoModel.from_pretrained(pretrained_model_name_or_path, **kwargs)
|
||||
inst.config = inst.model.config
|
||||
|
||||
inst.feature_extractor = (
|
||||
AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path) if load_feature_extractor else None
|
||||
)
|
||||
|
||||
inst.device = getattr(inst.model, "device", None)
|
||||
if inst.device is None:
|
||||
# fallback: infer from first parameter device
|
||||
@@ -212,12 +217,7 @@ class Qwen3TTSTokenizer:
|
||||
audios: AudioInput,
|
||||
sr: int | None = None,
|
||||
return_dict: bool = True,
|
||||
) -> (
|
||||
Qwen3TTSTokenizerV1EncoderOutput
|
||||
| Qwen3TTSTokenizerV2EncoderOutput
|
||||
| tuple[list[torch.Tensor], list[torch.Tensor] | None, list[torch.Tensor] | None]
|
||||
| tuple[list[torch.Tensor]]
|
||||
):
|
||||
) -> Qwen3TTSTokenizerV1EncoderOutput | Qwen3TTSTokenizerV2EncoderOutput | tuple:
|
||||
"""
|
||||
Batch-encode audio into discrete codes (and optional conditioning, depending on 25Hz/12Hz).
|
||||
|
||||
|
||||
@@ -50,8 +50,18 @@ _OMNI_MODELS = {
|
||||
),
|
||||
"Qwen3TTSForConditionalGeneration": (
|
||||
"qwen3_tts",
|
||||
"qwen3_tts_talker",
|
||||
"Qwen3TTSTalkerForConditionalGeneration",
|
||||
),
|
||||
"Qwen3TTSTalkerForConditionalGeneration": (
|
||||
"qwen3_tts",
|
||||
"Qwen3TTSModelForGeneration",
|
||||
"qwen3_tts_talker",
|
||||
"Qwen3TTSTalkerForConditionalGeneration",
|
||||
),
|
||||
"Qwen3TTSCode2Wav": (
|
||||
"qwen3_tts",
|
||||
"qwen3_tts_code2wav",
|
||||
"Qwen3TTSCode2Wav",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user