334 lines
15 KiB
Python
334 lines
15 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
"""Qwen3-TTS serving adapter."""
|
|
|
|
from collections.abc import Mapping
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from vllm.logger import init_logger
|
|
|
|
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
|
|
from vllm_omni.entrypoints.openai.tts_adapters.base import (
|
|
DEFAULT_TTS_LANGUAGES,
|
|
ARTTSAdapter,
|
|
PreparedRequest,
|
|
TTSGenerationError,
|
|
)
|
|
from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_precomputed_speakers
|
|
from vllm_omni.utils.speaker_cache import validate_qwen3_tts_profile
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
QWEN3_TTS_EFFECTIVE_MAX_TOKENS_KEY = "_qwen3_tts_effective_max_tokens"
|
|
_MIN_CODEC_FRAMES = 192
|
|
_MAX_CODEC_FRAMES_PER_TEXT_TOKEN = 12
|
|
|
|
|
|
class Qwen3TTSCodecLimitError(TTSGenerationError):
|
|
"""Qwen3-TTS Base exhausted its codec budget without emitting EOS."""
|
|
|
|
def __init__(self, message: str) -> None:
|
|
super().__init__(message, retryable=True)
|
|
|
|
|
|
@register_tts_adapter
|
|
class Qwen3TTSAdapter(ARTTSAdapter):
|
|
"""Adapter for Qwen3-TTS (AR ``engine_client`` backend)."""
|
|
|
|
validates_generation = True
|
|
stage_keys = frozenset({"qwen3_tts"})
|
|
name = "qwen3_tts"
|
|
|
|
def normalize(self, request: "OpenAICreateSpeechRequest") -> None:
|
|
"""Qwen3-TTS normalization (Base-task inference, voice lowercasing) is
|
|
performed inside ``validate`` today; kept fused for a strict behaviour
|
|
match."""
|
|
|
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
|
"""Validate Qwen TTS request parameters. Returns error message or None."""
|
|
# Infer Base task when ref_audio or ref_text is provided without explicit task_type.
|
|
server = self.ctx.server
|
|
if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None):
|
|
request.task_type = "Base"
|
|
|
|
# Normalize voice to lowercase for case-insensitive matching
|
|
if request.voice is not None:
|
|
request.voice = request.voice.lower()
|
|
if request.task_type is None and request.voice in self.capabilities.precomputed_speakers:
|
|
request.task_type = "Base"
|
|
task_type = request.task_type or "CustomVoice"
|
|
|
|
# Validate input is not empty
|
|
if not request.input or not request.input.strip():
|
|
return "Input text cannot be empty"
|
|
|
|
# Validate language (case-insensitive; normalized to the title-cased config form)
|
|
if request.language is not None:
|
|
request.language = request.language.title()
|
|
if request.language not in self.capabilities.supported_languages:
|
|
return (
|
|
f"Invalid language '{request.language}'. Supported: "
|
|
f"{', '.join(sorted(self.capabilities.supported_languages))}"
|
|
)
|
|
|
|
# Validate speaker for CustomVoice task
|
|
if task_type == "CustomVoice":
|
|
available_speakers = server._get_available_speakers()
|
|
if not available_speakers:
|
|
return (
|
|
"This model does not support CustomVoice task (no speakers configured). "
|
|
"Use task_type='Base' with ref_audio/ref_text for voice cloning, "
|
|
"or use a CustomVoice model."
|
|
)
|
|
if request.voice is not None and request.voice not in available_speakers:
|
|
return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(available_speakers))}"
|
|
|
|
# Validate speaker_embedding constraints
|
|
if request.speaker_embedding is not None:
|
|
if task_type != "Base":
|
|
return "'speaker_embedding' is only valid for Base task"
|
|
if not request.speaker_embedding:
|
|
return "'speaker_embedding' must be a non-empty list of floats"
|
|
# speaker_embedding implies x_vector_only_mode — set it before
|
|
# Base task validation so callers don't need to pass it explicitly.
|
|
request.x_vector_only_mode = True
|
|
emb_len = len(request.speaker_embedding)
|
|
dim_err = self.validate_tts_embedding_dim(emb_len)
|
|
if dim_err is not None:
|
|
return dim_err
|
|
# Validate Base task requirements
|
|
if task_type == "Base":
|
|
if request.voice is None:
|
|
# 1. Ensure a voice source is provided
|
|
if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None:
|
|
return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning"
|
|
# 2. Validate ref_audio format if it exists (using the helper from main)
|
|
if request.ref_audio is not None:
|
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
|
if fmt_err:
|
|
return fmt_err
|
|
# 3. Validate text requirements based on the mode
|
|
if not getattr(request, "x_vector_only_mode", False):
|
|
if not request.ref_text or not request.ref_text.strip():
|
|
return (
|
|
"Base task requires non-empty 'ref_text' (transcript of "
|
|
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
|
)
|
|
else:
|
|
voice_lower = request.voice.lower()
|
|
if voice_lower in server.uploaded_speakers:
|
|
# Check if data file exists for uploaded speaker
|
|
speaker_info = server.uploaded_speakers[voice_lower]
|
|
file_path = Path(speaker_info["file_path"])
|
|
if not file_path.exists():
|
|
return f"Data file for uploaded speaker '{request.voice}' not found on disk"
|
|
elif voice_lower in self.capabilities.precomputed_speakers:
|
|
profile = self.capabilities.precomputed_speakers[voice_lower]
|
|
mode = str(profile.get("mode") or "xvec").lower()
|
|
ref_text = request.ref_text or profile.get("ref_text")
|
|
if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()):
|
|
return (
|
|
f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in "
|
|
"the request or custom voice manifest"
|
|
)
|
|
else:
|
|
# need ref_audio for built-in speaker
|
|
if request.ref_audio is None:
|
|
return (
|
|
f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning"
|
|
)
|
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
|
if fmt_err:
|
|
return fmt_err
|
|
if not getattr(request, "x_vector_only_mode", False) and (
|
|
not request.ref_text or not request.ref_text.strip()
|
|
):
|
|
return (
|
|
"Base task requires non-empty 'ref_text' (transcript of "
|
|
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
|
)
|
|
|
|
# Validate cross-parameter dependencies
|
|
if task_type != "Base":
|
|
if request.ref_text is not None:
|
|
return "'ref_text' is only valid for Base task"
|
|
if request.x_vector_only_mode is not None:
|
|
return "'x_vector_only_mode' is only valid for Base task"
|
|
|
|
# Validate VoiceDesign task requirements
|
|
if task_type == "VoiceDesign" and not request.instructions:
|
|
return "VoiceDesign task requires 'instructions' to describe the voice"
|
|
|
|
# Validate instructions length (using cached value from initialization)
|
|
if request.instructions and len(request.instructions) > server._max_instructions_length:
|
|
return f"Instructions too long (max {server._max_instructions_length} characters)"
|
|
|
|
# Validate max_new_tokens range
|
|
if request.max_new_tokens is not None:
|
|
if request.max_new_tokens < self.max_new_tokens_min:
|
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
|
if request.max_new_tokens > self.max_new_tokens_max:
|
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
|
|
|
return None
|
|
|
|
async def build(
|
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
|
) -> PreparedRequest:
|
|
prompt, tts_params, warmup_key = await self.ctx.server._build_qwen3_tts_request(request)
|
|
return PreparedRequest(
|
|
prompt=prompt,
|
|
tts_params=tts_params,
|
|
model_type=tts_params.get("task_type", ["unknown"])[0],
|
|
warmup_artifact_key=warmup_key,
|
|
)
|
|
|
|
def _get_expected_speaker_embedding_dim(self) -> int:
|
|
"""Return the loaded Qwen3-TTS speaker embedding dim, if known.
|
|
|
|
The user-provided speaker embedding is concatenated directly with
|
|
talker codec embeddings, so the real compatibility requirement is the
|
|
talker hidden size.
|
|
"""
|
|
hf_config = self.ctx.engine_client.model_config.hf_config
|
|
talker_config = hf_config.talker_config
|
|
return int(talker_config.hidden_size)
|
|
|
|
def _load_precomputed_speakers(self) -> dict[str, dict]:
|
|
return load_precomputed_speakers(
|
|
self.ctx.engine_client,
|
|
expected_model_type=self.name,
|
|
validate_profile=lambda profile, tensors: validate_qwen3_tts_profile(
|
|
profile,
|
|
tensors,
|
|
expected_embedding_dim=self._get_expected_speaker_embedding_dim(),
|
|
),
|
|
)
|
|
|
|
def _load_supported_languages(self) -> frozenset[str]:
|
|
try:
|
|
config = self.ctx.engine_client.model_config.hf_config.talker_config
|
|
|
|
if isinstance(config, dict):
|
|
codec_language_id = config.get("codec_language_id")
|
|
else:
|
|
codec_language_id = getattr(config, "codec_language_id", None)
|
|
|
|
if codec_language_id and isinstance(codec_language_id, Mapping):
|
|
return frozenset(str(language).title() for language in codec_language_id) | {"Auto"}
|
|
|
|
logger.warning("No codec_language_id found in talker_config; falling back to default languages")
|
|
except Exception as exc:
|
|
logger.warning("Could not load languages from model config: %s", exc)
|
|
return DEFAULT_TTS_LANGUAGES
|
|
|
|
def validate_tts_embedding_dim(self, emb_dim: int) -> str | None:
|
|
expected_dim = self._get_expected_speaker_embedding_dim()
|
|
if emb_dim != expected_dim:
|
|
return f"speaker_embedding has {emb_dim} dimensions; expected {expected_dim} for the loaded Qwen3-TTS model"
|
|
return None
|
|
|
|
def apply_sampling_overrides(
|
|
self,
|
|
sampling_params_list: list,
|
|
request: "OpenAICreateSpeechRequest",
|
|
prompt: dict[str, Any] | None = None,
|
|
request_id: str | None = None,
|
|
) -> list:
|
|
"""Apply a text-scaled safety ceiling to Base codec generation.
|
|
|
|
Qwen3-TTS can rarely enter a repetitive state in which codec EOS is no
|
|
longer reachable through top-k sampling. A fixed 4096-frame ceiling
|
|
turns that into several minutes of unusable audio. Bound the default
|
|
Base-task budget by text length, while preserving an explicit caller
|
|
``max_new_tokens`` override and the configured budget for other tasks.
|
|
"""
|
|
import copy
|
|
|
|
del request_id
|
|
server = self.ctx.server
|
|
# Only scalar fields on stage 0 are changed below. Shallow-copy each
|
|
# stage so the shared defaults stay immutable without deep-copying the
|
|
# complete sampling configuration on every request.
|
|
sampling_params_list = [copy.copy(params) for params in sampling_params_list]
|
|
configured_cap = getattr(sampling_params_list[0], "max_tokens", None)
|
|
task_type = request.task_type or "CustomVoice"
|
|
text_tokens = None
|
|
dynamic_cap = None
|
|
effective_cap: int | None
|
|
|
|
if request.max_new_tokens is not None:
|
|
# An explicit request budget is an opt-out from the automatic
|
|
# ceiling. It remains an upper bound, and a length finish is still
|
|
# surfaced as an incomplete generation rather than valid audio.
|
|
effective_cap = int(request.max_new_tokens)
|
|
elif task_type == "Base":
|
|
counted_text_tokens = server._count_usage_text_tokens(request.input)
|
|
if counted_text_tokens > 0:
|
|
text_tokens = counted_text_tokens
|
|
dynamic_cap = max(_MIN_CODEC_FRAMES, text_tokens * _MAX_CODEC_FRAMES_PER_TEXT_TOKEN)
|
|
effective_cap = min(dynamic_cap, int(configured_cap)) if configured_cap is not None else dynamic_cap
|
|
else:
|
|
# Token counting is best-effort. If the tokenizer is missing or
|
|
# rejects the input, preserve the configured budget instead of
|
|
# truncating an otherwise valid request at the minimum ceiling.
|
|
effective_cap = int(configured_cap) if configured_cap is not None else None
|
|
else:
|
|
effective_cap = int(configured_cap) if configured_cap is not None else None
|
|
|
|
if effective_cap is not None:
|
|
effective_cap = max(1, effective_cap)
|
|
sampling_params_list[0].max_tokens = effective_cap
|
|
sampling_params_list[0].min_tokens = min(
|
|
int(getattr(sampling_params_list[0], "min_tokens", 0) or 0),
|
|
effective_cap,
|
|
)
|
|
|
|
if isinstance(prompt, dict):
|
|
additional_information = prompt.get("additional_information")
|
|
if isinstance(additional_information, dict) and effective_cap is not None:
|
|
additional_information[QWEN3_TTS_EFFECTIVE_MAX_TOKENS_KEY] = [effective_cap]
|
|
|
|
logger.debug(
|
|
"Qwen3-TTS codec budget: task_type=%s text_tokens=%s dynamic_cap=%s "
|
|
"configured_cap=%s request_cap=%s effective_cap=%s",
|
|
task_type,
|
|
text_tokens,
|
|
dynamic_cap,
|
|
configured_cap,
|
|
request.max_new_tokens,
|
|
effective_cap,
|
|
)
|
|
return sampling_params_list
|
|
|
|
def validate_generation(
|
|
self,
|
|
tts_params: Mapping[str, object],
|
|
*,
|
|
stage0_finish_reason: str | None,
|
|
output_tokens: int,
|
|
) -> None:
|
|
if QWEN3_TTS_EFFECTIVE_MAX_TOKENS_KEY not in tts_params:
|
|
return
|
|
task_type = tts_params.get("task_type")
|
|
if isinstance(task_type, (list, tuple)):
|
|
task_type = task_type[0] if task_type else None
|
|
if task_type != "Base" or stage0_finish_reason != "length":
|
|
return
|
|
|
|
raw_limit = tts_params.get(QWEN3_TTS_EFFECTIVE_MAX_TOKENS_KEY)
|
|
if isinstance(raw_limit, (list, tuple)):
|
|
raw_limit = raw_limit[0] if raw_limit else None
|
|
try:
|
|
limit = int(raw_limit) if isinstance(raw_limit, (str, bytes, bytearray, int, float)) else 0
|
|
except (TypeError, ValueError):
|
|
limit = 0
|
|
raise Qwen3TTSCodecLimitError(
|
|
"Qwen3-TTS Base did not emit codec EOS before its token budget "
|
|
f"({output_tokens}/{limit} codec tokens); the generated audio is incomplete."
|
|
)
|