anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Refactor] Move model-specific capability metadata into TTS adapters (#6138)

Browse Source 
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
boatmanandHongsheng Liu authored and GitHub committed 2026-08-22 14:26:49 +08:00
1 parent 42059f19d5
commit 3781214675
15 files changed
+722 -427
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pyproject.toml
+1
View File
@@ -192,6 +192,7 @@ select = [
"N", # pep8-naming
"UP", # pyupgrade
"TID251", # flake8-tidy-imports.banned-api
"TC004", # flake8-type-checking: runtime-used imports
]
ignore = [
"E203", # whitespace before ':' (conflicts with black)
tests/entrypoints/openai_api/test_serving_speech.py
+317 -108
View File
@@ -8,6 +8,7 @@ import logging
import os
import struct
import wave
from dataclasses import FrozenInstanceError, replace
from inspect import Signature, signature
from pathlib import Path
from types import SimpleNamespace
@@ -38,12 +39,13 @@ from vllm_omni.entrypoints.openai.protocol.audio import (
StreamingSpeechSessionConfig,
)
from vllm_omni.entrypoints.openai.serving_speech import (
_TTS_LANGUAGES,
OmniOpenAIServingSpeech,
_create_wav_header,
)
from vllm_omni.entrypoints.openai.tts_adapters.base import PreparedRequest, SpeechServingContext
from vllm_omni.entrypoints.openai.tts_adapters.base import DEFAULT_TTS_LANGUAGES, PreparedRequest, SpeechServingContext
from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_supported_speakers
from vllm_omni.entrypoints.openai.tts_adapters.ming_tts import MingTTSAdapter
from vllm_omni.entrypoints.openai.tts_adapters.voxtral import VoxtralTTSAdapter
from vllm_omni.model_executor.models.fish_speech.prompt_utils import (
FISH_TEXT_ONLY_SYSTEM_PROMPT,
build_fish_voice_clone_prompt_ids,
@@ -218,7 +220,8 @@ def _wav_data_url(samples: np.ndarray, sample_rate: int) -> str:
@pytest.fixture
def test_app(mocker: MockerFixture):
def test_app(mocker: MockerFixture, tmp_path, monkeypatch):
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
# Mock the engine client
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -241,6 +244,11 @@ def test_app(mocker: MockerFixture):
models=mock_models,
request_logger=mock_request_logger,
)
# Voice endpoint tests exercise generic embedding parsing/persistence with
# multiple valid dimensions. Supply the adapter contract without imposing
# one real model's fixed embedding dimension on this shared fixture.
speech_server._adapter = mocker.MagicMock()
speech_server._adapter.validate_tts_embedding_dim.return_value = None
speech_server._tts_tokenizer = lambda text, padding=False: {"input_ids": text.split()}
# Skip TTS validation in tests (mock doesn't set up supported_speakers)
@@ -263,31 +271,9 @@ def test_app(mocker: MockerFixture):
speech_server.create_speech = awaitable_patched_create_speech
app = FastAPI()
app.state.openai_serving_speech = speech_server
app.add_api_route("/v1/audio/speech", speech_server.create_speech, methods=["POST"], response_model=None)
# Add list_voices endpoint
async def list_voices():
speakers = sorted(speech_server.supported_speakers) if speech_server.supported_speakers else []
uploaded_voices = []
if hasattr(speech_server, "uploaded_speakers"):
for voice_name, info in speech_server.uploaded_speakers.items():
voice_entry = {
"name": info.get("name", voice_name),
"consent": info.get("consent", ""),
"created_at": info.get("created_at", 0),
"file_size": info.get("file_size", 0),
"mime_type": info.get("mime_type", ""),
"embedding_source": info.get("embedding_source", "audio"),
"embedding_dim": info.get("embedding_dim"),
}
if info.get("ref_text"):
voice_entry["ref_text"] = info["ref_text"]
if info.get("speaker_description"):
voice_entry["speaker_description"] = info["speaker_description"]
uploaded_voices.append(voice_entry)
return {"voices": speakers, "uploaded_voices": uploaded_voices}
app.add_api_route("/v1/audio/voices", list_voices, methods=["GET"])
app.add_api_route("/v1/audio/voices", api_server_module.list_voices, methods=["GET"])
app.add_api_route("/v1/audio/speech/batch", speech_server.create_speech_batch, methods=["POST"])
# Add upload_voice endpoint
@@ -459,9 +445,31 @@ class TestSpeechAPI:
assert audio_obj.speed == 2.5
def test_list_voices_endpoint(self, client):
handler = client.app.state.openai_serving_speech
handler._adapter.capabilities.supported_speakers = frozenset({"alice"})
handler._adapter.capabilities.precomputed_speakers = {"bob": {"name": "Bob"}}
handler.uploaded_speakers = {
"carol": {
"name": "Carol",
"created_at": 1,
}
}
response = client.get("/v1/audio/voices")
assert response.status_code == 200
assert "voices" in response.json()
assert response.json()["voices"] == ["alice", "bob", "carol", "default"]
assert response.json()["uploaded_voices"] == [
{
"name": "Carol",
"consent": "",
"created_at": 1,
"file_size": 0,
"mime_type": "",
"embedding_source": "audio",
"embedding_dim": None,
}
]
def test_upload_voice_success(self, client, tmp_path):
"""Test successful voice upload without ref_text."""
@@ -719,6 +727,20 @@ class TestSpeechAPI:
assert response.status_code == 400
assert "finite" in response.json()["detail"]
@pytest.mark.asyncio
async def test_diffusion_create_speech_with_unknown_voice(self, mocker: MockerFixture):
engine_client = mocker.MagicMock()
server = OmniOpenAIServingSpeech.for_diffusion(
diffusion_engine=engine_client,
model_name="test-model",
)
assert server._adapter is None
response = await server.create_speech(OpenAICreateSpeechRequest(input="test-input", voice="test-voice"))
assert response.status_code == 400
assert b"Invalid voice" in response.body
@pytest.mark.asyncio
async def test_create_diffusion_speech_extra_params(self, mocker: MockerFixture):
"""Test public diffusion speech success and extra_params propagation."""
@@ -767,7 +789,8 @@ class TestTTSMethods:
"""Unit tests for TTS validation and parameter building."""
@pytest.fixture
def speech_server(self, mocker: MockerFixture):
def speech_server(self, mocker, tmp_path, monkeypatch):
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_configs = []
@@ -990,12 +1013,18 @@ class TestTTSMethods:
def test_speaker_embedding_valid_base_task(self, speech_server):
"""speaker_embedding with Base task, x_vector_only_mode, and no ref_audio is accepted."""
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(talker_config=SimpleNamespace(hidden_size=1024))
)
emb = [0.1] * 1024
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
assert speech_server._validate_tts_request(req) is None
def test_speaker_embedding_auto_sets_x_vector_only_mode(self, speech_server):
"""speaker_embedding auto-implies x_vector_only_mode, so validation passes."""
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(talker_config=SimpleNamespace(hidden_size=1024))
)
emb = [0.1] * 1024
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb)
result = speech_server._validate_tts_request(req)
@@ -1067,6 +1096,7 @@ class TestTTSMethods:
"""Embedding uploads must match the loaded Qwen3-TTS model before being stored."""
speech_server._tts_model_type = "qwen3_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(
talker_config=SimpleNamespace(hidden_size=2048),
@@ -1085,6 +1115,7 @@ class TestTTSMethods:
def test_upload_ming_voice_embedding_wrong_dims_rejected_without_replacing_existing(self, speech_server):
"""Ming embedding uploads must match the model dimension before replacement."""
speech_server._tts_model_type = "ming_tts"
speech_server._adapter = speech_server._get_tts_adapter()
invalid_dim = SPEAKER_EMBEDDING_DIM - 1
existing = {
"name": "bad_emb_voice",
@@ -1112,9 +1143,13 @@ class TestTTSMethods:
def test_upload_ming_audio_voice_defers_speaker_extraction(self, speech_server, mocker: MockerFixture, tmp_path):
"""Ming audio uploads stay model-agnostic and defer extraction to use."""
speech_server._tts_model_type = "ming_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers_dir = tmp_path
speech_server.uploaded_speakers = {}
speech_server.supported_speakers = set()
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
supported_speakers=frozenset(),
)
speech_server._speaker_cache.clear()
left = np.full(32000, 0.5, dtype=np.float32)
@@ -1209,16 +1244,20 @@ class TestTTSMethods:
def test_precomputed_qwen3_voice_infers_base_without_ref_audio(self, speech_server):
"""Precomputed Qwen3 voices are reusable by name without per-request ref_audio."""
speech_server._tts_model_type = "qwen3_tts"
speech_server.precomputed_speakers = {
"alice": {
"name": "Alice",
"model_type": "qwen3_tts",
"mode": "icl",
"ref_text": "reference transcript",
"ref_code_length": 3,
}
}
speech_server.supported_speakers = {"alice"}
speech_server._adapter = speech_server._get_tts_adapter()
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={
"alice": {
"name": "Alice",
"model_type": "qwen3_tts",
"mode": "icl",
"ref_text": "reference transcript",
"ref_code_length": 3,
}
},
supported_speakers=frozenset({"alice"}),
)
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
assert speech_server._validate_tts_request(req) is None
@@ -1239,6 +1278,7 @@ class TestTTSMethods:
uploaded_path = tmp_path / "alice.safetensors"
save_file({"speaker_embedding": torch.tensor([0.1] * 4)}, str(uploaded_path))
speech_server._tts_model_type = "qwen3_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers_dir = tmp_path
speech_server.uploaded_speakers = {
"alice": {
@@ -1248,15 +1288,18 @@ class TestTTSMethods:
"embedding_source": "direct",
}
}
speech_server.precomputed_speakers = {
"alice": {
"name": "Alice",
"model_type": "qwen3_tts",
"mode": "icl",
"ref_text": "precomputed transcript",
"ref_code_length": 3,
}
}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={
"alice": {
"name": "Alice",
"model_type": "qwen3_tts",
"mode": "icl",
"ref_text": "precomputed transcript",
"ref_code_length": 3,
}
},
)
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
assert speech_server._validate_tts_request(req) is None
@@ -1283,13 +1326,16 @@ class TestTTSMethods:
talker_config=SimpleNamespace(hidden_size=4),
)
)
speech_server._adapter = speech_server._get_tts_adapter()
profiles = speech_server._load_precomputed_speakers()
profiles = speech_server._adapter._load_precomputed_speakers()
assert profiles == {}
speech_server.precomputed_speakers = profiles
speech_server.supported_speakers = set(profiles)
assert "alice" not in speech_server.supported_speakers
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers=profiles,
)
assert "alice" not in speech_server._adapter.capabilities.precomputed_speakers
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
assert speech_server._validate_tts_request(req) is not None
@@ -1317,13 +1363,16 @@ class TestTTSMethods:
talker_config=SimpleNamespace(hidden_size=4),
)
)
speech_server._adapter = speech_server._get_tts_adapter()
profiles = speech_server._load_precomputed_speakers()
profiles = speech_server._adapter._load_precomputed_speakers()
assert profiles == {}
speech_server.precomputed_speakers = profiles
speech_server.supported_speakers = set(profiles)
assert "alice" not in speech_server.supported_speakers
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers=profiles,
)
assert "alice" not in speech_server._adapter.capabilities.precomputed_speakers
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
assert speech_server._validate_tts_request(req) is not None
@@ -1338,13 +1387,16 @@ class TestTTSMethods:
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(custom_voice_dir=str(tmp_path))
)
speech_server._adapter = speech_server._get_tts_adapter()
profiles = speech_server._load_precomputed_speakers()
profiles = speech_server._adapter._load_precomputed_speakers()
assert profiles == {}
speech_server.precomputed_speakers = profiles
speech_server.supported_speakers = set(profiles)
assert "bob" not in speech_server.supported_speakers
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers=profiles,
)
assert "bob" not in speech_server._adapter.capabilities.precomputed_speakers
req = OpenAICreateSpeechRequest(input="Hello", voice="Bob")
assert speech_server._validate_tts_request(req) is not None
assert speech_server._validate_tts_request(OpenAICreateSpeechRequest(input="Hello")) is None
@@ -1352,9 +1404,12 @@ class TestTTSMethods:
def test_prepare_voxcpm2_rejects_supported_speaker_without_voice_profile(self, speech_server, mocker):
"""VoxCPM2 named voices must be uploaded or precomputed on the real request path."""
speech_server._tts_model_type = "voxcpm2"
speech_server.supported_speakers = {"bob", "default"}
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers = {}
speech_server.precomputed_speakers = {}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={},
)
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
speech_server.engine_client.generate = mocker.MagicMock(return_value="generator")
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
@@ -1370,9 +1425,12 @@ class TestTTSMethods:
def test_prepare_voxcpm2_accepts_default_voice(self, speech_server, mocker):
"""VoxCPM2 default voice preserves the built-in zero-shot request path."""
speech_server._tts_model_type = "voxcpm2"
speech_server.supported_speakers = {"default"}
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers = {}
speech_server.precomputed_speakers = {}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={},
)
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
@@ -1387,23 +1445,26 @@ class TestTTSMethods:
def test_prepare_voxcpm2_precomputed_voice_sets_model_cache_key(self, speech_server, mocker):
"""VoxCPM2 precomputed voices must carry voice metadata to the model cache lookup."""
speech_server._tts_model_type = "voxcpm2"
speech_server.supported_speakers = {"alice"}
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers = {}
speech_server.precomputed_speakers = {
"alice": {
"name": "Alice",
"model_type": "voxcpm2",
"mode": "reference",
"ref_audio_feat_len": 2,
}
}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={
"alice": {
"name": "Alice",
"model_type": "voxcpm2",
"mode": "reference",
"ref_audio_feat_len": 2,
}
},
)
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
return_value={
"prompt_token_ids": [1],
"additional_information": {
"voice_profile": speech_server.precomputed_speakers["alice"],
"voice_profile": speech_server._adapter.capabilities.precomputed_speakers["alice"],
},
}
)
@@ -1482,6 +1543,10 @@ class TestTTSMethods:
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
mock_stage = mocker.MagicMock()
mock_stage.engine_args.model_stage = "qwen3_tts"
mock_engine_client.stage_configs = [mock_stage]
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
@@ -1489,7 +1554,10 @@ class TestTTSMethods:
)
# Verify speakers are normalized to lowercase
assert server.supported_speakers == {"ryan", "vivian", "aiden"}
assert server._adapter.capabilities.supported_speakers == {"ryan", "vivian", "aiden"}
assert isinstance(server._adapter.capabilities.supported_speakers, frozenset)
with pytest.raises(FrozenInstanceError):
setattr(server._adapter.capabilities, "codec_frame_rate", 25.0)
def test_load_supported_speakers_skips_non_tts_omni_model(
self,
@@ -1502,7 +1570,7 @@ class TestTTSMethods:
)
assert speech_server._is_tts is False
assert speech_server._load_supported_speakers() == set()
assert load_supported_speakers(speech_server.engine_client) == set()
warning.assert_not_called()
def test_load_supported_languages_from_config(self, speech_server):
@@ -1515,7 +1583,8 @@ class TestTTSMethods:
)
)
)
assert speech_server._load_supported_languages() == {
adapter = speech_server._get_tts_adapter()
assert adapter._load_supported_languages() == {
"Chinese",
"English",
"Beijing_Dialect",
@@ -1528,11 +1597,17 @@ class TestTTSMethods:
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(talker_config={"codec_language_id": {"chinese": 2055, "english": 2050}})
)
assert speech_server._load_supported_languages() == {"Chinese", "English", "Auto"}
adapter = speech_server._get_tts_adapter()
assert adapter._load_supported_languages() == {"Chinese", "English", "Auto"}
def test_validate_language_custom_dialect_accepted(self, speech_server):
"""A language present in the model config passes validation, case-insensitively."""
speech_server.supported_languages = {"Chinese", "English", "Beijing_Dialect", "Auto"}
speech_server._tts_model_type = "qwen3_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
supported_languages=frozenset({"Chinese", "English", "Beijing_Dialect", "Auto"}),
)
for language in ("Beijing_Dialect", "beijing_dialect", "English", "english", "Auto", "AUTO"):
req = OpenAICreateSpeechRequest(input="Hello", language=language)
result = speech_server._validate_tts_request(req)
@@ -1542,7 +1617,12 @@ class TestTTSMethods:
def test_validate_language_unknown_rejected(self, speech_server):
"""A language not in the configured set is rejected."""
speech_server.supported_languages = {"Chinese", "English", "Auto"}
speech_server._tts_model_type = "qwen3_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
supported_languages=frozenset({"Chinese", "English", "Auto"}),
)
for language in ("Klingon", "klingon"):
req = OpenAICreateSpeechRequest(input="Hello", language=language)
assert "Invalid language" in speech_server._validate_tts_request(req)
@@ -1553,18 +1633,21 @@ class TestTTSMethods:
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(talker_config=SimpleNamespace(codec_language_id={}))
)
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
adapter = speech_server._get_tts_adapter()
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
def test_load_supported_languages_default_on_config_error(self, speech_server):
"""If the model config cannot be read, fall back to the default list."""
speech_server._tts_model_type = "qwen3_tts"
speech_server.engine_client = SimpleNamespace() # no model_config -> AttributeError
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
speech_server._adapter = None
adapter = speech_server._get_tts_adapter()
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
def test_load_supported_languages_default_for_non_qwen(self, speech_server):
"""Non-qwen3_tts model types get the default language set."""
speech_server._tts_model_type = None
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
adapter = MingTTSAdapter(SpeechServingContext(server=speech_server, engine_client=speech_server.engine_client))
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
def test_build_tts_params_with_uploaded_voice(self, speech_server, mocker: MockerFixture):
"""Test _build_tts_params auto-sets ref_audio for uploaded voices (x_vector only)."""
@@ -1577,7 +1660,6 @@ class TestTTSMethods:
"created_at": 1711234567,
}
}
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
mock_get_audio = mocker.patch.object(speech_server, "_get_uploaded_audio_data")
mock_get_audio.return_value = "data:audio/wav;base64,ZmFrZWF1ZGlv"
@@ -1601,7 +1683,6 @@ class TestTTSMethods:
"created_at": 1711234567,
}
}
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
mock_get_audio = mocker.patch.object(speech_server, "_get_uploaded_audio_data")
mock_get_audio.return_value = "data:audio/wav;base64,ZmFrZWF1ZGlv"
@@ -1618,7 +1699,6 @@ class TestTTSMethods:
"""Test _build_tts_params does not auto-set ref_audio for non-uploaded voices."""
# No uploaded speakers
speech_server.uploaded_speakers = {}
speech_server.supported_speakers = {"ryan", "vivian"}
req = OpenAICreateSpeechRequest(input="Hello", voice="ryan", task_type="Base")
@@ -1638,7 +1718,6 @@ class TestTTSMethods:
"mime_type": "audio/wav",
}
}
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
req = OpenAICreateSpeechRequest(
input="Hello", voice="custom_voice", task_type="Base", ref_audio="data:audio/wav;base64,ZXhwbGljaXQ="
@@ -1744,7 +1823,6 @@ class TestTTSMethods:
"cache_file": "/tmp/voice_samples/emb_voice.safetensors",
}
}
speech_server.supported_speakers = {"ryan", "vivian", "emb_voice"}
fake_embedding = [0.1] * 1024
mock_get_emb = mocker.patch.object(speech_server, "_get_uploaded_speaker_embedding")
@@ -2203,6 +2281,28 @@ class TestTTSMethods:
assert request.ref_audio is not None
assert call.kwargs["has_inline_ref_audio"] is False
@pytest.mark.asyncio
async def test_warmup_with_no_adapter(self, speech_server):
assert speech_server._adapter is None
await speech_server.warmup()
@pytest.mark.asyncio
async def test_voxcpm2_warmup_successfully(self, speech_server, mocker):
speech_server._tts_model_type = "voxcpm2"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers = {}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={},
)
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
return_value={"prompt_token_ids": [1], "additional_information": {}}
)
await speech_server.warmup()
class TestFileValidationFunctions:
"""Unit tests for file validation helper functions."""
@@ -2973,6 +3073,39 @@ def test_api_server_upload_voice_exception_returns_500(mocker: MockerFixture):
)
def test_api_server_upload_voice_with_no_adapter(
mocker: MockerFixture,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
):
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
_patch_api_server_base(mocker)
engine_client = mocker.MagicMock()
engine_client.errored = False
engine_client.stage_configs = []
models = mocker.MagicMock()
models.is_base_model.return_value = True
handler = OmniOpenAIServingSpeech(
engine_client=engine_client,
models=models,
request_logger=mocker.MagicMock(),
)
assert handler._adapter is None
raw_request = _make_api_server_request(handler)
response = asyncio.run(
api_server_module.upload_voice(
raw_request,
audio_sample=None,
speaker_embedding=json.dumps([0.1] * 1024),
consent="cons_test",
name="probe",
)
)
assert response.status_code == 200
def test_api_server_delete_voice_without_speech_handler_returns_404(mocker: MockerFixture):
_patch_api_server_base(mocker)
raw_request = _make_api_server_request(None, method="DELETE", path="/v1/audio/voices/probe")
@@ -3302,8 +3435,14 @@ class _FakeFishTokenizer:
@pytest.fixture
def fish_speech_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3573,8 +3712,14 @@ class TestWAVStreaming:
@pytest.fixture
def cosyvoice3_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3709,8 +3854,14 @@ class TestCosyVoice3Serving:
@pytest.fixture
def glm_tts_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3732,7 +3883,6 @@ def glm_tts_server(mocker: MockerFixture):
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
return OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
@@ -3809,8 +3959,14 @@ class TestGLMTTSServing:
@pytest.fixture
def ming_tts_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3868,8 +4024,14 @@ class TestMingTTSServing:
@pytest.fixture
def ming_flash_omni_tts_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3932,8 +4094,14 @@ class TestTTSAsyncOffloading:
@pytest.fixture
def voxtral_server(self, mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.voxtral.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.model_config = mocker.MagicMock(model="mistralai/Voxtral")
@@ -3958,8 +4126,14 @@ class TestTTSAsyncOffloading:
@pytest.fixture
def qwen3_tts_server(self, mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.model_config = mocker.MagicMock(model="Qwen/Qwen3-TTS", hf_config=mocker.MagicMock())
@@ -3983,6 +4157,32 @@ class TestTTSAsyncOffloading:
yield server
server.shutdown()
def test_voxtral_loads_supported_speakers(self, mocker: MockerFixture):
engine_client = mocker.MagicMock()
engine_client.model_config.hf_config.audio_config = {
"speaker_id": {
"Alice": 0,
"Bob": 1,
}
}
server = mocker.MagicMock()
adapter = VoxtralTTSAdapter(
SpeechServingContext(
server=server,
engine_client=engine_client,
)
)
adapter.load_capabilities()
assert adapter.capabilities.supported_speakers == frozenset(
{
"alice",
"bob",
}
)
def test_prepare_speech_generation_awaits_voxtral_async(self, voxtral_server, mocker: MockerFixture):
"""Voxtral path in _prepare_speech_generation should call the async wrapper."""
voxtral_server._build_voxtral_prompt_async = mocker.AsyncMock(
@@ -4214,7 +4414,10 @@ class TestTTSAsyncOffloading:
artifact_key,
)
qwen3_tts_server._ref_audio_model_artifact_ready.add((artifact_key, False))
qwen3_tts_server._codec_frame_rate = 25.0
qwen3_tts_server._adapter.capabilities = replace(
qwen3_tts_server._adapter.capabilities,
codec_frame_rate=25.0,
)
qwen3_tts_server._tts_tokenizer = lambda _text, padding=False: {"input_ids": list(range(10))}
qwen3_tts_server.engine_client.model_config.hf_config.talker_config = SimpleNamespace(
codec_language_id={},
@@ -4319,8 +4522,14 @@ class TestTTSAsyncOffloading:
def test_shutdown_is_idempotent(self, mocker: MockerFixture):
"""Calling shutdown() twice should not raise."""
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_configs = []
tools/pre_commit/check_tts_adapter.py
+1 -1
View File
@@ -28,7 +28,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2]
SERVING_SPEECH = Path("vllm_omni/entrypoints/openai/serving_speech.py")
# Budgets. These may only go DOWN. See the module docstring.
MAX_MODEL_TYPE_BRANCHES = 20
MAX_MODEL_TYPE_BRANCHES = 9
_GUIDANCE = """
Move the per-model behaviour into its adapter under
vllm_omni/entrypoints/openai/serving_speech.py
+52 -291
View File
@@ -9,7 +9,6 @@ import re
import struct
import time
from collections import OrderedDict
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from http import HTTPStatus
from pathlib import Path
@@ -20,7 +19,6 @@ import soundfile as sf
import torch
from fastapi import HTTPException, Request, UploadFile
from fastapi.responses import Response, StreamingResponse
from transformers.utils.hub import cached_file
from vllm.entrypoints.generate.base.serving import GenerateBaseServing as OpenAIServing
from vllm.entrypoints.launcher import terminate_if_errored
from vllm.entrypoints.openai.engine.protocol import (
@@ -70,13 +68,8 @@ from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import (
from vllm_omni.model_executor.models.ming_flash_omni.prompt_utils import (
create_instruction as ming_create_instruction,
)
from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.utils.speaker_cache import (
get_speaker_cache,
iter_custom_voice_profiles,
load_validated_profile_tensors,
)
from vllm_omni.utils.speaker_cache import get_speaker_cache
logger = init_logger(__name__)
@@ -103,21 +96,6 @@ _COSYVOICE3_PROMPT_PREFIX = f"You are a helpful assistant.{_COSYVOICE3_PROMPT_DE
# payloads and must fail the request, never serialize as a successful empty
# WAV. Covers both the TTS ("audex") and TTA ("audex_tta") pipelines.
_AUDEX_NO_AUDIO_GUARD_MODEL_TYPES = frozenset({"audex", "audex_tta"})
_TTS_LANGUAGES = frozenset(
{
"Auto",
"Chinese",
"English",
"Japanese",
"Korean",
"German",
"French",
"Russian",
"Portuguese",
"Spanish",
"Italian",
}
)
_REF_AUDIO_MIN_DURATION = 1.0 # seconds
_REF_AUDIO_MAX_DURATION = 30.0 # seconds
_REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES = 256
@@ -297,9 +275,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
except ValueError:
logger.warning("Invalid SPEAKER_MAX_UPLOADED=%r; using default 1000", _raw_cap)
self._max_uploaded_speakers = 1000
self.uploaded_speakers: dict[str, dict] = {}
self.precomputed_speakers: dict[str, dict[str, Any]] = {}
self.supported_speakers: set[str] = set()
self.uploaded_speakers: dict[str, dict[str, Any]] = {}
self._ref_audio_data_url_cache: dict[str, str] = {}
self._ref_audio_resolve_cache: OrderedDict[str, tuple[list[float], int, int, str]] = OrderedDict()
self._ref_audio_resolve_cache_bytes = 0
@@ -391,7 +367,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
speaker_data.setdefault("name", voice_name_lower)
speaker_data.setdefault("file_size", int(path.stat().st_size))
self.uploaded_speakers[voice_name_lower] = speaker_data
self.supported_speakers.add(voice_name_lower)
self._last_upload_ts = max(self._last_upload_ts, int(speaker_data.get("created_at", 0)))
restored += 1
if restored:
@@ -419,6 +394,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# Diffusion-only instances don't have a TTS stage; set None so any
# ``_is_tts_model()`` / ``_tts_stage`` access doesn't raise AttributeError.
instance._tts_stage = None
instance._adapter = None
instance._init_speaker_storage()
return instance
@@ -443,7 +419,24 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# Determine TTS model type or None
self._tts_model_type = self._detect_tts_model_type()
self.precomputed_speakers = self._load_precomputed_speakers()
# Resolve the per-model serving adapter (RFC #4327), keyed on the
# detected model-type. Every dedicated TTS model has an adapter; the
# adapter owns request validation, prompt/param building, capability
# metadata, and sampling overrides. The model-type label remains in the
# orchestrator for compatibility during this incremental migration.
self._adapter = None
if self._tts_stage is not None:
adapter_cls = resolve_adapter(self._tts_model_type)
if adapter_cls is not None:
ctx = SpeechServingContext(server=self, engine_client=self.engine_client)
self._adapter = adapter_cls(ctx)
logger.info("Resolved TTS serving adapter: %s", adapter_cls.__name__)
adapter = self._adapter
if adapter is not None:
adapter.load_capabilities()
available_speakers = self._get_available_speakers()
logger.info("Loaded %d supported speakers: %s", len(available_speakers), sorted(available_speakers))
# Sub-variant inside the full MOSS-TTS family. We collapse all five
# variants onto the same _tts_model_type="moss_tts" because they share
@@ -459,33 +452,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# Cache TTS configuration values (computed once, reused per request)
self._max_instructions_length = self._compute_max_instructions_length()
# Merge built-in speakers into the set initialized by _init_speaker_storage.
self.supported_speakers |= self._load_supported_speakers()
self.supported_speakers |= set(self.precomputed_speakers)
self.supported_languages = self._load_supported_languages()
self._tts_tokenizer = None
self._voxcpm2_tokenizer = None
self._audex_tokenizer = None
self._audex_tta_rvq = None
self._voxcpm2_split_map: dict[int, list[int]] = {}
if self.supported_speakers:
logger.info(
"Loaded %d supported speakers: %s", len(self.supported_speakers), sorted(self.supported_speakers)
)
else:
logger.info(
"No built-in speakers configured; only '%s' and uploaded voices are available", _DEFAULT_VOICE_NAME
)
# Batch configuration
self._batch_max_items: int = getattr(self.engine_client, "tts_batch_max_items", 32)
# Load speech tokenizer codec parameters for prompt length estimation
self._codec_frame_rate: float | None = self._load_codec_frame_rate()
# Shared thread pool executor for blocking TTS preprocessing
# operations. max_workers=1 serializes tokenizer access to avoid
# Rust RefCell "Already borrowed" errors from concurrent use.
@@ -494,18 +469,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
self._build_fish_speech_prompt_async = make_async(self._build_fish_speech_prompt, executor=self._tts_executor)
self._estimate_prompt_len_async = make_async(self._estimate_prompt_len, executor=self._tts_executor)
# Resolve the per-model serving adapter (RFC #4327), keyed on the
# detected model-type. Every dedicated TTS model has an adapter; the
# adapter owns request validation, prompt/param building, and sampling
# overrides; the model-type label remains available for compatibility.
self._adapter = None
if self._tts_stage is not None:
adapter_cls = resolve_adapter(self._tts_model_type)
if adapter_cls is not None:
ctx = SpeechServingContext(server=self, engine_client=self.engine_client)
self._adapter = adapter_cls(ctx)
logger.info("Resolved TTS serving adapter: %s", adapter_cls.__name__)
def _get_tts_adapter(self):
"""Return the per-model serving adapter for the current ``_tts_model_type``.
@@ -527,145 +490,44 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
if self._adapter is None or type(self._adapter) is not adapter_cls:
ctx = SpeechServingContext(server=self, engine_client=self.engine_client)
self._adapter = adapter_cls(ctx)
self._adapter.load_capabilities()
return self._adapter
def _uses_native_speed_control(self) -> bool:
adapter = self._get_tts_adapter()
return bool(adapter is not None and adapter.native_speed_control)
def _get_available_speakers(self) -> set[str]:
"""Return all built-in, precomputed, and runtime-uploaded speakers."""
available_speakers = set(self.uploaded_speakers)
if self._adapter is not None:
available_speakers.update(self._adapter.capabilities.supported_speakers)
available_speakers.update(self._adapter.capabilities.precomputed_speakers)
return available_speakers
def _audio_encode_speed(self, request: OpenAICreateSpeechRequest) -> float:
if self._uses_native_speed_control():
return 1.0
return float(request.speed or 1.0)
async def warmup(self) -> None:
"""Run a synthetic speech request to trigger all first-request warmup.
"""Run model-specific startup warmup through the resolved adapter.
Unlike qwen3-tts, whose CUDA Graph warmup targets a standalone tokenizer
decoder (no vLLM dependencies) and can complete entirely at model-init
time, VoxCPM2 needs to warm up PagedAttention scaffold/residual LLMs.
Their CUDA Graph capture requires a vLLM ``ForwardContext``
(attn_metadata, slot_mapping, etc.) that only exists during real
inference steps. The same request also pays the one-time torch.compile
JIT tax for the LocDiT estimator, feat_encoder, AudioVAE decoder, and
projection helpers.
For VoxCPM2 this shifts ~15s of torch.compile + CUDA Graph capture from
the first user request to server startup.
Warmup requirements differ by model. For example, Qwen3-TTS can warm up
its standalone tokenizer decoder during model initialization, while
VoxCPM2 requires a real inference request with an active vLLM
``ForwardContext``. The adapter owns that model-specific lifecycle logic.
"""
if self._tts_model_type != "voxcpm2":
if self._adapter is None:
return
t0 = time.time()
logger.info("Running warmup speech request for model_type=%s", self._tts_model_type)
# VoxCPM2 has no predefined speaker presets — "default" means zero-shot
# mode (no voice cloning). The voice field is required by the OpenAI
# API schema but semantically ignored by the model.
warmup_req = OpenAICreateSpeechRequest(
input="Warmup.",
voice="default",
response_format="wav",
speed=1.0,
stream=False,
model=self.model_name,
)
try:
_audio_bytes, _media_type = await self._generate_audio_bytes(warmup_req, request_id="speech-warmup")
except Exception as exc:
logger.warning("Speech warmup failed (non-fatal): %s", exc)
return
elapsed = time.time() - t0
logger.info("Speech warmup complete in %.1fs", elapsed)
def _get_qwen_tts_expected_speaker_embedding_dim(self) -> int | None:
"""Return the loaded Qwen3-TTS speaker embedding dim, if known.
The user-provided speaker embedding is concatenated directly with
talker codec embeddings, so the real compatibility requirement is the
talker hidden size.
"""
if self._tts_model_type != "qwen3_tts":
return None
hf_config = self.engine_client.model_config.hf_config
talker_config = hf_config.talker_config
return int(talker_config.hidden_size)
def _validate_qwen_tts_speaker_embedding_dim(self, emb_dim: int) -> str | None:
expected_dim = self._get_qwen_tts_expected_speaker_embedding_dim()
if expected_dim is None:
return None
if emb_dim != expected_dim:
return f"speaker_embedding has {emb_dim} dimensions; expected {expected_dim} for the loaded Qwen3-TTS model"
return None
def _load_codec_frame_rate(self) -> float | None:
"""Load codec frame rate from speech tokenizer config for prompt length estimation."""
if self._tts_model_type == "ming_tts":
try:
from vllm_omni.model_executor.models.ming_tts.config_ming_tts import MingTTSConfig
hf_config = self.engine_client.model_config.hf_config
ming_cfg = MingTTSConfig.from_hf_config(hf_config)
patch_size = int(ming_cfg.patch_size)
audio_frame_hop = int(ming_cfg.audio_frame_hop)
sample_rate = int(ming_cfg.sample_rate)
if patch_size <= 0 or audio_frame_hop <= 0 or sample_rate <= 0:
raise ValueError(
"Ming config has invalid tokenizer timing values: "
f"patch_size={patch_size}, audio_frame_hop={audio_frame_hop}, sample_rate={sample_rate}"
)
rate = float(sample_rate) / float(audio_frame_hop * patch_size)
logger.info(
"Derived Ming codec frame rate: %.1f Hz (sample_rate=%s, audio_frame_hop=%s, patch_size=%s)",
rate,
sample_rate,
audio_frame_hop,
patch_size,
)
return rate
except Exception as e:
logger.warning(f"Failed to derive Ming codec frame rate from hf_config: {e}")
try:
model_path = self.engine_client.model_config.model
st_config_path = os.path.join(model_path, "speech_tokenizer", "config.json")
if not os.path.exists(st_config_path):
st_config_path = cached_file(model_path, "speech_tokenizer/config.json")
if st_config_path is not None and os.path.exists(st_config_path):
with open(st_config_path) as f:
st_config = json.load(f)
output_sr = st_config.get("output_sample_rate")
downsample = st_config.get("encode_downsample_rate")
if output_sr and downsample and downsample > 0:
rate = float(output_sr) / float(downsample)
logger.info(
"Loaded codec frame rate: %.1f Hz (output_sample_rate=%s, encode_downsample_rate=%s)",
rate,
output_sr,
downsample,
)
return rate
except Exception as e:
logger.warning("Failed to load codec frame rate from speech tokenizer config: %s", e)
# Fallback: try codec_frame_rate_hz from hf_config
try:
hf_config = self.engine_client.model_config.hf_config
rate = getattr(hf_config, "codec_frame_rate_hz", None)
if rate is not None:
logger.info("Using codec frame rate from hf_config: %s Hz", rate)
return float(rate)
except Exception:
pass
return None
await self._adapter.warmup()
def shutdown(self) -> None:
"""Shut down the TTS thread pool executor."""
if self._tts_executor is not None:
self._tts_executor.shutdown(wait=False, cancel_futures=True)
self._tts_executor = None
for name in list(self.uploaded_speakers.keys()):
for name in list(self.uploaded_speakers):
self._speaker_cache.clear(name)
def _find_tts_stage(self):
@@ -708,45 +570,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
getattr(self._tts_stage.engine_args, "model_arch", None),
)
def _get_custom_voice_dir(self) -> str | None:
try:
value = getattr(self.engine_client.model_config.hf_config, "custom_voice_dir", None)
except AttributeError:
return None
if isinstance(value, os.PathLike):
return os.fspath(value)
if isinstance(value, str) and value:
return value
return None
def _load_precomputed_speakers(self) -> dict[str, dict[str, Any]]:
"""Load precomputed voice names from ``custom_voice_dir`` for API validation."""
if self._tts_model_type not in ("qwen3_tts", "voxcpm2"):
return {}
custom_voice_dir = self._get_custom_voice_dir()
if not custom_voice_dir:
return {}
profiles: dict[str, dict[str, Any]] = {}
qwen3_embedding_dim = self._get_qwen_tts_expected_speaker_embedding_dim()
for profile in iter_custom_voice_profiles(custom_voice_dir, expected_model_type=self._tts_model_type):
tensors = load_validated_profile_tensors(
profile,
expected_model_type=self._tts_model_type,
qwen3_embedding_dim=qwen3_embedding_dim,
)
if tensors is None:
continue
profiles[profile["voice_name_lower"]] = profile
if profiles:
logger.info(
"Loaded %d precomputed %s voice profile(s) from %s",
len(profiles),
self._tts_model_type,
custom_voice_dir,
)
return profiles
def _compute_max_instructions_length(self) -> int:
"""Compute max instructions length with precedence: CLI > stage config > default.
@@ -768,64 +591,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
def _get_available_voices(self) -> set[str]:
"""Get all voice names accepted by the API, including the placeholder default."""
return self.supported_speakers | self.uploaded_speakers.keys() | {_DEFAULT_VOICE_NAME}
def _load_supported_speakers(self) -> set[str]:
"""Load supported speakers (case-insensitive) from the model configuration."""
if self._tts_model_type == "ming_flash_omni_tts":
# Ming-flash-omni drives speaker selection via the caption JSON
# (audio_sequence[0]["说话人"]) rather than a spk_id table, so there
# is no static speaker list to surface here.
return set()
try:
if self._tts_model_type == "glm_tts":
return set()
if self._tts_model_type == "ming_tts":
return set()
if self._tts_model_type == "voxcpm2":
return {"default"}
if self._tts_model_type == "voxtral_tts":
config = self.engine_client.model_config.hf_config.audio_config
else:
# Default is qwen3_tts path
config = getattr(self.engine_client.model_config.hf_config, "talker_config", None)
if config is None:
return set()
# Check for speakers in either spk_id or speaker_id
for attr_name in ["spk_id", "speaker_id"]:
if isinstance(config, dict):
speakers_dict = config.get(attr_name)
else:
speakers_dict = getattr(config, attr_name, None)
if speakers_dict and isinstance(speakers_dict, dict):
return {speaker.lower() for speaker in speakers_dict.keys()}
logger.warning("No speakers found in config (checked spk_id and speaker_id)")
except Exception as e:
logger.warning("Could not load speakers from model config: %s", e)
return set()
def _load_supported_languages(self) -> frozenset[str]:
"""Load supported languages (title-cased) from the model configuration"""
if self._tts_model_type != "qwen3_tts":
return _TTS_LANGUAGES
try:
config = self.engine_client.model_config.hf_config.talker_config
if isinstance(config, dict):
codec_language_id = config.get("codec_language_id")
else:
codec_language_id = getattr(config, "codec_language_id", None)
if codec_language_id and isinstance(codec_language_id, Mapping):
return frozenset(str(language).title() for language in codec_language_id) | {"Auto"}
logger.warning("No codec_language_id found in talker_config; falling back to default languages")
except Exception as e:
logger.warning("Could not load languages from model config: %s", e)
return _TTS_LANGUAGES
return self._get_available_speakers() | {_DEFAULT_VOICE_NAME}
def _estimate_ref_code_len(self, ref_audio: object) -> int | None:
"""Estimate ref_code length from ref_audio waveform without running the codec.
@@ -833,7 +599,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
The codec produces one frame per (output_sample_rate / encode_downsample_rate)
audio samples, so ref_code_len = ceil(duration_seconds * codec_frame_rate).
"""
if self._codec_frame_rate is None:
codec_frame_rate = self._adapter.capabilities.codec_frame_rate
if codec_frame_rate is None:
return None
try:
# ref_audio comes from tts_params as [[wav_array, sr]] or similar nested structure
@@ -858,7 +625,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
if sr <= 0 or n_samples <= 0:
return None
duration = n_samples / sr
return math.ceil(duration * self._codec_frame_rate)
return math.ceil(duration * codec_frame_rate)
except Exception:
return None
@@ -1041,7 +808,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
wav_np, ref_sr = uploaded_ref
ref_audio = wav_np.tolist()
elif request.voice is not None:
voice_profile = self.precomputed_speakers.get(request.voice.lower())
voice_profile = self._adapter.capabilities.precomputed_speakers.get(request.voice.lower())
return build_voxcpm2_prompt(
hf_config=self.engine_client.model_config.hf_config,
tokenizer=self._voxcpm2_tokenizer,
@@ -1204,7 +971,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
if voice_name_lower not in self.uploaded_speakers:
return
old = self.uploaded_speakers.pop(voice_name_lower)
self.supported_speakers.discard(voice_name_lower)
self._ref_audio_data_url_cache.pop(voice_name_lower, None)
old_path = old.get("file_path")
if old_path:
@@ -1346,7 +1112,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
raise ValueError(f"Failed to save voice file: {e}")
self.uploaded_speakers[voice_name_lower] = speaker_data
self.supported_speakers.add(voice_name_lower)
logger.info("Uploaded new voice '%s' with consent ID '%s'", name, consent)
@@ -1397,13 +1162,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
raise ValueError("'speaker_embedding' values must be finite (no NaN or Inf)")
emb_dim = len(embedding)
if self._tts_model_type == "ming_tts":
if emb_dim != SPEAKER_EMBEDDING_DIM:
raise ValueError(f"Ming speaker embedding must have {SPEAKER_EMBEDDING_DIM} dims, got {emb_dim}")
else:
dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_dim)
if dim_err is not None:
raise ValueError(dim_err)
dim_err = self._adapter.validate_tts_embedding_dim(emb_dim) if self._adapter is not None else None
if dim_err is not None:
raise ValueError(dim_err)
async with self._upload_lock:
voice_name_lower = name.lower()
@@ -1443,7 +1204,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
speaker_data["file_size"] = file_path.stat().st_size
self.uploaded_speakers[voice_name_lower] = speaker_data
self.supported_speakers.add(voice_name_lower)
logger.info("Uploaded voice '%s' from speaker embedding (%d-dim)", name, emb_dim)
@@ -1473,7 +1233,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return False
speaker_info = self.uploaded_speakers.pop(voice_name_lower)
self.supported_speakers.discard(voice_name_lower)
self._ref_audio_data_url_cache.pop(voice_name_lower, None)
file_path = speaker_info.get("file_path")
@@ -2507,6 +2266,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# Speaker (voice)
if request.voice is not None:
voice_lower = request.voice.lower()
precomputed_speakers = self._adapter.capabilities.precomputed_speakers if self._adapter is not None else {}
params["speaker"] = [request.voice]
params["voice_created_at"] = [self._voice_created_at(voice_lower)]
@@ -2540,8 +2300,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
logger.info(
"Auto-set ref_audio for uploaded voice: %s (icl=%s)", request.voice, bool(stored_ref_text)
)
elif voice_lower in self.precomputed_speakers and request.ref_audio is None:
profile = self.precomputed_speakers[voice_lower]
elif voice_lower in precomputed_speakers and request.ref_audio is None:
profile = precomputed_speakers[voice_lower]
mode = str(profile.get("mode") or "xvec").lower()
params["speaker"] = [voice_lower]
params["task_type"] = ["Base"]
@@ -3349,7 +3109,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"""
if voice is not None:
voice = voice.lower()
if voice not in self.uploaded_speakers and voice not in self.supported_speakers:
available_speakers = self._get_available_speakers()
if voice not in available_speakers:
raise ValueError(
f"Invalid voice '{voice}'. Supported: {', '.join(sorted(self._get_available_voices()))}"
)
@@ -3534,7 +3295,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
decoded. Raw WAV streaming emits a header with placeholder size values first.
"""
if request.voice is not None:
if _is_default_voice(request.voice.lower(), self.supported_speakers):
if _is_default_voice(request.voice.lower(), self._get_available_speakers()):
request.voice = None
if self._diffusion_mode:
vllm_omni/entrypoints/openai/tts_adapters/base.py
+54
View File
@@ -15,9 +15,26 @@ from collections.abc import Callable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar
from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_codec_frame_rate, load_supported_speakers
if TYPE_CHECKING:
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
DEFAULT_TTS_LANGUAGES = frozenset(
{
"Auto",
"Chinese",
"English",
"Japanese",
"Korean",
"German",
"French",
"Russian",
"Portuguese",
"Spanish",
"Italian",
}
)
_conditioning_cache_salt_fn: "Callable[..., str] | None" = None
@@ -101,6 +118,14 @@ class SpeechServingContext:
diffusion_engine: Any | None = None
@dataclass(frozen=True)
class TTSCapabilities:
precomputed_speakers: dict[str, dict[str, Any]] = field(default_factory=dict)
supported_speakers: frozenset[str] = frozenset()
supported_languages: frozenset[str] = DEFAULT_TTS_LANGUAGES
codec_frame_rate: float | None = None
class TTSModelAdapter(ABC):
"""Mandatory base class for a TTS model served via ``/v1/audio/speech``.
@@ -137,6 +162,7 @@ class TTSModelAdapter(ABC):
def __init__(self, ctx: SpeechServingContext) -> None:
self.ctx = ctx
self.capabilities = TTSCapabilities()
@classmethod
def matches(cls, model_stage: str | None, model_arch: str | None) -> bool:
@@ -207,6 +233,34 @@ class TTSModelAdapter(ABC):
"""
return sampling_params_list
async def warmup(self) -> None:
return
def validate_tts_embedding_dim(self, emb_dim: int) -> str | None:
return None
def load_capabilities(self) -> TTSCapabilities:
self.capabilities = TTSCapabilities(
precomputed_speakers=self._load_precomputed_speakers(),
supported_speakers=frozenset(self._load_supported_speakers()),
supported_languages=self._load_supported_languages(),
codec_frame_rate=self._load_codec_frame_rate(),
)
return self.capabilities
def _load_precomputed_speakers(self) -> dict[str, dict[str, Any]]:
return {}
def _load_supported_speakers(self) -> set[str]:
# Preserve the legacy default path, which reads talker_config.
return load_supported_speakers(self.ctx.engine_client)
def _load_supported_languages(self) -> frozenset[str]:
return DEFAULT_TTS_LANGUAGES
def _load_codec_frame_rate(self) -> float | None:
return load_codec_frame_rate(self.ctx.engine_client)
class ARTTSAdapter(TTSModelAdapter):
"""Adapter for models served through the AR ``engine_client`` path.
vllm_omni/entrypoints/openai/tts_adapters/capabilities.py
+104
View File
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: Apache-2.0
"""Shared loading mechanisms for TTS model capability metadata."""
import json
import os
from collections.abc import Callable
from pathlib import Path
from typing import Any
import torch
from transformers.utils.hub import cached_file
from vllm.logger import init_logger
from vllm_omni.utils.speaker_cache import iter_custom_voice_profiles, load_validated_profile_tensors
logger = init_logger(__name__)
def load_supported_speakers(engine_client: Any, config: Any = None) -> set[str]:
"""Extract supported speaker names from a model configuration."""
try:
if config is None:
config = getattr(engine_client.model_config.hf_config, "talker_config", None)
if config is None:
return set()
for attr_name in ("spk_id", "speaker_id"):
speakers = config.get(attr_name) if isinstance(config, dict) else getattr(config, attr_name, None)
if speakers and isinstance(speakers, dict):
return {speaker.lower() for speaker in speakers}
logger.warning("No speakers found in config (checked spk_id and speaker_id)")
except Exception as exc:
logger.warning("Could not load speakers from model config: %s", exc)
return set()
def load_codec_frame_rate(engine_client: Any) -> float | None:
"""Load codec frame rate from speech-tokenizer or HF configuration."""
try:
model_path = engine_client.model_config.model
config_path = os.path.join(model_path, "speech_tokenizer", "config.json")
if not os.path.exists(config_path):
config_path = cached_file(model_path, "speech_tokenizer/config.json")
if config_path is not None and os.path.exists(config_path):
with open(config_path) as file:
config = json.load(file)
output_sr = config.get("output_sample_rate")
downsample = config.get("encode_downsample_rate")
if output_sr and downsample and downsample > 0:
rate = float(output_sr) / float(downsample)
logger.info(
"Loaded codec frame rate: %.1f Hz (output_sample_rate=%s, encode_downsample_rate=%s)",
rate,
output_sr,
downsample,
)
return rate
except Exception as exc:
logger.warning("Failed to load codec frame rate from speech tokenizer config: %s", exc)
try:
rate = getattr(engine_client.model_config.hf_config, "codec_frame_rate_hz", None)
if rate is not None:
logger.info("Using codec frame rate from hf_config: %s Hz", rate)
return float(rate)
except Exception:
pass
return None
def load_precomputed_speakers(
engine_client: Any,
*,
expected_model_type: str,
validate_profile: Callable[[dict[str, Any], dict[str, torch.Tensor]], str | None],
) -> dict[str, dict[str, Any]]:
"""Load and validate precomputed profiles from ``custom_voice_dir``."""
try:
custom_voice_dir = getattr(engine_client.model_config.hf_config, "custom_voice_dir", None)
except AttributeError:
return {}
if isinstance(custom_voice_dir, os.PathLike):
custom_voice_dir = os.fspath(custom_voice_dir)
if not isinstance(custom_voice_dir, str) or not custom_voice_dir:
return {}
profiles: dict[str, dict[str, Any]] = {}
for profile in iter_custom_voice_profiles(custom_voice_dir, expected_model_type=expected_model_type):
tensors = load_validated_profile_tensors(
profile,
expected_model_type=expected_model_type,
validate_profile=validate_profile,
)
if tensors is not None:
profiles[profile["voice_name_lower"]] = profile
if profiles:
logger.info(
"Loaded %d precomputed %s voice profile(s) from %s",
len(profiles),
expected_model_type,
Path(custom_voice_dir).expanduser(),
)
return profiles
vllm_omni/entrypoints/openai/tts_adapters/glm_tts.py
+3
View File
@@ -49,6 +49,9 @@ class GlmTTSAdapter(ARTTSAdapter):
prompt = await self.ctx.server._build_glm_tts_prompt(request, has_inline_ref_audio=has_inline_ref_audio)
return PreparedRequest(prompt=prompt, tts_params={}, model_type="glm_tts")
def _load_supported_speakers(self) -> set[str]:
return set()
def apply_sampling_overrides(
self,
sampling_params_list: list,
vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py
+7
View File
@@ -16,6 +16,9 @@ logger = init_logger(__name__)
@register_tts_adapter
class MingFlashOmniTTSAdapter(ARTTSAdapter):
# Ming-flash-omni drives speaker selection via the caption JSON
# (audio_sequence[0]["说话人"]) rather than a spk_id table, so there
# is no static speaker list to surface here.
stage_keys = frozenset({"ming_tts"})
name = "ming_flash_omni_tts"
@@ -61,3 +64,7 @@ class MingFlashOmniTTSAdapter(ARTTSAdapter):
server = self.ctx.server
prompt = server._build_ming_flash_omni_prompt(request)
return PreparedRequest(prompt=prompt, tts_params={}, model_type="ming_flash_omni_tts")
def _load_supported_speakers(self) -> set[str]:
# Speaker selection is driven by caption JSON rather than a static table.
return set()
vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py
+41
View File
@@ -184,3 +184,44 @@ class MingTTSAdapter(ARTTSAdapter):
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
return None
def validate_tts_embedding_dim(self, emb_dim: int) -> str | None:
if emb_dim != SPEAKER_EMBEDDING_DIM:
return f"Ming speaker embedding must have {SPEAKER_EMBEDDING_DIM} dims, got {emb_dim}"
return None
def _load_ming_tts_codec_frame_rate(self) -> float | None:
server = self.ctx.server
try:
from vllm_omni.model_executor.models.ming_tts.config_ming_tts import MingTTSConfig
hf_config = server.engine_client.model_config.hf_config
ming_cfg = MingTTSConfig.from_hf_config(hf_config)
patch_size = int(ming_cfg.patch_size)
audio_frame_hop = int(ming_cfg.audio_frame_hop)
sample_rate = int(ming_cfg.sample_rate)
if patch_size <= 0 or audio_frame_hop <= 0 or sample_rate <= 0:
raise ValueError(
"Ming config has invalid tokenizer timing values: "
f"patch_size={patch_size}, audio_frame_hop={audio_frame_hop}, sample_rate={sample_rate}"
)
rate = float(sample_rate) / float(audio_frame_hop * patch_size)
logger.info(
"Derived Ming codec frame rate: %.1f Hz (sample_rate=%s, audio_frame_hop=%s, patch_size=%s)",
rate,
sample_rate,
audio_frame_hop,
patch_size,
)
return rate
except Exception as e:
logger.warning(f"Failed to derive Ming codec frame rate from hf_config: {e}")
def _load_supported_speakers(self) -> set[str]:
return set()
def _load_codec_frame_rate(self) -> float | None:
codec_frame_rate = self._load_ming_tts_codec_frame_rate()
if codec_frame_rate is None:
codec_frame_rate = super()._load_codec_frame_rate()
return codec_frame_rate
vllm_omni/entrypoints/openai/tts_adapters/qwen3_tts.py
+65 -10
View File
@@ -1,13 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
"""Qwen3-TTS serving adapter."""
from collections.abc import Mapping
from pathlib import Path
from typing import TYPE_CHECKING, Any
from vllm.logger import init_logger
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens
from vllm_omni.entrypoints.openai.tts_adapters.base import (
DEFAULT_TTS_LANGUAGES,
ARTTSAdapter,
PreparedRequest,
apply_max_new_tokens,
)
from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_precomputed_speakers
from vllm_omni.utils.speaker_cache import validate_qwen3_tts_profile
if TYPE_CHECKING:
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
@@ -37,7 +45,7 @@ class Qwen3TTSAdapter(ARTTSAdapter):
# Normalize voice to lowercase for case-insensitive matching
if request.voice is not None:
request.voice = request.voice.lower()
if request.task_type is None and request.voice in server.precomputed_speakers:
if request.task_type is None and request.voice in self.capabilities.precomputed_speakers:
request.task_type = "Base"
task_type = request.task_type or "CustomVoice"
@@ -48,21 +56,23 @@ class Qwen3TTSAdapter(ARTTSAdapter):
# Validate language (case-insensitive; normalized to the title-cased config form)
if request.language is not None:
request.language = request.language.title()
if request.language not in server.supported_languages:
if request.language not in self.capabilities.supported_languages:
return (
f"Invalid language '{request.language}'. Supported: {', '.join(sorted(server.supported_languages))}"
f"Invalid language '{request.language}'. Supported: "
f"{', '.join(sorted(self.capabilities.supported_languages))}"
)
# Validate speaker for CustomVoice task
if task_type == "CustomVoice":
if not server.supported_speakers:
available_speakers = server._get_available_speakers()
if not available_speakers:
return (
"This model does not support CustomVoice task (no speakers configured). "
"Use task_type='Base' with ref_audio/ref_text for voice cloning, "
"or use a CustomVoice model."
)
if request.voice is not None and request.voice not in server.supported_speakers:
return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}"
if request.voice is not None and request.voice not in available_speakers:
return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(available_speakers))}"
# Validate speaker_embedding constraints
if request.speaker_embedding is not None:
@@ -74,7 +84,7 @@ class Qwen3TTSAdapter(ARTTSAdapter):
# Base task validation so callers don't need to pass it explicitly.
request.x_vector_only_mode = True
emb_len = len(request.speaker_embedding)
dim_err = server._validate_qwen_tts_speaker_embedding_dim(emb_len)
dim_err = self.validate_tts_embedding_dim(emb_len)
if dim_err is not None:
return dim_err
# Validate Base task requirements
@@ -103,8 +113,8 @@ class Qwen3TTSAdapter(ARTTSAdapter):
file_path = Path(speaker_info["file_path"])
if not file_path.exists():
return f"Data file for uploaded speaker '{request.voice}' not found on disk"
elif voice_lower in server.precomputed_speakers:
profile = server.precomputed_speakers[voice_lower]
elif voice_lower in self.capabilities.precomputed_speakers:
profile = self.capabilities.precomputed_speakers[voice_lower]
mode = str(profile.get("mode") or "xvec").lower()
ref_text = request.ref_text or profile.get("ref_text")
if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()):
@@ -164,6 +174,51 @@ class Qwen3TTSAdapter(ARTTSAdapter):
warmup_artifact_key=warmup_key,
)
def _get_expected_speaker_embedding_dim(self) -> int:
"""Return the loaded Qwen3-TTS speaker embedding dim, if known.
The user-provided speaker embedding is concatenated directly with
talker codec embeddings, so the real compatibility requirement is the
talker hidden size.
"""
hf_config = self.ctx.engine_client.model_config.hf_config
talker_config = hf_config.talker_config
return int(talker_config.hidden_size)
def _load_precomputed_speakers(self) -> dict[str, dict]:
return load_precomputed_speakers(
self.ctx.engine_client,
expected_model_type=self.name,
validate_profile=lambda profile, tensors: validate_qwen3_tts_profile(
profile,
tensors,
expected_embedding_dim=self._get_expected_speaker_embedding_dim(),
),
)
def _load_supported_languages(self) -> frozenset[str]:
try:
config = self.ctx.engine_client.model_config.hf_config.talker_config
if isinstance(config, dict):
codec_language_id = config.get("codec_language_id")
else:
codec_language_id = getattr(config, "codec_language_id", None)
if codec_language_id and isinstance(codec_language_id, Mapping):
return frozenset(str(language).title() for language in codec_language_id) | {"Auto"}
logger.warning("No codec_language_id found in talker_config; falling back to default languages")
except Exception as exc:
logger.warning("Could not load languages from model config: %s", exc)
return DEFAULT_TTS_LANGUAGES
def validate_tts_embedding_dim(self, emb_dim: int) -> str | None:
expected_dim = self._get_expected_speaker_embedding_dim()
if emb_dim != expected_dim:
return f"speaker_embedding has {emb_dim} dimensions; expected {expected_dim} for the loaded Qwen3-TTS model"
return None
def apply_sampling_overrides(
self,
sampling_params_list: list,
vllm_omni/entrypoints/openai/tts_adapters/voxcpm2.py
+52 -5
View File
@@ -1,13 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
"""VoxCPM2 serving adapter (AR base-LM + diffusion side-computation)."""
from typing import TYPE_CHECKING, Any
import time
from typing import Any
from vllm.logger import init_logger
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens
from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_precomputed_speakers
from vllm_omni.utils.speaker_cache import validate_voxcpm2_profile
if TYPE_CHECKING:
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
logger = init_logger(__name__)
@register_tts_adapter
@@ -31,7 +36,7 @@ class VoxCPM2Adapter(ARTTSAdapter):
if request.voice is not None:
request.voice = request.voice.lower()
available_voices = set(server.uploaded_speakers) | set(server.precomputed_speakers) | {"default"}
available_voices = server._get_available_speakers()
if request.voice not in available_voices:
supported = ", ".join(sorted(available_voices)) or "none"
return f"Invalid voice '{request.voice}'. Supported: {supported}"
@@ -65,12 +70,54 @@ class VoxCPM2Adapter(ARTTSAdapter):
tts_params = {}
if request.voice:
voice_lower = request.voice.lower()
if voice_lower in server.uploaded_speakers or voice_lower in server.precomputed_speakers:
if voice_lower in server.uploaded_speakers or voice_lower in self.capabilities.precomputed_speakers:
additional = prompt.setdefault("additional_information", {})
additional["voice_name"] = voice_lower
additional["voice_created_at"] = server._voice_created_at(voice_lower)
return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="voxcpm2")
async def warmup(self) -> None:
"""Warm up VoxCPM2 through a synthetic serving request.
VoxCPM2 needs to warm up its PagedAttention scaffold/residual LLMs. CUDA
Graph capture requires a vLLM ``ForwardContext`` containing attention
metadata and slot mappings, which is only available during inference. The
request also pays the one-time torch.compile cost for LocDiT, feat_encoder,
AudioVAE, and projection helpers.
"""
server = self.ctx.server
t0 = time.time()
logger.info("Running warmup speech request for model_type=%s", self.name)
# VoxCPM2 has no predefined speaker presets — "default" means zero-shot
# mode (no voice cloning). The voice field is required by the OpenAI
# API schema but semantically ignored by the model.
warmup_req = OpenAICreateSpeechRequest(
input="Warmup.",
voice="default",
response_format="wav",
speed=1.0,
stream=False,
model=server.model_name,
)
try:
_audio_bytes, _media_type = await server._generate_audio_bytes(warmup_req, request_id="speech-warmup")
except Exception as exc:
logger.warning("Speech warmup failed (non-fatal): %s", exc)
return
elapsed = time.time() - t0
logger.info("Speech warmup complete in %.1fs", elapsed)
def _load_precomputed_speakers(self) -> dict[str, dict]:
return load_precomputed_speakers(
self.ctx.engine_client,
expected_model_type=self.name,
validate_profile=validate_voxcpm2_profile,
)
def _load_supported_speakers(self) -> set[str]:
return {"default"}
def apply_sampling_overrides(
self,
sampling_params_list: list,
vllm_omni/entrypoints/openai/tts_adapters/voxtral.py
+8 -2
View File
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens
from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_supported_speakers
if TYPE_CHECKING:
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
@@ -32,8 +33,9 @@ class VoxtralTTSAdapter(ARTTSAdapter):
if request.voice is not None:
request.voice = request.voice.lower()
if server.supported_speakers and request.voice not in server.supported_speakers:
return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}"
available_speakers = server._get_available_speakers()
if available_speakers and request.voice not in available_speakers:
return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(available_speakers))}"
if request.max_new_tokens is not None:
if request.max_new_tokens < self.max_new_tokens_min:
@@ -49,6 +51,10 @@ class VoxtralTTSAdapter(ARTTSAdapter):
prompt = await self.ctx.server._build_voxtral_prompt_async(request)
return PreparedRequest(prompt=prompt, tts_params={}, model_type="voxtral_tts")
def _load_supported_speakers(self) -> set[str]:
config = self.ctx.engine_client.model_config.hf_config.audio_config
return load_supported_speakers(self.ctx.engine_client, config)
def apply_sampling_overrides(
self,
sampling_params_list: list,
vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py
+6 -1
View File
@@ -27,6 +27,7 @@ from vllm_omni.utils.speaker_cache import (
get_speaker_cache,
iter_custom_voice_profiles,
load_validated_profile_tensors,
validate_qwen3_tts_profile,
)
from .configuration_qwen3_tts import Qwen3TTSConfig, Qwen3TTSSpeakerEncoderConfig, Qwen3TTSTalkerConfig
@@ -527,7 +528,11 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
tensors = load_validated_profile_tensors(
profile,
expected_model_type="qwen3_tts",
qwen3_embedding_dim=expected_dim,
validate_profile=lambda profile, tensors: validate_qwen3_tts_profile(
profile,
tensors,
expected_embedding_dim=expected_dim,
),
)
if tensors is None:
continue
vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py
+6 -1
View File
@@ -42,6 +42,7 @@ from vllm_omni.utils.speaker_cache import (
get_speaker_cache,
iter_custom_voice_profiles,
load_validated_profile_tensors,
validate_voxcpm2_profile,
)
from vllm_omni.worker.runner_assisted_metadata import RunnerAssistedFullAttentionMetadataRequest
@@ -963,7 +964,11 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
loaded = 0
for profile in iter_custom_voice_profiles(custom_voice_dir, expected_model_type="voxcpm2"):
tensors = load_validated_profile_tensors(profile, expected_model_type="voxcpm2")
tensors = load_validated_profile_tensors(
profile,
expected_model_type="voxcpm2",
validate_profile=validate_voxcpm2_profile,
)
if tensors is None:
continue
vllm_omni/utils/speaker_cache.py
+5 -8
View File
@@ -10,6 +10,7 @@ import json
import os
import threading
from collections import OrderedDict
from collections.abc import Callable
from pathlib import Path
from typing import Any
@@ -131,7 +132,7 @@ def _first_dim(tensor: torch.Tensor | None) -> int:
return int(tensor.shape[0])
def _validate_qwen3_tts_profile(
def validate_qwen3_tts_profile(
profile: dict[str, Any],
tensors: dict[str, torch.Tensor],
*,
@@ -164,7 +165,7 @@ def _validate_qwen3_tts_profile(
return None
def _validate_voxcpm2_profile(profile: dict[str, Any], tensors: dict[str, torch.Tensor]) -> str | None:
def validate_voxcpm2_profile(profile: dict[str, Any], tensors: dict[str, torch.Tensor]) -> str | None:
# Successful validation also normalizes/augments `profile` for serving metadata.
ref_audio_feat = tensors.get("ref_audio_feat")
audio_feat = tensors.get("audio_feat")
@@ -207,7 +208,7 @@ def load_validated_profile_tensors(
profile: dict[str, Any],
*,
expected_model_type: str,
qwen3_embedding_dim: int | None = None,
validate_profile: Callable[[dict[str, Any], dict[str, torch.Tensor]], str | None],
) -> dict[str, torch.Tensor] | None:
tensors = _load_profile_tensors(profile)
error = None
@@ -215,12 +216,8 @@ def load_validated_profile_tensors(
error = "safetensors file could not be loaded"
elif str(profile.get("model_type") or expected_model_type or "") != expected_model_type:
error = f"model_type={profile.get('model_type')!r}, expected={expected_model_type!r}"
elif expected_model_type == "qwen3_tts":
error = _validate_qwen3_tts_profile(profile, tensors, expected_embedding_dim=qwen3_embedding_dim)
elif expected_model_type == "voxcpm2":
error = _validate_voxcpm2_profile(profile, tensors)
else:
error = f"unsupported custom voice model_type={expected_model_type!r}"
error = validate_profile(profile, tensors)
if error is not None:
logger.warning("Skipping custom %s voice %s: %s", expected_model_type, profile.get("name"), error)