[Refactor] Move model-specific capability metadata into TTS adapters (#6138)

Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
boatmanandHongsheng Liu authored and GitHub committed 2026-08-22 14:26:49 +08:00
1 parent 42059f19d5
commit 3781214675
15 files changed
+722 -427

No files matched your search

@@ -8,6 +8,7 @@ import logging
import os
import struct
import wave
from dataclasses import FrozenInstanceError, replace
from inspect import Signature, signature
from pathlib import Path
from types import SimpleNamespace
@@ -38,12 +39,13 @@ from vllm_omni.entrypoints.openai.protocol.audio import (
StreamingSpeechSessionConfig,
)
from vllm_omni.entrypoints.openai.serving_speech import (
_TTS_LANGUAGES,
OmniOpenAIServingSpeech,
_create_wav_header,
)
from vllm_omni.entrypoints.openai.tts_adapters.base import PreparedRequest, SpeechServingContext
from vllm_omni.entrypoints.openai.tts_adapters.base import DEFAULT_TTS_LANGUAGES, PreparedRequest, SpeechServingContext
from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_supported_speakers
from vllm_omni.entrypoints.openai.tts_adapters.ming_tts import MingTTSAdapter
from vllm_omni.entrypoints.openai.tts_adapters.voxtral import VoxtralTTSAdapter
from vllm_omni.model_executor.models.fish_speech.prompt_utils import (
FISH_TEXT_ONLY_SYSTEM_PROMPT,
build_fish_voice_clone_prompt_ids,
@@ -218,7 +220,8 @@ def _wav_data_url(samples: np.ndarray, sample_rate: int) -> str:
@pytest.fixture
def test_app(mocker: MockerFixture):
def test_app(mocker: MockerFixture, tmp_path, monkeypatch):
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
# Mock the engine client
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -241,6 +244,11 @@ def test_app(mocker: MockerFixture):
models=mock_models,
request_logger=mock_request_logger,
)
# Voice endpoint tests exercise generic embedding parsing/persistence with
# multiple valid dimensions. Supply the adapter contract without imposing
# one real model's fixed embedding dimension on this shared fixture.
speech_server._adapter = mocker.MagicMock()
speech_server._adapter.validate_tts_embedding_dim.return_value = None
speech_server._tts_tokenizer = lambda text, padding=False: {"input_ids": text.split()}
# Skip TTS validation in tests (mock doesn't set up supported_speakers)
@@ -263,31 +271,9 @@ def test_app(mocker: MockerFixture):
speech_server.create_speech = awaitable_patched_create_speech
app = FastAPI()
app.state.openai_serving_speech = speech_server
app.add_api_route("/v1/audio/speech", speech_server.create_speech, methods=["POST"], response_model=None)
# Add list_voices endpoint
async def list_voices():
speakers = sorted(speech_server.supported_speakers) if speech_server.supported_speakers else []
uploaded_voices = []
if hasattr(speech_server, "uploaded_speakers"):
for voice_name, info in speech_server.uploaded_speakers.items():
voice_entry = {
"name": info.get("name", voice_name),
"consent": info.get("consent", ""),
"created_at": info.get("created_at", 0),
"file_size": info.get("file_size", 0),
"mime_type": info.get("mime_type", ""),
"embedding_source": info.get("embedding_source", "audio"),
"embedding_dim": info.get("embedding_dim"),
}
if info.get("ref_text"):
voice_entry["ref_text"] = info["ref_text"]
if info.get("speaker_description"):
voice_entry["speaker_description"] = info["speaker_description"]
uploaded_voices.append(voice_entry)
return {"voices": speakers, "uploaded_voices": uploaded_voices}
app.add_api_route("/v1/audio/voices", list_voices, methods=["GET"])
app.add_api_route("/v1/audio/voices", api_server_module.list_voices, methods=["GET"])
app.add_api_route("/v1/audio/speech/batch", speech_server.create_speech_batch, methods=["POST"])
# Add upload_voice endpoint
@@ -459,9 +445,31 @@ class TestSpeechAPI:
assert audio_obj.speed == 2.5
def test_list_voices_endpoint(self, client):
handler = client.app.state.openai_serving_speech
handler._adapter.capabilities.supported_speakers = frozenset({"alice"})
handler._adapter.capabilities.precomputed_speakers = {"bob": {"name": "Bob"}}
handler.uploaded_speakers = {
"carol": {
"name": "Carol",
"created_at": 1,
}
}
response = client.get("/v1/audio/voices")
assert response.status_code == 200
assert "voices" in response.json()
assert response.json()["voices"] == ["alice", "bob", "carol", "default"]
assert response.json()["uploaded_voices"] == [
{
"name": "Carol",
"consent": "",
"created_at": 1,
"file_size": 0,
"mime_type": "",
"embedding_source": "audio",
"embedding_dim": None,
}
]
def test_upload_voice_success(self, client, tmp_path):
"""Test successful voice upload without ref_text."""
@@ -719,6 +727,20 @@ class TestSpeechAPI:
assert response.status_code == 400
assert "finite" in response.json()["detail"]
@pytest.mark.asyncio
async def test_diffusion_create_speech_with_unknown_voice(self, mocker: MockerFixture):
engine_client = mocker.MagicMock()
server = OmniOpenAIServingSpeech.for_diffusion(
diffusion_engine=engine_client,
model_name="test-model",
)
assert server._adapter is None
response = await server.create_speech(OpenAICreateSpeechRequest(input="test-input", voice="test-voice"))
assert response.status_code == 400
assert b"Invalid voice" in response.body
@pytest.mark.asyncio
async def test_create_diffusion_speech_extra_params(self, mocker: MockerFixture):
"""Test public diffusion speech success and extra_params propagation."""
@@ -767,7 +789,8 @@ class TestTTSMethods:
"""Unit tests for TTS validation and parameter building."""
@pytest.fixture
def speech_server(self, mocker: MockerFixture):
def speech_server(self, mocker, tmp_path, monkeypatch):
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_configs = []
@@ -990,12 +1013,18 @@ class TestTTSMethods:
def test_speaker_embedding_valid_base_task(self, speech_server):
"""speaker_embedding with Base task, x_vector_only_mode, and no ref_audio is accepted."""
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(talker_config=SimpleNamespace(hidden_size=1024))
)
emb = [0.1] * 1024
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
assert speech_server._validate_tts_request(req) is None
def test_speaker_embedding_auto_sets_x_vector_only_mode(self, speech_server):
"""speaker_embedding auto-implies x_vector_only_mode, so validation passes."""
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(talker_config=SimpleNamespace(hidden_size=1024))
)
emb = [0.1] * 1024
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb)
result = speech_server._validate_tts_request(req)
@@ -1067,6 +1096,7 @@ class TestTTSMethods:
"""Embedding uploads must match the loaded Qwen3-TTS model before being stored."""
speech_server._tts_model_type = "qwen3_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(
talker_config=SimpleNamespace(hidden_size=2048),
@@ -1085,6 +1115,7 @@ class TestTTSMethods:
def test_upload_ming_voice_embedding_wrong_dims_rejected_without_replacing_existing(self, speech_server):
"""Ming embedding uploads must match the model dimension before replacement."""
speech_server._tts_model_type = "ming_tts"
speech_server._adapter = speech_server._get_tts_adapter()
invalid_dim = SPEAKER_EMBEDDING_DIM - 1
existing = {
"name": "bad_emb_voice",
@@ -1112,9 +1143,13 @@ class TestTTSMethods:
def test_upload_ming_audio_voice_defers_speaker_extraction(self, speech_server, mocker: MockerFixture, tmp_path):
"""Ming audio uploads stay model-agnostic and defer extraction to use."""
speech_server._tts_model_type = "ming_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers_dir = tmp_path
speech_server.uploaded_speakers = {}
speech_server.supported_speakers = set()
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
supported_speakers=frozenset(),
)
speech_server._speaker_cache.clear()
left = np.full(32000, 0.5, dtype=np.float32)
@@ -1209,16 +1244,20 @@ class TestTTSMethods:
def test_precomputed_qwen3_voice_infers_base_without_ref_audio(self, speech_server):
"""Precomputed Qwen3 voices are reusable by name without per-request ref_audio."""
speech_server._tts_model_type = "qwen3_tts"
speech_server.precomputed_speakers = {
"alice": {
"name": "Alice",
"model_type": "qwen3_tts",
"mode": "icl",
"ref_text": "reference transcript",
"ref_code_length": 3,
}
}
speech_server.supported_speakers = {"alice"}
speech_server._adapter = speech_server._get_tts_adapter()
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={
"alice": {
"name": "Alice",
"model_type": "qwen3_tts",
"mode": "icl",
"ref_text": "reference transcript",
"ref_code_length": 3,
}
},
supported_speakers=frozenset({"alice"}),
)
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
assert speech_server._validate_tts_request(req) is None
@@ -1239,6 +1278,7 @@ class TestTTSMethods:
uploaded_path = tmp_path / "alice.safetensors"
save_file({"speaker_embedding": torch.tensor([0.1] * 4)}, str(uploaded_path))
speech_server._tts_model_type = "qwen3_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers_dir = tmp_path
speech_server.uploaded_speakers = {
"alice": {
@@ -1248,15 +1288,18 @@ class TestTTSMethods:
"embedding_source": "direct",
}
}
speech_server.precomputed_speakers = {
"alice": {
"name": "Alice",
"model_type": "qwen3_tts",
"mode": "icl",
"ref_text": "precomputed transcript",
"ref_code_length": 3,
}
}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={
"alice": {
"name": "Alice",
"model_type": "qwen3_tts",
"mode": "icl",
"ref_text": "precomputed transcript",
"ref_code_length": 3,
}
},
)
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
assert speech_server._validate_tts_request(req) is None
@@ -1283,13 +1326,16 @@ class TestTTSMethods:
talker_config=SimpleNamespace(hidden_size=4),
)
)
speech_server._adapter = speech_server._get_tts_adapter()
profiles = speech_server._load_precomputed_speakers()
profiles = speech_server._adapter._load_precomputed_speakers()
assert profiles == {}
speech_server.precomputed_speakers = profiles
speech_server.supported_speakers = set(profiles)
assert "alice" not in speech_server.supported_speakers
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers=profiles,
)
assert "alice" not in speech_server._adapter.capabilities.precomputed_speakers
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
assert speech_server._validate_tts_request(req) is not None
@@ -1317,13 +1363,16 @@ class TestTTSMethods:
talker_config=SimpleNamespace(hidden_size=4),
)
)
speech_server._adapter = speech_server._get_tts_adapter()
profiles = speech_server._load_precomputed_speakers()
profiles = speech_server._adapter._load_precomputed_speakers()
assert profiles == {}
speech_server.precomputed_speakers = profiles
speech_server.supported_speakers = set(profiles)
assert "alice" not in speech_server.supported_speakers
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers=profiles,
)
assert "alice" not in speech_server._adapter.capabilities.precomputed_speakers
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
assert speech_server._validate_tts_request(req) is not None
@@ -1338,13 +1387,16 @@ class TestTTSMethods:
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(custom_voice_dir=str(tmp_path))
)
speech_server._adapter = speech_server._get_tts_adapter()
profiles = speech_server._load_precomputed_speakers()
profiles = speech_server._adapter._load_precomputed_speakers()
assert profiles == {}
speech_server.precomputed_speakers = profiles
speech_server.supported_speakers = set(profiles)
assert "bob" not in speech_server.supported_speakers
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers=profiles,
)
assert "bob" not in speech_server._adapter.capabilities.precomputed_speakers
req = OpenAICreateSpeechRequest(input="Hello", voice="Bob")
assert speech_server._validate_tts_request(req) is not None
assert speech_server._validate_tts_request(OpenAICreateSpeechRequest(input="Hello")) is None
@@ -1352,9 +1404,12 @@ class TestTTSMethods:
def test_prepare_voxcpm2_rejects_supported_speaker_without_voice_profile(self, speech_server, mocker):
"""VoxCPM2 named voices must be uploaded or precomputed on the real request path."""
speech_server._tts_model_type = "voxcpm2"
speech_server.supported_speakers = {"bob", "default"}
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers = {}
speech_server.precomputed_speakers = {}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={},
)
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
speech_server.engine_client.generate = mocker.MagicMock(return_value="generator")
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
@@ -1370,9 +1425,12 @@ class TestTTSMethods:
def test_prepare_voxcpm2_accepts_default_voice(self, speech_server, mocker):
"""VoxCPM2 default voice preserves the built-in zero-shot request path."""
speech_server._tts_model_type = "voxcpm2"
speech_server.supported_speakers = {"default"}
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers = {}
speech_server.precomputed_speakers = {}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={},
)
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
@@ -1387,23 +1445,26 @@ class TestTTSMethods:
def test_prepare_voxcpm2_precomputed_voice_sets_model_cache_key(self, speech_server, mocker):
"""VoxCPM2 precomputed voices must carry voice metadata to the model cache lookup."""
speech_server._tts_model_type = "voxcpm2"
speech_server.supported_speakers = {"alice"}
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers = {}
speech_server.precomputed_speakers = {
"alice": {
"name": "Alice",
"model_type": "voxcpm2",
"mode": "reference",
"ref_audio_feat_len": 2,
}
}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={
"alice": {
"name": "Alice",
"model_type": "voxcpm2",
"mode": "reference",
"ref_audio_feat_len": 2,
}
},
)
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
return_value={
"prompt_token_ids": [1],
"additional_information": {
"voice_profile": speech_server.precomputed_speakers["alice"],
"voice_profile": speech_server._adapter.capabilities.precomputed_speakers["alice"],
},
}
)
@@ -1482,6 +1543,10 @@ class TestTTSMethods:
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
mock_stage = mocker.MagicMock()
mock_stage.engine_args.model_stage = "qwen3_tts"
mock_engine_client.stage_configs = [mock_stage]
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
@@ -1489,7 +1554,10 @@ class TestTTSMethods:
)
# Verify speakers are normalized to lowercase
assert server.supported_speakers == {"ryan", "vivian", "aiden"}
assert server._adapter.capabilities.supported_speakers == {"ryan", "vivian", "aiden"}
assert isinstance(server._adapter.capabilities.supported_speakers, frozenset)
with pytest.raises(FrozenInstanceError):
setattr(server._adapter.capabilities, "codec_frame_rate", 25.0)
def test_load_supported_speakers_skips_non_tts_omni_model(
self,
@@ -1502,7 +1570,7 @@ class TestTTSMethods:
)
assert speech_server._is_tts is False
assert speech_server._load_supported_speakers() == set()
assert load_supported_speakers(speech_server.engine_client) == set()
warning.assert_not_called()
def test_load_supported_languages_from_config(self, speech_server):
@@ -1515,7 +1583,8 @@ class TestTTSMethods:
)
)
)
assert speech_server._load_supported_languages() == {
adapter = speech_server._get_tts_adapter()
assert adapter._load_supported_languages() == {
"Chinese",
"English",
"Beijing_Dialect",
@@ -1528,11 +1597,17 @@ class TestTTSMethods:
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(talker_config={"codec_language_id": {"chinese": 2055, "english": 2050}})
)
assert speech_server._load_supported_languages() == {"Chinese", "English", "Auto"}
adapter = speech_server._get_tts_adapter()
assert adapter._load_supported_languages() == {"Chinese", "English", "Auto"}
def test_validate_language_custom_dialect_accepted(self, speech_server):
"""A language present in the model config passes validation, case-insensitively."""
speech_server.supported_languages = {"Chinese", "English", "Beijing_Dialect", "Auto"}
speech_server._tts_model_type = "qwen3_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
supported_languages=frozenset({"Chinese", "English", "Beijing_Dialect", "Auto"}),
)
for language in ("Beijing_Dialect", "beijing_dialect", "English", "english", "Auto", "AUTO"):
req = OpenAICreateSpeechRequest(input="Hello", language=language)
result = speech_server._validate_tts_request(req)
@@ -1542,7 +1617,12 @@ class TestTTSMethods:
def test_validate_language_unknown_rejected(self, speech_server):
"""A language not in the configured set is rejected."""
speech_server.supported_languages = {"Chinese", "English", "Auto"}
speech_server._tts_model_type = "qwen3_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
supported_languages=frozenset({"Chinese", "English", "Auto"}),
)
for language in ("Klingon", "klingon"):
req = OpenAICreateSpeechRequest(input="Hello", language=language)
assert "Invalid language" in speech_server._validate_tts_request(req)
@@ -1553,18 +1633,21 @@ class TestTTSMethods:
speech_server.engine_client.model_config = SimpleNamespace(
hf_config=SimpleNamespace(talker_config=SimpleNamespace(codec_language_id={}))
)
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
adapter = speech_server._get_tts_adapter()
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
def test_load_supported_languages_default_on_config_error(self, speech_server):
"""If the model config cannot be read, fall back to the default list."""
speech_server._tts_model_type = "qwen3_tts"
speech_server.engine_client = SimpleNamespace() # no model_config -> AttributeError
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
speech_server._adapter = None
adapter = speech_server._get_tts_adapter()
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
def test_load_supported_languages_default_for_non_qwen(self, speech_server):
"""Non-qwen3_tts model types get the default language set."""
speech_server._tts_model_type = None
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
adapter = MingTTSAdapter(SpeechServingContext(server=speech_server, engine_client=speech_server.engine_client))
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
def test_build_tts_params_with_uploaded_voice(self, speech_server, mocker: MockerFixture):
"""Test _build_tts_params auto-sets ref_audio for uploaded voices (x_vector only)."""
@@ -1577,7 +1660,6 @@ class TestTTSMethods:
"created_at": 1711234567,
}
}
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
mock_get_audio = mocker.patch.object(speech_server, "_get_uploaded_audio_data")
mock_get_audio.return_value = "data:audio/wav;base64,ZmFrZWF1ZGlv"
@@ -1601,7 +1683,6 @@ class TestTTSMethods:
"created_at": 1711234567,
}
}
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
mock_get_audio = mocker.patch.object(speech_server, "_get_uploaded_audio_data")
mock_get_audio.return_value = "data:audio/wav;base64,ZmFrZWF1ZGlv"
@@ -1618,7 +1699,6 @@ class TestTTSMethods:
"""Test _build_tts_params does not auto-set ref_audio for non-uploaded voices."""
# No uploaded speakers
speech_server.uploaded_speakers = {}
speech_server.supported_speakers = {"ryan", "vivian"}
req = OpenAICreateSpeechRequest(input="Hello", voice="ryan", task_type="Base")
@@ -1638,7 +1718,6 @@ class TestTTSMethods:
"mime_type": "audio/wav",
}
}
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
req = OpenAICreateSpeechRequest(
input="Hello", voice="custom_voice", task_type="Base", ref_audio="data:audio/wav;base64,ZXhwbGljaXQ="
@@ -1744,7 +1823,6 @@ class TestTTSMethods:
"cache_file": "/tmp/voice_samples/emb_voice.safetensors",
}
}
speech_server.supported_speakers = {"ryan", "vivian", "emb_voice"}
fake_embedding = [0.1] * 1024
mock_get_emb = mocker.patch.object(speech_server, "_get_uploaded_speaker_embedding")
@@ -2203,6 +2281,28 @@ class TestTTSMethods:
assert request.ref_audio is not None
assert call.kwargs["has_inline_ref_audio"] is False
@pytest.mark.asyncio
async def test_warmup_with_no_adapter(self, speech_server):
assert speech_server._adapter is None
await speech_server.warmup()
@pytest.mark.asyncio
async def test_voxcpm2_warmup_successfully(self, speech_server, mocker):
speech_server._tts_model_type = "voxcpm2"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server.uploaded_speakers = {}
speech_server._adapter.capabilities = replace(
speech_server._adapter.capabilities,
precomputed_speakers={},
)
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
return_value={"prompt_token_ids": [1], "additional_information": {}}
)
await speech_server.warmup()
class TestFileValidationFunctions:
"""Unit tests for file validation helper functions."""
@@ -2973,6 +3073,39 @@ def test_api_server_upload_voice_exception_returns_500(mocker: MockerFixture):
)
def test_api_server_upload_voice_with_no_adapter(
mocker: MockerFixture,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
):
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
_patch_api_server_base(mocker)
engine_client = mocker.MagicMock()
engine_client.errored = False
engine_client.stage_configs = []
models = mocker.MagicMock()
models.is_base_model.return_value = True
handler = OmniOpenAIServingSpeech(
engine_client=engine_client,
models=models,
request_logger=mocker.MagicMock(),
)
assert handler._adapter is None
raw_request = _make_api_server_request(handler)
response = asyncio.run(
api_server_module.upload_voice(
raw_request,
audio_sample=None,
speaker_embedding=json.dumps([0.1] * 1024),
consent="cons_test",
name="probe",
)
)
assert response.status_code == 200
def test_api_server_delete_voice_without_speech_handler_returns_404(mocker: MockerFixture):
_patch_api_server_base(mocker)
raw_request = _make_api_server_request(None, method="DELETE", path="/v1/audio/voices/probe")
@@ -3302,8 +3435,14 @@ class _FakeFishTokenizer:
@pytest.fixture
def fish_speech_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3573,8 +3712,14 @@ class TestWAVStreaming:
@pytest.fixture
def cosyvoice3_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3709,8 +3854,14 @@ class TestCosyVoice3Serving:
@pytest.fixture
def glm_tts_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3732,7 +3883,6 @@ def glm_tts_server(mocker: MockerFixture):
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
return OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
@@ -3809,8 +3959,14 @@ class TestGLMTTSServing:
@pytest.fixture
def ming_tts_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3868,8 +4024,14 @@ class TestMingTTSServing:
@pytest.fixture
def ming_flash_omni_tts_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
@@ -3932,8 +4094,14 @@ class TestTTSAsyncOffloading:
@pytest.fixture
def voxtral_server(self, mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.voxtral.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.model_config = mocker.MagicMock(model="mistralai/Voxtral")
@@ -3958,8 +4126,14 @@ class TestTTSAsyncOffloading:
@pytest.fixture
def qwen3_tts_server(self, mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.model_config = mocker.MagicMock(model="Qwen/Qwen3-TTS", hf_config=mocker.MagicMock())
@@ -3983,6 +4157,32 @@ class TestTTSAsyncOffloading:
yield server
server.shutdown()
def test_voxtral_loads_supported_speakers(self, mocker: MockerFixture):
engine_client = mocker.MagicMock()
engine_client.model_config.hf_config.audio_config = {
"speaker_id": {
"Alice": 0,
"Bob": 1,
}
}
server = mocker.MagicMock()
adapter = VoxtralTTSAdapter(
SpeechServingContext(
server=server,
engine_client=engine_client,
)
)
adapter.load_capabilities()
assert adapter.capabilities.supported_speakers == frozenset(
{
"alice",
"bob",
}
)
def test_prepare_speech_generation_awaits_voxtral_async(self, voxtral_server, mocker: MockerFixture):
"""Voxtral path in _prepare_speech_generation should call the async wrapper."""
voxtral_server._build_voxtral_prompt_async = mocker.AsyncMock(
@@ -4214,7 +4414,10 @@ class TestTTSAsyncOffloading:
artifact_key,
)
qwen3_tts_server._ref_audio_model_artifact_ready.add((artifact_key, False))
qwen3_tts_server._codec_frame_rate = 25.0
qwen3_tts_server._adapter.capabilities = replace(
qwen3_tts_server._adapter.capabilities,
codec_frame_rate=25.0,
)
qwen3_tts_server._tts_tokenizer = lambda _text, padding=False: {"input_ids": list(range(10))}
qwen3_tts_server.engine_client.model_config.hf_config.talker_config = SimpleNamespace(
codec_language_id={},
@@ -4319,8 +4522,14 @@ class TestTTSAsyncOffloading:
def test_shutdown_is_idempotent(self, mocker: MockerFixture):
"""Calling shutdown() twice should not raise."""
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
return_value=set(),
)
mocker.patch(
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
return_value=None,
)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_configs = []