[Refactor] Move model-specific capability metadata into TTS adapters (#6138)
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
42059f19d5
commit
3781214675
15 files changed
+722
-427
No files matched your search
@@ -8,6 +8,7 @@ import logging
|
||||
import os
|
||||
import struct
|
||||
import wave
|
||||
from dataclasses import FrozenInstanceError, replace
|
||||
from inspect import Signature, signature
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -38,12 +39,13 @@ from vllm_omni.entrypoints.openai.protocol.audio import (
|
||||
StreamingSpeechSessionConfig,
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.serving_speech import (
|
||||
_TTS_LANGUAGES,
|
||||
OmniOpenAIServingSpeech,
|
||||
_create_wav_header,
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import PreparedRequest, SpeechServingContext
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import DEFAULT_TTS_LANGUAGES, PreparedRequest, SpeechServingContext
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.capabilities import load_supported_speakers
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.ming_tts import MingTTSAdapter
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.voxtral import VoxtralTTSAdapter
|
||||
from vllm_omni.model_executor.models.fish_speech.prompt_utils import (
|
||||
FISH_TEXT_ONLY_SYSTEM_PROMPT,
|
||||
build_fish_voice_clone_prompt_ids,
|
||||
@@ -218,7 +220,8 @@ def _wav_data_url(samples: np.ndarray, sample_rate: int) -> str:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_app(mocker: MockerFixture):
|
||||
def test_app(mocker: MockerFixture, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
|
||||
# Mock the engine client
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
@@ -241,6 +244,11 @@ def test_app(mocker: MockerFixture):
|
||||
models=mock_models,
|
||||
request_logger=mock_request_logger,
|
||||
)
|
||||
# Voice endpoint tests exercise generic embedding parsing/persistence with
|
||||
# multiple valid dimensions. Supply the adapter contract without imposing
|
||||
# one real model's fixed embedding dimension on this shared fixture.
|
||||
speech_server._adapter = mocker.MagicMock()
|
||||
speech_server._adapter.validate_tts_embedding_dim.return_value = None
|
||||
speech_server._tts_tokenizer = lambda text, padding=False: {"input_ids": text.split()}
|
||||
|
||||
# Skip TTS validation in tests (mock doesn't set up supported_speakers)
|
||||
@@ -263,31 +271,9 @@ def test_app(mocker: MockerFixture):
|
||||
speech_server.create_speech = awaitable_patched_create_speech
|
||||
|
||||
app = FastAPI()
|
||||
app.state.openai_serving_speech = speech_server
|
||||
app.add_api_route("/v1/audio/speech", speech_server.create_speech, methods=["POST"], response_model=None)
|
||||
|
||||
# Add list_voices endpoint
|
||||
async def list_voices():
|
||||
speakers = sorted(speech_server.supported_speakers) if speech_server.supported_speakers else []
|
||||
uploaded_voices = []
|
||||
if hasattr(speech_server, "uploaded_speakers"):
|
||||
for voice_name, info in speech_server.uploaded_speakers.items():
|
||||
voice_entry = {
|
||||
"name": info.get("name", voice_name),
|
||||
"consent": info.get("consent", ""),
|
||||
"created_at": info.get("created_at", 0),
|
||||
"file_size": info.get("file_size", 0),
|
||||
"mime_type": info.get("mime_type", ""),
|
||||
"embedding_source": info.get("embedding_source", "audio"),
|
||||
"embedding_dim": info.get("embedding_dim"),
|
||||
}
|
||||
if info.get("ref_text"):
|
||||
voice_entry["ref_text"] = info["ref_text"]
|
||||
if info.get("speaker_description"):
|
||||
voice_entry["speaker_description"] = info["speaker_description"]
|
||||
uploaded_voices.append(voice_entry)
|
||||
return {"voices": speakers, "uploaded_voices": uploaded_voices}
|
||||
|
||||
app.add_api_route("/v1/audio/voices", list_voices, methods=["GET"])
|
||||
app.add_api_route("/v1/audio/voices", api_server_module.list_voices, methods=["GET"])
|
||||
app.add_api_route("/v1/audio/speech/batch", speech_server.create_speech_batch, methods=["POST"])
|
||||
|
||||
# Add upload_voice endpoint
|
||||
@@ -459,9 +445,31 @@ class TestSpeechAPI:
|
||||
assert audio_obj.speed == 2.5
|
||||
|
||||
def test_list_voices_endpoint(self, client):
|
||||
handler = client.app.state.openai_serving_speech
|
||||
handler._adapter.capabilities.supported_speakers = frozenset({"alice"})
|
||||
handler._adapter.capabilities.precomputed_speakers = {"bob": {"name": "Bob"}}
|
||||
handler.uploaded_speakers = {
|
||||
"carol": {
|
||||
"name": "Carol",
|
||||
"created_at": 1,
|
||||
}
|
||||
}
|
||||
|
||||
response = client.get("/v1/audio/voices")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "voices" in response.json()
|
||||
assert response.json()["voices"] == ["alice", "bob", "carol", "default"]
|
||||
assert response.json()["uploaded_voices"] == [
|
||||
{
|
||||
"name": "Carol",
|
||||
"consent": "",
|
||||
"created_at": 1,
|
||||
"file_size": 0,
|
||||
"mime_type": "",
|
||||
"embedding_source": "audio",
|
||||
"embedding_dim": None,
|
||||
}
|
||||
]
|
||||
|
||||
def test_upload_voice_success(self, client, tmp_path):
|
||||
"""Test successful voice upload without ref_text."""
|
||||
@@ -719,6 +727,20 @@ class TestSpeechAPI:
|
||||
assert response.status_code == 400
|
||||
assert "finite" in response.json()["detail"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diffusion_create_speech_with_unknown_voice(self, mocker: MockerFixture):
|
||||
engine_client = mocker.MagicMock()
|
||||
server = OmniOpenAIServingSpeech.for_diffusion(
|
||||
diffusion_engine=engine_client,
|
||||
model_name="test-model",
|
||||
)
|
||||
assert server._adapter is None
|
||||
|
||||
response = await server.create_speech(OpenAICreateSpeechRequest(input="test-input", voice="test-voice"))
|
||||
|
||||
assert response.status_code == 400
|
||||
assert b"Invalid voice" in response.body
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_diffusion_speech_extra_params(self, mocker: MockerFixture):
|
||||
"""Test public diffusion speech success and extra_params propagation."""
|
||||
@@ -767,7 +789,8 @@ class TestTTSMethods:
|
||||
"""Unit tests for TTS validation and parameter building."""
|
||||
|
||||
@pytest.fixture
|
||||
def speech_server(self, mocker: MockerFixture):
|
||||
def speech_server(self, mocker, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.stage_configs = []
|
||||
@@ -990,12 +1013,18 @@ class TestTTSMethods:
|
||||
|
||||
def test_speaker_embedding_valid_base_task(self, speech_server):
|
||||
"""speaker_embedding with Base task, x_vector_only_mode, and no ref_audio is accepted."""
|
||||
speech_server.engine_client.model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(talker_config=SimpleNamespace(hidden_size=1024))
|
||||
)
|
||||
emb = [0.1] * 1024
|
||||
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
|
||||
assert speech_server._validate_tts_request(req) is None
|
||||
|
||||
def test_speaker_embedding_auto_sets_x_vector_only_mode(self, speech_server):
|
||||
"""speaker_embedding auto-implies x_vector_only_mode, so validation passes."""
|
||||
speech_server.engine_client.model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(talker_config=SimpleNamespace(hidden_size=1024))
|
||||
)
|
||||
emb = [0.1] * 1024
|
||||
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb)
|
||||
result = speech_server._validate_tts_request(req)
|
||||
@@ -1067,6 +1096,7 @@ class TestTTSMethods:
|
||||
"""Embedding uploads must match the loaded Qwen3-TTS model before being stored."""
|
||||
|
||||
speech_server._tts_model_type = "qwen3_tts"
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server.engine_client.model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
talker_config=SimpleNamespace(hidden_size=2048),
|
||||
@@ -1085,6 +1115,7 @@ class TestTTSMethods:
|
||||
def test_upload_ming_voice_embedding_wrong_dims_rejected_without_replacing_existing(self, speech_server):
|
||||
"""Ming embedding uploads must match the model dimension before replacement."""
|
||||
speech_server._tts_model_type = "ming_tts"
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
invalid_dim = SPEAKER_EMBEDDING_DIM - 1
|
||||
existing = {
|
||||
"name": "bad_emb_voice",
|
||||
@@ -1112,9 +1143,13 @@ class TestTTSMethods:
|
||||
def test_upload_ming_audio_voice_defers_speaker_extraction(self, speech_server, mocker: MockerFixture, tmp_path):
|
||||
"""Ming audio uploads stay model-agnostic and defer extraction to use."""
|
||||
speech_server._tts_model_type = "ming_tts"
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server.uploaded_speakers_dir = tmp_path
|
||||
speech_server.uploaded_speakers = {}
|
||||
speech_server.supported_speakers = set()
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
supported_speakers=frozenset(),
|
||||
)
|
||||
speech_server._speaker_cache.clear()
|
||||
|
||||
left = np.full(32000, 0.5, dtype=np.float32)
|
||||
@@ -1209,16 +1244,20 @@ class TestTTSMethods:
|
||||
def test_precomputed_qwen3_voice_infers_base_without_ref_audio(self, speech_server):
|
||||
"""Precomputed Qwen3 voices are reusable by name without per-request ref_audio."""
|
||||
speech_server._tts_model_type = "qwen3_tts"
|
||||
speech_server.precomputed_speakers = {
|
||||
"alice": {
|
||||
"name": "Alice",
|
||||
"model_type": "qwen3_tts",
|
||||
"mode": "icl",
|
||||
"ref_text": "reference transcript",
|
||||
"ref_code_length": 3,
|
||||
}
|
||||
}
|
||||
speech_server.supported_speakers = {"alice"}
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers={
|
||||
"alice": {
|
||||
"name": "Alice",
|
||||
"model_type": "qwen3_tts",
|
||||
"mode": "icl",
|
||||
"ref_text": "reference transcript",
|
||||
"ref_code_length": 3,
|
||||
}
|
||||
},
|
||||
supported_speakers=frozenset({"alice"}),
|
||||
)
|
||||
|
||||
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
|
||||
assert speech_server._validate_tts_request(req) is None
|
||||
@@ -1239,6 +1278,7 @@ class TestTTSMethods:
|
||||
uploaded_path = tmp_path / "alice.safetensors"
|
||||
save_file({"speaker_embedding": torch.tensor([0.1] * 4)}, str(uploaded_path))
|
||||
speech_server._tts_model_type = "qwen3_tts"
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server.uploaded_speakers_dir = tmp_path
|
||||
speech_server.uploaded_speakers = {
|
||||
"alice": {
|
||||
@@ -1248,15 +1288,18 @@ class TestTTSMethods:
|
||||
"embedding_source": "direct",
|
||||
}
|
||||
}
|
||||
speech_server.precomputed_speakers = {
|
||||
"alice": {
|
||||
"name": "Alice",
|
||||
"model_type": "qwen3_tts",
|
||||
"mode": "icl",
|
||||
"ref_text": "precomputed transcript",
|
||||
"ref_code_length": 3,
|
||||
}
|
||||
}
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers={
|
||||
"alice": {
|
||||
"name": "Alice",
|
||||
"model_type": "qwen3_tts",
|
||||
"mode": "icl",
|
||||
"ref_text": "precomputed transcript",
|
||||
"ref_code_length": 3,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
|
||||
assert speech_server._validate_tts_request(req) is None
|
||||
@@ -1283,13 +1326,16 @@ class TestTTSMethods:
|
||||
talker_config=SimpleNamespace(hidden_size=4),
|
||||
)
|
||||
)
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
|
||||
profiles = speech_server._load_precomputed_speakers()
|
||||
profiles = speech_server._adapter._load_precomputed_speakers()
|
||||
assert profiles == {}
|
||||
|
||||
speech_server.precomputed_speakers = profiles
|
||||
speech_server.supported_speakers = set(profiles)
|
||||
assert "alice" not in speech_server.supported_speakers
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers=profiles,
|
||||
)
|
||||
assert "alice" not in speech_server._adapter.capabilities.precomputed_speakers
|
||||
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
|
||||
assert speech_server._validate_tts_request(req) is not None
|
||||
|
||||
@@ -1317,13 +1363,16 @@ class TestTTSMethods:
|
||||
talker_config=SimpleNamespace(hidden_size=4),
|
||||
)
|
||||
)
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
|
||||
profiles = speech_server._load_precomputed_speakers()
|
||||
profiles = speech_server._adapter._load_precomputed_speakers()
|
||||
assert profiles == {}
|
||||
|
||||
speech_server.precomputed_speakers = profiles
|
||||
speech_server.supported_speakers = set(profiles)
|
||||
assert "alice" not in speech_server.supported_speakers
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers=profiles,
|
||||
)
|
||||
assert "alice" not in speech_server._adapter.capabilities.precomputed_speakers
|
||||
req = OpenAICreateSpeechRequest(input="Hello", voice="Alice")
|
||||
assert speech_server._validate_tts_request(req) is not None
|
||||
|
||||
@@ -1338,13 +1387,16 @@ class TestTTSMethods:
|
||||
speech_server.engine_client.model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(custom_voice_dir=str(tmp_path))
|
||||
)
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
|
||||
profiles = speech_server._load_precomputed_speakers()
|
||||
profiles = speech_server._adapter._load_precomputed_speakers()
|
||||
assert profiles == {}
|
||||
|
||||
speech_server.precomputed_speakers = profiles
|
||||
speech_server.supported_speakers = set(profiles)
|
||||
assert "bob" not in speech_server.supported_speakers
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers=profiles,
|
||||
)
|
||||
assert "bob" not in speech_server._adapter.capabilities.precomputed_speakers
|
||||
req = OpenAICreateSpeechRequest(input="Hello", voice="Bob")
|
||||
assert speech_server._validate_tts_request(req) is not None
|
||||
assert speech_server._validate_tts_request(OpenAICreateSpeechRequest(input="Hello")) is None
|
||||
@@ -1352,9 +1404,12 @@ class TestTTSMethods:
|
||||
def test_prepare_voxcpm2_rejects_supported_speaker_without_voice_profile(self, speech_server, mocker):
|
||||
"""VoxCPM2 named voices must be uploaded or precomputed on the real request path."""
|
||||
speech_server._tts_model_type = "voxcpm2"
|
||||
speech_server.supported_speakers = {"bob", "default"}
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server.uploaded_speakers = {}
|
||||
speech_server.precomputed_speakers = {}
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers={},
|
||||
)
|
||||
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||
speech_server.engine_client.generate = mocker.MagicMock(return_value="generator")
|
||||
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
|
||||
@@ -1370,9 +1425,12 @@ class TestTTSMethods:
|
||||
def test_prepare_voxcpm2_accepts_default_voice(self, speech_server, mocker):
|
||||
"""VoxCPM2 default voice preserves the built-in zero-shot request path."""
|
||||
speech_server._tts_model_type = "voxcpm2"
|
||||
speech_server.supported_speakers = {"default"}
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server.uploaded_speakers = {}
|
||||
speech_server.precomputed_speakers = {}
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers={},
|
||||
)
|
||||
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
|
||||
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
|
||||
@@ -1387,23 +1445,26 @@ class TestTTSMethods:
|
||||
def test_prepare_voxcpm2_precomputed_voice_sets_model_cache_key(self, speech_server, mocker):
|
||||
"""VoxCPM2 precomputed voices must carry voice metadata to the model cache lookup."""
|
||||
speech_server._tts_model_type = "voxcpm2"
|
||||
speech_server.supported_speakers = {"alice"}
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server.uploaded_speakers = {}
|
||||
speech_server.precomputed_speakers = {
|
||||
"alice": {
|
||||
"name": "Alice",
|
||||
"model_type": "voxcpm2",
|
||||
"mode": "reference",
|
||||
"ref_audio_feat_len": 2,
|
||||
}
|
||||
}
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers={
|
||||
"alice": {
|
||||
"name": "Alice",
|
||||
"model_type": "voxcpm2",
|
||||
"mode": "reference",
|
||||
"ref_audio_feat_len": 2,
|
||||
}
|
||||
},
|
||||
)
|
||||
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
|
||||
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
|
||||
return_value={
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": {
|
||||
"voice_profile": speech_server.precomputed_speakers["alice"],
|
||||
"voice_profile": speech_server._adapter.capabilities.precomputed_speakers["alice"],
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -1482,6 +1543,10 @@ class TestTTSMethods:
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
mock_stage = mocker.MagicMock()
|
||||
mock_stage.engine_args.model_stage = "qwen3_tts"
|
||||
mock_engine_client.stage_configs = [mock_stage]
|
||||
|
||||
server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
@@ -1489,7 +1554,10 @@ class TestTTSMethods:
|
||||
)
|
||||
|
||||
# Verify speakers are normalized to lowercase
|
||||
assert server.supported_speakers == {"ryan", "vivian", "aiden"}
|
||||
assert server._adapter.capabilities.supported_speakers == {"ryan", "vivian", "aiden"}
|
||||
assert isinstance(server._adapter.capabilities.supported_speakers, frozenset)
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
setattr(server._adapter.capabilities, "codec_frame_rate", 25.0)
|
||||
|
||||
def test_load_supported_speakers_skips_non_tts_omni_model(
|
||||
self,
|
||||
@@ -1502,7 +1570,7 @@ class TestTTSMethods:
|
||||
)
|
||||
|
||||
assert speech_server._is_tts is False
|
||||
assert speech_server._load_supported_speakers() == set()
|
||||
assert load_supported_speakers(speech_server.engine_client) == set()
|
||||
warning.assert_not_called()
|
||||
|
||||
def test_load_supported_languages_from_config(self, speech_server):
|
||||
@@ -1515,7 +1583,8 @@ class TestTTSMethods:
|
||||
)
|
||||
)
|
||||
)
|
||||
assert speech_server._load_supported_languages() == {
|
||||
adapter = speech_server._get_tts_adapter()
|
||||
assert adapter._load_supported_languages() == {
|
||||
"Chinese",
|
||||
"English",
|
||||
"Beijing_Dialect",
|
||||
@@ -1528,11 +1597,17 @@ class TestTTSMethods:
|
||||
speech_server.engine_client.model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(talker_config={"codec_language_id": {"chinese": 2055, "english": 2050}})
|
||||
)
|
||||
assert speech_server._load_supported_languages() == {"Chinese", "English", "Auto"}
|
||||
adapter = speech_server._get_tts_adapter()
|
||||
assert adapter._load_supported_languages() == {"Chinese", "English", "Auto"}
|
||||
|
||||
def test_validate_language_custom_dialect_accepted(self, speech_server):
|
||||
"""A language present in the model config passes validation, case-insensitively."""
|
||||
speech_server.supported_languages = {"Chinese", "English", "Beijing_Dialect", "Auto"}
|
||||
speech_server._tts_model_type = "qwen3_tts"
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
supported_languages=frozenset({"Chinese", "English", "Beijing_Dialect", "Auto"}),
|
||||
)
|
||||
for language in ("Beijing_Dialect", "beijing_dialect", "English", "english", "Auto", "AUTO"):
|
||||
req = OpenAICreateSpeechRequest(input="Hello", language=language)
|
||||
result = speech_server._validate_tts_request(req)
|
||||
@@ -1542,7 +1617,12 @@ class TestTTSMethods:
|
||||
|
||||
def test_validate_language_unknown_rejected(self, speech_server):
|
||||
"""A language not in the configured set is rejected."""
|
||||
speech_server.supported_languages = {"Chinese", "English", "Auto"}
|
||||
speech_server._tts_model_type = "qwen3_tts"
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
supported_languages=frozenset({"Chinese", "English", "Auto"}),
|
||||
)
|
||||
for language in ("Klingon", "klingon"):
|
||||
req = OpenAICreateSpeechRequest(input="Hello", language=language)
|
||||
assert "Invalid language" in speech_server._validate_tts_request(req)
|
||||
@@ -1553,18 +1633,21 @@ class TestTTSMethods:
|
||||
speech_server.engine_client.model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(talker_config=SimpleNamespace(codec_language_id={}))
|
||||
)
|
||||
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
|
||||
adapter = speech_server._get_tts_adapter()
|
||||
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
|
||||
|
||||
def test_load_supported_languages_default_on_config_error(self, speech_server):
|
||||
"""If the model config cannot be read, fall back to the default list."""
|
||||
speech_server._tts_model_type = "qwen3_tts"
|
||||
speech_server.engine_client = SimpleNamespace() # no model_config -> AttributeError
|
||||
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
|
||||
speech_server._adapter = None
|
||||
adapter = speech_server._get_tts_adapter()
|
||||
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
|
||||
|
||||
def test_load_supported_languages_default_for_non_qwen(self, speech_server):
|
||||
"""Non-qwen3_tts model types get the default language set."""
|
||||
speech_server._tts_model_type = None
|
||||
assert speech_server._load_supported_languages() == _TTS_LANGUAGES
|
||||
adapter = MingTTSAdapter(SpeechServingContext(server=speech_server, engine_client=speech_server.engine_client))
|
||||
assert adapter._load_supported_languages() == DEFAULT_TTS_LANGUAGES
|
||||
|
||||
def test_build_tts_params_with_uploaded_voice(self, speech_server, mocker: MockerFixture):
|
||||
"""Test _build_tts_params auto-sets ref_audio for uploaded voices (x_vector only)."""
|
||||
@@ -1577,7 +1660,6 @@ class TestTTSMethods:
|
||||
"created_at": 1711234567,
|
||||
}
|
||||
}
|
||||
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
|
||||
|
||||
mock_get_audio = mocker.patch.object(speech_server, "_get_uploaded_audio_data")
|
||||
mock_get_audio.return_value = "data:audio/wav;base64,ZmFrZWF1ZGlv"
|
||||
@@ -1601,7 +1683,6 @@ class TestTTSMethods:
|
||||
"created_at": 1711234567,
|
||||
}
|
||||
}
|
||||
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
|
||||
|
||||
mock_get_audio = mocker.patch.object(speech_server, "_get_uploaded_audio_data")
|
||||
mock_get_audio.return_value = "data:audio/wav;base64,ZmFrZWF1ZGlv"
|
||||
@@ -1618,7 +1699,6 @@ class TestTTSMethods:
|
||||
"""Test _build_tts_params does not auto-set ref_audio for non-uploaded voices."""
|
||||
# No uploaded speakers
|
||||
speech_server.uploaded_speakers = {}
|
||||
speech_server.supported_speakers = {"ryan", "vivian"}
|
||||
|
||||
req = OpenAICreateSpeechRequest(input="Hello", voice="ryan", task_type="Base")
|
||||
|
||||
@@ -1638,7 +1718,6 @@ class TestTTSMethods:
|
||||
"mime_type": "audio/wav",
|
||||
}
|
||||
}
|
||||
speech_server.supported_speakers = {"ryan", "vivian", "custom_voice"}
|
||||
|
||||
req = OpenAICreateSpeechRequest(
|
||||
input="Hello", voice="custom_voice", task_type="Base", ref_audio="data:audio/wav;base64,ZXhwbGljaXQ="
|
||||
@@ -1744,7 +1823,6 @@ class TestTTSMethods:
|
||||
"cache_file": "/tmp/voice_samples/emb_voice.safetensors",
|
||||
}
|
||||
}
|
||||
speech_server.supported_speakers = {"ryan", "vivian", "emb_voice"}
|
||||
|
||||
fake_embedding = [0.1] * 1024
|
||||
mock_get_emb = mocker.patch.object(speech_server, "_get_uploaded_speaker_embedding")
|
||||
@@ -2203,6 +2281,28 @@ class TestTTSMethods:
|
||||
assert request.ref_audio is not None
|
||||
assert call.kwargs["has_inline_ref_audio"] is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_warmup_with_no_adapter(self, speech_server):
|
||||
assert speech_server._adapter is None
|
||||
await speech_server.warmup()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_voxcpm2_warmup_successfully(self, speech_server, mocker):
|
||||
speech_server._tts_model_type = "voxcpm2"
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server.uploaded_speakers = {}
|
||||
speech_server._adapter.capabilities = replace(
|
||||
speech_server._adapter.capabilities,
|
||||
precomputed_speakers={},
|
||||
)
|
||||
speech_server.engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||
speech_server.engine_client.generate = mocker.MagicMock(return_value=iter(()))
|
||||
speech_server._build_voxcpm2_prompt = mocker.AsyncMock(
|
||||
return_value={"prompt_token_ids": [1], "additional_information": {}}
|
||||
)
|
||||
|
||||
await speech_server.warmup()
|
||||
|
||||
|
||||
class TestFileValidationFunctions:
|
||||
"""Unit tests for file validation helper functions."""
|
||||
@@ -2973,6 +3073,39 @@ def test_api_server_upload_voice_exception_returns_500(mocker: MockerFixture):
|
||||
)
|
||||
|
||||
|
||||
def test_api_server_upload_voice_with_no_adapter(
|
||||
mocker: MockerFixture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
):
|
||||
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
|
||||
_patch_api_server_base(mocker)
|
||||
engine_client = mocker.MagicMock()
|
||||
engine_client.errored = False
|
||||
engine_client.stage_configs = []
|
||||
models = mocker.MagicMock()
|
||||
models.is_base_model.return_value = True
|
||||
handler = OmniOpenAIServingSpeech(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
assert handler._adapter is None
|
||||
raw_request = _make_api_server_request(handler)
|
||||
|
||||
response = asyncio.run(
|
||||
api_server_module.upload_voice(
|
||||
raw_request,
|
||||
audio_sample=None,
|
||||
speaker_embedding=json.dumps([0.1] * 1024),
|
||||
consent="cons_test",
|
||||
name="probe",
|
||||
)
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
def test_api_server_delete_voice_without_speech_handler_returns_404(mocker: MockerFixture):
|
||||
_patch_api_server_base(mocker)
|
||||
raw_request = _make_api_server_request(None, method="DELETE", path="/v1/audio/voices/probe")
|
||||
@@ -3302,8 +3435,14 @@ class _FakeFishTokenizer:
|
||||
|
||||
@pytest.fixture
|
||||
def fish_speech_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
@@ -3573,8 +3712,14 @@ class TestWAVStreaming:
|
||||
|
||||
@pytest.fixture
|
||||
def cosyvoice3_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
@@ -3709,8 +3854,14 @@ class TestCosyVoice3Serving:
|
||||
|
||||
@pytest.fixture
|
||||
def glm_tts_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
@@ -3732,7 +3883,6 @@ def glm_tts_server(mocker: MockerFixture):
|
||||
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
return OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
@@ -3809,8 +3959,14 @@ class TestGLMTTSServing:
|
||||
|
||||
@pytest.fixture
|
||||
def ming_tts_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
@@ -3868,8 +4024,14 @@ class TestMingTTSServing:
|
||||
|
||||
@pytest.fixture
|
||||
def ming_flash_omni_tts_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
@@ -3932,8 +4094,14 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
@pytest.fixture
|
||||
def voxtral_server(self, mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.voxtral.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.model_config = mocker.MagicMock(model="mistralai/Voxtral")
|
||||
@@ -3958,8 +4126,14 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
@pytest.fixture
|
||||
def qwen3_tts_server(self, mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.model_config = mocker.MagicMock(model="Qwen/Qwen3-TTS", hf_config=mocker.MagicMock())
|
||||
@@ -3983,6 +4157,32 @@ class TestTTSAsyncOffloading:
|
||||
yield server
|
||||
server.shutdown()
|
||||
|
||||
def test_voxtral_loads_supported_speakers(self, mocker: MockerFixture):
|
||||
engine_client = mocker.MagicMock()
|
||||
engine_client.model_config.hf_config.audio_config = {
|
||||
"speaker_id": {
|
||||
"Alice": 0,
|
||||
"Bob": 1,
|
||||
}
|
||||
}
|
||||
|
||||
server = mocker.MagicMock()
|
||||
adapter = VoxtralTTSAdapter(
|
||||
SpeechServingContext(
|
||||
server=server,
|
||||
engine_client=engine_client,
|
||||
)
|
||||
)
|
||||
|
||||
adapter.load_capabilities()
|
||||
|
||||
assert adapter.capabilities.supported_speakers == frozenset(
|
||||
{
|
||||
"alice",
|
||||
"bob",
|
||||
}
|
||||
)
|
||||
|
||||
def test_prepare_speech_generation_awaits_voxtral_async(self, voxtral_server, mocker: MockerFixture):
|
||||
"""Voxtral path in _prepare_speech_generation should call the async wrapper."""
|
||||
voxtral_server._build_voxtral_prompt_async = mocker.AsyncMock(
|
||||
@@ -4214,7 +4414,10 @@ class TestTTSAsyncOffloading:
|
||||
artifact_key,
|
||||
)
|
||||
qwen3_tts_server._ref_audio_model_artifact_ready.add((artifact_key, False))
|
||||
qwen3_tts_server._codec_frame_rate = 25.0
|
||||
qwen3_tts_server._adapter.capabilities = replace(
|
||||
qwen3_tts_server._adapter.capabilities,
|
||||
codec_frame_rate=25.0,
|
||||
)
|
||||
qwen3_tts_server._tts_tokenizer = lambda _text, padding=False: {"input_ids": list(range(10))}
|
||||
qwen3_tts_server.engine_client.model_config.hf_config.talker_config = SimpleNamespace(
|
||||
codec_language_id={},
|
||||
@@ -4319,8 +4522,14 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
def test_shutdown_is_idempotent(self, mocker: MockerFixture):
|
||||
"""Calling shutdown() twice should not raise."""
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.stage_configs = []
|
||||
|
||||
Reference in new issue
Block a user