[Bugfix] Accept 'speaker' as alias for 'voice' in TTS speech API (#2424)
Signed-off-by: marksverdhei <marksverdhei@hotmail.com> Signed-off-by: marksverdhai <249650165+marksverdhai@users.noreply.github.com> Co-authored-by: marksverdhai <249650165+marksverdhai@users.noreply.github.com>
This commit is contained in:
1 parent
408365fa0d
commit
feefdaee9f
4 files changed
+255
-16
No files matched your search
@@ -71,7 +71,7 @@ def run_tts_generation(args) -> None:
|
|||||||
payload = {
|
payload = {
|
||||||
"model": args.model,
|
"model": args.model,
|
||||||
"input": args.text,
|
"input": args.text,
|
||||||
"speaker": args.speaker,
|
"voice": args.speaker,
|
||||||
"response_format": args.response_format,
|
"response_format": args.response_format,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1028,6 +1028,181 @@ class TestTTSMethods:
|
|||||||
|
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
|
# ── speaker field alias ──
|
||||||
|
|
||||||
|
def test_speaker_alias_accepted_as_voice(self):
|
||||||
|
"""The 'speaker' JSON key should be accepted as an alias for 'voice'."""
|
||||||
|
req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "custom_voice"})
|
||||||
|
assert req.voice == "custom_voice"
|
||||||
|
|
||||||
|
def test_voice_field_still_accepted(self):
|
||||||
|
"""The canonical 'voice' JSON key should still work."""
|
||||||
|
req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "voice": "custom_voice"})
|
||||||
|
assert req.voice == "custom_voice"
|
||||||
|
|
||||||
|
def test_speaker_alias_in_base_task_with_uploaded_voice(self, speech_server):
|
||||||
|
"""Using 'speaker' key with an uploaded voice should work for Base task."""
|
||||||
|
speech_server.uploaded_speakers = {
|
||||||
|
"utesf": {
|
||||||
|
"name": "UTESF",
|
||||||
|
"file_path": "/tmp/voice_samples/utesf.wav",
|
||||||
|
"mime_type": "audio/wav",
|
||||||
|
"ref_text": None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "UTESF", "task_type": "Base"})
|
||||||
|
assert req.voice == "UTESF"
|
||||||
|
with patch("pathlib.Path.exists", return_value=True):
|
||||||
|
result = speech_server._validate_qwen_tts_request(req)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
# ── uploaded voice with embedding ──
|
||||||
|
|
||||||
|
def test_build_tts_params_with_uploaded_voice_embedding(self, speech_server):
|
||||||
|
"""Test _build_tts_params loads embedding for embedding-uploaded voices."""
|
||||||
|
speech_server.uploaded_speakers = {
|
||||||
|
"emb_voice": {
|
||||||
|
"name": "emb_voice",
|
||||||
|
"file_path": "/tmp/voice_samples/emb_voice.safetensors",
|
||||||
|
"mime_type": "application/x-safetensors",
|
||||||
|
"embedding_source": "direct",
|
||||||
|
"embedding_dim": 1024,
|
||||||
|
"cache_status": "ready",
|
||||||
|
"cache_file": "/tmp/voice_samples/emb_voice.safetensors",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
speech_server.supported_speakers = {"ryan", "vivian", "emb_voice"}
|
||||||
|
|
||||||
|
fake_embedding = [0.1] * 1024
|
||||||
|
with patch.object(speech_server, "_get_uploaded_speaker_embedding") as mock_get_emb:
|
||||||
|
mock_get_emb.return_value = fake_embedding
|
||||||
|
req = OpenAICreateSpeechRequest(input="Hello", voice="emb_voice")
|
||||||
|
params = speech_server._build_tts_params(req)
|
||||||
|
|
||||||
|
assert "voice_clone_prompt" in params
|
||||||
|
assert params["voice_clone_prompt"][0]["ref_spk_embedding"] == fake_embedding
|
||||||
|
assert params["task_type"] == ["Base"]
|
||||||
|
assert params["x_vector_only_mode"] == [True]
|
||||||
|
assert "ref_audio" not in params
|
||||||
|
|
||||||
|
# ── regression: full flow from issue #1603 ──
|
||||||
|
|
||||||
|
def test_regression_1603_speaker_key_with_uploaded_audio_voice(self, speech_server):
|
||||||
|
"""Regression test for #1603: upload audio voice, then invoke TTS with 'speaker' key.
|
||||||
|
|
||||||
|
Verifies the full validate → build_params pipeline works end-to-end.
|
||||||
|
"""
|
||||||
|
speech_server.uploaded_speakers = {
|
||||||
|
"utesf": {
|
||||||
|
"name": "UTESF",
|
||||||
|
"file_path": "/tmp/voice_samples/utesf.wav",
|
||||||
|
"mime_type": "audio/wav",
|
||||||
|
"ref_text": "Hola, esta es una prueba.",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# Parse with 'speaker' alias (the key users actually send)
|
||||||
|
req = OpenAICreateSpeechRequest.model_validate(
|
||||||
|
{"input": "Hello world", "speaker": "UTESF", "task_type": "Base"}
|
||||||
|
)
|
||||||
|
assert req.voice == "UTESF"
|
||||||
|
|
||||||
|
# Validation should pass (file exists)
|
||||||
|
with patch("pathlib.Path.exists", return_value=True):
|
||||||
|
err = speech_server._validate_qwen_tts_request(req)
|
||||||
|
assert err is None, f"Validation failed: {err}"
|
||||||
|
|
||||||
|
# Build params should auto-set ref_audio from stored file
|
||||||
|
with patch.object(speech_server, "_get_uploaded_audio_data") as mock_audio:
|
||||||
|
mock_audio.return_value = "data:audio/wav;base64,ZmFrZQ=="
|
||||||
|
params = speech_server._build_tts_params(req)
|
||||||
|
|
||||||
|
assert params["task_type"] == ["Base"]
|
||||||
|
assert params["ref_audio"] == ["data:audio/wav;base64,ZmFrZQ=="]
|
||||||
|
assert params["ref_text"] == ["Hola, esta es una prueba."]
|
||||||
|
assert params["x_vector_only_mode"] == [False]
|
||||||
|
assert params["speaker"] == ["utesf"]
|
||||||
|
|
||||||
|
def test_regression_1603_speaker_key_with_uploaded_embedding_voice(self, speech_server):
|
||||||
|
"""Regression test for #1603: upload embedding voice, then invoke TTS with 'speaker' key.
|
||||||
|
|
||||||
|
Verifies embedding-uploaded voices are loaded as voice_clone_prompt, not as audio.
|
||||||
|
"""
|
||||||
|
speech_server.uploaded_speakers = {
|
||||||
|
"myvoice": {
|
||||||
|
"name": "myvoice",
|
||||||
|
"file_path": "/tmp/voice_samples/myvoice.safetensors",
|
||||||
|
"mime_type": "application/x-safetensors",
|
||||||
|
"embedding_source": "direct",
|
||||||
|
"embedding_dim": 1024,
|
||||||
|
"cache_status": "ready",
|
||||||
|
"cache_file": "/tmp/voice_samples/myvoice.safetensors",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# Parse with 'speaker' alias
|
||||||
|
req = OpenAICreateSpeechRequest.model_validate(
|
||||||
|
{"input": "Hello world", "speaker": "myvoice", "task_type": "Base"}
|
||||||
|
)
|
||||||
|
assert req.voice == "myvoice"
|
||||||
|
|
||||||
|
# Validation should pass
|
||||||
|
with patch("pathlib.Path.exists", return_value=True):
|
||||||
|
err = speech_server._validate_qwen_tts_request(req)
|
||||||
|
assert err is None, f"Validation failed: {err}"
|
||||||
|
|
||||||
|
# Build params should use embedding, NOT audio
|
||||||
|
fake_emb = [0.1] * 1024
|
||||||
|
with patch.object(speech_server, "_get_uploaded_speaker_embedding") as mock_emb:
|
||||||
|
mock_emb.return_value = fake_emb
|
||||||
|
params = speech_server._build_tts_params(req)
|
||||||
|
|
||||||
|
assert params["task_type"] == ["Base"]
|
||||||
|
assert params["x_vector_only_mode"] == [True]
|
||||||
|
assert "voice_clone_prompt" in params
|
||||||
|
assert params["voice_clone_prompt"][0]["ref_spk_embedding"] == fake_emb
|
||||||
|
# Must NOT have ref_audio — that would fail for safetensors files
|
||||||
|
assert "ref_audio" not in params
|
||||||
|
|
||||||
|
def test_validate_rejects_embedding_voice_with_pending_cache(self, speech_server):
|
||||||
|
"""Validation should reject embedding voices whose cache is not yet ready."""
|
||||||
|
speech_server.uploaded_speakers = {
|
||||||
|
"myvoice": {
|
||||||
|
"name": "myvoice",
|
||||||
|
"file_path": "/tmp/myvoice.safetensors",
|
||||||
|
"mime_type": "application/x-safetensors",
|
||||||
|
"embedding_source": "direct",
|
||||||
|
"cache_status": "pending",
|
||||||
|
"cache_file": None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "myvoice", "task_type": "Base"})
|
||||||
|
with patch("pathlib.Path.exists", return_value=True):
|
||||||
|
err = speech_server._validate_qwen_tts_request(req)
|
||||||
|
assert err is not None
|
||||||
|
assert "not yet ready" in err
|
||||||
|
|
||||||
|
def test_x_vector_only_mode_not_overwritten_for_uploaded_embedding(self, speech_server):
|
||||||
|
"""x_vector_only_mode set by uploaded embedding must not be overwritten by request field."""
|
||||||
|
speech_server.uploaded_speakers = {
|
||||||
|
"emb_voice": {
|
||||||
|
"name": "emb_voice",
|
||||||
|
"file_path": "/tmp/emb_voice.safetensors",
|
||||||
|
"mime_type": "application/x-safetensors",
|
||||||
|
"embedding_source": "direct",
|
||||||
|
"embedding_dim": 1024,
|
||||||
|
"cache_status": "ready",
|
||||||
|
"cache_file": "/tmp/emb_voice.safetensors",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fake_emb = [0.1] * 1024
|
||||||
|
with patch.object(speech_server, "_get_uploaded_speaker_embedding") as mock_emb:
|
||||||
|
mock_emb.return_value = fake_emb
|
||||||
|
# Client explicitly sends x_vector_only_mode=False, but embedding requires True
|
||||||
|
req = OpenAICreateSpeechRequest(input="Hello", voice="emb_voice", x_vector_only_mode=False)
|
||||||
|
params = speech_server._build_tts_params(req)
|
||||||
|
|
||||||
|
assert params["x_vector_only_mode"] == [True]
|
||||||
|
assert "voice_clone_prompt" in params
|
||||||
|
|
||||||
def test_max_instructions_length_default(self, speech_server):
|
def test_max_instructions_length_default(self, speech_server):
|
||||||
"""Test default max instructions length (500) when no config provided."""
|
"""Test default max instructions length (500) when no config provided."""
|
||||||
# Fixture creates server with no CLI override and no TTS stage
|
# Fixture creates server with no CLI override and no TTS stage
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import math
|
|||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
|
||||||
|
|
||||||
_MAX_EMBEDDING_DIM = 8192
|
_MAX_EMBEDDING_DIM = 8192
|
||||||
|
|
||||||
@@ -10,8 +10,12 @@ _MAX_EMBEDDING_DIM = 8192
|
|||||||
class OpenAICreateSpeechRequest(BaseModel):
|
class OpenAICreateSpeechRequest(BaseModel):
|
||||||
input: str
|
input: str
|
||||||
model: str | None = None
|
model: str | None = None
|
||||||
|
# Accept both "voice" (OpenAI convention) and "speaker" (model/internal
|
||||||
|
# convention) as input keys. Intentionally global — all TTS backends
|
||||||
|
# (Qwen3-TTS, Voxtral, Fish Speech) use this field for the speaker name.
|
||||||
voice: str | None = Field(
|
voice: str | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
|
validation_alias=AliasChoices("voice", "speaker"),
|
||||||
description="Speaker/voice to use. For Qwen3-TTS: vivian, ryan, aiden, etc.",
|
description="Speaker/voice to use. For Qwen3-TTS: vivian, ryan, aiden, etc.",
|
||||||
)
|
)
|
||||||
instructions: str | None = Field(
|
instructions: str | None = Field(
|
||||||
|
|||||||
@@ -467,6 +467,48 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
logger.error(f"Could not read audio file for voice {voice_name}: {e}")
|
logger.error(f"Could not read audio file for voice {voice_name}: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _get_uploaded_speaker_embedding(self, voice_name: str) -> list[float] | None:
|
||||||
|
"""Load pre-computed speaker embedding for an uploaded voice.
|
||||||
|
|
||||||
|
Returns the embedding as a list of floats, or None if the voice
|
||||||
|
was not uploaded with an embedding (i.e. it has audio instead).
|
||||||
|
"""
|
||||||
|
voice_name_lower = voice_name.lower()
|
||||||
|
if voice_name_lower not in self.uploaded_speakers:
|
||||||
|
return None
|
||||||
|
|
||||||
|
speaker_info = self.uploaded_speakers[voice_name_lower]
|
||||||
|
if speaker_info.get("embedding_source") != "direct":
|
||||||
|
return None
|
||||||
|
|
||||||
|
cache_file = speaker_info.get("cache_file")
|
||||||
|
if not cache_file or not Path(cache_file).exists():
|
||||||
|
logger.warning("Embedding file not found for voice %s: %s", voice_name, cache_file)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not _validate_path_within_directory(Path(cache_file), self.uploaded_speakers_dir):
|
||||||
|
logger.error("Cache file path traversal detected for voice %s: %s", voice_name, cache_file)
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
from safetensors.torch import load_file
|
||||||
|
except ImportError:
|
||||||
|
logger.error(
|
||||||
|
"The 'safetensors' package is required to load speaker embeddings. "
|
||||||
|
"Install it with: pip install safetensors"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
tensors = load_file(cache_file)
|
||||||
|
if "speaker_embedding" not in tensors:
|
||||||
|
logger.warning("Key 'speaker_embedding' not found in %s for voice %s", cache_file, voice_name)
|
||||||
|
return None
|
||||||
|
return tensors["speaker_embedding"].squeeze().tolist()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Could not load embedding for voice %s: %s", voice_name, e)
|
||||||
|
return None
|
||||||
|
|
||||||
async def upload_voice(
|
async def upload_voice(
|
||||||
self,
|
self,
|
||||||
audio_file: UploadFile,
|
audio_file: UploadFile,
|
||||||
@@ -858,11 +900,17 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
# voice is not None
|
# voice is not None
|
||||||
voice_lower = request.voice.lower()
|
voice_lower = request.voice.lower()
|
||||||
if voice_lower in self.uploaded_speakers:
|
if voice_lower in self.uploaded_speakers:
|
||||||
# Check if audio file exists for uploaded speaker
|
# Check if data file exists for uploaded speaker
|
||||||
speaker_info = self.uploaded_speakers[voice_lower]
|
speaker_info = self.uploaded_speakers[voice_lower]
|
||||||
file_path = Path(speaker_info["file_path"])
|
file_path = Path(speaker_info["file_path"])
|
||||||
if not file_path.exists():
|
if not file_path.exists():
|
||||||
return f"Audio file for uploaded speaker '{request.voice}' not found on disk"
|
return f"Data file for uploaded speaker '{request.voice}' not found on disk"
|
||||||
|
# For embedding-uploaded voices, verify the cache is ready
|
||||||
|
if speaker_info.get("embedding_source") == "direct":
|
||||||
|
cache_file = speaker_info.get("cache_file")
|
||||||
|
if not cache_file or not Path(cache_file).exists():
|
||||||
|
status = speaker_info.get("cache_status", "unknown")
|
||||||
|
return f"Speaker embedding for '{request.voice}' is not yet ready (cache_status='{status}')"
|
||||||
else:
|
else:
|
||||||
# need ref_audio for built-in speaker
|
# need ref_audio for built-in speaker
|
||||||
if request.ref_audio is None:
|
if request.ref_audio is None:
|
||||||
@@ -1107,20 +1155,32 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
# Uploaded voices use task_type="Base" (CustomVoice requires built-in spk_id).
|
# Uploaded voices use task_type="Base" (CustomVoice requires built-in spk_id).
|
||||||
# If ref_text was provided at upload time, use in-context cloning; otherwise x_vector only.
|
# If ref_text was provided at upload time, use in-context cloning; otherwise x_vector only.
|
||||||
if request.voice.lower() in self.uploaded_speakers and request.ref_audio is None:
|
if request.voice.lower() in self.uploaded_speakers and request.ref_audio is None:
|
||||||
audio_data = self._get_uploaded_audio_data(request.voice)
|
|
||||||
if not audio_data:
|
|
||||||
raise ValueError(f"Audio file for uploaded voice '{request.voice}' is missing or corrupted")
|
|
||||||
speaker_info = self.uploaded_speakers[request.voice.lower()]
|
speaker_info = self.uploaded_speakers[request.voice.lower()]
|
||||||
stored_ref_text = speaker_info.get("ref_text")
|
|
||||||
params["ref_audio"] = [audio_data]
|
# Check if this voice was uploaded with a pre-computed embedding.
|
||||||
params["task_type"] = ["Base"]
|
# Populate request.speaker_embedding so the existing code path
|
||||||
params["voice_created_at"] = [speaker_info.get("created_at", 0)]
|
# (below) handles voice_clone_prompt and x_vector_only_mode.
|
||||||
if stored_ref_text:
|
embedding = self._get_uploaded_speaker_embedding(request.voice)
|
||||||
params["ref_text"] = [stored_ref_text]
|
if embedding is not None:
|
||||||
params["x_vector_only_mode"] = [False]
|
request.speaker_embedding = embedding
|
||||||
|
params["task_type"] = ["Base"]
|
||||||
|
logger.info("Auto-set speaker_embedding for uploaded voice: %s", request.voice)
|
||||||
else:
|
else:
|
||||||
params["x_vector_only_mode"] = [True]
|
audio_data = self._get_uploaded_audio_data(request.voice)
|
||||||
logger.info("Auto-set ref_audio for uploaded voice: %s (icl=%s)", request.voice, bool(stored_ref_text))
|
if not audio_data:
|
||||||
|
raise ValueError(f"Audio file for uploaded voice '{request.voice}' is missing or corrupted")
|
||||||
|
stored_ref_text = speaker_info.get("ref_text")
|
||||||
|
params["ref_audio"] = [audio_data]
|
||||||
|
params["task_type"] = ["Base"]
|
||||||
|
params["voice_created_at"] = [speaker_info.get("created_at", 0)]
|
||||||
|
if stored_ref_text:
|
||||||
|
params["ref_text"] = [stored_ref_text]
|
||||||
|
params["x_vector_only_mode"] = [False]
|
||||||
|
else:
|
||||||
|
params["x_vector_only_mode"] = [True]
|
||||||
|
logger.info(
|
||||||
|
"Auto-set ref_audio for uploaded voice: %s (icl=%s)", request.voice, bool(stored_ref_text)
|
||||||
|
)
|
||||||
|
|
||||||
elif params["task_type"][0] == "CustomVoice":
|
elif params["task_type"][0] == "CustomVoice":
|
||||||
params["speaker"] = ["Vivian"] # Default for CustomVoice
|
params["speaker"] = ["Vivian"] # Default for CustomVoice
|
||||||
|
|||||||
Reference in new issue
Block a user