[Cleanup]Migrate model validation to adapters and clean up serving_speech.py (#4833)
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
boatmanandYueqian Lin authored and GitHub committed
1 parent
10d3a25916
commit
32dddfcef4
16 files changed
+547
-550
No files matched your search
@@ -6,7 +6,7 @@ E2E Online expansion tests for higgs-audio v2 against /v1/audio/speech.
|
||||
v1 scope is plain text -> 24 kHz speech plus shallow voice clone via
|
||||
ref_audio + ref_text (inline) or voice=<name> (after POST /v1/audio/voices).
|
||||
The model-aware request validator
|
||||
(vllm_omni/entrypoints/openai/serving_speech.py::_validate_higgs_audio_v2_request)
|
||||
(vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2::validate)
|
||||
rejects multi-speaker tags, language overrides, task_type, and bare
|
||||
voice=<name> for names that do not match an uploaded speaker — this suite
|
||||
exercises both the happy path (plain text in, audio bytes out) and the
|
||||
|
||||
@@ -1549,7 +1549,7 @@ class TestTTSMethods:
|
||||
req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "UTESF", "task_type": "Base"})
|
||||
assert req.voice == "UTESF"
|
||||
mocker.patch("pathlib.Path.exists", return_value=True)
|
||||
result = speech_server._validate_qwen_tts_request(req)
|
||||
result = speech_server._validate_tts_request(req)
|
||||
assert result is None
|
||||
|
||||
# ── uploaded voice with embedding ──
|
||||
@@ -1684,7 +1684,7 @@ class TestTTSMethods:
|
||||
|
||||
# Validation should pass (file exists)
|
||||
mocker.patch("pathlib.Path.exists", return_value=True)
|
||||
err = speech_server._validate_qwen_tts_request(req)
|
||||
err = speech_server._validate_tts_request(req)
|
||||
assert err is None, f"Validation failed: {err}"
|
||||
|
||||
# Build params should auto-set ref_audio from stored file
|
||||
@@ -1722,7 +1722,7 @@ class TestTTSMethods:
|
||||
|
||||
# Validation should pass
|
||||
mocker.patch("pathlib.Path.exists", return_value=True)
|
||||
err = speech_server._validate_qwen_tts_request(req)
|
||||
err = speech_server._validate_tts_request(req)
|
||||
assert err is None, f"Validation failed: {err}"
|
||||
|
||||
# Build params should use embedding, NOT audio
|
||||
@@ -1866,6 +1866,35 @@ class TestTTSMethods:
|
||||
assert error is not None
|
||||
assert "max 10 characters" in error
|
||||
|
||||
def test_batch_voice_not_treated_as_inline_ref_audio(
|
||||
self,
|
||||
speech_server,
|
||||
mocker: MockerFixture,
|
||||
):
|
||||
speech_server._check_model = mocker.AsyncMock(return_value=None)
|
||||
speech_server._batch_max_items = 32
|
||||
|
||||
def validate_and_resolve_uploaded_voice(request):
|
||||
assert request.voice == "alice"
|
||||
assert request.ref_audio is None
|
||||
|
||||
request.ref_audio = "data:audio/wav;base64,dGVzdA=="
|
||||
request.ref_text = "reference text"
|
||||
return None
|
||||
|
||||
speech_server._validate_tts_request = mocker.Mock(side_effect=validate_and_resolve_uploaded_voice)
|
||||
speech_server._generate_audio_bytes = mocker.AsyncMock(return_value=("YWJj", "audio/wav"))
|
||||
|
||||
batch = BatchSpeechRequest(voice="alice", items=[SpeechBatchItem(input="hello")])
|
||||
|
||||
response = asyncio.run(speech_server.create_speech_batch(batch))
|
||||
call = speech_server._generate_audio_bytes.await_args
|
||||
request = call.args[0]
|
||||
|
||||
assert response.results[0].status == "success"
|
||||
assert request.ref_audio is not None
|
||||
assert call.kwargs["has_inline_ref_audio"] is False
|
||||
|
||||
|
||||
class TestFileValidationFunctions:
|
||||
"""Unit tests for file validation helper functions."""
|
||||
@@ -3221,25 +3250,25 @@ class TestCosyVoice3Serving:
|
||||
|
||||
def test_validate_cosyvoice3_empty_input(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(input="", ref_audio="data:audio/wav;base64,abc", ref_text="hello")
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
error = cosyvoice3_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "empty" in error.lower()
|
||||
|
||||
def test_validate_cosyvoice3_missing_ref_audio(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_text="hello")
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
error = cosyvoice3_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "ref_audio" in error.lower()
|
||||
|
||||
def test_validate_cosyvoice3_missing_ref_text(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
error = cosyvoice3_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "ref_text" in error.lower()
|
||||
|
||||
def test_validate_cosyvoice3_invalid_ref_audio_format(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="/local/path.wav", ref_text="hello")
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
error = cosyvoice3_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "url" in error.lower() or "format" in error.lower()
|
||||
|
||||
@@ -3249,7 +3278,7 @@ class TestCosyVoice3Serving:
|
||||
ref_audio="data:audio/wav;base64,abc123",
|
||||
ref_text="Reference transcript",
|
||||
)
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
error = cosyvoice3_server._validate_tts_request(request)
|
||||
assert error is None
|
||||
|
||||
def test_validate_cosyvoice3_max_new_tokens_range(self, cosyvoice3_server):
|
||||
@@ -3259,7 +3288,7 @@ class TestCosyVoice3Serving:
|
||||
ref_text="hello",
|
||||
max_new_tokens=0,
|
||||
)
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
error = cosyvoice3_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "max_new_tokens" in error
|
||||
|
||||
@@ -3325,13 +3354,13 @@ def glm_tts_server(mocker: MockerFixture):
|
||||
class TestGLMTTSServing:
|
||||
def test_validate_glm_tts_requires_ref_audio(self, glm_tts_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_text="Reference transcript")
|
||||
error = glm_tts_server._validate_glm_tts_request(request)
|
||||
error = glm_tts_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "ref_audio" in error
|
||||
|
||||
def test_validate_glm_tts_requires_ref_text(self, glm_tts_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
|
||||
error = glm_tts_server._validate_glm_tts_request(request)
|
||||
error = glm_tts_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "ref_text" in error
|
||||
|
||||
@@ -3438,7 +3467,7 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
def test_prepare_speech_generation_awaits_qwen3_tts_async(self, qwen3_tts_server, mocker: MockerFixture):
|
||||
"""Qwen3 TTS path should call _estimate_prompt_len_async."""
|
||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||
)
|
||||
@@ -3455,7 +3484,7 @@ class TestTTSAsyncOffloading:
|
||||
qwen3_tts_server.engine_client.default_sampling_params_list = [
|
||||
SimpleNamespace(max_tokens=2048, seed=42, extra_args=None)
|
||||
]
|
||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||
)
|
||||
@@ -3505,7 +3534,7 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
def test_prepare_speech_generation_treats_sse_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
||||
"""stream_format=sse should request delta-style multimodal outputs."""
|
||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||
)
|
||||
@@ -3522,7 +3551,7 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
def test_prepare_speech_generation_treats_stream_true_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
||||
"""stream=True should request delta-style multimodal outputs for SSE streaming."""
|
||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||
)
|
||||
@@ -3539,7 +3568,7 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
def test_prepare_speech_generation_treats_audio_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
||||
"""stream_format=audio should request delta-style multimodal outputs."""
|
||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||
)
|
||||
@@ -3559,7 +3588,7 @@ class TestTTSAsyncOffloading:
|
||||
):
|
||||
"""Full-payload TTS streaming should not request delta multimodal outputs."""
|
||||
qwen3_tts_server.engine_client.model_config.async_chunk = False
|
||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||
)
|
||||
|
||||
@@ -150,7 +150,6 @@ _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256
|
||||
_HIGGS_V3_REF_CODE_CACHE_MAX_BYTES = 64 * 1024 * 1024
|
||||
_QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key"
|
||||
_TTS_MAX_INSTRUCTIONS_LENGTH = 500
|
||||
_TTS_MAX_NEW_TOKENS_MIN = 1
|
||||
_TTS_MAX_NEW_TOKENS_MAX = 4096
|
||||
_MING_DEFAULT_PROMPT = MING_DEFAULT_PROMPT
|
||||
|
||||
@@ -1490,48 +1489,22 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate TTS request parameters. Returns error message or None."""
|
||||
if self._tts_model_type == "step_audio2":
|
||||
# StepAudio2 only requires non-empty input text
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
return None
|
||||
if self._tts_model_type == "voxtral_tts":
|
||||
return self._validate_voxtral_tts_request(request)
|
||||
if self._tts_model_type == "fish_tts":
|
||||
return self._validate_fish_tts_request(request)
|
||||
if self._tts_model_type == "cosyvoice3":
|
||||
return self._validate_cosyvoice3_request(request)
|
||||
if self._tts_model_type == "voxcpm2":
|
||||
return self._validate_voxcpm2_request(request)
|
||||
if self._tts_model_type == "ming_flash_omni_tts":
|
||||
return self._validate_ming_flash_omni_tts_request(request)
|
||||
if self._tts_model_type == "ming_tts":
|
||||
return self._validate_ming_tts_request(request)
|
||||
if self._tts_model_type in ("moss_tts_nano", "moss_tts"):
|
||||
return self._validate_moss_tts_request(request)
|
||||
if self._tts_model_type == "glm_tts":
|
||||
return self._validate_glm_tts_request(request)
|
||||
return self._validate_qwen_tts_request(request)
|
||||
|
||||
def _validate_voxcpm2_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate VoxCPM2 request parameters. Returns error message or None."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
adapter = self._get_tts_adapter()
|
||||
if adapter is not None:
|
||||
return adapter.validate(request)
|
||||
|
||||
if request.voice is not None:
|
||||
request.voice = request.voice.lower()
|
||||
available_voices = set(self.uploaded_speakers) | set(self.precomputed_speakers) | {"default"}
|
||||
if request.voice not in available_voices:
|
||||
supported = ", ".join(sorted(available_voices)) or "none"
|
||||
return f"Invalid voice '{request.voice}'. Supported: {supported}"
|
||||
adapter_cls = resolve_adapter("qwen3_tts")
|
||||
if adapter_cls is None:
|
||||
raise ValueError("Qwen3-TTS adapter is not registered")
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
ctx = SpeechServingContext(
|
||||
server=self,
|
||||
engine_client=self.engine_client,
|
||||
)
|
||||
return adapter_cls(ctx).validate(request)
|
||||
|
||||
def _voxcpm2_encode(self, text: str) -> list[int]:
|
||||
"""Tokenize text for VoxCPM2, splitting multichar Chinese tokens."""
|
||||
@@ -1599,158 +1572,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
|
||||
return None
|
||||
|
||||
def _validate_voxtral_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate Voxtral TTS request parameters. Returns error message or None."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
# Voxtral TTS requires either a preset voice or ref_audio for voice cloning.
|
||||
if request.voice is None and request.ref_audio is None:
|
||||
return "Either 'voice' (preset speaker) or 'ref_audio' (voice cloning) must be provided"
|
||||
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if request.voice is not None:
|
||||
request.voice = request.voice.lower()
|
||||
if self.supported_speakers and request.voice not in self.supported_speakers:
|
||||
return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
|
||||
def _validate_qwen_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate Qwen TTS request parameters. Returns error message or None."""
|
||||
# Infer Base task when ref_audio or ref_text is provided without explicit task_type.
|
||||
if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None):
|
||||
request.task_type = "Base"
|
||||
|
||||
# Normalize voice to lowercase for case-insensitive matching
|
||||
if request.voice is not None:
|
||||
request.voice = request.voice.lower()
|
||||
if request.task_type is None and request.voice in self.precomputed_speakers:
|
||||
request.task_type = "Base"
|
||||
task_type = request.task_type or "CustomVoice"
|
||||
|
||||
# Validate input is not empty
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
# Validate language (case-insensitive; normalized to the title-cased config form)
|
||||
if request.language is not None:
|
||||
request.language = request.language.title()
|
||||
if request.language not in self.supported_languages:
|
||||
return (
|
||||
f"Invalid language '{request.language}'. Supported: {', '.join(sorted(self.supported_languages))}"
|
||||
)
|
||||
|
||||
# Validate speaker for CustomVoice task
|
||||
if task_type == "CustomVoice":
|
||||
if not self.supported_speakers:
|
||||
return (
|
||||
"This model does not support CustomVoice task (no speakers configured). "
|
||||
"Use task_type='Base' with ref_audio/ref_text for voice cloning, "
|
||||
"or use a CustomVoice model."
|
||||
)
|
||||
if request.voice is not None and request.voice not in self.supported_speakers:
|
||||
return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}"
|
||||
|
||||
# Validate speaker_embedding constraints
|
||||
if request.speaker_embedding is not None:
|
||||
if task_type != "Base":
|
||||
return "'speaker_embedding' is only valid for Base task"
|
||||
if not request.speaker_embedding:
|
||||
return "'speaker_embedding' must be a non-empty list of floats"
|
||||
# speaker_embedding implies x_vector_only_mode — set it before
|
||||
# Base task validation so callers don't need to pass it explicitly.
|
||||
request.x_vector_only_mode = True
|
||||
emb_len = len(request.speaker_embedding)
|
||||
dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_len)
|
||||
if dim_err is not None:
|
||||
return dim_err
|
||||
# Validate Base task requirements
|
||||
if task_type == "Base":
|
||||
if request.voice is None:
|
||||
# 1. Ensure a voice source is provided
|
||||
if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None:
|
||||
return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning"
|
||||
# 2. Validate ref_audio format if it exists (using the helper from main)
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
# 3. Validate text requirements based on the mode
|
||||
if not getattr(request, "x_vector_only_mode", False):
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return (
|
||||
"Base task requires non-empty 'ref_text' (transcript of "
|
||||
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
||||
)
|
||||
else:
|
||||
voice_lower = request.voice.lower()
|
||||
if voice_lower in self.uploaded_speakers:
|
||||
# Check if data file exists for uploaded speaker
|
||||
speaker_info = self.uploaded_speakers[voice_lower]
|
||||
file_path = Path(speaker_info["file_path"])
|
||||
if not file_path.exists():
|
||||
return f"Data file for uploaded speaker '{request.voice}' not found on disk"
|
||||
elif voice_lower in self.precomputed_speakers:
|
||||
profile = self.precomputed_speakers[voice_lower]
|
||||
mode = str(profile.get("mode") or "xvec").lower()
|
||||
ref_text = request.ref_text or profile.get("ref_text")
|
||||
if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()):
|
||||
return (
|
||||
f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in "
|
||||
"the request or custom voice manifest"
|
||||
)
|
||||
else:
|
||||
# need ref_audio for built-in speaker
|
||||
if request.ref_audio is None:
|
||||
return (
|
||||
f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning"
|
||||
)
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if not getattr(request, "x_vector_only_mode", False) and (
|
||||
not request.ref_text or not request.ref_text.strip()
|
||||
):
|
||||
return (
|
||||
"Base task requires non-empty 'ref_text' (transcript of "
|
||||
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
||||
)
|
||||
|
||||
# Validate cross-parameter dependencies
|
||||
if task_type != "Base":
|
||||
if request.ref_text is not None:
|
||||
return "'ref_text' is only valid for Base task"
|
||||
if request.x_vector_only_mode is not None:
|
||||
return "'x_vector_only_mode' is only valid for Base task"
|
||||
|
||||
# Validate VoiceDesign task requirements
|
||||
if task_type == "VoiceDesign" and not request.instructions:
|
||||
return "VoiceDesign task requires 'instructions' to describe the voice"
|
||||
|
||||
# Validate instructions length (using cached value from initialization)
|
||||
if request.instructions and len(request.instructions) > self._max_instructions_length:
|
||||
return f"Instructions too long (max {self._max_instructions_length} characters)"
|
||||
|
||||
# Validate max_new_tokens range
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
|
||||
def _detect_moss_variant(self) -> str:
|
||||
"""Sub-classify a ``moss_tts``-stage server into the actual MOSS-TTS
|
||||
variant family (tts, ttsd, sound_effect, voice_generator, realtime).
|
||||
@@ -1776,66 +1597,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
return "voice_generator"
|
||||
return "tts"
|
||||
|
||||
def _validate_moss_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate any MOSS-TTS-family request (nano + 5 full variants).
|
||||
|
||||
Dispatches by ``self._moss_variant``:
|
||||
- ``tts``/``realtime``: require ``ref_audio`` (voice cloning).
|
||||
- ``ttsd``: require ``ref_audio`` (speaker 1); ``ref_audio_2``
|
||||
optional (defaults to the same ref for both speakers).
|
||||
- ``sound_effect``: require ``ambient_sound`` (no ref_audio).
|
||||
- ``voice_generator``: require ``instructions`` (no ref_audio).
|
||||
- For the legacy moss_tts_nano model_type the variant is None and
|
||||
we fall through to the original nano contract (ref_audio only).
|
||||
"""
|
||||
if not request.input or not request.input.strip():
|
||||
# SoundEffect can legitimately have empty input (just ambient_sound).
|
||||
if self._moss_variant != "sound_effect":
|
||||
return "Input text cannot be empty"
|
||||
|
||||
v = self._moss_variant
|
||||
if v in (None, "tts", "realtime", "local"):
|
||||
if request.ref_audio is None:
|
||||
label = (
|
||||
"MOSS-TTS-Nano"
|
||||
if v is None
|
||||
else (
|
||||
"MOSS-TTS-Realtime"
|
||||
if v == "realtime"
|
||||
else ("MOSS-TTS-Local-Transformer" if v == "local" else "MOSS-TTS")
|
||||
)
|
||||
)
|
||||
return f"{label} requires 'ref_audio' (reference audio for voice cloning)."
|
||||
return self._validate_ref_audio_format(request.ref_audio)
|
||||
|
||||
if v == "ttsd":
|
||||
if request.ref_audio is None:
|
||||
return "MOSS-TTSD requires 'ref_audio' (speaker 1 reference)."
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if request.ref_audio_2 is not None:
|
||||
return self._validate_ref_audio_format(request.ref_audio_2)
|
||||
return None
|
||||
|
||||
if v == "sound_effect":
|
||||
if not request.ambient_sound or not request.ambient_sound.strip():
|
||||
return (
|
||||
"MOSS-SoundEffect requires 'ambient_sound' (natural language "
|
||||
"description of the sound effect to synthesise)."
|
||||
)
|
||||
return None
|
||||
|
||||
if v == "voice_generator":
|
||||
if not request.instructions or not request.instructions.strip():
|
||||
return (
|
||||
"MOSS-VoiceGenerator requires 'instructions' (natural language "
|
||||
"voice description, e.g. 'a warm female voice with an American accent')."
|
||||
)
|
||||
return None
|
||||
|
||||
return None # unreachable
|
||||
|
||||
def _get_moss_processor(self):
|
||||
"""Lazily load the upstream MOSS-TTS processor once per server.
|
||||
|
||||
@@ -1994,93 +1755,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
params["max_new_frames"] = [request.max_new_tokens]
|
||||
return params
|
||||
|
||||
def _validate_higgs_audio_v2_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate higgs_audio_v2 request parameters. Returns error message or None.
|
||||
|
||||
Accepted: plain text -> speech, or shallow voice clone via ``ref_audio``
|
||||
+ ``ref_text`` (both required together). Still out of scope: preset
|
||||
``voice``/``speaker`` selection, ``x_vector_only_mode`` /
|
||||
``speaker_embedding`` helpers, ``task_type``/``language``/
|
||||
``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]``
|
||||
tags inside the input body.
|
||||
"""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import (
|
||||
MULTI_SPEAKER_TAG_PATTERN,
|
||||
)
|
||||
|
||||
if not request.input or not request.input.strip():
|
||||
return "higgs_audio_v2: input text cannot be empty"
|
||||
|
||||
# Voice clone: ref_audio and ref_text must come together.
|
||||
if request.ref_audio is not None and not request.ref_text:
|
||||
return (
|
||||
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
||||
"'ref_text'; received ref_audio without ref_text"
|
||||
)
|
||||
if request.ref_text and request.ref_audio is None:
|
||||
return (
|
||||
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
||||
"'ref_text'; received ref_text without ref_audio"
|
||||
)
|
||||
|
||||
if request.x_vector_only_mode is not None:
|
||||
return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)"
|
||||
if request.speaker_embedding is not None:
|
||||
return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)"
|
||||
if request.voice and request.ref_audio is None:
|
||||
# _apply_uploaded_speaker runs before this validator; if voice was
|
||||
# an uploaded speaker, ref_audio is now populated and ref_text is
|
||||
# backfilled from the speaker entry. A bare voice= with no
|
||||
# ref_audio means the name didn't resolve to an uploaded speaker
|
||||
# (and higgs has no built-in preset voices).
|
||||
return (
|
||||
"higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; "
|
||||
f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. "
|
||||
f"Got voice={request.voice!r}"
|
||||
)
|
||||
if request.instructions:
|
||||
return (
|
||||
"higgs_audio_v2 v1 does not support 'instructions' (voice "
|
||||
"style/emotion control); supply plain text instead"
|
||||
)
|
||||
if request.task_type is not None:
|
||||
return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech"
|
||||
if request.language is not None:
|
||||
return (
|
||||
"higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text"
|
||||
)
|
||||
if request.speed is not None and request.speed != 1.0:
|
||||
return (
|
||||
"higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)"
|
||||
)
|
||||
|
||||
if MULTI_SPEAKER_TAG_PATTERN.search(request.input):
|
||||
return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
|
||||
def _validate_glm_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate GLM-TTS request — requires ref_audio for voice cloning."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
if request.ref_audio is None:
|
||||
return "GLM-TTS requires 'ref_audio' for zero-shot voice cloning"
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "GLM-TTS voice cloning requires 'ref_text' (transcript of the reference audio)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be >= {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
return None
|
||||
|
||||
async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest):
|
||||
"""Build prompt_token_ids for higgs_audio_v2 via the upstream processor.
|
||||
|
||||
@@ -2157,18 +1831,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
# ---- higgs-audio v3 ----
|
||||
|
||||
def _validate_higgs_audio_v3_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate higgs_audio_v3 request parameters."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "higgs_audio_v3: input text cannot be empty"
|
||||
if request.ref_audio is not None and not request.ref_text:
|
||||
# Voice clone ref_text is optional for v3 (improves fidelity but not required)
|
||||
pass
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
return None
|
||||
|
||||
async def _build_higgs_audio_v3_params(self, request: OpenAICreateSpeechRequest):
|
||||
"""Build prompt_token_ids for higgs_audio_v3.
|
||||
|
||||
@@ -2302,143 +1964,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
self._higgs_audio_v3_adapter = adapter
|
||||
return adapter
|
||||
|
||||
def _validate_fish_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate Fish Speech request parameters. Returns error message or None."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "Voice cloning requires 'ref_text' (transcript of the reference audio)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
|
||||
def _validate_cosyvoice3_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate CosyVoice3 request parameters. Returns error message or None."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
# CosyVoice3 requires reference audio for voice cloning
|
||||
if request.ref_audio is None:
|
||||
return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)"
|
||||
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
|
||||
def _validate_ming_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate Ming TTS request parameters. Returns error message or None."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
if isinstance(request.ref_audio, list):
|
||||
return self._validate_ming_tts_podcast_request(request)
|
||||
return self._validate_ming_tts_single_speaker_request(request)
|
||||
|
||||
def _validate_ming_tts_single_speaker_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if request.speaker_embedding is not None:
|
||||
if not request.speaker_embedding:
|
||||
return "'speaker_embedding' must be a non-empty list of floats"
|
||||
emb_len = len(request.speaker_embedding)
|
||||
if emb_len != 192:
|
||||
logger.warning(
|
||||
"speaker_embedding has %d dimensions; Ming dense expects 192. "
|
||||
"Wrong dimensions will likely fail or degrade output.",
|
||||
emb_len,
|
||||
)
|
||||
|
||||
voice_lower = request.voice.lower() if isinstance(request.voice, str) else None
|
||||
uploaded_voice = bool(voice_lower and voice_lower in self.uploaded_speakers)
|
||||
clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice
|
||||
|
||||
if request.task_type == "Base" and not clone_source_present:
|
||||
return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample"
|
||||
|
||||
if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip():
|
||||
return "'ref_text' must be non-empty when provided with 'ref_audio'"
|
||||
|
||||
# Ming offline ref-audio cases use prompt_waveform without prompt_text;
|
||||
# keep the transcript requirement for other TTS models.
|
||||
if request.ref_audio is not None and request.speaker_embedding is None and not self._is_ming_tts_model():
|
||||
uploaded_ref_text = self.uploaded_speakers[voice_lower].get("ref_text") if uploaded_voice else None
|
||||
if not (request.ref_text and request.ref_text.strip()) and not uploaded_ref_text:
|
||||
return "Reference-audio cloning requires non-empty 'ref_text'"
|
||||
|
||||
if request.ref_text is not None and request.ref_audio is None and not uploaded_voice:
|
||||
return "'ref_text' requires 'ref_audio' or an uploaded voice sample"
|
||||
|
||||
if request.instructions and len(request.instructions) > self._max_instructions_length:
|
||||
return f"Instructions too long (max {self._max_instructions_length} characters)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
|
||||
def _validate_ming_tts_podcast_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
if len(request.ref_audio) < 2:
|
||||
return "Podcast-style Ming requests require at least two 'ref_audio' clips"
|
||||
|
||||
for ref_audio in request.ref_audio:
|
||||
fmt_err = self._validate_ref_audio_format(ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "Podcast-style Ming requests require non-empty 'ref_text'"
|
||||
|
||||
if request.speaker_embedding is not None:
|
||||
embeddings = request.speaker_embedding
|
||||
embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1
|
||||
if embedding_count != len(request.ref_audio):
|
||||
return (
|
||||
"Podcast-style Ming requests require one speaker embedding per ref_audio clip; "
|
||||
f"got {embedding_count} embeddings for {len(request.ref_audio)} clips"
|
||||
)
|
||||
if embeddings and isinstance(embeddings[0], list):
|
||||
for item in embeddings:
|
||||
if len(item) != 192:
|
||||
return "Podcast-style Ming speaker embeddings must each have 192 dimensions"
|
||||
|
||||
if request.instructions and len(request.instructions) > self._max_instructions_length:
|
||||
return f"Instructions too long (max {self._max_instructions_length} characters)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
|
||||
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
|
||||
"""Resolve ref_audio to (wav_samples, sample_rate).
|
||||
|
||||
@@ -2573,9 +2098,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
# ---- Ming TTS helpers ----
|
||||
|
||||
def _is_ming_tts_model(self) -> bool:
|
||||
return self._tts_model_type == "ming_tts"
|
||||
|
||||
def _coerce_ming_prompt_waveform(self, wav_samples, sample_rate):
|
||||
from torchaudio.functional import resample as resample_audio
|
||||
|
||||
@@ -3488,6 +3010,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
self,
|
||||
request: OpenAICreateSpeechRequest,
|
||||
request_id: str | None = None,
|
||||
has_inline_ref_audio: bool | None = None,
|
||||
) -> tuple[str, Any, dict[str, Any]]:
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
@@ -3520,7 +3043,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
# in place. The builders need to know whether the caller supplied audio
|
||||
# inline vs. via an uploaded voice.
|
||||
model_type: str | None = None
|
||||
has_inline_ref_audio = request.ref_audio is not None
|
||||
has_inline_ref_audio = (request.ref_audio is not None) if has_inline_ref_audio is None else has_inline_ref_audio
|
||||
if self._tts_model_type == "ming_flash_omni_tts":
|
||||
# ming_flash_omni is intentionally NOT migrated onto the adapter
|
||||
# framework in this PR (it has no registered adapter); keep it on the
|
||||
@@ -3743,12 +3266,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
base64_encode: bool = False,
|
||||
request_id: str | None = None,
|
||||
usage_out: list[SpeechTokenUsage] | None = None,
|
||||
has_inline_ref_audio: bool | None = None,
|
||||
) -> tuple[bytes | str, str]:
|
||||
# ``usage_out`` is an opt-in output channel: when a list is passed, the
|
||||
# computed SpeechTokenUsage is appended to it. The return stays a
|
||||
# 2-tuple so existing callers (and their test mocks) are unaffected;
|
||||
# only the batch path, which surfaces per-item usage, opts in.
|
||||
request_id, generator, bytes_tts_params = await self._prepare_speech_generation(request, request_id=request_id)
|
||||
request_id, generator, bytes_tts_params = await self._prepare_speech_generation(
|
||||
request, request_id=request_id, has_inline_ref_audio=has_inline_ref_audio
|
||||
)
|
||||
artifact_ready = False
|
||||
|
||||
try:
|
||||
@@ -4205,17 +3731,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
merged_requests = [self._merge_batch_item(batch_request, item) for item in batch_request.items]
|
||||
|
||||
async def _run_item(idx: int, req: OpenAICreateSpeechRequest) -> SpeechBatchItemResult:
|
||||
# Batch validation still goes through _validate_tts_request directly
|
||||
# (not the adapter). The single-request path validates via the
|
||||
# adapter; both ultimately call the same per-model validators, so the
|
||||
# surfaces stay in sync. Routing batch through the adapter (and its
|
||||
# uploaded-speaker handling) is a follow-up (RFC #4327).
|
||||
has_inline_ref_audio = req.ref_audio is not None
|
||||
validation_error = self._validate_tts_request(req)
|
||||
if validation_error is not None:
|
||||
return SpeechBatchItemResult(index=idx, status="error", error=validation_error)
|
||||
usage_box: list[SpeechTokenUsage] = []
|
||||
try:
|
||||
audio_data, media_type = await self._generate_audio_bytes(req, base64_encode=True, usage_out=usage_box)
|
||||
audio_data, media_type = await self._generate_audio_bytes(
|
||||
req, base64_encode=True, usage_out=usage_box, has_inline_ref_audio=has_inline_ref_audio
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Batch item %d failed: %s", idx, e)
|
||||
return SpeechBatchItemResult(index=idx, status="error", error=str(e))
|
||||
|
||||
@@ -104,6 +104,10 @@ class TTSModelAdapter(ABC):
|
||||
#: Serving backend: ``"ar"`` (engine_client) or ``"diffusion"``.
|
||||
backend: ClassVar[str] = "ar"
|
||||
|
||||
max_new_tokens_min = 1
|
||||
|
||||
max_new_tokens_max = 4096
|
||||
|
||||
def __init__(self, ctx: SpeechServingContext) -> None:
|
||||
self.ctx = ctx
|
||||
|
||||
|
||||
@@ -16,10 +16,32 @@ class CosyVoice3Adapter(ARTTSAdapter):
|
||||
name = "cosyvoice3"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
||||
"""Validate CosyVoice3 request parameters. Returns error message or None."""
|
||||
server = self.ctx.server
|
||||
err = server._apply_uploaded_speaker(request)
|
||||
if err:
|
||||
return err
|
||||
return self.ctx.server._validate_cosyvoice3_request(request)
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
# CosyVoice3 requires reference audio for voice cloning
|
||||
if request.ref_audio is None:
|
||||
return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)"
|
||||
|
||||
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
if request.max_new_tokens > self.max_new_tokens_max:
|
||||
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -15,6 +15,11 @@ class CovoAudioAdapter(ARTTSAdapter):
|
||||
stage_keys = frozenset({"fused_thinker_talker"})
|
||||
name = "covo_audio"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
) -> PreparedRequest:
|
||||
|
||||
@@ -16,10 +16,28 @@ class FishSpeechAdapter(ARTTSAdapter):
|
||||
name = "fish_tts"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
||||
"""Validate Fish Speech request parameters. Returns error message or None."""
|
||||
server = self.ctx.server
|
||||
err = server._apply_uploaded_speaker(request)
|
||||
if err:
|
||||
return err
|
||||
return self.ctx.server._validate_fish_tts_request(request)
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "Voice cloning requires 'ref_text' (transcript of the reference audio)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
if request.max_new_tokens > self.max_new_tokens_max:
|
||||
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -16,10 +16,28 @@ class GlmTTSAdapter(ARTTSAdapter):
|
||||
name = "glm_tts"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
||||
"""Validate GLM-TTS request — requires ref_audio for voice cloning."""
|
||||
server = self.ctx.server
|
||||
err = server._apply_uploaded_speaker(request)
|
||||
if err:
|
||||
return err
|
||||
return self.ctx.server._validate_glm_tts_request(request)
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
if request.ref_audio is None:
|
||||
return "GLM-TTS requires 'ref_audio' for zero-shot voice cloning"
|
||||
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "GLM-TTS voice cloning requires 'ref_text' (transcript of the reference audio)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be >= {self.max_new_tokens_min}"
|
||||
if request.max_new_tokens > self.max_new_tokens_max:
|
||||
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -16,10 +16,76 @@ class HiggsAudioV2Adapter(ARTTSAdapter):
|
||||
name = "higgs_audio_v2"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
||||
"""Validate higgs_audio_v2 request parameters. Returns error message or None.
|
||||
|
||||
Accepted: plain text -> speech, or shallow voice clone via ``ref_audio``
|
||||
+ ``ref_text`` (both required together). Still out of scope: preset
|
||||
``voice``/``speaker`` selection, ``x_vector_only_mode`` /
|
||||
``speaker_embedding`` helpers, ``task_type``/``language``/
|
||||
``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]``
|
||||
tags inside the input body.
|
||||
"""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import (
|
||||
MULTI_SPEAKER_TAG_PATTERN,
|
||||
)
|
||||
|
||||
server = self.ctx.server
|
||||
err = server._apply_uploaded_speaker(request)
|
||||
if err:
|
||||
return err
|
||||
return self.ctx.server._validate_higgs_audio_v2_request(request)
|
||||
|
||||
if not request.input or not request.input.strip():
|
||||
return "higgs_audio_v2: input text cannot be empty"
|
||||
|
||||
# Voice clone: ref_audio and ref_text must come together.
|
||||
if request.ref_audio is not None and not request.ref_text:
|
||||
return (
|
||||
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
||||
"'ref_text'; received ref_audio without ref_text"
|
||||
)
|
||||
if request.ref_text and request.ref_audio is None:
|
||||
return (
|
||||
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
||||
"'ref_text'; received ref_text without ref_audio"
|
||||
)
|
||||
|
||||
if request.x_vector_only_mode is not None:
|
||||
return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)"
|
||||
if request.speaker_embedding is not None:
|
||||
return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)"
|
||||
if request.voice and request.ref_audio is None:
|
||||
# _apply_uploaded_speaker runs before this validator; if voice was
|
||||
# an uploaded speaker, ref_audio is now populated and ref_text is
|
||||
# backfilled from the speaker entry. A bare voice= with no
|
||||
# ref_audio means the name didn't resolve to an uploaded speaker
|
||||
# (and higgs has no built-in preset voices).
|
||||
return (
|
||||
"higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; "
|
||||
f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. "
|
||||
f"Got voice={request.voice!r}"
|
||||
)
|
||||
if request.instructions:
|
||||
return (
|
||||
"higgs_audio_v2 v1 does not support 'instructions' (voice "
|
||||
"style/emotion control); supply plain text instead"
|
||||
)
|
||||
if request.task_type is not None:
|
||||
return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech"
|
||||
if request.language is not None:
|
||||
return (
|
||||
"higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text"
|
||||
)
|
||||
if request.speed is not None and request.speed != 1.0:
|
||||
return (
|
||||
"higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)"
|
||||
)
|
||||
|
||||
if MULTI_SPEAKER_TAG_PATTERN.search(request.input):
|
||||
return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -16,10 +16,20 @@ class HiggsAudioV3Adapter(ARTTSAdapter):
|
||||
name = "higgs_audio_v3"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
||||
"""Validate higgs_audio_v3 request parameters."""
|
||||
server = self.ctx.server
|
||||
err = server._apply_uploaded_speaker(request)
|
||||
if err:
|
||||
return err
|
||||
return self.ctx.server._validate_higgs_audio_v3_request(request)
|
||||
if not request.input or not request.input.strip():
|
||||
return "higgs_audio_v3: input text cannot be empty"
|
||||
if request.ref_audio is not None and not request.ref_text:
|
||||
# Voice clone ref_text is optional for v3 (improves fidelity but not required)
|
||||
pass
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -9,6 +9,10 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, Prepare
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@register_tts_adapter
|
||||
class MingTTSAdapter(ARTTSAdapter):
|
||||
@@ -16,7 +20,13 @@ class MingTTSAdapter(ARTTSAdapter):
|
||||
name = "ming_tts"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
return self.ctx.server._validate_ming_tts_request(request)
|
||||
"""Validate Ming TTS request parameters. Returns error message or None."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
if isinstance(request.ref_audio, list):
|
||||
return self._validate_ming_tts_podcast_request(request)
|
||||
return self._validate_ming_tts_single_speaker_request(request)
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
@@ -51,3 +61,82 @@ class MingTTSAdapter(ARTTSAdapter):
|
||||
tts_params = prompt.get("additional_information", {})
|
||||
# Ming stop-token / max_tokens sampling stays in the orchestrator tail.
|
||||
return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="ming_tts")
|
||||
|
||||
def _validate_ming_tts_single_speaker_request(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
server = self.ctx.server
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if request.speaker_embedding is not None:
|
||||
if not request.speaker_embedding:
|
||||
return "'speaker_embedding' must be a non-empty list of floats"
|
||||
emb_len = len(request.speaker_embedding)
|
||||
if emb_len != 192:
|
||||
logger.warning(
|
||||
"speaker_embedding has %d dimensions; Ming dense expects 192. "
|
||||
"Wrong dimensions will likely fail or degrade output.",
|
||||
emb_len,
|
||||
)
|
||||
|
||||
voice_lower = request.voice.lower() if isinstance(request.voice, str) else None
|
||||
uploaded_voice = bool(voice_lower and voice_lower in server.uploaded_speakers)
|
||||
clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice
|
||||
|
||||
if request.task_type == "Base" and not clone_source_present:
|
||||
return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample"
|
||||
|
||||
if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip():
|
||||
return "'ref_text' must be non-empty when provided with 'ref_audio'"
|
||||
|
||||
if request.ref_text is not None and request.ref_audio is None and not uploaded_voice:
|
||||
return "'ref_text' requires 'ref_audio' or an uploaded voice sample"
|
||||
|
||||
if request.instructions and len(request.instructions) > server._max_instructions_length:
|
||||
return f"Instructions too long (max {server._max_instructions_length} characters)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
if request.max_new_tokens > self.max_new_tokens_max:
|
||||
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||
|
||||
return None
|
||||
|
||||
def _validate_ming_tts_podcast_request(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
server = self.ctx.server
|
||||
if len(request.ref_audio) < 2:
|
||||
return "Podcast-style Ming requests require at least two 'ref_audio' clips"
|
||||
|
||||
for ref_audio in request.ref_audio:
|
||||
fmt_err = server._validate_ref_audio_format(ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "Podcast-style Ming requests require non-empty 'ref_text'"
|
||||
|
||||
if request.speaker_embedding is not None:
|
||||
embeddings = request.speaker_embedding
|
||||
embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1
|
||||
if embedding_count != len(request.ref_audio):
|
||||
return (
|
||||
"Podcast-style Ming requests require one speaker embedding per ref_audio clip; "
|
||||
f"got {embedding_count} embeddings for {len(request.ref_audio)} clips"
|
||||
)
|
||||
if embeddings and isinstance(embeddings[0], list):
|
||||
for item in embeddings:
|
||||
if len(item) != 192:
|
||||
return "Podcast-style Ming speaker embeddings must each have 192 dimensions"
|
||||
|
||||
if request.instructions and len(request.instructions) > server._max_instructions_length:
|
||||
return f"Instructions too long (max {server._max_instructions_length} characters)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
if request.max_new_tokens > self.max_new_tokens_max:
|
||||
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||
|
||||
return None
|
||||
@@ -18,10 +18,69 @@ if TYPE_CHECKING:
|
||||
|
||||
class _MossTTSAdapterBase(ARTTSAdapter):
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
||||
"""Validate any MOSS-TTS-family request (nano + 5 full variants).
|
||||
|
||||
Dispatches by ``self._moss_variant``:
|
||||
- ``tts``/``realtime``: require ``ref_audio`` (voice cloning).
|
||||
- ``ttsd``: require ``ref_audio`` (speaker 1); ``ref_audio_2``
|
||||
optional (defaults to the same ref for both speakers).
|
||||
- ``sound_effect``: require ``ambient_sound`` (no ref_audio).
|
||||
- ``voice_generator``: require ``instructions`` (no ref_audio).
|
||||
- For the legacy moss_tts_nano model_type the variant is None and
|
||||
we fall through to the original nano contract (ref_audio only).
|
||||
"""
|
||||
server = self.ctx.server
|
||||
err = server._apply_uploaded_speaker(request)
|
||||
if err:
|
||||
return err
|
||||
return self.ctx.server._validate_moss_tts_request(request)
|
||||
|
||||
if not request.input or not request.input.strip():
|
||||
# SoundEffect can legitimately have empty input (just ambient_sound).
|
||||
if server._moss_variant != "sound_effect":
|
||||
return "Input text cannot be empty"
|
||||
|
||||
v = server._moss_variant
|
||||
if v in (None, "tts", "realtime", "local"):
|
||||
if request.ref_audio is None:
|
||||
label = (
|
||||
"MOSS-TTS-Nano"
|
||||
if v is None
|
||||
else (
|
||||
"MOSS-TTS-Realtime"
|
||||
if v == "realtime"
|
||||
else ("MOSS-TTS-Local-Transformer" if v == "local" else "MOSS-TTS")
|
||||
)
|
||||
)
|
||||
return f"{label} requires 'ref_audio' (reference audio for voice cloning)."
|
||||
return server._validate_ref_audio_format(request.ref_audio)
|
||||
|
||||
if v == "ttsd":
|
||||
if request.ref_audio is None:
|
||||
return "MOSS-TTSD requires 'ref_audio' (speaker 1 reference)."
|
||||
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if request.ref_audio_2 is not None:
|
||||
return server._validate_ref_audio_format(request.ref_audio_2)
|
||||
return None
|
||||
|
||||
if v == "sound_effect":
|
||||
if not request.ambient_sound or not request.ambient_sound.strip():
|
||||
return (
|
||||
"MOSS-SoundEffect requires 'ambient_sound' (natural language "
|
||||
"description of the sound effect to synthesise)."
|
||||
)
|
||||
return None
|
||||
|
||||
if v == "voice_generator":
|
||||
if not request.instructions or not request.instructions.strip():
|
||||
return (
|
||||
"MOSS-VoiceGenerator requires 'instructions' (natural language "
|
||||
"voice description, e.g. 'a warm female voice with an American accent')."
|
||||
)
|
||||
return None
|
||||
|
||||
return None # unreachable
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -1,18 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Qwen3-TTS serving adapter.
|
||||
|
||||
First model migrated onto the adapter framework (RFC #4327). It is the dispatch
|
||||
surface for Qwen3-TTS: the serving orchestrator routes prompt/param building
|
||||
through ``build()`` instead of the inline ``else`` branch in
|
||||
``_prepare_speech_generation``.
|
||||
|
||||
To stay behaviour-identical and avoid divergence from the batch path (which
|
||||
shares ``_validate_qwen_tts_request`` and ``_build_tts_params``), the adapter
|
||||
reuses the single-source helper implementations on the serving instance through
|
||||
``ctx.server`` rather than copying them. Subsequent PRs relocate those bodies as
|
||||
more models migrate and the batch path also routes through adapters.
|
||||
"""
|
||||
"""Qwen3-TTS serving adapter."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -36,13 +25,133 @@ class Qwen3TTSAdapter(ARTTSAdapter):
|
||||
def normalize(self, request: "OpenAICreateSpeechRequest") -> None:
|
||||
"""Qwen3-TTS normalization (Base-task inference, voice lowercasing) is
|
||||
performed inside ``validate`` today; kept fused for a strict behaviour
|
||||
match. See ``_validate_qwen_tts_request``."""
|
||||
match."""
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
# Go through the shared dispatcher (which routes to
|
||||
# _validate_qwen_tts_request for this model-type) rather than the leaf
|
||||
# validator directly, matching the pre-adapter qwen3 path.
|
||||
return self.ctx.server._validate_tts_request(request)
|
||||
"""Validate Qwen TTS request parameters. Returns error message or None."""
|
||||
# Infer Base task when ref_audio or ref_text is provided without explicit task_type.
|
||||
server = self.ctx.server
|
||||
if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None):
|
||||
request.task_type = "Base"
|
||||
|
||||
# Normalize voice to lowercase for case-insensitive matching
|
||||
if request.voice is not None:
|
||||
request.voice = request.voice.lower()
|
||||
if request.task_type is None and request.voice in server.precomputed_speakers:
|
||||
request.task_type = "Base"
|
||||
task_type = request.task_type or "CustomVoice"
|
||||
|
||||
# Validate input is not empty
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
# Validate language (case-insensitive; normalized to the title-cased config form)
|
||||
if request.language is not None:
|
||||
request.language = request.language.title()
|
||||
if request.language not in server.supported_languages:
|
||||
return (
|
||||
f"Invalid language '{request.language}'. Supported: {', '.join(sorted(server.supported_languages))}"
|
||||
)
|
||||
|
||||
# Validate speaker for CustomVoice task
|
||||
if task_type == "CustomVoice":
|
||||
if not server.supported_speakers:
|
||||
return (
|
||||
"This model does not support CustomVoice task (no speakers configured). "
|
||||
"Use task_type='Base' with ref_audio/ref_text for voice cloning, "
|
||||
"or use a CustomVoice model."
|
||||
)
|
||||
if request.voice is not None and request.voice not in server.supported_speakers:
|
||||
return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}"
|
||||
|
||||
# Validate speaker_embedding constraints
|
||||
if request.speaker_embedding is not None:
|
||||
if task_type != "Base":
|
||||
return "'speaker_embedding' is only valid for Base task"
|
||||
if not request.speaker_embedding:
|
||||
return "'speaker_embedding' must be a non-empty list of floats"
|
||||
# speaker_embedding implies x_vector_only_mode — set it before
|
||||
# Base task validation so callers don't need to pass it explicitly.
|
||||
request.x_vector_only_mode = True
|
||||
emb_len = len(request.speaker_embedding)
|
||||
dim_err = server._validate_qwen_tts_speaker_embedding_dim(emb_len)
|
||||
if dim_err is not None:
|
||||
return dim_err
|
||||
# Validate Base task requirements
|
||||
if task_type == "Base":
|
||||
if request.voice is None:
|
||||
# 1. Ensure a voice source is provided
|
||||
if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None:
|
||||
return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning"
|
||||
# 2. Validate ref_audio format if it exists (using the helper from main)
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
# 3. Validate text requirements based on the mode
|
||||
if not getattr(request, "x_vector_only_mode", False):
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return (
|
||||
"Base task requires non-empty 'ref_text' (transcript of "
|
||||
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
||||
)
|
||||
else:
|
||||
voice_lower = request.voice.lower()
|
||||
if voice_lower in server.uploaded_speakers:
|
||||
# Check if data file exists for uploaded speaker
|
||||
speaker_info = server.uploaded_speakers[voice_lower]
|
||||
file_path = Path(speaker_info["file_path"])
|
||||
if not file_path.exists():
|
||||
return f"Data file for uploaded speaker '{request.voice}' not found on disk"
|
||||
elif voice_lower in server.precomputed_speakers:
|
||||
profile = server.precomputed_speakers[voice_lower]
|
||||
mode = str(profile.get("mode") or "xvec").lower()
|
||||
ref_text = request.ref_text or profile.get("ref_text")
|
||||
if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()):
|
||||
return (
|
||||
f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in "
|
||||
"the request or custom voice manifest"
|
||||
)
|
||||
else:
|
||||
# need ref_audio for built-in speaker
|
||||
if request.ref_audio is None:
|
||||
return (
|
||||
f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning"
|
||||
)
|
||||
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if not getattr(request, "x_vector_only_mode", False) and (
|
||||
not request.ref_text or not request.ref_text.strip()
|
||||
):
|
||||
return (
|
||||
"Base task requires non-empty 'ref_text' (transcript of "
|
||||
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
||||
)
|
||||
|
||||
# Validate cross-parameter dependencies
|
||||
if task_type != "Base":
|
||||
if request.ref_text is not None:
|
||||
return "'ref_text' is only valid for Base task"
|
||||
if request.x_vector_only_mode is not None:
|
||||
return "'x_vector_only_mode' is only valid for Base task"
|
||||
|
||||
# Validate VoiceDesign task requirements
|
||||
if task_type == "VoiceDesign" and not request.instructions:
|
||||
return "VoiceDesign task requires 'instructions' to describe the voice"
|
||||
|
||||
# Validate instructions length (using cached value from initialization)
|
||||
if request.instructions and len(request.instructions) > server._max_instructions_length:
|
||||
return f"Instructions too long (max {server._max_instructions_length} characters)"
|
||||
|
||||
# Validate max_new_tokens range
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
if request.max_new_tokens > self.max_new_tokens_max:
|
||||
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -23,8 +23,9 @@ class StepAudio2Adapter(ARTTSAdapter):
|
||||
name = "step_audio2"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
# Dispatcher routes to the step_audio2 case (non-empty input check).
|
||||
return self.ctx.server._validate_tts_request(request)
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -19,7 +19,25 @@ class VoxCPM2Adapter(ARTTSAdapter):
|
||||
name = "voxcpm2"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
return self.ctx.server._validate_voxcpm2_request(request)
|
||||
"""Validate VoxCPM2 request parameters. Returns error message or None."""
|
||||
server = self.ctx.server
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
if request.voice is not None:
|
||||
request.voice = request.voice.lower()
|
||||
available_voices = set(server.uploaded_speakers) | set(server.precomputed_speakers) | {"default"}
|
||||
if request.voice not in available_voices:
|
||||
supported = ", ".join(sorted(available_voices)) or "none"
|
||||
return f"Invalid voice '{request.voice}'. Supported: {supported}"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
if request.max_new_tokens > self.max_new_tokens_max:
|
||||
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
@@ -16,7 +16,32 @@ class VoxtralTTSAdapter(ARTTSAdapter):
|
||||
name = "voxtral_tts"
|
||||
|
||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||
return self.ctx.server._validate_voxtral_tts_request(request)
|
||||
"""Validate Voxtral TTS request parameters. Returns error message or None."""
|
||||
server = self.ctx.server
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
# Voxtral TTS requires either a preset voice or ref_audio for voice cloning.
|
||||
if request.voice is None and request.ref_audio is None:
|
||||
return "Either 'voice' (preset speaker) or 'ref_audio' (voice cloning) must be provided"
|
||||
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if request.voice is not None:
|
||||
request.voice = request.voice.lower()
|
||||
if server.supported_speakers and request.voice not in server.supported_speakers:
|
||||
return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < self.max_new_tokens_min:
|
||||
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||
if request.max_new_tokens > self.max_new_tokens_max:
|
||||
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||
|
||||
return None
|
||||
|
||||
async def build(
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
|
||||
Reference in new issue
Block a user