[Cleanup]Migrate model validation to adapters and clean up serving_speech.py (#4833)
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
1 parent
10d3a25916
commit
32dddfcef4
16 files changed
+547
-550
No files matched your search
@@ -6,7 +6,7 @@ E2E Online expansion tests for higgs-audio v2 against /v1/audio/speech.
|
|||||||
v1 scope is plain text -> 24 kHz speech plus shallow voice clone via
|
v1 scope is plain text -> 24 kHz speech plus shallow voice clone via
|
||||||
ref_audio + ref_text (inline) or voice=<name> (after POST /v1/audio/voices).
|
ref_audio + ref_text (inline) or voice=<name> (after POST /v1/audio/voices).
|
||||||
The model-aware request validator
|
The model-aware request validator
|
||||||
(vllm_omni/entrypoints/openai/serving_speech.py::_validate_higgs_audio_v2_request)
|
(vllm_omni/entrypoints/openai/tts_adapters/higgs_audio_v2::validate)
|
||||||
rejects multi-speaker tags, language overrides, task_type, and bare
|
rejects multi-speaker tags, language overrides, task_type, and bare
|
||||||
voice=<name> for names that do not match an uploaded speaker — this suite
|
voice=<name> for names that do not match an uploaded speaker — this suite
|
||||||
exercises both the happy path (plain text in, audio bytes out) and the
|
exercises both the happy path (plain text in, audio bytes out) and the
|
||||||
|
|||||||
@@ -1549,7 +1549,7 @@ class TestTTSMethods:
|
|||||||
req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "UTESF", "task_type": "Base"})
|
req = OpenAICreateSpeechRequest.model_validate({"input": "Hello", "speaker": "UTESF", "task_type": "Base"})
|
||||||
assert req.voice == "UTESF"
|
assert req.voice == "UTESF"
|
||||||
mocker.patch("pathlib.Path.exists", return_value=True)
|
mocker.patch("pathlib.Path.exists", return_value=True)
|
||||||
result = speech_server._validate_qwen_tts_request(req)
|
result = speech_server._validate_tts_request(req)
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
# ── uploaded voice with embedding ──
|
# ── uploaded voice with embedding ──
|
||||||
@@ -1684,7 +1684,7 @@ class TestTTSMethods:
|
|||||||
|
|
||||||
# Validation should pass (file exists)
|
# Validation should pass (file exists)
|
||||||
mocker.patch("pathlib.Path.exists", return_value=True)
|
mocker.patch("pathlib.Path.exists", return_value=True)
|
||||||
err = speech_server._validate_qwen_tts_request(req)
|
err = speech_server._validate_tts_request(req)
|
||||||
assert err is None, f"Validation failed: {err}"
|
assert err is None, f"Validation failed: {err}"
|
||||||
|
|
||||||
# Build params should auto-set ref_audio from stored file
|
# Build params should auto-set ref_audio from stored file
|
||||||
@@ -1722,7 +1722,7 @@ class TestTTSMethods:
|
|||||||
|
|
||||||
# Validation should pass
|
# Validation should pass
|
||||||
mocker.patch("pathlib.Path.exists", return_value=True)
|
mocker.patch("pathlib.Path.exists", return_value=True)
|
||||||
err = speech_server._validate_qwen_tts_request(req)
|
err = speech_server._validate_tts_request(req)
|
||||||
assert err is None, f"Validation failed: {err}"
|
assert err is None, f"Validation failed: {err}"
|
||||||
|
|
||||||
# Build params should use embedding, NOT audio
|
# Build params should use embedding, NOT audio
|
||||||
@@ -1866,6 +1866,35 @@ class TestTTSMethods:
|
|||||||
assert error is not None
|
assert error is not None
|
||||||
assert "max 10 characters" in error
|
assert "max 10 characters" in error
|
||||||
|
|
||||||
|
def test_batch_voice_not_treated_as_inline_ref_audio(
|
||||||
|
self,
|
||||||
|
speech_server,
|
||||||
|
mocker: MockerFixture,
|
||||||
|
):
|
||||||
|
speech_server._check_model = mocker.AsyncMock(return_value=None)
|
||||||
|
speech_server._batch_max_items = 32
|
||||||
|
|
||||||
|
def validate_and_resolve_uploaded_voice(request):
|
||||||
|
assert request.voice == "alice"
|
||||||
|
assert request.ref_audio is None
|
||||||
|
|
||||||
|
request.ref_audio = "data:audio/wav;base64,dGVzdA=="
|
||||||
|
request.ref_text = "reference text"
|
||||||
|
return None
|
||||||
|
|
||||||
|
speech_server._validate_tts_request = mocker.Mock(side_effect=validate_and_resolve_uploaded_voice)
|
||||||
|
speech_server._generate_audio_bytes = mocker.AsyncMock(return_value=("YWJj", "audio/wav"))
|
||||||
|
|
||||||
|
batch = BatchSpeechRequest(voice="alice", items=[SpeechBatchItem(input="hello")])
|
||||||
|
|
||||||
|
response = asyncio.run(speech_server.create_speech_batch(batch))
|
||||||
|
call = speech_server._generate_audio_bytes.await_args
|
||||||
|
request = call.args[0]
|
||||||
|
|
||||||
|
assert response.results[0].status == "success"
|
||||||
|
assert request.ref_audio is not None
|
||||||
|
assert call.kwargs["has_inline_ref_audio"] is False
|
||||||
|
|
||||||
|
|
||||||
class TestFileValidationFunctions:
|
class TestFileValidationFunctions:
|
||||||
"""Unit tests for file validation helper functions."""
|
"""Unit tests for file validation helper functions."""
|
||||||
@@ -3221,25 +3250,25 @@ class TestCosyVoice3Serving:
|
|||||||
|
|
||||||
def test_validate_cosyvoice3_empty_input(self, cosyvoice3_server):
|
def test_validate_cosyvoice3_empty_input(self, cosyvoice3_server):
|
||||||
request = OpenAICreateSpeechRequest(input="", ref_audio="data:audio/wav;base64,abc", ref_text="hello")
|
request = OpenAICreateSpeechRequest(input="", ref_audio="data:audio/wav;base64,abc", ref_text="hello")
|
||||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
error = cosyvoice3_server._validate_tts_request(request)
|
||||||
assert error is not None
|
assert error is not None
|
||||||
assert "empty" in error.lower()
|
assert "empty" in error.lower()
|
||||||
|
|
||||||
def test_validate_cosyvoice3_missing_ref_audio(self, cosyvoice3_server):
|
def test_validate_cosyvoice3_missing_ref_audio(self, cosyvoice3_server):
|
||||||
request = OpenAICreateSpeechRequest(input="Hello", ref_text="hello")
|
request = OpenAICreateSpeechRequest(input="Hello", ref_text="hello")
|
||||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
error = cosyvoice3_server._validate_tts_request(request)
|
||||||
assert error is not None
|
assert error is not None
|
||||||
assert "ref_audio" in error.lower()
|
assert "ref_audio" in error.lower()
|
||||||
|
|
||||||
def test_validate_cosyvoice3_missing_ref_text(self, cosyvoice3_server):
|
def test_validate_cosyvoice3_missing_ref_text(self, cosyvoice3_server):
|
||||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
|
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
|
||||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
error = cosyvoice3_server._validate_tts_request(request)
|
||||||
assert error is not None
|
assert error is not None
|
||||||
assert "ref_text" in error.lower()
|
assert "ref_text" in error.lower()
|
||||||
|
|
||||||
def test_validate_cosyvoice3_invalid_ref_audio_format(self, cosyvoice3_server):
|
def test_validate_cosyvoice3_invalid_ref_audio_format(self, cosyvoice3_server):
|
||||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="/local/path.wav", ref_text="hello")
|
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="/local/path.wav", ref_text="hello")
|
||||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
error = cosyvoice3_server._validate_tts_request(request)
|
||||||
assert error is not None
|
assert error is not None
|
||||||
assert "url" in error.lower() or "format" in error.lower()
|
assert "url" in error.lower() or "format" in error.lower()
|
||||||
|
|
||||||
@@ -3249,7 +3278,7 @@ class TestCosyVoice3Serving:
|
|||||||
ref_audio="data:audio/wav;base64,abc123",
|
ref_audio="data:audio/wav;base64,abc123",
|
||||||
ref_text="Reference transcript",
|
ref_text="Reference transcript",
|
||||||
)
|
)
|
||||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
error = cosyvoice3_server._validate_tts_request(request)
|
||||||
assert error is None
|
assert error is None
|
||||||
|
|
||||||
def test_validate_cosyvoice3_max_new_tokens_range(self, cosyvoice3_server):
|
def test_validate_cosyvoice3_max_new_tokens_range(self, cosyvoice3_server):
|
||||||
@@ -3259,7 +3288,7 @@ class TestCosyVoice3Serving:
|
|||||||
ref_text="hello",
|
ref_text="hello",
|
||||||
max_new_tokens=0,
|
max_new_tokens=0,
|
||||||
)
|
)
|
||||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
error = cosyvoice3_server._validate_tts_request(request)
|
||||||
assert error is not None
|
assert error is not None
|
||||||
assert "max_new_tokens" in error
|
assert "max_new_tokens" in error
|
||||||
|
|
||||||
@@ -3325,13 +3354,13 @@ def glm_tts_server(mocker: MockerFixture):
|
|||||||
class TestGLMTTSServing:
|
class TestGLMTTSServing:
|
||||||
def test_validate_glm_tts_requires_ref_audio(self, glm_tts_server):
|
def test_validate_glm_tts_requires_ref_audio(self, glm_tts_server):
|
||||||
request = OpenAICreateSpeechRequest(input="Hello", ref_text="Reference transcript")
|
request = OpenAICreateSpeechRequest(input="Hello", ref_text="Reference transcript")
|
||||||
error = glm_tts_server._validate_glm_tts_request(request)
|
error = glm_tts_server._validate_tts_request(request)
|
||||||
assert error is not None
|
assert error is not None
|
||||||
assert "ref_audio" in error
|
assert "ref_audio" in error
|
||||||
|
|
||||||
def test_validate_glm_tts_requires_ref_text(self, glm_tts_server):
|
def test_validate_glm_tts_requires_ref_text(self, glm_tts_server):
|
||||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
|
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
|
||||||
error = glm_tts_server._validate_glm_tts_request(request)
|
error = glm_tts_server._validate_tts_request(request)
|
||||||
assert error is not None
|
assert error is not None
|
||||||
assert "ref_text" in error
|
assert "ref_text" in error
|
||||||
|
|
||||||
@@ -3438,7 +3467,7 @@ class TestTTSAsyncOffloading:
|
|||||||
|
|
||||||
def test_prepare_speech_generation_awaits_qwen3_tts_async(self, qwen3_tts_server, mocker: MockerFixture):
|
def test_prepare_speech_generation_awaits_qwen3_tts_async(self, qwen3_tts_server, mocker: MockerFixture):
|
||||||
"""Qwen3 TTS path should call _estimate_prompt_len_async."""
|
"""Qwen3 TTS path should call _estimate_prompt_len_async."""
|
||||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||||
)
|
)
|
||||||
@@ -3455,7 +3484,7 @@ class TestTTSAsyncOffloading:
|
|||||||
qwen3_tts_server.engine_client.default_sampling_params_list = [
|
qwen3_tts_server.engine_client.default_sampling_params_list = [
|
||||||
SimpleNamespace(max_tokens=2048, seed=42, extra_args=None)
|
SimpleNamespace(max_tokens=2048, seed=42, extra_args=None)
|
||||||
]
|
]
|
||||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||||
)
|
)
|
||||||
@@ -3505,7 +3534,7 @@ class TestTTSAsyncOffloading:
|
|||||||
|
|
||||||
def test_prepare_speech_generation_treats_sse_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
def test_prepare_speech_generation_treats_sse_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
||||||
"""stream_format=sse should request delta-style multimodal outputs."""
|
"""stream_format=sse should request delta-style multimodal outputs."""
|
||||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||||
)
|
)
|
||||||
@@ -3522,7 +3551,7 @@ class TestTTSAsyncOffloading:
|
|||||||
|
|
||||||
def test_prepare_speech_generation_treats_stream_true_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
def test_prepare_speech_generation_treats_stream_true_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
||||||
"""stream=True should request delta-style multimodal outputs for SSE streaming."""
|
"""stream=True should request delta-style multimodal outputs for SSE streaming."""
|
||||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||||
)
|
)
|
||||||
@@ -3539,7 +3568,7 @@ class TestTTSAsyncOffloading:
|
|||||||
|
|
||||||
def test_prepare_speech_generation_treats_audio_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
def test_prepare_speech_generation_treats_audio_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
||||||
"""stream_format=audio should request delta-style multimodal outputs."""
|
"""stream_format=audio should request delta-style multimodal outputs."""
|
||||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||||
)
|
)
|
||||||
@@ -3559,7 +3588,7 @@ class TestTTSAsyncOffloading:
|
|||||||
):
|
):
|
||||||
"""Full-payload TTS streaming should not request delta multimodal outputs."""
|
"""Full-payload TTS streaming should not request delta multimodal outputs."""
|
||||||
qwen3_tts_server.engine_client.model_config.async_chunk = False
|
qwen3_tts_server.engine_client.model_config.async_chunk = False
|
||||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||||
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
qwen3_tts_server._build_tts_params = mocker.MagicMock(
|
||||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -150,7 +150,6 @@ _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256
|
|||||||
_HIGGS_V3_REF_CODE_CACHE_MAX_BYTES = 64 * 1024 * 1024
|
_HIGGS_V3_REF_CODE_CACHE_MAX_BYTES = 64 * 1024 * 1024
|
||||||
_QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key"
|
_QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key"
|
||||||
_TTS_MAX_INSTRUCTIONS_LENGTH = 500
|
_TTS_MAX_INSTRUCTIONS_LENGTH = 500
|
||||||
_TTS_MAX_NEW_TOKENS_MIN = 1
|
|
||||||
_TTS_MAX_NEW_TOKENS_MAX = 4096
|
_TTS_MAX_NEW_TOKENS_MAX = 4096
|
||||||
_MING_DEFAULT_PROMPT = MING_DEFAULT_PROMPT
|
_MING_DEFAULT_PROMPT = MING_DEFAULT_PROMPT
|
||||||
|
|
||||||
@@ -1490,48 +1489,22 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
|
|
||||||
def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||||
"""Validate TTS request parameters. Returns error message or None."""
|
"""Validate TTS request parameters. Returns error message or None."""
|
||||||
if self._tts_model_type == "step_audio2":
|
|
||||||
# StepAudio2 only requires non-empty input text
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "Input text cannot be empty"
|
|
||||||
return None
|
|
||||||
if self._tts_model_type == "voxtral_tts":
|
|
||||||
return self._validate_voxtral_tts_request(request)
|
|
||||||
if self._tts_model_type == "fish_tts":
|
|
||||||
return self._validate_fish_tts_request(request)
|
|
||||||
if self._tts_model_type == "cosyvoice3":
|
|
||||||
return self._validate_cosyvoice3_request(request)
|
|
||||||
if self._tts_model_type == "voxcpm2":
|
|
||||||
return self._validate_voxcpm2_request(request)
|
|
||||||
if self._tts_model_type == "ming_flash_omni_tts":
|
if self._tts_model_type == "ming_flash_omni_tts":
|
||||||
return self._validate_ming_flash_omni_tts_request(request)
|
return self._validate_ming_flash_omni_tts_request(request)
|
||||||
if self._tts_model_type == "ming_tts":
|
|
||||||
return self._validate_ming_tts_request(request)
|
|
||||||
if self._tts_model_type in ("moss_tts_nano", "moss_tts"):
|
|
||||||
return self._validate_moss_tts_request(request)
|
|
||||||
if self._tts_model_type == "glm_tts":
|
|
||||||
return self._validate_glm_tts_request(request)
|
|
||||||
return self._validate_qwen_tts_request(request)
|
|
||||||
|
|
||||||
def _validate_voxcpm2_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
adapter = self._get_tts_adapter()
|
||||||
"""Validate VoxCPM2 request parameters. Returns error message or None."""
|
if adapter is not None:
|
||||||
if not request.input or not request.input.strip():
|
return adapter.validate(request)
|
||||||
return "Input text cannot be empty"
|
|
||||||
|
|
||||||
if request.voice is not None:
|
adapter_cls = resolve_adapter("qwen3_tts")
|
||||||
request.voice = request.voice.lower()
|
if adapter_cls is None:
|
||||||
available_voices = set(self.uploaded_speakers) | set(self.precomputed_speakers) | {"default"}
|
raise ValueError("Qwen3-TTS adapter is not registered")
|
||||||
if request.voice not in available_voices:
|
|
||||||
supported = ", ".join(sorted(available_voices)) or "none"
|
|
||||||
return f"Invalid voice '{request.voice}'. Supported: {supported}"
|
|
||||||
|
|
||||||
if request.max_new_tokens is not None:
|
ctx = SpeechServingContext(
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
server=self,
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
engine_client=self.engine_client,
|
||||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
)
|
||||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
return adapter_cls(ctx).validate(request)
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _voxcpm2_encode(self, text: str) -> list[int]:
|
def _voxcpm2_encode(self, text: str) -> list[int]:
|
||||||
"""Tokenize text for VoxCPM2, splitting multichar Chinese tokens."""
|
"""Tokenize text for VoxCPM2, splitting multichar Chinese tokens."""
|
||||||
@@ -1599,158 +1572,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
|
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _validate_voxtral_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate Voxtral TTS request parameters. Returns error message or None."""
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "Input text cannot be empty"
|
|
||||||
|
|
||||||
# Voxtral TTS requires either a preset voice or ref_audio for voice cloning.
|
|
||||||
if request.voice is None and request.ref_audio is None:
|
|
||||||
return "Either 'voice' (preset speaker) or 'ref_audio' (voice cloning) must be provided"
|
|
||||||
|
|
||||||
if request.ref_audio is not None:
|
|
||||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
|
|
||||||
if request.voice is not None:
|
|
||||||
request.voice = request.voice.lower()
|
|
||||||
if self.supported_speakers and request.voice not in self.supported_speakers:
|
|
||||||
return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}"
|
|
||||||
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
|
||||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _validate_qwen_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate Qwen TTS request parameters. Returns error message or None."""
|
|
||||||
# Infer Base task when ref_audio or ref_text is provided without explicit task_type.
|
|
||||||
if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None):
|
|
||||||
request.task_type = "Base"
|
|
||||||
|
|
||||||
# Normalize voice to lowercase for case-insensitive matching
|
|
||||||
if request.voice is not None:
|
|
||||||
request.voice = request.voice.lower()
|
|
||||||
if request.task_type is None and request.voice in self.precomputed_speakers:
|
|
||||||
request.task_type = "Base"
|
|
||||||
task_type = request.task_type or "CustomVoice"
|
|
||||||
|
|
||||||
# Validate input is not empty
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "Input text cannot be empty"
|
|
||||||
|
|
||||||
# Validate language (case-insensitive; normalized to the title-cased config form)
|
|
||||||
if request.language is not None:
|
|
||||||
request.language = request.language.title()
|
|
||||||
if request.language not in self.supported_languages:
|
|
||||||
return (
|
|
||||||
f"Invalid language '{request.language}'. Supported: {', '.join(sorted(self.supported_languages))}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Validate speaker for CustomVoice task
|
|
||||||
if task_type == "CustomVoice":
|
|
||||||
if not self.supported_speakers:
|
|
||||||
return (
|
|
||||||
"This model does not support CustomVoice task (no speakers configured). "
|
|
||||||
"Use task_type='Base' with ref_audio/ref_text for voice cloning, "
|
|
||||||
"or use a CustomVoice model."
|
|
||||||
)
|
|
||||||
if request.voice is not None and request.voice not in self.supported_speakers:
|
|
||||||
return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}"
|
|
||||||
|
|
||||||
# Validate speaker_embedding constraints
|
|
||||||
if request.speaker_embedding is not None:
|
|
||||||
if task_type != "Base":
|
|
||||||
return "'speaker_embedding' is only valid for Base task"
|
|
||||||
if not request.speaker_embedding:
|
|
||||||
return "'speaker_embedding' must be a non-empty list of floats"
|
|
||||||
# speaker_embedding implies x_vector_only_mode — set it before
|
|
||||||
# Base task validation so callers don't need to pass it explicitly.
|
|
||||||
request.x_vector_only_mode = True
|
|
||||||
emb_len = len(request.speaker_embedding)
|
|
||||||
dim_err = self._validate_qwen_tts_speaker_embedding_dim(emb_len)
|
|
||||||
if dim_err is not None:
|
|
||||||
return dim_err
|
|
||||||
# Validate Base task requirements
|
|
||||||
if task_type == "Base":
|
|
||||||
if request.voice is None:
|
|
||||||
# 1. Ensure a voice source is provided
|
|
||||||
if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None:
|
|
||||||
return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning"
|
|
||||||
# 2. Validate ref_audio format if it exists (using the helper from main)
|
|
||||||
if request.ref_audio is not None:
|
|
||||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
# 3. Validate text requirements based on the mode
|
|
||||||
if not getattr(request, "x_vector_only_mode", False):
|
|
||||||
if not request.ref_text or not request.ref_text.strip():
|
|
||||||
return (
|
|
||||||
"Base task requires non-empty 'ref_text' (transcript of "
|
|
||||||
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
voice_lower = request.voice.lower()
|
|
||||||
if voice_lower in self.uploaded_speakers:
|
|
||||||
# Check if data file exists for uploaded speaker
|
|
||||||
speaker_info = self.uploaded_speakers[voice_lower]
|
|
||||||
file_path = Path(speaker_info["file_path"])
|
|
||||||
if not file_path.exists():
|
|
||||||
return f"Data file for uploaded speaker '{request.voice}' not found on disk"
|
|
||||||
elif voice_lower in self.precomputed_speakers:
|
|
||||||
profile = self.precomputed_speakers[voice_lower]
|
|
||||||
mode = str(profile.get("mode") or "xvec").lower()
|
|
||||||
ref_text = request.ref_text or profile.get("ref_text")
|
|
||||||
if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()):
|
|
||||||
return (
|
|
||||||
f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in "
|
|
||||||
"the request or custom voice manifest"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# need ref_audio for built-in speaker
|
|
||||||
if request.ref_audio is None:
|
|
||||||
return (
|
|
||||||
f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning"
|
|
||||||
)
|
|
||||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
if not getattr(request, "x_vector_only_mode", False) and (
|
|
||||||
not request.ref_text or not request.ref_text.strip()
|
|
||||||
):
|
|
||||||
return (
|
|
||||||
"Base task requires non-empty 'ref_text' (transcript of "
|
|
||||||
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Validate cross-parameter dependencies
|
|
||||||
if task_type != "Base":
|
|
||||||
if request.ref_text is not None:
|
|
||||||
return "'ref_text' is only valid for Base task"
|
|
||||||
if request.x_vector_only_mode is not None:
|
|
||||||
return "'x_vector_only_mode' is only valid for Base task"
|
|
||||||
|
|
||||||
# Validate VoiceDesign task requirements
|
|
||||||
if task_type == "VoiceDesign" and not request.instructions:
|
|
||||||
return "VoiceDesign task requires 'instructions' to describe the voice"
|
|
||||||
|
|
||||||
# Validate instructions length (using cached value from initialization)
|
|
||||||
if request.instructions and len(request.instructions) > self._max_instructions_length:
|
|
||||||
return f"Instructions too long (max {self._max_instructions_length} characters)"
|
|
||||||
|
|
||||||
# Validate max_new_tokens range
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
|
||||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _detect_moss_variant(self) -> str:
|
def _detect_moss_variant(self) -> str:
|
||||||
"""Sub-classify a ``moss_tts``-stage server into the actual MOSS-TTS
|
"""Sub-classify a ``moss_tts``-stage server into the actual MOSS-TTS
|
||||||
variant family (tts, ttsd, sound_effect, voice_generator, realtime).
|
variant family (tts, ttsd, sound_effect, voice_generator, realtime).
|
||||||
@@ -1776,66 +1597,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
return "voice_generator"
|
return "voice_generator"
|
||||||
return "tts"
|
return "tts"
|
||||||
|
|
||||||
def _validate_moss_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate any MOSS-TTS-family request (nano + 5 full variants).
|
|
||||||
|
|
||||||
Dispatches by ``self._moss_variant``:
|
|
||||||
- ``tts``/``realtime``: require ``ref_audio`` (voice cloning).
|
|
||||||
- ``ttsd``: require ``ref_audio`` (speaker 1); ``ref_audio_2``
|
|
||||||
optional (defaults to the same ref for both speakers).
|
|
||||||
- ``sound_effect``: require ``ambient_sound`` (no ref_audio).
|
|
||||||
- ``voice_generator``: require ``instructions`` (no ref_audio).
|
|
||||||
- For the legacy moss_tts_nano model_type the variant is None and
|
|
||||||
we fall through to the original nano contract (ref_audio only).
|
|
||||||
"""
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
# SoundEffect can legitimately have empty input (just ambient_sound).
|
|
||||||
if self._moss_variant != "sound_effect":
|
|
||||||
return "Input text cannot be empty"
|
|
||||||
|
|
||||||
v = self._moss_variant
|
|
||||||
if v in (None, "tts", "realtime", "local"):
|
|
||||||
if request.ref_audio is None:
|
|
||||||
label = (
|
|
||||||
"MOSS-TTS-Nano"
|
|
||||||
if v is None
|
|
||||||
else (
|
|
||||||
"MOSS-TTS-Realtime"
|
|
||||||
if v == "realtime"
|
|
||||||
else ("MOSS-TTS-Local-Transformer" if v == "local" else "MOSS-TTS")
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return f"{label} requires 'ref_audio' (reference audio for voice cloning)."
|
|
||||||
return self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
|
|
||||||
if v == "ttsd":
|
|
||||||
if request.ref_audio is None:
|
|
||||||
return "MOSS-TTSD requires 'ref_audio' (speaker 1 reference)."
|
|
||||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
if request.ref_audio_2 is not None:
|
|
||||||
return self._validate_ref_audio_format(request.ref_audio_2)
|
|
||||||
return None
|
|
||||||
|
|
||||||
if v == "sound_effect":
|
|
||||||
if not request.ambient_sound or not request.ambient_sound.strip():
|
|
||||||
return (
|
|
||||||
"MOSS-SoundEffect requires 'ambient_sound' (natural language "
|
|
||||||
"description of the sound effect to synthesise)."
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
if v == "voice_generator":
|
|
||||||
if not request.instructions or not request.instructions.strip():
|
|
||||||
return (
|
|
||||||
"MOSS-VoiceGenerator requires 'instructions' (natural language "
|
|
||||||
"voice description, e.g. 'a warm female voice with an American accent')."
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
return None # unreachable
|
|
||||||
|
|
||||||
def _get_moss_processor(self):
|
def _get_moss_processor(self):
|
||||||
"""Lazily load the upstream MOSS-TTS processor once per server.
|
"""Lazily load the upstream MOSS-TTS processor once per server.
|
||||||
|
|
||||||
@@ -1994,93 +1755,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
params["max_new_frames"] = [request.max_new_tokens]
|
params["max_new_frames"] = [request.max_new_tokens]
|
||||||
return params
|
return params
|
||||||
|
|
||||||
def _validate_higgs_audio_v2_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate higgs_audio_v2 request parameters. Returns error message or None.
|
|
||||||
|
|
||||||
Accepted: plain text -> speech, or shallow voice clone via ``ref_audio``
|
|
||||||
+ ``ref_text`` (both required together). Still out of scope: preset
|
|
||||||
``voice``/``speaker`` selection, ``x_vector_only_mode`` /
|
|
||||||
``speaker_embedding`` helpers, ``task_type``/``language``/
|
|
||||||
``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]``
|
|
||||||
tags inside the input body.
|
|
||||||
"""
|
|
||||||
from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import (
|
|
||||||
MULTI_SPEAKER_TAG_PATTERN,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "higgs_audio_v2: input text cannot be empty"
|
|
||||||
|
|
||||||
# Voice clone: ref_audio and ref_text must come together.
|
|
||||||
if request.ref_audio is not None and not request.ref_text:
|
|
||||||
return (
|
|
||||||
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
|
||||||
"'ref_text'; received ref_audio without ref_text"
|
|
||||||
)
|
|
||||||
if request.ref_text and request.ref_audio is None:
|
|
||||||
return (
|
|
||||||
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
|
||||||
"'ref_text'; received ref_text without ref_audio"
|
|
||||||
)
|
|
||||||
|
|
||||||
if request.x_vector_only_mode is not None:
|
|
||||||
return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)"
|
|
||||||
if request.speaker_embedding is not None:
|
|
||||||
return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)"
|
|
||||||
if request.voice and request.ref_audio is None:
|
|
||||||
# _apply_uploaded_speaker runs before this validator; if voice was
|
|
||||||
# an uploaded speaker, ref_audio is now populated and ref_text is
|
|
||||||
# backfilled from the speaker entry. A bare voice= with no
|
|
||||||
# ref_audio means the name didn't resolve to an uploaded speaker
|
|
||||||
# (and higgs has no built-in preset voices).
|
|
||||||
return (
|
|
||||||
"higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; "
|
|
||||||
f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. "
|
|
||||||
f"Got voice={request.voice!r}"
|
|
||||||
)
|
|
||||||
if request.instructions:
|
|
||||||
return (
|
|
||||||
"higgs_audio_v2 v1 does not support 'instructions' (voice "
|
|
||||||
"style/emotion control); supply plain text instead"
|
|
||||||
)
|
|
||||||
if request.task_type is not None:
|
|
||||||
return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech"
|
|
||||||
if request.language is not None:
|
|
||||||
return (
|
|
||||||
"higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text"
|
|
||||||
)
|
|
||||||
if request.speed is not None and request.speed != 1.0:
|
|
||||||
return (
|
|
||||||
"higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)"
|
|
||||||
)
|
|
||||||
|
|
||||||
if MULTI_SPEAKER_TAG_PATTERN.search(request.input):
|
|
||||||
return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input"
|
|
||||||
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
|
|
||||||
def _validate_glm_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate GLM-TTS request — requires ref_audio for voice cloning."""
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "Input text cannot be empty"
|
|
||||||
|
|
||||||
if request.ref_audio is None:
|
|
||||||
return "GLM-TTS requires 'ref_audio' for zero-shot voice cloning"
|
|
||||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
if not request.ref_text or not request.ref_text.strip():
|
|
||||||
return "GLM-TTS voice cloning requires 'ref_text' (transcript of the reference audio)"
|
|
||||||
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be >= {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
|
||||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest):
|
async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest):
|
||||||
"""Build prompt_token_ids for higgs_audio_v2 via the upstream processor.
|
"""Build prompt_token_ids for higgs_audio_v2 via the upstream processor.
|
||||||
|
|
||||||
@@ -2157,18 +1831,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
|
|
||||||
# ---- higgs-audio v3 ----
|
# ---- higgs-audio v3 ----
|
||||||
|
|
||||||
def _validate_higgs_audio_v3_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate higgs_audio_v3 request parameters."""
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "higgs_audio_v3: input text cannot be empty"
|
|
||||||
if request.ref_audio is not None and not request.ref_text:
|
|
||||||
# Voice clone ref_text is optional for v3 (improves fidelity but not required)
|
|
||||||
pass
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def _build_higgs_audio_v3_params(self, request: OpenAICreateSpeechRequest):
|
async def _build_higgs_audio_v3_params(self, request: OpenAICreateSpeechRequest):
|
||||||
"""Build prompt_token_ids for higgs_audio_v3.
|
"""Build prompt_token_ids for higgs_audio_v3.
|
||||||
|
|
||||||
@@ -2302,143 +1964,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
self._higgs_audio_v3_adapter = adapter
|
self._higgs_audio_v3_adapter = adapter
|
||||||
return adapter
|
return adapter
|
||||||
|
|
||||||
def _validate_fish_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate Fish Speech request parameters. Returns error message or None."""
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "Input text cannot be empty"
|
|
||||||
|
|
||||||
if request.ref_audio is not None:
|
|
||||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
if not request.ref_text or not request.ref_text.strip():
|
|
||||||
return "Voice cloning requires 'ref_text' (transcript of the reference audio)"
|
|
||||||
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
|
||||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _validate_cosyvoice3_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate CosyVoice3 request parameters. Returns error message or None."""
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "Input text cannot be empty"
|
|
||||||
|
|
||||||
# CosyVoice3 requires reference audio for voice cloning
|
|
||||||
if request.ref_audio is None:
|
|
||||||
return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)"
|
|
||||||
|
|
||||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
|
|
||||||
if not request.ref_text or not request.ref_text.strip():
|
|
||||||
return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)"
|
|
||||||
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
|
||||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _validate_ming_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
"""Validate Ming TTS request parameters. Returns error message or None."""
|
|
||||||
if not request.input or not request.input.strip():
|
|
||||||
return "Input text cannot be empty"
|
|
||||||
|
|
||||||
if isinstance(request.ref_audio, list):
|
|
||||||
return self._validate_ming_tts_podcast_request(request)
|
|
||||||
return self._validate_ming_tts_single_speaker_request(request)
|
|
||||||
|
|
||||||
def _validate_ming_tts_single_speaker_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
if request.ref_audio is not None:
|
|
||||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
|
|
||||||
if request.speaker_embedding is not None:
|
|
||||||
if not request.speaker_embedding:
|
|
||||||
return "'speaker_embedding' must be a non-empty list of floats"
|
|
||||||
emb_len = len(request.speaker_embedding)
|
|
||||||
if emb_len != 192:
|
|
||||||
logger.warning(
|
|
||||||
"speaker_embedding has %d dimensions; Ming dense expects 192. "
|
|
||||||
"Wrong dimensions will likely fail or degrade output.",
|
|
||||||
emb_len,
|
|
||||||
)
|
|
||||||
|
|
||||||
voice_lower = request.voice.lower() if isinstance(request.voice, str) else None
|
|
||||||
uploaded_voice = bool(voice_lower and voice_lower in self.uploaded_speakers)
|
|
||||||
clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice
|
|
||||||
|
|
||||||
if request.task_type == "Base" and not clone_source_present:
|
|
||||||
return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample"
|
|
||||||
|
|
||||||
if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip():
|
|
||||||
return "'ref_text' must be non-empty when provided with 'ref_audio'"
|
|
||||||
|
|
||||||
# Ming offline ref-audio cases use prompt_waveform without prompt_text;
|
|
||||||
# keep the transcript requirement for other TTS models.
|
|
||||||
if request.ref_audio is not None and request.speaker_embedding is None and not self._is_ming_tts_model():
|
|
||||||
uploaded_ref_text = self.uploaded_speakers[voice_lower].get("ref_text") if uploaded_voice else None
|
|
||||||
if not (request.ref_text and request.ref_text.strip()) and not uploaded_ref_text:
|
|
||||||
return "Reference-audio cloning requires non-empty 'ref_text'"
|
|
||||||
|
|
||||||
if request.ref_text is not None and request.ref_audio is None and not uploaded_voice:
|
|
||||||
return "'ref_text' requires 'ref_audio' or an uploaded voice sample"
|
|
||||||
|
|
||||||
if request.instructions and len(request.instructions) > self._max_instructions_length:
|
|
||||||
return f"Instructions too long (max {self._max_instructions_length} characters)"
|
|
||||||
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
|
||||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _validate_ming_tts_podcast_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
|
||||||
if len(request.ref_audio) < 2:
|
|
||||||
return "Podcast-style Ming requests require at least two 'ref_audio' clips"
|
|
||||||
|
|
||||||
for ref_audio in request.ref_audio:
|
|
||||||
fmt_err = self._validate_ref_audio_format(ref_audio)
|
|
||||||
if fmt_err:
|
|
||||||
return fmt_err
|
|
||||||
|
|
||||||
if not request.ref_text or not request.ref_text.strip():
|
|
||||||
return "Podcast-style Ming requests require non-empty 'ref_text'"
|
|
||||||
|
|
||||||
if request.speaker_embedding is not None:
|
|
||||||
embeddings = request.speaker_embedding
|
|
||||||
embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1
|
|
||||||
if embedding_count != len(request.ref_audio):
|
|
||||||
return (
|
|
||||||
"Podcast-style Ming requests require one speaker embedding per ref_audio clip; "
|
|
||||||
f"got {embedding_count} embeddings for {len(request.ref_audio)} clips"
|
|
||||||
)
|
|
||||||
if embeddings and isinstance(embeddings[0], list):
|
|
||||||
for item in embeddings:
|
|
||||||
if len(item) != 192:
|
|
||||||
return "Podcast-style Ming speaker embeddings must each have 192 dimensions"
|
|
||||||
|
|
||||||
if request.instructions and len(request.instructions) > self._max_instructions_length:
|
|
||||||
return f"Instructions too long (max {self._max_instructions_length} characters)"
|
|
||||||
|
|
||||||
if request.max_new_tokens is not None:
|
|
||||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
|
||||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
|
||||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
|
||||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
|
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
|
||||||
"""Resolve ref_audio to (wav_samples, sample_rate).
|
"""Resolve ref_audio to (wav_samples, sample_rate).
|
||||||
|
|
||||||
@@ -2573,9 +2098,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
|
|
||||||
# ---- Ming TTS helpers ----
|
# ---- Ming TTS helpers ----
|
||||||
|
|
||||||
def _is_ming_tts_model(self) -> bool:
|
|
||||||
return self._tts_model_type == "ming_tts"
|
|
||||||
|
|
||||||
def _coerce_ming_prompt_waveform(self, wav_samples, sample_rate):
|
def _coerce_ming_prompt_waveform(self, wav_samples, sample_rate):
|
||||||
from torchaudio.functional import resample as resample_audio
|
from torchaudio.functional import resample as resample_audio
|
||||||
|
|
||||||
@@ -3488,6 +3010,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
self,
|
self,
|
||||||
request: OpenAICreateSpeechRequest,
|
request: OpenAICreateSpeechRequest,
|
||||||
request_id: str | None = None,
|
request_id: str | None = None,
|
||||||
|
has_inline_ref_audio: bool | None = None,
|
||||||
) -> tuple[str, Any, dict[str, Any]]:
|
) -> tuple[str, Any, dict[str, Any]]:
|
||||||
if self.engine_client.errored:
|
if self.engine_client.errored:
|
||||||
raise self.engine_client.dead_error
|
raise self.engine_client.dead_error
|
||||||
@@ -3520,7 +3043,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
# in place. The builders need to know whether the caller supplied audio
|
# in place. The builders need to know whether the caller supplied audio
|
||||||
# inline vs. via an uploaded voice.
|
# inline vs. via an uploaded voice.
|
||||||
model_type: str | None = None
|
model_type: str | None = None
|
||||||
has_inline_ref_audio = request.ref_audio is not None
|
has_inline_ref_audio = (request.ref_audio is not None) if has_inline_ref_audio is None else has_inline_ref_audio
|
||||||
if self._tts_model_type == "ming_flash_omni_tts":
|
if self._tts_model_type == "ming_flash_omni_tts":
|
||||||
# ming_flash_omni is intentionally NOT migrated onto the adapter
|
# ming_flash_omni is intentionally NOT migrated onto the adapter
|
||||||
# framework in this PR (it has no registered adapter); keep it on the
|
# framework in this PR (it has no registered adapter); keep it on the
|
||||||
@@ -3743,12 +3266,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
base64_encode: bool = False,
|
base64_encode: bool = False,
|
||||||
request_id: str | None = None,
|
request_id: str | None = None,
|
||||||
usage_out: list[SpeechTokenUsage] | None = None,
|
usage_out: list[SpeechTokenUsage] | None = None,
|
||||||
|
has_inline_ref_audio: bool | None = None,
|
||||||
) -> tuple[bytes | str, str]:
|
) -> tuple[bytes | str, str]:
|
||||||
# ``usage_out`` is an opt-in output channel: when a list is passed, the
|
# ``usage_out`` is an opt-in output channel: when a list is passed, the
|
||||||
# computed SpeechTokenUsage is appended to it. The return stays a
|
# computed SpeechTokenUsage is appended to it. The return stays a
|
||||||
# 2-tuple so existing callers (and their test mocks) are unaffected;
|
# 2-tuple so existing callers (and their test mocks) are unaffected;
|
||||||
# only the batch path, which surfaces per-item usage, opts in.
|
# only the batch path, which surfaces per-item usage, opts in.
|
||||||
request_id, generator, bytes_tts_params = await self._prepare_speech_generation(request, request_id=request_id)
|
request_id, generator, bytes_tts_params = await self._prepare_speech_generation(
|
||||||
|
request, request_id=request_id, has_inline_ref_audio=has_inline_ref_audio
|
||||||
|
)
|
||||||
artifact_ready = False
|
artifact_ready = False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -4205,17 +3731,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
merged_requests = [self._merge_batch_item(batch_request, item) for item in batch_request.items]
|
merged_requests = [self._merge_batch_item(batch_request, item) for item in batch_request.items]
|
||||||
|
|
||||||
async def _run_item(idx: int, req: OpenAICreateSpeechRequest) -> SpeechBatchItemResult:
|
async def _run_item(idx: int, req: OpenAICreateSpeechRequest) -> SpeechBatchItemResult:
|
||||||
# Batch validation still goes through _validate_tts_request directly
|
has_inline_ref_audio = req.ref_audio is not None
|
||||||
# (not the adapter). The single-request path validates via the
|
|
||||||
# adapter; both ultimately call the same per-model validators, so the
|
|
||||||
# surfaces stay in sync. Routing batch through the adapter (and its
|
|
||||||
# uploaded-speaker handling) is a follow-up (RFC #4327).
|
|
||||||
validation_error = self._validate_tts_request(req)
|
validation_error = self._validate_tts_request(req)
|
||||||
if validation_error is not None:
|
if validation_error is not None:
|
||||||
return SpeechBatchItemResult(index=idx, status="error", error=validation_error)
|
return SpeechBatchItemResult(index=idx, status="error", error=validation_error)
|
||||||
usage_box: list[SpeechTokenUsage] = []
|
usage_box: list[SpeechTokenUsage] = []
|
||||||
try:
|
try:
|
||||||
audio_data, media_type = await self._generate_audio_bytes(req, base64_encode=True, usage_out=usage_box)
|
audio_data, media_type = await self._generate_audio_bytes(
|
||||||
|
req, base64_encode=True, usage_out=usage_box, has_inline_ref_audio=has_inline_ref_audio
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Batch item %d failed: %s", idx, e)
|
logger.exception("Batch item %d failed: %s", idx, e)
|
||||||
return SpeechBatchItemResult(index=idx, status="error", error=str(e))
|
return SpeechBatchItemResult(index=idx, status="error", error=str(e))
|
||||||
|
|||||||
@@ -104,6 +104,10 @@ class TTSModelAdapter(ABC):
|
|||||||
#: Serving backend: ``"ar"`` (engine_client) or ``"diffusion"``.
|
#: Serving backend: ``"ar"`` (engine_client) or ``"diffusion"``.
|
||||||
backend: ClassVar[str] = "ar"
|
backend: ClassVar[str] = "ar"
|
||||||
|
|
||||||
|
max_new_tokens_min = 1
|
||||||
|
|
||||||
|
max_new_tokens_max = 4096
|
||||||
|
|
||||||
def __init__(self, ctx: SpeechServingContext) -> None:
|
def __init__(self, ctx: SpeechServingContext) -> None:
|
||||||
self.ctx = ctx
|
self.ctx = ctx
|
||||||
|
|
||||||
|
|||||||
@@ -16,10 +16,32 @@ class CosyVoice3Adapter(ARTTSAdapter):
|
|||||||
name = "cosyvoice3"
|
name = "cosyvoice3"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
"""Validate CosyVoice3 request parameters. Returns error message or None."""
|
||||||
|
server = self.ctx.server
|
||||||
|
err = server._apply_uploaded_speaker(request)
|
||||||
if err:
|
if err:
|
||||||
return err
|
return err
|
||||||
return self.ctx.server._validate_cosyvoice3_request(request)
|
if not request.input or not request.input.strip():
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
|
||||||
|
# CosyVoice3 requires reference audio for voice cloning
|
||||||
|
if request.ref_audio is None:
|
||||||
|
return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)"
|
||||||
|
|
||||||
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
|
||||||
|
if not request.ref_text or not request.ref_text.strip():
|
||||||
|
return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)"
|
||||||
|
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
if request.max_new_tokens > self.max_new_tokens_max:
|
||||||
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -15,6 +15,11 @@ class CovoAudioAdapter(ARTTSAdapter):
|
|||||||
stage_keys = frozenset({"fused_thinker_talker"})
|
stage_keys = frozenset({"fused_thinker_talker"})
|
||||||
name = "covo_audio"
|
name = "covo_audio"
|
||||||
|
|
||||||
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
|
if not request.input or not request.input.strip():
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
) -> PreparedRequest:
|
) -> PreparedRequest:
|
||||||
|
|||||||
@@ -16,10 +16,28 @@ class FishSpeechAdapter(ARTTSAdapter):
|
|||||||
name = "fish_tts"
|
name = "fish_tts"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
"""Validate Fish Speech request parameters. Returns error message or None."""
|
||||||
|
server = self.ctx.server
|
||||||
|
err = server._apply_uploaded_speaker(request)
|
||||||
if err:
|
if err:
|
||||||
return err
|
return err
|
||||||
return self.ctx.server._validate_fish_tts_request(request)
|
if not request.input or not request.input.strip():
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
|
||||||
|
if request.ref_audio is not None:
|
||||||
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
if not request.ref_text or not request.ref_text.strip():
|
||||||
|
return "Voice cloning requires 'ref_text' (transcript of the reference audio)"
|
||||||
|
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
if request.max_new_tokens > self.max_new_tokens_max:
|
||||||
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -16,10 +16,28 @@ class GlmTTSAdapter(ARTTSAdapter):
|
|||||||
name = "glm_tts"
|
name = "glm_tts"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
"""Validate GLM-TTS request — requires ref_audio for voice cloning."""
|
||||||
|
server = self.ctx.server
|
||||||
|
err = server._apply_uploaded_speaker(request)
|
||||||
if err:
|
if err:
|
||||||
return err
|
return err
|
||||||
return self.ctx.server._validate_glm_tts_request(request)
|
if not request.input or not request.input.strip():
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
|
||||||
|
if request.ref_audio is None:
|
||||||
|
return "GLM-TTS requires 'ref_audio' for zero-shot voice cloning"
|
||||||
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
if not request.ref_text or not request.ref_text.strip():
|
||||||
|
return "GLM-TTS voice cloning requires 'ref_text' (transcript of the reference audio)"
|
||||||
|
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be >= {self.max_new_tokens_min}"
|
||||||
|
if request.max_new_tokens > self.max_new_tokens_max:
|
||||||
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -16,10 +16,76 @@ class HiggsAudioV2Adapter(ARTTSAdapter):
|
|||||||
name = "higgs_audio_v2"
|
name = "higgs_audio_v2"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
"""Validate higgs_audio_v2 request parameters. Returns error message or None.
|
||||||
|
|
||||||
|
Accepted: plain text -> speech, or shallow voice clone via ``ref_audio``
|
||||||
|
+ ``ref_text`` (both required together). Still out of scope: preset
|
||||||
|
``voice``/``speaker`` selection, ``x_vector_only_mode`` /
|
||||||
|
``speaker_embedding`` helpers, ``task_type``/``language``/
|
||||||
|
``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]``
|
||||||
|
tags inside the input body.
|
||||||
|
"""
|
||||||
|
from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import (
|
||||||
|
MULTI_SPEAKER_TAG_PATTERN,
|
||||||
|
)
|
||||||
|
|
||||||
|
server = self.ctx.server
|
||||||
|
err = server._apply_uploaded_speaker(request)
|
||||||
if err:
|
if err:
|
||||||
return err
|
return err
|
||||||
return self.ctx.server._validate_higgs_audio_v2_request(request)
|
|
||||||
|
if not request.input or not request.input.strip():
|
||||||
|
return "higgs_audio_v2: input text cannot be empty"
|
||||||
|
|
||||||
|
# Voice clone: ref_audio and ref_text must come together.
|
||||||
|
if request.ref_audio is not None and not request.ref_text:
|
||||||
|
return (
|
||||||
|
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
||||||
|
"'ref_text'; received ref_audio without ref_text"
|
||||||
|
)
|
||||||
|
if request.ref_text and request.ref_audio is None:
|
||||||
|
return (
|
||||||
|
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
||||||
|
"'ref_text'; received ref_text without ref_audio"
|
||||||
|
)
|
||||||
|
|
||||||
|
if request.x_vector_only_mode is not None:
|
||||||
|
return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)"
|
||||||
|
if request.speaker_embedding is not None:
|
||||||
|
return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)"
|
||||||
|
if request.voice and request.ref_audio is None:
|
||||||
|
# _apply_uploaded_speaker runs before this validator; if voice was
|
||||||
|
# an uploaded speaker, ref_audio is now populated and ref_text is
|
||||||
|
# backfilled from the speaker entry. A bare voice= with no
|
||||||
|
# ref_audio means the name didn't resolve to an uploaded speaker
|
||||||
|
# (and higgs has no built-in preset voices).
|
||||||
|
return (
|
||||||
|
"higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; "
|
||||||
|
f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. "
|
||||||
|
f"Got voice={request.voice!r}"
|
||||||
|
)
|
||||||
|
if request.instructions:
|
||||||
|
return (
|
||||||
|
"higgs_audio_v2 v1 does not support 'instructions' (voice "
|
||||||
|
"style/emotion control); supply plain text instead"
|
||||||
|
)
|
||||||
|
if request.task_type is not None:
|
||||||
|
return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech"
|
||||||
|
if request.language is not None:
|
||||||
|
return (
|
||||||
|
"higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text"
|
||||||
|
)
|
||||||
|
if request.speed is not None and request.speed != 1.0:
|
||||||
|
return (
|
||||||
|
"higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)"
|
||||||
|
)
|
||||||
|
|
||||||
|
if MULTI_SPEAKER_TAG_PATTERN.search(request.input):
|
||||||
|
return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input"
|
||||||
|
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -16,10 +16,20 @@ class HiggsAudioV3Adapter(ARTTSAdapter):
|
|||||||
name = "higgs_audio_v3"
|
name = "higgs_audio_v3"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
"""Validate higgs_audio_v3 request parameters."""
|
||||||
|
server = self.ctx.server
|
||||||
|
err = server._apply_uploaded_speaker(request)
|
||||||
if err:
|
if err:
|
||||||
return err
|
return err
|
||||||
return self.ctx.server._validate_higgs_audio_v3_request(request)
|
if not request.input or not request.input.strip():
|
||||||
|
return "higgs_audio_v3: input text cannot be empty"
|
||||||
|
if request.ref_audio is not None and not request.ref_text:
|
||||||
|
# Voice clone ref_text is optional for v3 (improves fidelity but not required)
|
||||||
|
pass
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -9,6 +9,10 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, Prepare
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||||
|
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@register_tts_adapter
|
@register_tts_adapter
|
||||||
class MingTTSAdapter(ARTTSAdapter):
|
class MingTTSAdapter(ARTTSAdapter):
|
||||||
@@ -16,7 +20,13 @@ class MingTTSAdapter(ARTTSAdapter):
|
|||||||
name = "ming_tts"
|
name = "ming_tts"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
return self.ctx.server._validate_ming_tts_request(request)
|
"""Validate Ming TTS request parameters. Returns error message or None."""
|
||||||
|
if not request.input or not request.input.strip():
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
|
||||||
|
if isinstance(request.ref_audio, list):
|
||||||
|
return self._validate_ming_tts_podcast_request(request)
|
||||||
|
return self._validate_ming_tts_single_speaker_request(request)
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
@@ -51,3 +61,82 @@ class MingTTSAdapter(ARTTSAdapter):
|
|||||||
tts_params = prompt.get("additional_information", {})
|
tts_params = prompt.get("additional_information", {})
|
||||||
# Ming stop-token / max_tokens sampling stays in the orchestrator tail.
|
# Ming stop-token / max_tokens sampling stays in the orchestrator tail.
|
||||||
return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="ming_tts")
|
return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="ming_tts")
|
||||||
|
|
||||||
|
def _validate_ming_tts_single_speaker_request(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
|
server = self.ctx.server
|
||||||
|
if request.ref_audio is not None:
|
||||||
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
|
||||||
|
if request.speaker_embedding is not None:
|
||||||
|
if not request.speaker_embedding:
|
||||||
|
return "'speaker_embedding' must be a non-empty list of floats"
|
||||||
|
emb_len = len(request.speaker_embedding)
|
||||||
|
if emb_len != 192:
|
||||||
|
logger.warning(
|
||||||
|
"speaker_embedding has %d dimensions; Ming dense expects 192. "
|
||||||
|
"Wrong dimensions will likely fail or degrade output.",
|
||||||
|
emb_len,
|
||||||
|
)
|
||||||
|
|
||||||
|
voice_lower = request.voice.lower() if isinstance(request.voice, str) else None
|
||||||
|
uploaded_voice = bool(voice_lower and voice_lower in server.uploaded_speakers)
|
||||||
|
clone_source_present = request.ref_audio is not None or request.speaker_embedding is not None or uploaded_voice
|
||||||
|
|
||||||
|
if request.task_type == "Base" and not clone_source_present:
|
||||||
|
return "Base task requires 'ref_audio', 'speaker_embedding', or an uploaded voice sample"
|
||||||
|
|
||||||
|
if request.ref_audio is not None and request.ref_text is not None and not request.ref_text.strip():
|
||||||
|
return "'ref_text' must be non-empty when provided with 'ref_audio'"
|
||||||
|
|
||||||
|
if request.ref_text is not None and request.ref_audio is None and not uploaded_voice:
|
||||||
|
return "'ref_text' requires 'ref_audio' or an uploaded voice sample"
|
||||||
|
|
||||||
|
if request.instructions and len(request.instructions) > server._max_instructions_length:
|
||||||
|
return f"Instructions too long (max {server._max_instructions_length} characters)"
|
||||||
|
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
if request.max_new_tokens > self.max_new_tokens_max:
|
||||||
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _validate_ming_tts_podcast_request(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
|
server = self.ctx.server
|
||||||
|
if len(request.ref_audio) < 2:
|
||||||
|
return "Podcast-style Ming requests require at least two 'ref_audio' clips"
|
||||||
|
|
||||||
|
for ref_audio in request.ref_audio:
|
||||||
|
fmt_err = server._validate_ref_audio_format(ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
|
||||||
|
if not request.ref_text or not request.ref_text.strip():
|
||||||
|
return "Podcast-style Ming requests require non-empty 'ref_text'"
|
||||||
|
|
||||||
|
if request.speaker_embedding is not None:
|
||||||
|
embeddings = request.speaker_embedding
|
||||||
|
embedding_count = len(embeddings) if embeddings and isinstance(embeddings[0], list) else 1
|
||||||
|
if embedding_count != len(request.ref_audio):
|
||||||
|
return (
|
||||||
|
"Podcast-style Ming requests require one speaker embedding per ref_audio clip; "
|
||||||
|
f"got {embedding_count} embeddings for {len(request.ref_audio)} clips"
|
||||||
|
)
|
||||||
|
if embeddings and isinstance(embeddings[0], list):
|
||||||
|
for item in embeddings:
|
||||||
|
if len(item) != 192:
|
||||||
|
return "Podcast-style Ming speaker embeddings must each have 192 dimensions"
|
||||||
|
|
||||||
|
if request.instructions and len(request.instructions) > server._max_instructions_length:
|
||||||
|
return f"Instructions too long (max {server._max_instructions_length} characters)"
|
||||||
|
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
if request.max_new_tokens > self.max_new_tokens_max:
|
||||||
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||||
|
|
||||||
|
return None
|
||||||
@@ -18,10 +18,69 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
class _MossTTSAdapterBase(ARTTSAdapter):
|
class _MossTTSAdapterBase(ARTTSAdapter):
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
err = self.ctx.server._apply_uploaded_speaker(request)
|
"""Validate any MOSS-TTS-family request (nano + 5 full variants).
|
||||||
|
|
||||||
|
Dispatches by ``self._moss_variant``:
|
||||||
|
- ``tts``/``realtime``: require ``ref_audio`` (voice cloning).
|
||||||
|
- ``ttsd``: require ``ref_audio`` (speaker 1); ``ref_audio_2``
|
||||||
|
optional (defaults to the same ref for both speakers).
|
||||||
|
- ``sound_effect``: require ``ambient_sound`` (no ref_audio).
|
||||||
|
- ``voice_generator``: require ``instructions`` (no ref_audio).
|
||||||
|
- For the legacy moss_tts_nano model_type the variant is None and
|
||||||
|
we fall through to the original nano contract (ref_audio only).
|
||||||
|
"""
|
||||||
|
server = self.ctx.server
|
||||||
|
err = server._apply_uploaded_speaker(request)
|
||||||
if err:
|
if err:
|
||||||
return err
|
return err
|
||||||
return self.ctx.server._validate_moss_tts_request(request)
|
|
||||||
|
if not request.input or not request.input.strip():
|
||||||
|
# SoundEffect can legitimately have empty input (just ambient_sound).
|
||||||
|
if server._moss_variant != "sound_effect":
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
|
||||||
|
v = server._moss_variant
|
||||||
|
if v in (None, "tts", "realtime", "local"):
|
||||||
|
if request.ref_audio is None:
|
||||||
|
label = (
|
||||||
|
"MOSS-TTS-Nano"
|
||||||
|
if v is None
|
||||||
|
else (
|
||||||
|
"MOSS-TTS-Realtime"
|
||||||
|
if v == "realtime"
|
||||||
|
else ("MOSS-TTS-Local-Transformer" if v == "local" else "MOSS-TTS")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return f"{label} requires 'ref_audio' (reference audio for voice cloning)."
|
||||||
|
return server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
|
||||||
|
if v == "ttsd":
|
||||||
|
if request.ref_audio is None:
|
||||||
|
return "MOSS-TTSD requires 'ref_audio' (speaker 1 reference)."
|
||||||
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
if request.ref_audio_2 is not None:
|
||||||
|
return server._validate_ref_audio_format(request.ref_audio_2)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if v == "sound_effect":
|
||||||
|
if not request.ambient_sound or not request.ambient_sound.strip():
|
||||||
|
return (
|
||||||
|
"MOSS-SoundEffect requires 'ambient_sound' (natural language "
|
||||||
|
"description of the sound effect to synthesise)."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if v == "voice_generator":
|
||||||
|
if not request.instructions or not request.instructions.strip():
|
||||||
|
return (
|
||||||
|
"MOSS-VoiceGenerator requires 'instructions' (natural language "
|
||||||
|
"voice description, e.g. 'a warm female voice with an American accent')."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
return None # unreachable
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -1,18 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
"""Qwen3-TTS serving adapter.
|
"""Qwen3-TTS serving adapter."""
|
||||||
|
|
||||||
First model migrated onto the adapter framework (RFC #4327). It is the dispatch
|
|
||||||
surface for Qwen3-TTS: the serving orchestrator routes prompt/param building
|
|
||||||
through ``build()`` instead of the inline ``else`` branch in
|
|
||||||
``_prepare_speech_generation``.
|
|
||||||
|
|
||||||
To stay behaviour-identical and avoid divergence from the batch path (which
|
|
||||||
shares ``_validate_qwen_tts_request`` and ``_build_tts_params``), the adapter
|
|
||||||
reuses the single-source helper implementations on the serving instance through
|
|
||||||
``ctx.server`` rather than copying them. Subsequent PRs relocate those bodies as
|
|
||||||
more models migrate and the batch path also routes through adapters.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -36,13 +25,133 @@ class Qwen3TTSAdapter(ARTTSAdapter):
|
|||||||
def normalize(self, request: "OpenAICreateSpeechRequest") -> None:
|
def normalize(self, request: "OpenAICreateSpeechRequest") -> None:
|
||||||
"""Qwen3-TTS normalization (Base-task inference, voice lowercasing) is
|
"""Qwen3-TTS normalization (Base-task inference, voice lowercasing) is
|
||||||
performed inside ``validate`` today; kept fused for a strict behaviour
|
performed inside ``validate`` today; kept fused for a strict behaviour
|
||||||
match. See ``_validate_qwen_tts_request``."""
|
match."""
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
# Go through the shared dispatcher (which routes to
|
"""Validate Qwen TTS request parameters. Returns error message or None."""
|
||||||
# _validate_qwen_tts_request for this model-type) rather than the leaf
|
# Infer Base task when ref_audio or ref_text is provided without explicit task_type.
|
||||||
# validator directly, matching the pre-adapter qwen3 path.
|
server = self.ctx.server
|
||||||
return self.ctx.server._validate_tts_request(request)
|
if request.task_type is None and (request.ref_audio is not None or request.ref_text is not None):
|
||||||
|
request.task_type = "Base"
|
||||||
|
|
||||||
|
# Normalize voice to lowercase for case-insensitive matching
|
||||||
|
if request.voice is not None:
|
||||||
|
request.voice = request.voice.lower()
|
||||||
|
if request.task_type is None and request.voice in server.precomputed_speakers:
|
||||||
|
request.task_type = "Base"
|
||||||
|
task_type = request.task_type or "CustomVoice"
|
||||||
|
|
||||||
|
# Validate input is not empty
|
||||||
|
if not request.input or not request.input.strip():
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
|
||||||
|
# Validate language (case-insensitive; normalized to the title-cased config form)
|
||||||
|
if request.language is not None:
|
||||||
|
request.language = request.language.title()
|
||||||
|
if request.language not in server.supported_languages:
|
||||||
|
return (
|
||||||
|
f"Invalid language '{request.language}'. Supported: {', '.join(sorted(server.supported_languages))}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate speaker for CustomVoice task
|
||||||
|
if task_type == "CustomVoice":
|
||||||
|
if not server.supported_speakers:
|
||||||
|
return (
|
||||||
|
"This model does not support CustomVoice task (no speakers configured). "
|
||||||
|
"Use task_type='Base' with ref_audio/ref_text for voice cloning, "
|
||||||
|
"or use a CustomVoice model."
|
||||||
|
)
|
||||||
|
if request.voice is not None and request.voice not in server.supported_speakers:
|
||||||
|
return f"Invalid voice '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}"
|
||||||
|
|
||||||
|
# Validate speaker_embedding constraints
|
||||||
|
if request.speaker_embedding is not None:
|
||||||
|
if task_type != "Base":
|
||||||
|
return "'speaker_embedding' is only valid for Base task"
|
||||||
|
if not request.speaker_embedding:
|
||||||
|
return "'speaker_embedding' must be a non-empty list of floats"
|
||||||
|
# speaker_embedding implies x_vector_only_mode — set it before
|
||||||
|
# Base task validation so callers don't need to pass it explicitly.
|
||||||
|
request.x_vector_only_mode = True
|
||||||
|
emb_len = len(request.speaker_embedding)
|
||||||
|
dim_err = server._validate_qwen_tts_speaker_embedding_dim(emb_len)
|
||||||
|
if dim_err is not None:
|
||||||
|
return dim_err
|
||||||
|
# Validate Base task requirements
|
||||||
|
if task_type == "Base":
|
||||||
|
if request.voice is None:
|
||||||
|
# 1. Ensure a voice source is provided
|
||||||
|
if request.ref_audio is None and getattr(request, "speaker_embedding", None) is None:
|
||||||
|
return "Base task requires 'ref_audio' or 'speaker_embedding' for voice cloning"
|
||||||
|
# 2. Validate ref_audio format if it exists (using the helper from main)
|
||||||
|
if request.ref_audio is not None:
|
||||||
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
# 3. Validate text requirements based on the mode
|
||||||
|
if not getattr(request, "x_vector_only_mode", False):
|
||||||
|
if not request.ref_text or not request.ref_text.strip():
|
||||||
|
return (
|
||||||
|
"Base task requires non-empty 'ref_text' (transcript of "
|
||||||
|
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
voice_lower = request.voice.lower()
|
||||||
|
if voice_lower in server.uploaded_speakers:
|
||||||
|
# Check if data file exists for uploaded speaker
|
||||||
|
speaker_info = server.uploaded_speakers[voice_lower]
|
||||||
|
file_path = Path(speaker_info["file_path"])
|
||||||
|
if not file_path.exists():
|
||||||
|
return f"Data file for uploaded speaker '{request.voice}' not found on disk"
|
||||||
|
elif voice_lower in server.precomputed_speakers:
|
||||||
|
profile = server.precomputed_speakers[voice_lower]
|
||||||
|
mode = str(profile.get("mode") or "xvec").lower()
|
||||||
|
ref_text = request.ref_text or profile.get("ref_text")
|
||||||
|
if mode == "icl" and (not isinstance(ref_text, str) or not ref_text.strip()):
|
||||||
|
return (
|
||||||
|
f"Precomputed voice '{request.voice}' uses ICL mode but has no ref_text in "
|
||||||
|
"the request or custom voice manifest"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# need ref_audio for built-in speaker
|
||||||
|
if request.ref_audio is None:
|
||||||
|
return (
|
||||||
|
f"Base task with built-in speaker '{request.voice}' requires 'ref_audio' for voice cloning"
|
||||||
|
)
|
||||||
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
if not getattr(request, "x_vector_only_mode", False) and (
|
||||||
|
not request.ref_text or not request.ref_text.strip()
|
||||||
|
):
|
||||||
|
return (
|
||||||
|
"Base task requires non-empty 'ref_text' (transcript of "
|
||||||
|
"the reference audio) unless 'x_vector_only_mode' is enabled"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate cross-parameter dependencies
|
||||||
|
if task_type != "Base":
|
||||||
|
if request.ref_text is not None:
|
||||||
|
return "'ref_text' is only valid for Base task"
|
||||||
|
if request.x_vector_only_mode is not None:
|
||||||
|
return "'x_vector_only_mode' is only valid for Base task"
|
||||||
|
|
||||||
|
# Validate VoiceDesign task requirements
|
||||||
|
if task_type == "VoiceDesign" and not request.instructions:
|
||||||
|
return "VoiceDesign task requires 'instructions' to describe the voice"
|
||||||
|
|
||||||
|
# Validate instructions length (using cached value from initialization)
|
||||||
|
if request.instructions and len(request.instructions) > server._max_instructions_length:
|
||||||
|
return f"Instructions too long (max {server._max_instructions_length} characters)"
|
||||||
|
|
||||||
|
# Validate max_new_tokens range
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
if request.max_new_tokens > self.max_new_tokens_max:
|
||||||
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -23,8 +23,9 @@ class StepAudio2Adapter(ARTTSAdapter):
|
|||||||
name = "step_audio2"
|
name = "step_audio2"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
# Dispatcher routes to the step_audio2 case (non-empty input check).
|
if not request.input or not request.input.strip():
|
||||||
return self.ctx.server._validate_tts_request(request)
|
return "Input text cannot be empty"
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -19,7 +19,25 @@ class VoxCPM2Adapter(ARTTSAdapter):
|
|||||||
name = "voxcpm2"
|
name = "voxcpm2"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
return self.ctx.server._validate_voxcpm2_request(request)
|
"""Validate VoxCPM2 request parameters. Returns error message or None."""
|
||||||
|
server = self.ctx.server
|
||||||
|
if not request.input or not request.input.strip():
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
|
||||||
|
if request.voice is not None:
|
||||||
|
request.voice = request.voice.lower()
|
||||||
|
available_voices = set(server.uploaded_speakers) | set(server.precomputed_speakers) | {"default"}
|
||||||
|
if request.voice not in available_voices:
|
||||||
|
supported = ", ".join(sorted(available_voices)) or "none"
|
||||||
|
return f"Invalid voice '{request.voice}'. Supported: {supported}"
|
||||||
|
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
if request.max_new_tokens > self.max_new_tokens_max:
|
||||||
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
@@ -16,7 +16,32 @@ class VoxtralTTSAdapter(ARTTSAdapter):
|
|||||||
name = "voxtral_tts"
|
name = "voxtral_tts"
|
||||||
|
|
||||||
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
||||||
return self.ctx.server._validate_voxtral_tts_request(request)
|
"""Validate Voxtral TTS request parameters. Returns error message or None."""
|
||||||
|
server = self.ctx.server
|
||||||
|
if not request.input or not request.input.strip():
|
||||||
|
return "Input text cannot be empty"
|
||||||
|
|
||||||
|
# Voxtral TTS requires either a preset voice or ref_audio for voice cloning.
|
||||||
|
if request.voice is None and request.ref_audio is None:
|
||||||
|
return "Either 'voice' (preset speaker) or 'ref_audio' (voice cloning) must be provided"
|
||||||
|
|
||||||
|
if request.ref_audio is not None:
|
||||||
|
fmt_err = server._validate_ref_audio_format(request.ref_audio)
|
||||||
|
if fmt_err:
|
||||||
|
return fmt_err
|
||||||
|
|
||||||
|
if request.voice is not None:
|
||||||
|
request.voice = request.voice.lower()
|
||||||
|
if server.supported_speakers and request.voice not in server.supported_speakers:
|
||||||
|
return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(server.supported_speakers))}"
|
||||||
|
|
||||||
|
if request.max_new_tokens is not None:
|
||||||
|
if request.max_new_tokens < self.max_new_tokens_min:
|
||||||
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
||||||
|
if request.max_new_tokens > self.max_new_tokens_max:
|
||||||
|
return f"max_new_tokens cannot exceed {self.max_new_tokens_max}"
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
async def build(
|
async def build(
|
||||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||||
|
|||||||
Reference in new issue
Block a user