111 lines
5.1 KiB
Python
111 lines
5.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""Higgs-Audio v2 serving adapter."""
|
|
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
|
|
from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
|
|
|
|
|
@register_tts_adapter
|
|
class HiggsAudioV2Adapter(ARTTSAdapter):
|
|
stage_keys = frozenset({"higgs_audio_v2"})
|
|
name = "higgs_audio_v2"
|
|
|
|
def validate(self, request: "OpenAICreateSpeechRequest") -> str | None:
|
|
"""Validate higgs_audio_v2 request parameters. Returns error message or None.
|
|
|
|
Accepted: plain text -> speech, or shallow voice clone via ``ref_audio``
|
|
+ ``ref_text`` (both required together). Still out of scope: preset
|
|
``voice``/``speaker`` selection, ``x_vector_only_mode`` /
|
|
``speaker_embedding`` helpers, ``task_type``/``language``/
|
|
``instructions``/``speed`` overrides, and multi-speaker ``[SPEAKERn]``
|
|
tags inside the input body.
|
|
"""
|
|
from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import (
|
|
MULTI_SPEAKER_TAG_PATTERN,
|
|
)
|
|
|
|
server = self.ctx.server
|
|
err = server._apply_uploaded_speaker(request)
|
|
if err:
|
|
return err
|
|
|
|
if not request.input or not request.input.strip():
|
|
return "higgs_audio_v2: input text cannot be empty"
|
|
|
|
# Voice clone: ref_audio and ref_text must come together.
|
|
if request.ref_audio is not None and not request.ref_text:
|
|
return (
|
|
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
|
"'ref_text'; received ref_audio without ref_text"
|
|
)
|
|
if request.ref_text and request.ref_audio is None:
|
|
return (
|
|
"higgs_audio_v2 voice clone requires both 'ref_audio' and "
|
|
"'ref_text'; received ref_text without ref_audio"
|
|
)
|
|
|
|
if request.x_vector_only_mode is not None:
|
|
return "higgs_audio_v2 v1 does not support 'x_vector_only_mode' (voice-cloning helper field)"
|
|
if request.speaker_embedding is not None:
|
|
return "higgs_audio_v2 v1 does not support 'speaker_embedding' (voice-cloning helper field)"
|
|
if request.voice and request.ref_audio is None:
|
|
# _apply_uploaded_speaker runs before this validator; if voice was
|
|
# an uploaded speaker, ref_audio is now populated and ref_text is
|
|
# backfilled from the speaker entry. A bare voice= with no
|
|
# ref_audio means the name didn't resolve to an uploaded speaker
|
|
# (and higgs has no built-in preset voices).
|
|
return (
|
|
"higgs_audio_v2 v1 does not support 'voice'/'speaker' selection for built-in voices; "
|
|
f"upload a voice first via POST /v1/audio/voices, or use ref_audio + ref_text. "
|
|
f"Got voice={request.voice!r}"
|
|
)
|
|
if request.instructions:
|
|
return (
|
|
"higgs_audio_v2 v1 does not support 'instructions' (voice "
|
|
"style/emotion control); supply plain text instead"
|
|
)
|
|
if request.task_type is not None:
|
|
return "higgs_audio_v2 v1 does not support 'task_type'; the model is single-mode plain text -> speech"
|
|
if request.language is not None:
|
|
return (
|
|
"higgs_audio_v2 v1 does not accept 'language' overrides; the model infers language from the input text"
|
|
)
|
|
if request.speed is not None and request.speed != 1.0:
|
|
return (
|
|
"higgs_audio_v2 v1 does not support 'speed' adjustments; the audio is rendered at native rate (24 kHz)"
|
|
)
|
|
|
|
if MULTI_SPEAKER_TAG_PATTERN.search(request.input):
|
|
return "higgs_audio_v2 v1 does not support multi-speaker [SPEAKERn] tags; remove the tag from the input"
|
|
|
|
if request.max_new_tokens is not None:
|
|
if request.max_new_tokens < self.max_new_tokens_min:
|
|
return f"max_new_tokens must be at least {self.max_new_tokens_min}"
|
|
|
|
async def build(
|
|
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
|
) -> PreparedRequest:
|
|
server = self.ctx.server
|
|
prompt = await server._build_higgs_audio_v2_params(request)
|
|
if request.voice:
|
|
voice_lower = request.voice.lower()
|
|
if voice_lower in server.uploaded_speakers and not has_inline_ref_audio:
|
|
additional = prompt.setdefault("additional_information", {})
|
|
additional["voice_name"] = voice_lower
|
|
additional["voice_created_at"] = server._voice_created_at(voice_lower)
|
|
return PreparedRequest(prompt=prompt, tts_params={}, model_type="higgs_audio_v2")
|
|
|
|
def apply_sampling_overrides(
|
|
self,
|
|
sampling_params_list: list,
|
|
request: "OpenAICreateSpeechRequest",
|
|
prompt: dict[str, Any] | None = None,
|
|
request_id: str | None = None,
|
|
) -> list:
|
|
return apply_max_new_tokens(sampling_params_list, request)
|