[Bugfix] Invalidate stale ref_audio cache on local file modification (#4873) (#5670)

Signed-off-by: pranavthakur0-0 <pranavthakurdew@gmail.com>
Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
This commit is contained in:
Pranav ThakurandZhou Taichang authored and GitHub committed 2026-08-25 10:06:32 +08:00
1 parent 50b8a150f9
commit 53b60bd9bc
10 files changed
+675 -103

No files matched your search

+256 -75
View File
@@ -13,6 +13,8 @@ from concurrent.futures import ThreadPoolExecutor
from http import HTTPStatus
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
from urllib.request import url2pathname
import numpy as np
import soundfile as sf
@@ -98,6 +100,8 @@ _COSYVOICE3_PROMPT_PREFIX = f"You are a helpful assistant.{_COSYVOICE3_PROMPT_DE
_AUDEX_NO_AUDIO_GUARD_MODEL_TYPES = frozenset({"audex", "audex_tta"})
_REF_AUDIO_MIN_DURATION = 1.0 # seconds
_REF_AUDIO_MAX_DURATION = 30.0 # seconds
_REF_AUDIO_METADATA_FETCH_ATTEMPTS = 3
_REMOTE_REF_AUDIO_SCHEMES = frozenset({"http", "https", "data"})
_REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES = 256
_REF_AUDIO_RESOLVE_CACHE_MAX_BYTES = 256 * 1024 * 1024
_HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256
@@ -228,6 +232,13 @@ def _conditioning_cache_salt(request, tts_params: dict | None = None) -> str:
must also be folded in. ``voice_created_at`` bumps on every (re-)upload,
which uniquely identifies the resolved reference artifact together with
the voice name; the decoded ref_audio array itself need not be hashed.
Local ``ref_audio`` locators are the same class of gap: the raw request
hashes the path/URI string, so an on-disk rewrite is invisible to the
salt. Callers stash the content-aware key from ``_resolve_ref_audio`` as
``ref_audio_cache_key`` (and ``ref_audio_2_cache_key`` / emotion keys
when those clips exist) so a file edit cannot reuse KV from the previous
reference.
"""
h = hashlib.sha256()
for part in (
@@ -245,13 +256,15 @@ def _conditioning_cache_salt(request, tts_params: dict | None = None) -> str:
if part is not None:
h.update(repr(part).encode("utf-8"))
# Fold resolved conditioning that is auto-derived for uploaded voices and
# absent from the raw request.
# absent from the raw request (or that a locator string cannot see).
for key in (
"voice_created_at",
"task_type",
"speaker",
"ref_text",
"x_vector_only_mode",
"ref_audio_cache_key",
"ref_audio_2_cache_key",
):
h.update(b"\x00")
value = tts_params.get(key) if tts_params is not None else None
@@ -803,7 +816,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
ref_sr = None
voice_profile = None
if request.ref_audio is not None:
ref_audio, ref_sr = await self._resolve_ref_audio(request.ref_audio)
ref_audio, ref_sr, _ = await self._resolve_ref_audio(request.ref_audio)
elif uploaded_ref is not None:
wav_np, ref_sr = uploaded_ref
ref_audio = wav_np.tolist()
@@ -1290,11 +1303,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"""Validate ref_audio is a supported URI format. Returns error or None."""
if not isinstance(ref_audio, str):
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
if not (
ref_audio.startswith(("http://", "https://"))
or ref_audio.startswith("data:")
or ref_audio.startswith("file://")
):
scheme = (urlparse(ref_audio).scheme or "").lower()
if scheme not in {"http", "https", "data", "file"}:
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
return None
@@ -1343,7 +1353,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
self._moss_processor_cache = proc
return proc
async def _build_moss_tts_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]:
async def _build_moss_tts_params(
self,
request: OpenAICreateSpeechRequest,
*,
has_inline_ref_audio: bool = False,
) -> dict[str, Any]:
"""Build the talker prompt + ``additional_information`` payload for any
MOSS-TTS-family request (nano + 5 full variants).
@@ -1371,8 +1386,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
}
if request.max_new_tokens is not None:
params["max_new_frames"] = [request.max_new_tokens]
wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio)
params["prompt_audio_array"] = [[wav_list, sr]]
params["ref_audio_cache_key"] = cache_key
return params
# ---- MOSS-TTS-Realtime: keep the old prompt_audio_array path ----
@@ -1390,8 +1406,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
}
if request.max_new_tokens is not None:
params["max_new_frames"] = [request.max_new_tokens]
wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio)
params["prompt_audio_array"] = [[wav_list, sr]]
params["ref_audio_cache_key"] = cache_key
return params
# ---- MossTTSDelay family (tts/ttsd/sound_effect/voice_generator)
@@ -1422,30 +1439,47 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# only pulls it on the delay-family path (alongside the upstream proc).
from vllm_omni.model_executor.models.moss_tts.reference_encoder import encode_reference_codes
_voice = getattr(request, "voice", None)
_voice = _voice.strip() if isinstance(_voice, str) else ""
_voice_created = self._voice_created_at(_voice.lower()) if _voice else 0
# Named-voice speaker cache is only valid for uploaded speakers
# without an inline ref_audio. ``request.voice`` plus a file/URL
# otherwise keys on (name, created_at=0) and skips the content-aware
# resolve — the adapter already applies this guard when tagging
# ``voice_name`` on tts_params.
_raw_voice = getattr(request, "voice", None)
_raw_voice = _raw_voice.strip() if isinstance(_raw_voice, str) else ""
_voice_lower = _raw_voice.lower() if _raw_voice else ""
_use_named_voice = bool(_voice_lower) and _voice_lower in self.uploaded_speakers and not has_inline_ref_audio
_voice = _voice_lower if _use_named_voice else ""
_voice_created = self._voice_created_at(_voice) if _voice else 0
_resolve_keys: list[str] = []
async def _encode_ref(ref_str: str) -> torch.Tensor:
async def _tracked_resolve(ref_str: str) -> tuple[list, int, str]:
wav_list, sr, cache_key = await self._resolve_ref_audio(ref_str)
_resolve_keys.append(cache_key)
return wav_list, sr, cache_key
async def _encode_ref(ref_str: str, *, named_voice: bool) -> torch.Tensor:
# Named-voice cache is (voice_name, created_at). TTSD speaker 2 is a
# different clip, so it must stay anonymous or it reuses speaker 1.
use_named = named_voice and bool(_voice)
return await encode_reference_codes(
ref_str,
processor=proc,
resolve_ref_audio=self._resolve_ref_audio,
resolve_ref_audio=_tracked_resolve,
speaker_cache=self._speaker_cache,
variant=v,
n_vq=n_vq,
sr_target=sr_target,
voice_name=_voice or None,
voice_created_at=_voice_created,
voice_name=_voice if use_named else None,
voice_created_at=_voice_created if use_named else 0,
)
user_kwargs: dict[str, Any] = {"text": request.input or ""}
if v in ("tts", "local"):
user_kwargs["reference"] = [await _encode_ref(request.ref_audio)]
user_kwargs["reference"] = [await _encode_ref(request.ref_audio, named_voice=True)]
elif v == "ttsd":
refs = [await _encode_ref(request.ref_audio)]
refs = [await _encode_ref(request.ref_audio, named_voice=True)]
if request.ref_audio_2:
refs.append(await _encode_ref(request.ref_audio_2))
refs.append(await _encode_ref(request.ref_audio_2, named_voice=False))
user_kwargs["reference"] = refs
elif v == "sound_effect":
user_kwargs["text"] = request.input or "" # may be empty
@@ -1479,6 +1513,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
}
if request.max_new_tokens is not None:
params["max_new_frames"] = [request.max_new_tokens]
if _resolve_keys:
params["ref_audio_cache_key"] = _resolve_keys[0]
if len(_resolve_keys) > 1:
params["ref_audio_2_cache_key"] = _resolve_keys[1]
return params
async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest):
@@ -1507,7 +1545,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
prompt_token_ids = input_ids_to_python_list(inputs)
return tokens_input(prompt_token_ids=prompt_token_ids)
wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
wav_list, sr, _ = await self._resolve_ref_audio(request.ref_audio)
wav = np.asarray(wav_list, dtype=np.float32)
out = await asyncio.to_thread(
build_voice_clone_prompt,
@@ -1576,8 +1614,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
encode_reference_audio,
)
wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
artifact_key = self._get_resolved_ref_audio_artifact_key(request.ref_audio)
wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio)
artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key)
wav = np.asarray(wav_list, dtype=np.float32)
ref_codes_delayed, cache_hit, inflight_wait = await self._resolve_higgs_audio_v3_ref_codes(
artifact_key,
@@ -1599,7 +1637,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
prompt["additional_information"] = {
"audio_input_ids": ref_codes_delayed.to(torch.long),
"audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool),
"ref_audio_cache_key": cache_key,
}
# Placeholder prompt ids do not change when a same-path file is
# rewritten at the same duration; fold the content-aware resolve key
# into cache_salt so prefix caching cannot reuse the previous KV.
prompt["cache_salt"] = _conditioning_cache_salt(request, prompt["additional_information"])
return prompt
async def _resolve_higgs_audio_v3_ref_codes(
@@ -1690,43 +1733,100 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
self._higgs_audio_v3_adapter = adapter
return adapter
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
"""Resolve ref_audio to (wav_samples, sample_rate).
@staticmethod
def _local_ref_audio_stat_path(ref_audio_str: str) -> str | None:
"""Filesystem path to stat for a local locator, else ``None``.
Delegates to upstream vLLM's MediaConnector which handles http(s)
URLs, ``data:`` base64 URIs, and ``file:`` local paths (the latter
gated by ``--allowed-local-media-path``).
Scheme comparison is case-insensitive (``urlparse`` lowercases it), so
``FILE:///...`` is treated as a local file the same way ``file://`` is.
Remote ``http`` / ``https`` / ``data`` locators, including ``HTTP://``,
return ``None`` and stay string-keyed.
"""
cache_key = hashlib.sha1(ref_audio_str.encode("utf-8")).hexdigest()
cached = self._ref_audio_resolve_cache.get(cache_key)
if cached is not None:
self._ref_audio_resolve_cache.move_to_end(cache_key)
wav_list, sr, _, _ = cached
logger.debug(
"Resolved ref_audio from cache: samples=%d sr=%d duration_s=%.3f",
len(wav_list),
sr,
len(wav_list) / sr if sr > 0 else 0.0,
)
return wav_list, sr
parsed = urlparse(ref_audio_str)
scheme = (parsed.scheme or "").lower()
if scheme in _REMOTE_REF_AUDIO_SCHEMES:
return None
if scheme == "file":
netloc = parsed.netloc or ""
if netloc.lower() not in ("", "localhost"):
raise OSError(f"file:// URI with non-local authority {netloc!r} cannot be stat'd")
return url2pathname(parsed.path or "")
if scheme:
return None
return ref_audio_str
# In diffusion mode, model_config may not be available
if self._diffusion_mode:
connector = MediaConnector()
else:
model_config = self.model_config
connector = MediaConnector(
allowed_local_media_path=model_config.allowed_local_media_path,
allowed_media_domains=model_config.allowed_media_domains,
)
fetch_start_s = time.perf_counter()
wav_np, sr = await connector.fetch_audio_async(ref_audio_str)
fetch_decode_ms = (time.perf_counter() - fetch_start_s) * 1000.0
@staticmethod
def _get_ref_audio_cache_key(
ref_audio_str: str,
allowed_local_media_path: str | None = None,
) -> str:
"""Compute a cache key hash for *ref_audio_str*.
For local files (bare paths and ``file://`` URIs) the key folds in
``st_mtime_ns`` and ``st_size`` so that an on-disk edit automatically
invalidates the cached waveform without a server restart. Remote URLs
and ``data:`` URIs are keyed on the raw string alone — the server does
not re-fetch them to check for changes.
``os.stat`` runs only when *allowed_local_media_path* is a non-empty
path. vLLM's default is ``""`` (not ``None``); both mean local media
is refused, so a stat would be exposure for a request that cannot
load the file.
Note: when the key changes the *previous* entry stays in
``_ref_audio_resolve_cache`` until LRU eviction, so
``_discard_ref_audio_artifact_ready_if_unreferenced`` will not fire for
the replaced file and its stale artifact key remains in
``_ref_audio_model_artifact_ready``. This is functionally correct
(the stale entry is never *used*) but doubles memory until eviction.
"""
cache_key_source = ref_audio_str
try:
path = OmniOpenAIServingSpeech._local_ref_audio_stat_path(ref_audio_str)
except OSError as exc:
path = None
logger.debug("Skipping local-file cache metadata for %s: %s", ref_audio_str[:80], exc)
if path is not None:
try:
# Only stat paths the server operator has explicitly permitted
# via --allowed-local-media-path. The default is "" (and
# diffusion mode passes None); MediaConnector refuses local
# file loads in both cases, so a stat is exposure for zero
# benefit. ``if not`` covers "" and None; ``is None`` would
# treat "" as an allowlist of the process cwd.
if not allowed_local_media_path:
raise OSError("no allowed_local_media_path configured; skipping stat")
resolved = os.path.realpath(path)
allowed_base = os.path.realpath(allowed_local_media_path)
if os.path.commonpath([resolved, allowed_base]) != allowed_base:
raise OSError("path outside allowed_local_media_path; skipping stat")
st = os.stat(path)
cache_key_source = f"{ref_audio_str}:{st.st_mtime_ns}:{st.st_size}"
except (OSError, ValueError):
# Truncate the value to avoid dumping huge base64 blobs into
# the server log when a client omits the ``data:`` prefix.
display = ref_audio_str[:80]
if len(ref_audio_str) > 80:
display += f"... ({len(ref_audio_str)} chars)"
logger.debug(
"Failed to stat ref_audio path %s; falling back to string-only cache key (stale cache possible)",
display,
)
return hashlib.sha1(cache_key_source.encode("utf-8")).hexdigest()
async def _ref_audio_cache_key(self, ref_audio_str: str, allowed_local_media_path: str | None) -> str:
"""Stat local files off the event loop; remote locators stay a cheap hash."""
return await asyncio.to_thread(
self._get_ref_audio_cache_key,
ref_audio_str,
allowed_local_media_path,
)
def _finalize_fetched_ref_audio(self, wav_np: np.ndarray, sr: int) -> tuple[list[float], int, str, float]:
wav_np = np.asarray(wav_np, dtype=np.float32)
if wav_np.ndim > 1:
wav_np = np.mean(wav_np, axis=-1)
sr = int(sr)
artifact_key = self._make_ref_audio_artifact_cache_key(wav_np, sr)
duration = len(wav_np) / sr if sr > 0 else 0.0
if duration < _REF_AUDIO_MIN_DURATION:
raise ValueError(
@@ -1738,19 +1838,92 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
f"Reference audio too long ({duration:.1f}s). "
f"Maximum {_REF_AUDIO_MAX_DURATION:.0f}s supported — use a shorter clip."
)
tolist_start_s = time.perf_counter()
wav_list = wav_np.tolist()
tolist_ms = (time.perf_counter() - tolist_start_s) * 1000.0
logger.debug(
"Resolved ref_audio: fetch_decode_ms=%.3f tolist_ms=%.3f samples=%d sr=%d duration_s=%.3f",
fetch_decode_ms,
tolist_ms,
len(wav_np),
sr,
duration,
artifact_key = self._make_ref_audio_artifact_cache_key(wav_np, sr)
return wav_np.tolist(), sr, artifact_key, duration
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int, str]:
"""Resolve ref_audio to (wav_samples, sample_rate, cache_key).
Delegates to upstream vLLM's MediaConnector which handles http(s)
URLs, ``data:`` base64 URIs, and ``file:`` local paths (the latter
gated by ``--allowed-local-media-path``).
Local file references incorporate mtime and size into the cache key
so that modified files are automatically reloaded without a server
restart. Remote URLs remain cached by their original string locator.
The returned *cache_key* should be passed to
``_get_resolved_ref_audio_artifact_key`` when the caller needs the
artifact key, avoiding a redundant ``os.stat`` and the TOCTOU window.
After a cache miss the file is re-stat'd; a metadata change retries
the fetch so the stored waveform matches the key.
"""
# Pass the allowed-local-media-path so the stat is restricted to
# paths the server operator has explicitly permitted.
allowed_path = None
if not self._diffusion_mode:
allowed_path = getattr(self.model_config, "allowed_local_media_path", None)
wav_list: list[float] | None = None
sr = 0
artifact_key = ""
post_key = ""
connector: MediaConnector | None = None
for attempt in range(_REF_AUDIO_METADATA_FETCH_ATTEMPTS):
cache_key = await self._ref_audio_cache_key(ref_audio_str, allowed_path)
cached = self._ref_audio_resolve_cache.get(cache_key)
if cached is not None:
self._ref_audio_resolve_cache.move_to_end(cache_key)
wav_list, sr, _, _ = cached
logger.debug(
"Resolved ref_audio from cache: samples=%d sr=%d duration_s=%.3f",
len(wav_list),
sr,
len(wav_list) / sr if sr > 0 else 0.0,
)
return wav_list, sr, cache_key
if connector is None:
# In diffusion mode, model_config may not be available
if self._diffusion_mode:
connector = MediaConnector()
else:
model_config = self.model_config
connector = MediaConnector(
allowed_local_media_path=model_config.allowed_local_media_path,
allowed_media_domains=model_config.allowed_media_domains,
)
fetch_start_s = time.perf_counter()
wav_np, fetched_sr = await connector.fetch_audio_async(ref_audio_str)
fetch_decode_ms = (time.perf_counter() - fetch_start_s) * 1000.0
tolist_start_s = time.perf_counter()
wav_list, sr, artifact_key, duration = self._finalize_fetched_ref_audio(wav_np, fetched_sr)
tolist_ms = (time.perf_counter() - tolist_start_s) * 1000.0
logger.debug(
"Resolved ref_audio: fetch_decode_ms=%.3f tolist_ms=%.3f samples=%d sr=%d duration_s=%.3f",
fetch_decode_ms,
tolist_ms,
len(wav_list),
sr,
duration,
)
post_key = await self._ref_audio_cache_key(ref_audio_str, allowed_path)
if post_key == cache_key:
self._put_resolved_ref_audio(cache_key, wav_list, sr, artifact_key)
return wav_list, sr, cache_key
logger.debug(
"ref_audio metadata changed during fetch (attempt %d/%d); retrying",
attempt + 1,
_REF_AUDIO_METADATA_FETCH_ATTEMPTS,
)
logger.warning(
"ref_audio file changed during fetch after %d attempts; skipping resolve cache",
_REF_AUDIO_METADATA_FETCH_ATTEMPTS,
)
self._put_resolved_ref_audio(cache_key, wav_list, sr, artifact_key)
return wav_list, sr
assert wav_list is not None and post_key
return wav_list, sr, post_key
@staticmethod
def _make_ref_audio_artifact_cache_key(wav: np.ndarray, sr: int) -> str:
@@ -1761,12 +1934,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
h.update(wav_f32.tobytes(order="C"))
return h.hexdigest()
def _get_resolved_ref_audio_artifact_key(self, ref_audio_str: str) -> str | None:
source_key = hashlib.sha1(ref_audio_str.encode("utf-8")).hexdigest()
cached = self._ref_audio_resolve_cache.get(source_key)
def _get_resolved_ref_audio_artifact_key(self, cache_key: str) -> str | None:
"""Look up the artifact key for a previously resolved ref_audio.
*cache_key* must be the exact key returned by the preceding
``_resolve_ref_audio`` call. Passing the key explicitly avoids a
second ``os.stat`` syscall and eliminates the TOCTOU window that
would arise from recomputing the key independently.
"""
cached = self._ref_audio_resolve_cache.get(cache_key)
if cached is None:
return None
self._ref_audio_resolve_cache.move_to_end(source_key)
self._ref_audio_resolve_cache.move_to_end(cache_key)
return cached[3]
def _put_resolved_ref_audio(self, cache_key: str, wav_list: list[float], sr: int, artifact_key: str) -> None:
@@ -1831,7 +2010,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
async def _resolve_ref_audio_many(self, ref_audio_list: list[str]) -> list[tuple[list[float], int]]:
resolved = []
for ref_audio in ref_audio_list:
resolved.append(await self._resolve_ref_audio(ref_audio))
wav_list, sr, _ = await self._resolve_ref_audio(ref_audio)
resolved.append((wav_list, sr))
return resolved
# ---- Ming TTS helpers ----
@@ -2502,7 +2682,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
+ mm_processor_kwargs with prompt_text.
"""
# Resolve reference audio
wav_samples, sr = await self._resolve_ref_audio(request.ref_audio)
wav_samples, sr, _ = await self._resolve_ref_audio(request.ref_audio)
audio_data = (np.asarray(wav_samples, dtype=np.float32), sr)
# Wrap the reference transcript in the CosyVoice3 instruction template
@@ -2616,7 +2796,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"""
# Voice cloning requires ref_audio + ref_text
if request.ref_audio is not None and request.ref_text:
wav_samples, sr = await self._resolve_ref_audio(request.ref_audio)
wav_samples, sr, _ = await self._resolve_ref_audio(request.ref_audio)
audio_data = (np.asarray(wav_samples, dtype=np.float32), int(sr))
mm_kwargs: dict[str, Any] = {
@@ -2748,8 +2928,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# Uploaded voice: ref_audio was auto-set as [base64_data_url]
ref_audio_source = tts_params["ref_audio"][0]
if ref_audio_source is not None and isinstance(ref_audio_source, str):
wav_list, sr = await self._resolve_ref_audio(ref_audio_source)
artifact_key = self._get_resolved_ref_audio_artifact_key(ref_audio_source)
wav_list, sr, cache_key = await self._resolve_ref_audio(ref_audio_source)
tts_params["ref_audio_cache_key"] = cache_key
artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key)
if artifact_key:
tts_params[_QWEN3_TTS_REF_AUDIO_CACHE_KEY] = [artifact_key]
ref_code_length = self._estimate_ref_code_len([wav_list, sr])
@@ -3142,7 +3323,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
request_id = f"speech-{random_uuid()}"
prompt: dict[str, Any] = {"input": request.input}
if request.ref_audio:
wav, sr = await self._resolve_ref_audio(request.ref_audio)
wav, sr, _ = await self._resolve_ref_audio(request.ref_audio)
prompt["ref_audio"] = (np.asarray(wav, dtype=np.float32), sr)
if request.ref_text:
prompt["ref_text"] = request.ref_text
@@ -49,13 +49,14 @@ class FishSpeechAdapter(ARTTSAdapter):
) -> PreparedRequest:
server = self.ctx.server
ref_audio_data = None
tts_params: dict = {}
if request.ref_audio is not None:
wav_list, sr = await server._resolve_ref_audio(request.ref_audio)
wav_list, sr, cache_key = await server._resolve_ref_audio(request.ref_audio)
ref_audio_data = (wav_list, sr)
tts_params["ref_audio_cache_key"] = cache_key
prompt = await server._build_fish_speech_prompt_async(
request, ref_audio_data=ref_audio_data, has_inline_ref_audio=has_inline_ref_audio
)
tts_params = {}
prompt["cache_salt"] = conditioning_cache_salt(request, tts_params)
return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="fish_speech")
@@ -108,6 +108,8 @@ def indextts2_conditioning_cache_salt(
"use_random",
"lang",
"text_normalization",
"ref_audio_cache_key",
"emo_audio_cache_key",
):
h.update(b"\x00")
h.update(key.encode("utf-8"))
@@ -260,8 +262,9 @@ class IndexTTS2Adapter(ARTTSAdapter):
ref_audio_source = server._get_uploaded_audio_data(voice_lower)
using_uploaded_voice = ref_audio_source is not None
if ref_audio_source is not None and isinstance(ref_audio_source, str):
wav_list, sr = await server._resolve_ref_audio(ref_audio_source)
wav_list, sr, cache_key = await server._resolve_ref_audio(ref_audio_source)
params["voice"] = [[wav_list, sr]]
params["ref_audio_cache_key"] = [cache_key]
if using_uploaded_voice and voice_lower:
params["voice_name"] = [voice_lower]
params["voice_created_at"] = [server._voice_created_at(voice_lower)]
@@ -271,8 +274,9 @@ class IndexTTS2Adapter(ARTTSAdapter):
if key not in extras:
continue
if key == "emo_audio":
wav_list, sr = await server._resolve_ref_audio(extras[key])
wav_list, sr, emo_key = await server._resolve_ref_audio(extras[key])
params[key] = [[wav_list, sr]]
params["emo_audio_cache_key"] = [emo_key]
else:
params[key] = [extras[key]]
return params
@@ -64,7 +64,7 @@ class MingTTSAdapter(ARTTSAdapter):
if isinstance(ref_audio_source, list):
ref_audio_data = await server._resolve_ref_audio_many(ref_audio_source)
elif ref_audio_source is not None and isinstance(ref_audio_source, str):
wav_list, sr = await server._resolve_ref_audio(ref_audio_source)
wav_list, sr, _ = await server._resolve_ref_audio(ref_audio_source)
ref_audio_data = (wav_list, sr)
prompt = server._build_ming_dense_prompt(
request,
@@ -91,7 +91,7 @@ class _MossTTSAdapterBase(ARTTSAdapter):
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
) -> PreparedRequest:
server = self.ctx.server
tts_params = await server._build_moss_tts_params(request)
tts_params = await server._build_moss_tts_params(request, has_inline_ref_audio=has_inline_ref_audio)
if request.voice:
voice_lower = request.voice.lower()
if voice_lower in server.uploaded_speakers and not has_inline_ref_audio:
@@ -33,7 +33,7 @@ class OmniVoiceAdapter(ARTTSAdapter):
server = self.ctx.server
prompt: dict = {"input": request.input}
if request.ref_audio:
wav, sr = await server._resolve_ref_audio(request.ref_audio)
wav, sr, _ = await server._resolve_ref_audio(request.ref_audio)
prompt["ref_audio"] = (np.asarray(wav, dtype=np.float32), sr)
if request.ref_text:
prompt["ref_text"] = request.ref_text