Signed-off-by: pranavthakur0-0 <pranavthakurdew@gmail.com> Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
50b8a150f9
commit
53b60bd9bc
10 files changed
+675
-103
No files matched your search
@@ -13,6 +13,8 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import url2pathname
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
@@ -98,6 +100,8 @@ _COSYVOICE3_PROMPT_PREFIX = f"You are a helpful assistant.{_COSYVOICE3_PROMPT_DE
|
||||
_AUDEX_NO_AUDIO_GUARD_MODEL_TYPES = frozenset({"audex", "audex_tta"})
|
||||
_REF_AUDIO_MIN_DURATION = 1.0 # seconds
|
||||
_REF_AUDIO_MAX_DURATION = 30.0 # seconds
|
||||
_REF_AUDIO_METADATA_FETCH_ATTEMPTS = 3
|
||||
_REMOTE_REF_AUDIO_SCHEMES = frozenset({"http", "https", "data"})
|
||||
_REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES = 256
|
||||
_REF_AUDIO_RESOLVE_CACHE_MAX_BYTES = 256 * 1024 * 1024
|
||||
_HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256
|
||||
@@ -228,6 +232,13 @@ def _conditioning_cache_salt(request, tts_params: dict | None = None) -> str:
|
||||
must also be folded in. ``voice_created_at`` bumps on every (re-)upload,
|
||||
which uniquely identifies the resolved reference artifact together with
|
||||
the voice name; the decoded ref_audio array itself need not be hashed.
|
||||
|
||||
Local ``ref_audio`` locators are the same class of gap: the raw request
|
||||
hashes the path/URI string, so an on-disk rewrite is invisible to the
|
||||
salt. Callers stash the content-aware key from ``_resolve_ref_audio`` as
|
||||
``ref_audio_cache_key`` (and ``ref_audio_2_cache_key`` / emotion keys
|
||||
when those clips exist) so a file edit cannot reuse KV from the previous
|
||||
reference.
|
||||
"""
|
||||
h = hashlib.sha256()
|
||||
for part in (
|
||||
@@ -245,13 +256,15 @@ def _conditioning_cache_salt(request, tts_params: dict | None = None) -> str:
|
||||
if part is not None:
|
||||
h.update(repr(part).encode("utf-8"))
|
||||
# Fold resolved conditioning that is auto-derived for uploaded voices and
|
||||
# absent from the raw request.
|
||||
# absent from the raw request (or that a locator string cannot see).
|
||||
for key in (
|
||||
"voice_created_at",
|
||||
"task_type",
|
||||
"speaker",
|
||||
"ref_text",
|
||||
"x_vector_only_mode",
|
||||
"ref_audio_cache_key",
|
||||
"ref_audio_2_cache_key",
|
||||
):
|
||||
h.update(b"\x00")
|
||||
value = tts_params.get(key) if tts_params is not None else None
|
||||
@@ -803,7 +816,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
ref_sr = None
|
||||
voice_profile = None
|
||||
if request.ref_audio is not None:
|
||||
ref_audio, ref_sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
ref_audio, ref_sr, _ = await self._resolve_ref_audio(request.ref_audio)
|
||||
elif uploaded_ref is not None:
|
||||
wav_np, ref_sr = uploaded_ref
|
||||
ref_audio = wav_np.tolist()
|
||||
@@ -1290,11 +1303,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
"""Validate ref_audio is a supported URI format. Returns error or None."""
|
||||
if not isinstance(ref_audio, str):
|
||||
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
|
||||
if not (
|
||||
ref_audio.startswith(("http://", "https://"))
|
||||
or ref_audio.startswith("data:")
|
||||
or ref_audio.startswith("file://")
|
||||
):
|
||||
scheme = (urlparse(ref_audio).scheme or "").lower()
|
||||
if scheme not in {"http", "https", "data", "file"}:
|
||||
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
|
||||
return None
|
||||
|
||||
@@ -1343,7 +1353,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
self._moss_processor_cache = proc
|
||||
return proc
|
||||
|
||||
async def _build_moss_tts_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]:
|
||||
async def _build_moss_tts_params(
|
||||
self,
|
||||
request: OpenAICreateSpeechRequest,
|
||||
*,
|
||||
has_inline_ref_audio: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the talker prompt + ``additional_information`` payload for any
|
||||
MOSS-TTS-family request (nano + 5 full variants).
|
||||
|
||||
@@ -1371,8 +1386,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
}
|
||||
if request.max_new_tokens is not None:
|
||||
params["max_new_frames"] = [request.max_new_tokens]
|
||||
wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio)
|
||||
params["prompt_audio_array"] = [[wav_list, sr]]
|
||||
params["ref_audio_cache_key"] = cache_key
|
||||
return params
|
||||
|
||||
# ---- MOSS-TTS-Realtime: keep the old prompt_audio_array path ----
|
||||
@@ -1390,8 +1406,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
}
|
||||
if request.max_new_tokens is not None:
|
||||
params["max_new_frames"] = [request.max_new_tokens]
|
||||
wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio)
|
||||
params["prompt_audio_array"] = [[wav_list, sr]]
|
||||
params["ref_audio_cache_key"] = cache_key
|
||||
return params
|
||||
|
||||
# ---- MossTTSDelay family (tts/ttsd/sound_effect/voice_generator)
|
||||
@@ -1422,30 +1439,47 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
# only pulls it on the delay-family path (alongside the upstream proc).
|
||||
from vllm_omni.model_executor.models.moss_tts.reference_encoder import encode_reference_codes
|
||||
|
||||
_voice = getattr(request, "voice", None)
|
||||
_voice = _voice.strip() if isinstance(_voice, str) else ""
|
||||
_voice_created = self._voice_created_at(_voice.lower()) if _voice else 0
|
||||
# Named-voice speaker cache is only valid for uploaded speakers
|
||||
# without an inline ref_audio. ``request.voice`` plus a file/URL
|
||||
# otherwise keys on (name, created_at=0) and skips the content-aware
|
||||
# resolve — the adapter already applies this guard when tagging
|
||||
# ``voice_name`` on tts_params.
|
||||
_raw_voice = getattr(request, "voice", None)
|
||||
_raw_voice = _raw_voice.strip() if isinstance(_raw_voice, str) else ""
|
||||
_voice_lower = _raw_voice.lower() if _raw_voice else ""
|
||||
_use_named_voice = bool(_voice_lower) and _voice_lower in self.uploaded_speakers and not has_inline_ref_audio
|
||||
_voice = _voice_lower if _use_named_voice else ""
|
||||
_voice_created = self._voice_created_at(_voice) if _voice else 0
|
||||
_resolve_keys: list[str] = []
|
||||
|
||||
async def _encode_ref(ref_str: str) -> torch.Tensor:
|
||||
async def _tracked_resolve(ref_str: str) -> tuple[list, int, str]:
|
||||
wav_list, sr, cache_key = await self._resolve_ref_audio(ref_str)
|
||||
_resolve_keys.append(cache_key)
|
||||
return wav_list, sr, cache_key
|
||||
|
||||
async def _encode_ref(ref_str: str, *, named_voice: bool) -> torch.Tensor:
|
||||
# Named-voice cache is (voice_name, created_at). TTSD speaker 2 is a
|
||||
# different clip, so it must stay anonymous or it reuses speaker 1.
|
||||
use_named = named_voice and bool(_voice)
|
||||
return await encode_reference_codes(
|
||||
ref_str,
|
||||
processor=proc,
|
||||
resolve_ref_audio=self._resolve_ref_audio,
|
||||
resolve_ref_audio=_tracked_resolve,
|
||||
speaker_cache=self._speaker_cache,
|
||||
variant=v,
|
||||
n_vq=n_vq,
|
||||
sr_target=sr_target,
|
||||
voice_name=_voice or None,
|
||||
voice_created_at=_voice_created,
|
||||
voice_name=_voice if use_named else None,
|
||||
voice_created_at=_voice_created if use_named else 0,
|
||||
)
|
||||
|
||||
user_kwargs: dict[str, Any] = {"text": request.input or ""}
|
||||
if v in ("tts", "local"):
|
||||
user_kwargs["reference"] = [await _encode_ref(request.ref_audio)]
|
||||
user_kwargs["reference"] = [await _encode_ref(request.ref_audio, named_voice=True)]
|
||||
elif v == "ttsd":
|
||||
refs = [await _encode_ref(request.ref_audio)]
|
||||
refs = [await _encode_ref(request.ref_audio, named_voice=True)]
|
||||
if request.ref_audio_2:
|
||||
refs.append(await _encode_ref(request.ref_audio_2))
|
||||
refs.append(await _encode_ref(request.ref_audio_2, named_voice=False))
|
||||
user_kwargs["reference"] = refs
|
||||
elif v == "sound_effect":
|
||||
user_kwargs["text"] = request.input or "" # may be empty
|
||||
@@ -1479,6 +1513,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
}
|
||||
if request.max_new_tokens is not None:
|
||||
params["max_new_frames"] = [request.max_new_tokens]
|
||||
if _resolve_keys:
|
||||
params["ref_audio_cache_key"] = _resolve_keys[0]
|
||||
if len(_resolve_keys) > 1:
|
||||
params["ref_audio_2_cache_key"] = _resolve_keys[1]
|
||||
return params
|
||||
|
||||
async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest):
|
||||
@@ -1507,7 +1545,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
prompt_token_ids = input_ids_to_python_list(inputs)
|
||||
return tokens_input(prompt_token_ids=prompt_token_ids)
|
||||
|
||||
wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
wav_list, sr, _ = await self._resolve_ref_audio(request.ref_audio)
|
||||
wav = np.asarray(wav_list, dtype=np.float32)
|
||||
out = await asyncio.to_thread(
|
||||
build_voice_clone_prompt,
|
||||
@@ -1576,8 +1614,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
encode_reference_audio,
|
||||
)
|
||||
|
||||
wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
artifact_key = self._get_resolved_ref_audio_artifact_key(request.ref_audio)
|
||||
wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio)
|
||||
artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key)
|
||||
wav = np.asarray(wav_list, dtype=np.float32)
|
||||
ref_codes_delayed, cache_hit, inflight_wait = await self._resolve_higgs_audio_v3_ref_codes(
|
||||
artifact_key,
|
||||
@@ -1599,7 +1637,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
prompt["additional_information"] = {
|
||||
"audio_input_ids": ref_codes_delayed.to(torch.long),
|
||||
"audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool),
|
||||
"ref_audio_cache_key": cache_key,
|
||||
}
|
||||
# Placeholder prompt ids do not change when a same-path file is
|
||||
# rewritten at the same duration; fold the content-aware resolve key
|
||||
# into cache_salt so prefix caching cannot reuse the previous KV.
|
||||
prompt["cache_salt"] = _conditioning_cache_salt(request, prompt["additional_information"])
|
||||
return prompt
|
||||
|
||||
async def _resolve_higgs_audio_v3_ref_codes(
|
||||
@@ -1690,43 +1733,100 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
self._higgs_audio_v3_adapter = adapter
|
||||
return adapter
|
||||
|
||||
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
|
||||
"""Resolve ref_audio to (wav_samples, sample_rate).
|
||||
@staticmethod
|
||||
def _local_ref_audio_stat_path(ref_audio_str: str) -> str | None:
|
||||
"""Filesystem path to stat for a local locator, else ``None``.
|
||||
|
||||
Delegates to upstream vLLM's MediaConnector which handles http(s)
|
||||
URLs, ``data:`` base64 URIs, and ``file:`` local paths (the latter
|
||||
gated by ``--allowed-local-media-path``).
|
||||
Scheme comparison is case-insensitive (``urlparse`` lowercases it), so
|
||||
``FILE:///...`` is treated as a local file the same way ``file://`` is.
|
||||
Remote ``http`` / ``https`` / ``data`` locators, including ``HTTP://``,
|
||||
return ``None`` and stay string-keyed.
|
||||
"""
|
||||
cache_key = hashlib.sha1(ref_audio_str.encode("utf-8")).hexdigest()
|
||||
cached = self._ref_audio_resolve_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
self._ref_audio_resolve_cache.move_to_end(cache_key)
|
||||
wav_list, sr, _, _ = cached
|
||||
logger.debug(
|
||||
"Resolved ref_audio from cache: samples=%d sr=%d duration_s=%.3f",
|
||||
len(wav_list),
|
||||
sr,
|
||||
len(wav_list) / sr if sr > 0 else 0.0,
|
||||
)
|
||||
return wav_list, sr
|
||||
parsed = urlparse(ref_audio_str)
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
if scheme in _REMOTE_REF_AUDIO_SCHEMES:
|
||||
return None
|
||||
if scheme == "file":
|
||||
netloc = parsed.netloc or ""
|
||||
if netloc.lower() not in ("", "localhost"):
|
||||
raise OSError(f"file:// URI with non-local authority {netloc!r} cannot be stat'd")
|
||||
return url2pathname(parsed.path or "")
|
||||
if scheme:
|
||||
return None
|
||||
return ref_audio_str
|
||||
|
||||
# In diffusion mode, model_config may not be available
|
||||
if self._diffusion_mode:
|
||||
connector = MediaConnector()
|
||||
else:
|
||||
model_config = self.model_config
|
||||
connector = MediaConnector(
|
||||
allowed_local_media_path=model_config.allowed_local_media_path,
|
||||
allowed_media_domains=model_config.allowed_media_domains,
|
||||
)
|
||||
fetch_start_s = time.perf_counter()
|
||||
wav_np, sr = await connector.fetch_audio_async(ref_audio_str)
|
||||
fetch_decode_ms = (time.perf_counter() - fetch_start_s) * 1000.0
|
||||
@staticmethod
|
||||
def _get_ref_audio_cache_key(
|
||||
ref_audio_str: str,
|
||||
allowed_local_media_path: str | None = None,
|
||||
) -> str:
|
||||
"""Compute a cache key hash for *ref_audio_str*.
|
||||
|
||||
For local files (bare paths and ``file://`` URIs) the key folds in
|
||||
``st_mtime_ns`` and ``st_size`` so that an on-disk edit automatically
|
||||
invalidates the cached waveform without a server restart. Remote URLs
|
||||
and ``data:`` URIs are keyed on the raw string alone — the server does
|
||||
not re-fetch them to check for changes.
|
||||
|
||||
``os.stat`` runs only when *allowed_local_media_path* is a non-empty
|
||||
path. vLLM's default is ``""`` (not ``None``); both mean local media
|
||||
is refused, so a stat would be exposure for a request that cannot
|
||||
load the file.
|
||||
|
||||
Note: when the key changes the *previous* entry stays in
|
||||
``_ref_audio_resolve_cache`` until LRU eviction, so
|
||||
``_discard_ref_audio_artifact_ready_if_unreferenced`` will not fire for
|
||||
the replaced file and its stale artifact key remains in
|
||||
``_ref_audio_model_artifact_ready``. This is functionally correct
|
||||
(the stale entry is never *used*) but doubles memory until eviction.
|
||||
"""
|
||||
cache_key_source = ref_audio_str
|
||||
try:
|
||||
path = OmniOpenAIServingSpeech._local_ref_audio_stat_path(ref_audio_str)
|
||||
except OSError as exc:
|
||||
path = None
|
||||
logger.debug("Skipping local-file cache metadata for %s: %s", ref_audio_str[:80], exc)
|
||||
if path is not None:
|
||||
try:
|
||||
# Only stat paths the server operator has explicitly permitted
|
||||
# via --allowed-local-media-path. The default is "" (and
|
||||
# diffusion mode passes None); MediaConnector refuses local
|
||||
# file loads in both cases, so a stat is exposure for zero
|
||||
# benefit. ``if not`` covers "" and None; ``is None`` would
|
||||
# treat "" as an allowlist of the process cwd.
|
||||
if not allowed_local_media_path:
|
||||
raise OSError("no allowed_local_media_path configured; skipping stat")
|
||||
resolved = os.path.realpath(path)
|
||||
allowed_base = os.path.realpath(allowed_local_media_path)
|
||||
if os.path.commonpath([resolved, allowed_base]) != allowed_base:
|
||||
raise OSError("path outside allowed_local_media_path; skipping stat")
|
||||
st = os.stat(path)
|
||||
cache_key_source = f"{ref_audio_str}:{st.st_mtime_ns}:{st.st_size}"
|
||||
except (OSError, ValueError):
|
||||
# Truncate the value to avoid dumping huge base64 blobs into
|
||||
# the server log when a client omits the ``data:`` prefix.
|
||||
display = ref_audio_str[:80]
|
||||
if len(ref_audio_str) > 80:
|
||||
display += f"... ({len(ref_audio_str)} chars)"
|
||||
logger.debug(
|
||||
"Failed to stat ref_audio path %s; falling back to string-only cache key (stale cache possible)",
|
||||
display,
|
||||
)
|
||||
return hashlib.sha1(cache_key_source.encode("utf-8")).hexdigest()
|
||||
|
||||
async def _ref_audio_cache_key(self, ref_audio_str: str, allowed_local_media_path: str | None) -> str:
|
||||
"""Stat local files off the event loop; remote locators stay a cheap hash."""
|
||||
return await asyncio.to_thread(
|
||||
self._get_ref_audio_cache_key,
|
||||
ref_audio_str,
|
||||
allowed_local_media_path,
|
||||
)
|
||||
|
||||
def _finalize_fetched_ref_audio(self, wav_np: np.ndarray, sr: int) -> tuple[list[float], int, str, float]:
|
||||
wav_np = np.asarray(wav_np, dtype=np.float32)
|
||||
if wav_np.ndim > 1:
|
||||
wav_np = np.mean(wav_np, axis=-1)
|
||||
sr = int(sr)
|
||||
artifact_key = self._make_ref_audio_artifact_cache_key(wav_np, sr)
|
||||
duration = len(wav_np) / sr if sr > 0 else 0.0
|
||||
if duration < _REF_AUDIO_MIN_DURATION:
|
||||
raise ValueError(
|
||||
@@ -1738,19 +1838,92 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
f"Reference audio too long ({duration:.1f}s). "
|
||||
f"Maximum {_REF_AUDIO_MAX_DURATION:.0f}s supported — use a shorter clip."
|
||||
)
|
||||
tolist_start_s = time.perf_counter()
|
||||
wav_list = wav_np.tolist()
|
||||
tolist_ms = (time.perf_counter() - tolist_start_s) * 1000.0
|
||||
logger.debug(
|
||||
"Resolved ref_audio: fetch_decode_ms=%.3f tolist_ms=%.3f samples=%d sr=%d duration_s=%.3f",
|
||||
fetch_decode_ms,
|
||||
tolist_ms,
|
||||
len(wav_np),
|
||||
sr,
|
||||
duration,
|
||||
artifact_key = self._make_ref_audio_artifact_cache_key(wav_np, sr)
|
||||
return wav_np.tolist(), sr, artifact_key, duration
|
||||
|
||||
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int, str]:
|
||||
"""Resolve ref_audio to (wav_samples, sample_rate, cache_key).
|
||||
|
||||
Delegates to upstream vLLM's MediaConnector which handles http(s)
|
||||
URLs, ``data:`` base64 URIs, and ``file:`` local paths (the latter
|
||||
gated by ``--allowed-local-media-path``).
|
||||
|
||||
Local file references incorporate mtime and size into the cache key
|
||||
so that modified files are automatically reloaded without a server
|
||||
restart. Remote URLs remain cached by their original string locator.
|
||||
|
||||
The returned *cache_key* should be passed to
|
||||
``_get_resolved_ref_audio_artifact_key`` when the caller needs the
|
||||
artifact key, avoiding a redundant ``os.stat`` and the TOCTOU window.
|
||||
After a cache miss the file is re-stat'd; a metadata change retries
|
||||
the fetch so the stored waveform matches the key.
|
||||
"""
|
||||
# Pass the allowed-local-media-path so the stat is restricted to
|
||||
# paths the server operator has explicitly permitted.
|
||||
allowed_path = None
|
||||
if not self._diffusion_mode:
|
||||
allowed_path = getattr(self.model_config, "allowed_local_media_path", None)
|
||||
|
||||
wav_list: list[float] | None = None
|
||||
sr = 0
|
||||
artifact_key = ""
|
||||
post_key = ""
|
||||
connector: MediaConnector | None = None
|
||||
for attempt in range(_REF_AUDIO_METADATA_FETCH_ATTEMPTS):
|
||||
cache_key = await self._ref_audio_cache_key(ref_audio_str, allowed_path)
|
||||
cached = self._ref_audio_resolve_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
self._ref_audio_resolve_cache.move_to_end(cache_key)
|
||||
wav_list, sr, _, _ = cached
|
||||
logger.debug(
|
||||
"Resolved ref_audio from cache: samples=%d sr=%d duration_s=%.3f",
|
||||
len(wav_list),
|
||||
sr,
|
||||
len(wav_list) / sr if sr > 0 else 0.0,
|
||||
)
|
||||
return wav_list, sr, cache_key
|
||||
|
||||
if connector is None:
|
||||
# In diffusion mode, model_config may not be available
|
||||
if self._diffusion_mode:
|
||||
connector = MediaConnector()
|
||||
else:
|
||||
model_config = self.model_config
|
||||
connector = MediaConnector(
|
||||
allowed_local_media_path=model_config.allowed_local_media_path,
|
||||
allowed_media_domains=model_config.allowed_media_domains,
|
||||
)
|
||||
|
||||
fetch_start_s = time.perf_counter()
|
||||
wav_np, fetched_sr = await connector.fetch_audio_async(ref_audio_str)
|
||||
fetch_decode_ms = (time.perf_counter() - fetch_start_s) * 1000.0
|
||||
tolist_start_s = time.perf_counter()
|
||||
wav_list, sr, artifact_key, duration = self._finalize_fetched_ref_audio(wav_np, fetched_sr)
|
||||
tolist_ms = (time.perf_counter() - tolist_start_s) * 1000.0
|
||||
logger.debug(
|
||||
"Resolved ref_audio: fetch_decode_ms=%.3f tolist_ms=%.3f samples=%d sr=%d duration_s=%.3f",
|
||||
fetch_decode_ms,
|
||||
tolist_ms,
|
||||
len(wav_list),
|
||||
sr,
|
||||
duration,
|
||||
)
|
||||
post_key = await self._ref_audio_cache_key(ref_audio_str, allowed_path)
|
||||
if post_key == cache_key:
|
||||
self._put_resolved_ref_audio(cache_key, wav_list, sr, artifact_key)
|
||||
return wav_list, sr, cache_key
|
||||
logger.debug(
|
||||
"ref_audio metadata changed during fetch (attempt %d/%d); retrying",
|
||||
attempt + 1,
|
||||
_REF_AUDIO_METADATA_FETCH_ATTEMPTS,
|
||||
)
|
||||
|
||||
logger.warning(
|
||||
"ref_audio file changed during fetch after %d attempts; skipping resolve cache",
|
||||
_REF_AUDIO_METADATA_FETCH_ATTEMPTS,
|
||||
)
|
||||
self._put_resolved_ref_audio(cache_key, wav_list, sr, artifact_key)
|
||||
return wav_list, sr
|
||||
assert wav_list is not None and post_key
|
||||
return wav_list, sr, post_key
|
||||
|
||||
@staticmethod
|
||||
def _make_ref_audio_artifact_cache_key(wav: np.ndarray, sr: int) -> str:
|
||||
@@ -1761,12 +1934,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
h.update(wav_f32.tobytes(order="C"))
|
||||
return h.hexdigest()
|
||||
|
||||
def _get_resolved_ref_audio_artifact_key(self, ref_audio_str: str) -> str | None:
|
||||
source_key = hashlib.sha1(ref_audio_str.encode("utf-8")).hexdigest()
|
||||
cached = self._ref_audio_resolve_cache.get(source_key)
|
||||
def _get_resolved_ref_audio_artifact_key(self, cache_key: str) -> str | None:
|
||||
"""Look up the artifact key for a previously resolved ref_audio.
|
||||
|
||||
*cache_key* must be the exact key returned by the preceding
|
||||
``_resolve_ref_audio`` call. Passing the key explicitly avoids a
|
||||
second ``os.stat`` syscall and eliminates the TOCTOU window that
|
||||
would arise from recomputing the key independently.
|
||||
"""
|
||||
cached = self._ref_audio_resolve_cache.get(cache_key)
|
||||
if cached is None:
|
||||
return None
|
||||
self._ref_audio_resolve_cache.move_to_end(source_key)
|
||||
self._ref_audio_resolve_cache.move_to_end(cache_key)
|
||||
return cached[3]
|
||||
|
||||
def _put_resolved_ref_audio(self, cache_key: str, wav_list: list[float], sr: int, artifact_key: str) -> None:
|
||||
@@ -1831,7 +2010,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
async def _resolve_ref_audio_many(self, ref_audio_list: list[str]) -> list[tuple[list[float], int]]:
|
||||
resolved = []
|
||||
for ref_audio in ref_audio_list:
|
||||
resolved.append(await self._resolve_ref_audio(ref_audio))
|
||||
wav_list, sr, _ = await self._resolve_ref_audio(ref_audio)
|
||||
resolved.append((wav_list, sr))
|
||||
return resolved
|
||||
|
||||
# ---- Ming TTS helpers ----
|
||||
@@ -2502,7 +2682,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
+ mm_processor_kwargs with prompt_text.
|
||||
"""
|
||||
# Resolve reference audio
|
||||
wav_samples, sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
wav_samples, sr, _ = await self._resolve_ref_audio(request.ref_audio)
|
||||
audio_data = (np.asarray(wav_samples, dtype=np.float32), sr)
|
||||
|
||||
# Wrap the reference transcript in the CosyVoice3 instruction template
|
||||
@@ -2616,7 +2796,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
"""
|
||||
# Voice cloning requires ref_audio + ref_text
|
||||
if request.ref_audio is not None and request.ref_text:
|
||||
wav_samples, sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
wav_samples, sr, _ = await self._resolve_ref_audio(request.ref_audio)
|
||||
audio_data = (np.asarray(wav_samples, dtype=np.float32), int(sr))
|
||||
|
||||
mm_kwargs: dict[str, Any] = {
|
||||
@@ -2748,8 +2928,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
# Uploaded voice: ref_audio was auto-set as [base64_data_url]
|
||||
ref_audio_source = tts_params["ref_audio"][0]
|
||||
if ref_audio_source is not None and isinstance(ref_audio_source, str):
|
||||
wav_list, sr = await self._resolve_ref_audio(ref_audio_source)
|
||||
artifact_key = self._get_resolved_ref_audio_artifact_key(ref_audio_source)
|
||||
wav_list, sr, cache_key = await self._resolve_ref_audio(ref_audio_source)
|
||||
tts_params["ref_audio_cache_key"] = cache_key
|
||||
artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key)
|
||||
if artifact_key:
|
||||
tts_params[_QWEN3_TTS_REF_AUDIO_CACHE_KEY] = [artifact_key]
|
||||
ref_code_length = self._estimate_ref_code_len([wav_list, sr])
|
||||
@@ -3142,7 +3323,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
request_id = f"speech-{random_uuid()}"
|
||||
prompt: dict[str, Any] = {"input": request.input}
|
||||
if request.ref_audio:
|
||||
wav, sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
wav, sr, _ = await self._resolve_ref_audio(request.ref_audio)
|
||||
prompt["ref_audio"] = (np.asarray(wav, dtype=np.float32), sr)
|
||||
if request.ref_text:
|
||||
prompt["ref_text"] = request.ref_text
|
||||
|
||||
@@ -49,13 +49,14 @@ class FishSpeechAdapter(ARTTSAdapter):
|
||||
) -> PreparedRequest:
|
||||
server = self.ctx.server
|
||||
ref_audio_data = None
|
||||
tts_params: dict = {}
|
||||
if request.ref_audio is not None:
|
||||
wav_list, sr = await server._resolve_ref_audio(request.ref_audio)
|
||||
wav_list, sr, cache_key = await server._resolve_ref_audio(request.ref_audio)
|
||||
ref_audio_data = (wav_list, sr)
|
||||
tts_params["ref_audio_cache_key"] = cache_key
|
||||
prompt = await server._build_fish_speech_prompt_async(
|
||||
request, ref_audio_data=ref_audio_data, has_inline_ref_audio=has_inline_ref_audio
|
||||
)
|
||||
tts_params = {}
|
||||
prompt["cache_salt"] = conditioning_cache_salt(request, tts_params)
|
||||
return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="fish_speech")
|
||||
|
||||
|
||||
@@ -108,6 +108,8 @@ def indextts2_conditioning_cache_salt(
|
||||
"use_random",
|
||||
"lang",
|
||||
"text_normalization",
|
||||
"ref_audio_cache_key",
|
||||
"emo_audio_cache_key",
|
||||
):
|
||||
h.update(b"\x00")
|
||||
h.update(key.encode("utf-8"))
|
||||
@@ -260,8 +262,9 @@ class IndexTTS2Adapter(ARTTSAdapter):
|
||||
ref_audio_source = server._get_uploaded_audio_data(voice_lower)
|
||||
using_uploaded_voice = ref_audio_source is not None
|
||||
if ref_audio_source is not None and isinstance(ref_audio_source, str):
|
||||
wav_list, sr = await server._resolve_ref_audio(ref_audio_source)
|
||||
wav_list, sr, cache_key = await server._resolve_ref_audio(ref_audio_source)
|
||||
params["voice"] = [[wav_list, sr]]
|
||||
params["ref_audio_cache_key"] = [cache_key]
|
||||
if using_uploaded_voice and voice_lower:
|
||||
params["voice_name"] = [voice_lower]
|
||||
params["voice_created_at"] = [server._voice_created_at(voice_lower)]
|
||||
@@ -271,8 +274,9 @@ class IndexTTS2Adapter(ARTTSAdapter):
|
||||
if key not in extras:
|
||||
continue
|
||||
if key == "emo_audio":
|
||||
wav_list, sr = await server._resolve_ref_audio(extras[key])
|
||||
wav_list, sr, emo_key = await server._resolve_ref_audio(extras[key])
|
||||
params[key] = [[wav_list, sr]]
|
||||
params["emo_audio_cache_key"] = [emo_key]
|
||||
else:
|
||||
params[key] = [extras[key]]
|
||||
return params
|
||||
|
||||
@@ -64,7 +64,7 @@ class MingTTSAdapter(ARTTSAdapter):
|
||||
if isinstance(ref_audio_source, list):
|
||||
ref_audio_data = await server._resolve_ref_audio_many(ref_audio_source)
|
||||
elif ref_audio_source is not None and isinstance(ref_audio_source, str):
|
||||
wav_list, sr = await server._resolve_ref_audio(ref_audio_source)
|
||||
wav_list, sr, _ = await server._resolve_ref_audio(ref_audio_source)
|
||||
ref_audio_data = (wav_list, sr)
|
||||
prompt = server._build_ming_dense_prompt(
|
||||
request,
|
||||
|
||||
@@ -91,7 +91,7 @@ class _MossTTSAdapterBase(ARTTSAdapter):
|
||||
self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool
|
||||
) -> PreparedRequest:
|
||||
server = self.ctx.server
|
||||
tts_params = await server._build_moss_tts_params(request)
|
||||
tts_params = await server._build_moss_tts_params(request, has_inline_ref_audio=has_inline_ref_audio)
|
||||
if request.voice:
|
||||
voice_lower = request.voice.lower()
|
||||
if voice_lower in server.uploaded_speakers and not has_inline_ref_audio:
|
||||
|
||||
@@ -33,7 +33,7 @@ class OmniVoiceAdapter(ARTTSAdapter):
|
||||
server = self.ctx.server
|
||||
prompt: dict = {"input": request.input}
|
||||
if request.ref_audio:
|
||||
wav, sr = await server._resolve_ref_audio(request.ref_audio)
|
||||
wav, sr, _ = await server._resolve_ref_audio(request.ref_audio)
|
||||
prompt["ref_audio"] = (np.asarray(wav, dtype=np.float32), sr)
|
||||
if request.ref_text:
|
||||
prompt["ref_text"] = request.ref_text
|
||||
|
||||
Reference in new issue
Block a user