From 53b60bd9bcbbebc49fe75792dfc668580819ffe2 Mon Sep 17 00:00:00 2001 From: Pranav Thakur <103683515+pranavthakur0-0@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:36:32 +0530 Subject: [PATCH] [Bugfix] Invalidate stale ref_audio cache on local file modification (#4873) (#5670) Signed-off-by: pranavthakur0-0 Co-authored-by: Zhou Taichang --- docs/serving/speech_api.md | 2 +- .../openai_api/test_serving_speech.py | 362 +++++++++++++++++- .../openai_api/test_tts_adapter.py | 7 + .../entrypoints/openai/serving_speech.py | 331 ++++++++++++---- .../openai/tts_adapters/fish_speech.py | 5 +- .../openai/tts_adapters/indextts2.py | 8 +- .../openai/tts_adapters/ming_tts.py | 2 +- .../openai/tts_adapters/moss_tts.py | 2 +- .../openai/tts_adapters/omnivoice.py | 2 +- .../models/moss_tts/reference_encoder.py | 57 ++- 10 files changed, 675 insertions(+), 103 deletions(-) diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index e20390385..92321e536 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -131,7 +131,7 @@ Content-Type: application/json | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `ref_audio` | string | null | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) | +| `ref_audio` | string | null | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`). Local files fold `mtime_ns` and `size` into cache keys to automatically reload on-disk edits; HTTP URLs and base64 URIs remain cached by string locator. | | `ref_text` | string | null | Transcript of reference audio | | `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) | diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index dda02641f..281c5e24c 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -1237,10 +1237,364 @@ class TestTTSMethods: assert second[1] == 24000 assert first[0] is second[0] assert first[0][0] == pytest.approx(float(wav[0]), abs=1e-4) + cache_key = first[2] assert speech_server._get_resolved_ref_audio_artifact_key( - ref_audio + cache_key ) == speech_server._make_ref_audio_artifact_cache_key(np.asarray(first[0], dtype=np.float32), 24000) + # ── ref-audio cache key tests (static helper) ── + + def test_local_file_cache_key_invalidation(self): + """Modified file must produce a different cache key (mtime change).""" + import pathlib + import tempfile + import time as _time + + with tempfile.TemporaryDirectory() as tmp_dir: + test_file = pathlib.Path(tmp_dir) / "test_audio.wav" + test_file.write_text("dummy audio data 1") + file_uri = f"file://{test_file}" + key1 = OmniOpenAIServingSpeech._get_ref_audio_cache_key( + file_uri, + allowed_local_media_path=tmp_dir, + ) + _time.sleep(0.01) + test_file.write_text("dummy audio data 2") + key2 = OmniOpenAIServingSpeech._get_ref_audio_cache_key( + file_uri, + allowed_local_media_path=tmp_dir, + ) + assert key1 != key2, "Cache key should change when file is modified!" + + def test_remote_url_cache_key(self): + """Remote URLs must produce a stable, repeatable cache key.""" + url = "https://example.com/audio.wav" + key1 = OmniOpenAIServingSpeech._get_ref_audio_cache_key(url) + key2 = OmniOpenAIServingSpeech._get_ref_audio_cache_key(url) + assert key1 == key2, "Cache key for URLs should remain constant!" + + def test_missing_file_cache_key(self, caplog): + """A missing file falls back to the URI-string hash and emits a log + so the stale-cache risk is visible.""" + file_uri = "file:///path/to/nonexistent/file.wav" + target_logger = logging.getLogger("vllm_omni.entrypoints.openai.serving_speech") + target_logger.addHandler(caplog.handler) + prev_level = target_logger.level + target_logger.setLevel(logging.DEBUG) + try: + key = OmniOpenAIServingSpeech._get_ref_audio_cache_key( + file_uri, + allowed_local_media_path="/path/to/nonexistent", + ) + finally: + target_logger.removeHandler(caplog.handler) + target_logger.setLevel(prev_level) + expected_key = hashlib.sha1(file_uri.encode("utf-8")).hexdigest() + assert key == expected_key, "Missing files should fallback to the string hash." + assert any("stale cache" in r.getMessage() for r in caplog.records), ( + "A log message about stale cache must be emitted when os.stat fails" + ) + + def test_percent_encoded_file_uri(self): + """Percent-encoded file:// URIs must be decoded so os.stat hits the real file.""" + import pathlib + import tempfile + + with tempfile.TemporaryDirectory() as tmp_dir: + test_file = pathlib.Path(tmp_dir) / "my audio.wav" + test_file.write_text("dummy audio data") + encoded_uri = f"file://{str(test_file).replace(' ', '%20')}" + key = OmniOpenAIServingSpeech._get_ref_audio_cache_key( + encoded_uri, + allowed_local_media_path=tmp_dir, + ) + string_only_key = hashlib.sha1(encoded_uri.encode("utf-8")).hexdigest() + assert key != string_only_key, ( + "Percent-encoded URI should be decoded and stat'd, not fall back to string-only cache key!" + ) + + def test_no_allowlist_skips_stat_for_local_paths(self): + """When allowed_local_media_path is None, local paths must NOT be stat'd + — the key falls back to the string-only hash.""" + import pathlib + import tempfile + + with tempfile.TemporaryDirectory() as tmp_dir: + test_file = pathlib.Path(tmp_dir) / "test.wav" + test_file.write_text("data") + file_uri = f"file://{test_file}" + key = OmniOpenAIServingSpeech._get_ref_audio_cache_key( + file_uri, + allowed_local_media_path=None, + ) + # With no allowlist the stat is skipped, so the key is just sha1(uri) + expected = hashlib.sha1(file_uri.encode("utf-8")).hexdigest() + assert key == expected, "No allowlist should skip stat and use string-only key" + + def test_empty_string_allowlist_skips_stat_for_local_paths(self): + """vLLM's default allowed_local_media_path is '' not None; that must + also skip stat rather than treating cwd as the allowlist.""" + import pathlib + import tempfile + + with tempfile.TemporaryDirectory() as tmp_dir: + test_file = pathlib.Path(tmp_dir) / "test.wav" + test_file.write_text("data") + file_uri = f"file://{test_file}" + key = OmniOpenAIServingSpeech._get_ref_audio_cache_key( + file_uri, + allowed_local_media_path="", + ) + expected = hashlib.sha1(file_uri.encode("utf-8")).hexdigest() + assert key == expected, "Empty-string allowlist should skip stat and use string-only key" + + def test_uppercase_file_uri_uses_local_metadata(self): + """FILE:// must be treated as a local file, not a string-only locator.""" + import pathlib + import tempfile + + with tempfile.TemporaryDirectory() as tmp_dir: + test_file = pathlib.Path(tmp_dir) / "test.wav" + test_file.write_text("data") + upper_uri = f"FILE://{test_file}" + key = OmniOpenAIServingSpeech._get_ref_audio_cache_key( + upper_uri, + allowed_local_media_path=tmp_dir, + ) + string_only = hashlib.sha1(upper_uri.encode("utf-8")).hexdigest() + assert key != string_only + + def test_uppercase_http_uri_is_remote(self, speech_server): + url = "HTTP://example.com/audio.wav" + key = OmniOpenAIServingSpeech._get_ref_audio_cache_key(url) + assert key == hashlib.sha1(url.encode("utf-8")).hexdigest() + assert speech_server._validate_ref_audio_format(url) is None + assert speech_server._validate_ref_audio_format("FILE:///tmp/a.wav") is None + assert speech_server._validate_ref_audio_format("/tmp/a.wav") is not None + + @pytest.mark.asyncio + async def test_resolve_ref_audio_retries_when_file_changes_during_fetch(self, speech_server, mocker): + import pathlib + import tempfile + + wav = np.linspace(-0.5, 0.5, 48000, dtype=np.float32) + with tempfile.TemporaryDirectory() as tmp_dir: + path = pathlib.Path(tmp_dir) / "spk.wav" + path.write_bytes(b"v1") + uri = f"file://{path}" + speech_server.model_config.allowed_local_media_path = tmp_dir + speech_server.model_config.allowed_media_domains = None + speech_server._diffusion_mode = False + + fetches = {"n": 0} + + async def fake_fetch(_ref): + fetches["n"] += 1 + if fetches["n"] == 1: + path.write_bytes(b"v2-longer-content") + return wav, 24000 + + mock_connector = mocker.MagicMock() + mock_connector.fetch_audio_async = fake_fetch + mocker.patch( + "vllm_omni.entrypoints.openai.serving_speech.MediaConnector", + return_value=mock_connector, + ) + + wav_list, sr, cache_key = await speech_server._resolve_ref_audio(uri) + expected_key = OmniOpenAIServingSpeech._get_ref_audio_cache_key(uri, allowed_local_media_path=tmp_dir) + assert sr == 24000 + assert len(wav_list) == 48000 + assert fetches["n"] == 2 + assert cache_key == expected_key + assert cache_key in speech_server._ref_audio_resolve_cache + + def test_conditioning_cache_salt_changes_with_ref_audio_cache_key(self): + """Prefix-cache salt must fold the content-aware resolve key so a + same-path file rewrite cannot reuse KV from the previous reference.""" + from vllm_omni.entrypoints.openai.serving_speech import _conditioning_cache_salt + + req = OpenAICreateSpeechRequest(input="hello", ref_audio="file:///data/spk.wav") + salt_a = _conditioning_cache_salt(req, {"ref_audio_cache_key": "key_aaa"}) + salt_b = _conditioning_cache_salt(req, {"ref_audio_cache_key": "key_bbb"}) + salt_locator_only = _conditioning_cache_salt(req, {}) + assert salt_a != salt_b + assert salt_a != salt_locator_only + + @pytest.mark.asyncio + async def test_higgs_v3_cache_salt_changes_with_ref_audio_cache_key(self, speech_server, mocker): + """Higgs v3 voice clone uses placeholder prompt ids + prefix caching. + The salt must move when the resolve key moves, or a same-path rewrite + reuses KV from the previous reference.""" + from unittest.mock import AsyncMock, MagicMock + + adapter = MagicMock() + adapter.build_prompt.return_value = [1, 1, 1, 1] + speech_server._resolve_higgs_audio_v3_adapter = AsyncMock(return_value=adapter) + codes = torch.arange(8, dtype=torch.long).view(8, 1) + speech_server._resolve_higgs_audio_v3_ref_codes = AsyncMock(return_value=(codes, False, False)) + speech_server._get_resolved_ref_audio_artifact_key = MagicMock(return_value="art") + + req = OpenAICreateSpeechRequest( + input="hello", + ref_audio="file:///data/spk.wav", + ref_text="transcript", + ) + speech_server._resolve_ref_audio = AsyncMock(return_value=([0.1] * 48000, 24000, "key_aaa")) + prompt_a = await speech_server._build_higgs_audio_v3_params(req) + speech_server._resolve_ref_audio = AsyncMock(return_value=([0.9] * 48000, 24000, "key_bbb")) + prompt_b = await speech_server._build_higgs_audio_v3_params(req) + + assert prompt_a["prompt_token_ids"] == prompt_b["prompt_token_ids"] + assert prompt_a["cache_salt"] != prompt_b["cache_salt"] + assert prompt_a["additional_information"]["ref_audio_cache_key"] == "key_aaa" + assert prompt_b["additional_information"]["ref_audio_cache_key"] == "key_bbb" + + # ── encode_reference_codes: speaker cache invalidation ── + + @pytest.mark.asyncio + async def test_encode_reference_codes_reencodes_on_key_change(self): + """Changing the resolve_cache_key must trigger a fresh encode, not serve + the stale cached tensor. This is the user-visible fix for delay-family + voice clones that were reproducing the old speaker.""" + from unittest.mock import MagicMock + + from vllm_omni.model_executor.models.moss_tts.reference_encoder import encode_reference_codes + from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache + + speaker_cache = SpeakerEmbeddingCache(max_bytes=64 * 1024 * 1024) + + call_count = 0 + codes_v1 = torch.arange(10, dtype=torch.int64) + codes_v2 = torch.arange(10, 20, dtype=torch.int64) + + # Stub processor that tracks call count and returns different codes + processor = MagicMock() + + def fake_encode(wavs, sampling_rate, n_vq): + nonlocal call_count + call_count += 1 + return [codes_v1 if call_count == 1 else codes_v2] + + processor.encode_audios_from_wav = fake_encode + + # resolve_ref_audio returns different cache keys on successive calls + resolve_keys = iter(["key_aaa", "key_bbb"]) + + async def mock_resolve(ref_str): + return ([0.1, 0.2, 0.3], 24000, next(resolve_keys)) + + # First call: cold miss, encodes and stores + result1 = await encode_reference_codes( + "data:audio/wav;base64,fake", + processor=processor, + resolve_ref_audio=mock_resolve, + speaker_cache=speaker_cache, + variant="tts", + n_vq=32, + sr_target=24000, + ) + assert call_count == 1 + assert torch.equal(result1, codes_v1) + + # Second call: different resolve key → must re-encode, not serve cached + result2 = await encode_reference_codes( + "data:audio/wav;base64,fake", + processor=processor, + resolve_ref_audio=mock_resolve, + speaker_cache=speaker_cache, + variant="tts", + n_vq=32, + sr_target=24000, + ) + assert call_count == 2, "Different resolve key should trigger re-encode" + assert torch.equal(result2, codes_v2) + + @pytest.mark.asyncio + async def test_encode_reference_codes_named_voice_skips_resolve_on_hit(self): + """Named-voice cache hit must NOT call resolve_ref_audio.""" + from unittest.mock import MagicMock + + from vllm_omni.model_executor.models.moss_tts.reference_encoder import encode_reference_codes + from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache + + speaker_cache = SpeakerEmbeddingCache(max_bytes=64 * 1024 * 1024) + + processor = MagicMock() + processor.encode_audios_from_wav = lambda wavs, sampling_rate, n_vq: [torch.zeros(5)] + + resolve_called = 0 + + async def mock_resolve(ref_str): + nonlocal resolve_called + resolve_called += 1 + return ([0.1], 24000, "some_key") + + # First call: cold miss, resolves + encodes + await encode_reference_codes( + "data:audio/wav;base64,fake", + processor=processor, + resolve_ref_audio=mock_resolve, + speaker_cache=speaker_cache, + variant="tts", + n_vq=32, + sr_target=24000, + voice_name="alice", + voice_created_at=100, + ) + assert resolve_called == 1 + + # Second call: warm hit, must NOT resolve + await encode_reference_codes( + "data:audio/wav;base64,fake", + processor=processor, + resolve_ref_audio=mock_resolve, + speaker_cache=speaker_cache, + variant="tts", + n_vq=32, + sr_target=24000, + voice_name="alice", + voice_created_at=100, + ) + assert resolve_called == 1, "Named-voice cache hit should skip resolve_ref_audio" + + @pytest.mark.asyncio + async def test_ttsd_second_reference_does_not_use_named_voice(self, speech_server, mocker): + """Uploaded voice names speaker 1 only; speaker 2 must stay anonymous.""" + speech_server._moss_variant = "ttsd" + speech_server.uploaded_speakers = {"alice": {}} + mocker.patch.object(speech_server, "_voice_created_at", return_value=42) + + proc = mocker.MagicMock() + proc.model_config.n_vq = 8 + proc.model_config.sampling_rate = 24000 + proc.build_user_message.return_value = "msg" + proc.return_value = {"input_ids": [torch.zeros((4, 9), dtype=torch.int64)]} + mocker.patch.object(speech_server, "_get_moss_processor", return_value=proc) + + seen: list[tuple[str, str | None]] = [] + + async def fake_encode(ref_str, **kwargs): + seen.append((ref_str, kwargs.get("voice_name"))) + return torch.zeros(3, dtype=torch.int64) + + mocker.patch( + "vllm_omni.model_executor.models.moss_tts.reference_encoder.encode_reference_codes", + fake_encode, + ) + + req = OpenAICreateSpeechRequest( + input="hello", + voice="alice", + ref_audio="data:audio/wav;base64,aaa", + ref_audio_2="data:audio/wav;base64,bbb", + ) + await speech_server._build_moss_tts_params(req, has_inline_ref_audio=False) + assert seen == [ + ("data:audio/wav;base64,aaa", "alice"), + ("data:audio/wav;base64,bbb", None), + ] + def test_precomputed_qwen3_voice_infers_base_without_ref_audio(self, speech_server): """Precomputed Qwen3 voices are reusable by name without per-request ref_audio.""" speech_server._tts_model_type = "qwen3_tts" @@ -1896,7 +2250,7 @@ class TestTTSMethods: mocker.patch.object( speech_server, "_resolve_ref_audio", - new=mocker.AsyncMock(return_value=ref_audio_data), + new=mocker.AsyncMock(return_value=(*ref_audio_data, "fake_cache_key")), ) mock_prompt = mocker.patch.object( speech_server, @@ -2013,7 +2367,7 @@ class TestTTSMethods: mocker.patch.object( speech_server, "_resolve_ref_audio", - new=mocker.AsyncMock(return_value=ref_audio_data), + new=mocker.AsyncMock(return_value=(*ref_audio_data, "fake_cache_key")), ) mock_prompt = mocker.patch.object( speech_server, @@ -4382,7 +4736,7 @@ class TestTTSAsyncOffloading: ): """Base explicit true should reach the model prompt additional_information.""" qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None) - qwen3_tts_server._resolve_ref_audio = mocker.AsyncMock(return_value=([0.0] * 48000, 24000)) + qwen3_tts_server._resolve_ref_audio = mocker.AsyncMock(return_value=([0.0] * 48000, 24000, "fake_cache_key")) qwen3_tts_server._get_resolved_ref_audio_artifact_key = mocker.MagicMock(return_value=None) qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512) diff --git a/tests/entrypoints/openai_api/test_tts_adapter.py b/tests/entrypoints/openai_api/test_tts_adapter.py index 4f6ed2457..100f4fc06 100644 --- a/tests/entrypoints/openai_api/test_tts_adapter.py +++ b/tests/entrypoints/openai_api/test_tts_adapter.py @@ -183,6 +183,13 @@ def test_indextts25_speed_does_not_change_conditioning_cache_salt(): ) == indextts2_conditioning_cache_salt(slow_request, slow_params) +def test_indextts2_conditioning_cache_salt_changes_with_ref_audio_cache_key(): + request = SimpleNamespace(input="hello", ref_audio="file:///data/spk.wav") + salt_a = indextts2_conditioning_cache_salt(request, {"ref_audio_cache_key": ["key_aaa"]}) + salt_b = indextts2_conditioning_cache_salt(request, {"ref_audio_cache_key": ["key_bbb"]}) + assert salt_a != salt_b + + @pytest.mark.parametrize( ("hf_config", "expected_tokenizer_file"), [ diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index ca64c6181..c2c200039 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -13,6 +13,8 @@ from concurrent.futures import ThreadPoolExecutor from http import HTTPStatus from pathlib import Path from typing import Any +from urllib.parse import urlparse +from urllib.request import url2pathname import numpy as np import soundfile as sf @@ -98,6 +100,8 @@ _COSYVOICE3_PROMPT_PREFIX = f"You are a helpful assistant.{_COSYVOICE3_PROMPT_DE _AUDEX_NO_AUDIO_GUARD_MODEL_TYPES = frozenset({"audex", "audex_tta"}) _REF_AUDIO_MIN_DURATION = 1.0 # seconds _REF_AUDIO_MAX_DURATION = 30.0 # seconds +_REF_AUDIO_METADATA_FETCH_ATTEMPTS = 3 +_REMOTE_REF_AUDIO_SCHEMES = frozenset({"http", "https", "data"}) _REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES = 256 _REF_AUDIO_RESOLVE_CACHE_MAX_BYTES = 256 * 1024 * 1024 _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256 @@ -228,6 +232,13 @@ def _conditioning_cache_salt(request, tts_params: dict | None = None) -> str: must also be folded in. ``voice_created_at`` bumps on every (re-)upload, which uniquely identifies the resolved reference artifact together with the voice name; the decoded ref_audio array itself need not be hashed. + + Local ``ref_audio`` locators are the same class of gap: the raw request + hashes the path/URI string, so an on-disk rewrite is invisible to the + salt. Callers stash the content-aware key from ``_resolve_ref_audio`` as + ``ref_audio_cache_key`` (and ``ref_audio_2_cache_key`` / emotion keys + when those clips exist) so a file edit cannot reuse KV from the previous + reference. """ h = hashlib.sha256() for part in ( @@ -245,13 +256,15 @@ def _conditioning_cache_salt(request, tts_params: dict | None = None) -> str: if part is not None: h.update(repr(part).encode("utf-8")) # Fold resolved conditioning that is auto-derived for uploaded voices and - # absent from the raw request. + # absent from the raw request (or that a locator string cannot see). for key in ( "voice_created_at", "task_type", "speaker", "ref_text", "x_vector_only_mode", + "ref_audio_cache_key", + "ref_audio_2_cache_key", ): h.update(b"\x00") value = tts_params.get(key) if tts_params is not None else None @@ -803,7 +816,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ref_sr = None voice_profile = None if request.ref_audio is not None: - ref_audio, ref_sr = await self._resolve_ref_audio(request.ref_audio) + ref_audio, ref_sr, _ = await self._resolve_ref_audio(request.ref_audio) elif uploaded_ref is not None: wav_np, ref_sr = uploaded_ref ref_audio = wav_np.tolist() @@ -1290,11 +1303,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): """Validate ref_audio is a supported URI format. Returns error or None.""" if not isinstance(ref_audio, str): return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)" - if not ( - ref_audio.startswith(("http://", "https://")) - or ref_audio.startswith("data:") - or ref_audio.startswith("file://") - ): + scheme = (urlparse(ref_audio).scheme or "").lower() + if scheme not in {"http", "https", "data", "file"}: return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)" return None @@ -1343,7 +1353,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._moss_processor_cache = proc return proc - async def _build_moss_tts_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: + async def _build_moss_tts_params( + self, + request: OpenAICreateSpeechRequest, + *, + has_inline_ref_audio: bool = False, + ) -> dict[str, Any]: """Build the talker prompt + ``additional_information`` payload for any MOSS-TTS-family request (nano + 5 full variants). @@ -1371,8 +1386,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): } if request.max_new_tokens is not None: params["max_new_frames"] = [request.max_new_tokens] - wav_list, sr = await self._resolve_ref_audio(request.ref_audio) + wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) params["prompt_audio_array"] = [[wav_list, sr]] + params["ref_audio_cache_key"] = cache_key return params # ---- MOSS-TTS-Realtime: keep the old prompt_audio_array path ---- @@ -1390,8 +1406,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): } if request.max_new_tokens is not None: params["max_new_frames"] = [request.max_new_tokens] - wav_list, sr = await self._resolve_ref_audio(request.ref_audio) + wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) params["prompt_audio_array"] = [[wav_list, sr]] + params["ref_audio_cache_key"] = cache_key return params # ---- MossTTSDelay family (tts/ttsd/sound_effect/voice_generator) @@ -1422,30 +1439,47 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # only pulls it on the delay-family path (alongside the upstream proc). from vllm_omni.model_executor.models.moss_tts.reference_encoder import encode_reference_codes - _voice = getattr(request, "voice", None) - _voice = _voice.strip() if isinstance(_voice, str) else "" - _voice_created = self._voice_created_at(_voice.lower()) if _voice else 0 + # Named-voice speaker cache is only valid for uploaded speakers + # without an inline ref_audio. ``request.voice`` plus a file/URL + # otherwise keys on (name, created_at=0) and skips the content-aware + # resolve — the adapter already applies this guard when tagging + # ``voice_name`` on tts_params. + _raw_voice = getattr(request, "voice", None) + _raw_voice = _raw_voice.strip() if isinstance(_raw_voice, str) else "" + _voice_lower = _raw_voice.lower() if _raw_voice else "" + _use_named_voice = bool(_voice_lower) and _voice_lower in self.uploaded_speakers and not has_inline_ref_audio + _voice = _voice_lower if _use_named_voice else "" + _voice_created = self._voice_created_at(_voice) if _voice else 0 + _resolve_keys: list[str] = [] - async def _encode_ref(ref_str: str) -> torch.Tensor: + async def _tracked_resolve(ref_str: str) -> tuple[list, int, str]: + wav_list, sr, cache_key = await self._resolve_ref_audio(ref_str) + _resolve_keys.append(cache_key) + return wav_list, sr, cache_key + + async def _encode_ref(ref_str: str, *, named_voice: bool) -> torch.Tensor: + # Named-voice cache is (voice_name, created_at). TTSD speaker 2 is a + # different clip, so it must stay anonymous or it reuses speaker 1. + use_named = named_voice and bool(_voice) return await encode_reference_codes( ref_str, processor=proc, - resolve_ref_audio=self._resolve_ref_audio, + resolve_ref_audio=_tracked_resolve, speaker_cache=self._speaker_cache, variant=v, n_vq=n_vq, sr_target=sr_target, - voice_name=_voice or None, - voice_created_at=_voice_created, + voice_name=_voice if use_named else None, + voice_created_at=_voice_created if use_named else 0, ) user_kwargs: dict[str, Any] = {"text": request.input or ""} if v in ("tts", "local"): - user_kwargs["reference"] = [await _encode_ref(request.ref_audio)] + user_kwargs["reference"] = [await _encode_ref(request.ref_audio, named_voice=True)] elif v == "ttsd": - refs = [await _encode_ref(request.ref_audio)] + refs = [await _encode_ref(request.ref_audio, named_voice=True)] if request.ref_audio_2: - refs.append(await _encode_ref(request.ref_audio_2)) + refs.append(await _encode_ref(request.ref_audio_2, named_voice=False)) user_kwargs["reference"] = refs elif v == "sound_effect": user_kwargs["text"] = request.input or "" # may be empty @@ -1479,6 +1513,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): } if request.max_new_tokens is not None: params["max_new_frames"] = [request.max_new_tokens] + if _resolve_keys: + params["ref_audio_cache_key"] = _resolve_keys[0] + if len(_resolve_keys) > 1: + params["ref_audio_2_cache_key"] = _resolve_keys[1] return params async def _build_higgs_audio_v2_params(self, request: OpenAICreateSpeechRequest): @@ -1507,7 +1545,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): prompt_token_ids = input_ids_to_python_list(inputs) return tokens_input(prompt_token_ids=prompt_token_ids) - wav_list, sr = await self._resolve_ref_audio(request.ref_audio) + wav_list, sr, _ = await self._resolve_ref_audio(request.ref_audio) wav = np.asarray(wav_list, dtype=np.float32) out = await asyncio.to_thread( build_voice_clone_prompt, @@ -1576,8 +1614,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): encode_reference_audio, ) - wav_list, sr = await self._resolve_ref_audio(request.ref_audio) - artifact_key = self._get_resolved_ref_audio_artifact_key(request.ref_audio) + wav_list, sr, cache_key = await self._resolve_ref_audio(request.ref_audio) + artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key) wav = np.asarray(wav_list, dtype=np.float32) ref_codes_delayed, cache_hit, inflight_wait = await self._resolve_higgs_audio_v3_ref_codes( artifact_key, @@ -1599,7 +1637,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): prompt["additional_information"] = { "audio_input_ids": ref_codes_delayed.to(torch.long), "audio_input_ids_mask": torch.ones(ref_codes_delayed.shape[0], dtype=torch.bool), + "ref_audio_cache_key": cache_key, } + # Placeholder prompt ids do not change when a same-path file is + # rewritten at the same duration; fold the content-aware resolve key + # into cache_salt so prefix caching cannot reuse the previous KV. + prompt["cache_salt"] = _conditioning_cache_salt(request, prompt["additional_information"]) return prompt async def _resolve_higgs_audio_v3_ref_codes( @@ -1690,43 +1733,100 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._higgs_audio_v3_adapter = adapter return adapter - async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]: - """Resolve ref_audio to (wav_samples, sample_rate). + @staticmethod + def _local_ref_audio_stat_path(ref_audio_str: str) -> str | None: + """Filesystem path to stat for a local locator, else ``None``. - Delegates to upstream vLLM's MediaConnector which handles http(s) - URLs, ``data:`` base64 URIs, and ``file:`` local paths (the latter - gated by ``--allowed-local-media-path``). + Scheme comparison is case-insensitive (``urlparse`` lowercases it), so + ``FILE:///...`` is treated as a local file the same way ``file://`` is. + Remote ``http`` / ``https`` / ``data`` locators, including ``HTTP://``, + return ``None`` and stay string-keyed. """ - cache_key = hashlib.sha1(ref_audio_str.encode("utf-8")).hexdigest() - cached = self._ref_audio_resolve_cache.get(cache_key) - if cached is not None: - self._ref_audio_resolve_cache.move_to_end(cache_key) - wav_list, sr, _, _ = cached - logger.debug( - "Resolved ref_audio from cache: samples=%d sr=%d duration_s=%.3f", - len(wav_list), - sr, - len(wav_list) / sr if sr > 0 else 0.0, - ) - return wav_list, sr + parsed = urlparse(ref_audio_str) + scheme = (parsed.scheme or "").lower() + if scheme in _REMOTE_REF_AUDIO_SCHEMES: + return None + if scheme == "file": + netloc = parsed.netloc or "" + if netloc.lower() not in ("", "localhost"): + raise OSError(f"file:// URI with non-local authority {netloc!r} cannot be stat'd") + return url2pathname(parsed.path or "") + if scheme: + return None + return ref_audio_str - # In diffusion mode, model_config may not be available - if self._diffusion_mode: - connector = MediaConnector() - else: - model_config = self.model_config - connector = MediaConnector( - allowed_local_media_path=model_config.allowed_local_media_path, - allowed_media_domains=model_config.allowed_media_domains, - ) - fetch_start_s = time.perf_counter() - wav_np, sr = await connector.fetch_audio_async(ref_audio_str) - fetch_decode_ms = (time.perf_counter() - fetch_start_s) * 1000.0 + @staticmethod + def _get_ref_audio_cache_key( + ref_audio_str: str, + allowed_local_media_path: str | None = None, + ) -> str: + """Compute a cache key hash for *ref_audio_str*. + + For local files (bare paths and ``file://`` URIs) the key folds in + ``st_mtime_ns`` and ``st_size`` so that an on-disk edit automatically + invalidates the cached waveform without a server restart. Remote URLs + and ``data:`` URIs are keyed on the raw string alone — the server does + not re-fetch them to check for changes. + + ``os.stat`` runs only when *allowed_local_media_path* is a non-empty + path. vLLM's default is ``""`` (not ``None``); both mean local media + is refused, so a stat would be exposure for a request that cannot + load the file. + + Note: when the key changes the *previous* entry stays in + ``_ref_audio_resolve_cache`` until LRU eviction, so + ``_discard_ref_audio_artifact_ready_if_unreferenced`` will not fire for + the replaced file and its stale artifact key remains in + ``_ref_audio_model_artifact_ready``. This is functionally correct + (the stale entry is never *used*) but doubles memory until eviction. + """ + cache_key_source = ref_audio_str + try: + path = OmniOpenAIServingSpeech._local_ref_audio_stat_path(ref_audio_str) + except OSError as exc: + path = None + logger.debug("Skipping local-file cache metadata for %s: %s", ref_audio_str[:80], exc) + if path is not None: + try: + # Only stat paths the server operator has explicitly permitted + # via --allowed-local-media-path. The default is "" (and + # diffusion mode passes None); MediaConnector refuses local + # file loads in both cases, so a stat is exposure for zero + # benefit. ``if not`` covers "" and None; ``is None`` would + # treat "" as an allowlist of the process cwd. + if not allowed_local_media_path: + raise OSError("no allowed_local_media_path configured; skipping stat") + resolved = os.path.realpath(path) + allowed_base = os.path.realpath(allowed_local_media_path) + if os.path.commonpath([resolved, allowed_base]) != allowed_base: + raise OSError("path outside allowed_local_media_path; skipping stat") + st = os.stat(path) + cache_key_source = f"{ref_audio_str}:{st.st_mtime_ns}:{st.st_size}" + except (OSError, ValueError): + # Truncate the value to avoid dumping huge base64 blobs into + # the server log when a client omits the ``data:`` prefix. + display = ref_audio_str[:80] + if len(ref_audio_str) > 80: + display += f"... ({len(ref_audio_str)} chars)" + logger.debug( + "Failed to stat ref_audio path %s; falling back to string-only cache key (stale cache possible)", + display, + ) + return hashlib.sha1(cache_key_source.encode("utf-8")).hexdigest() + + async def _ref_audio_cache_key(self, ref_audio_str: str, allowed_local_media_path: str | None) -> str: + """Stat local files off the event loop; remote locators stay a cheap hash.""" + return await asyncio.to_thread( + self._get_ref_audio_cache_key, + ref_audio_str, + allowed_local_media_path, + ) + + def _finalize_fetched_ref_audio(self, wav_np: np.ndarray, sr: int) -> tuple[list[float], int, str, float]: wav_np = np.asarray(wav_np, dtype=np.float32) if wav_np.ndim > 1: wav_np = np.mean(wav_np, axis=-1) sr = int(sr) - artifact_key = self._make_ref_audio_artifact_cache_key(wav_np, sr) duration = len(wav_np) / sr if sr > 0 else 0.0 if duration < _REF_AUDIO_MIN_DURATION: raise ValueError( @@ -1738,19 +1838,92 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): f"Reference audio too long ({duration:.1f}s). " f"Maximum {_REF_AUDIO_MAX_DURATION:.0f}s supported — use a shorter clip." ) - tolist_start_s = time.perf_counter() - wav_list = wav_np.tolist() - tolist_ms = (time.perf_counter() - tolist_start_s) * 1000.0 - logger.debug( - "Resolved ref_audio: fetch_decode_ms=%.3f tolist_ms=%.3f samples=%d sr=%d duration_s=%.3f", - fetch_decode_ms, - tolist_ms, - len(wav_np), - sr, - duration, + artifact_key = self._make_ref_audio_artifact_cache_key(wav_np, sr) + return wav_np.tolist(), sr, artifact_key, duration + + async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int, str]: + """Resolve ref_audio to (wav_samples, sample_rate, cache_key). + + Delegates to upstream vLLM's MediaConnector which handles http(s) + URLs, ``data:`` base64 URIs, and ``file:`` local paths (the latter + gated by ``--allowed-local-media-path``). + + Local file references incorporate mtime and size into the cache key + so that modified files are automatically reloaded without a server + restart. Remote URLs remain cached by their original string locator. + + The returned *cache_key* should be passed to + ``_get_resolved_ref_audio_artifact_key`` when the caller needs the + artifact key, avoiding a redundant ``os.stat`` and the TOCTOU window. + After a cache miss the file is re-stat'd; a metadata change retries + the fetch so the stored waveform matches the key. + """ + # Pass the allowed-local-media-path so the stat is restricted to + # paths the server operator has explicitly permitted. + allowed_path = None + if not self._diffusion_mode: + allowed_path = getattr(self.model_config, "allowed_local_media_path", None) + + wav_list: list[float] | None = None + sr = 0 + artifact_key = "" + post_key = "" + connector: MediaConnector | None = None + for attempt in range(_REF_AUDIO_METADATA_FETCH_ATTEMPTS): + cache_key = await self._ref_audio_cache_key(ref_audio_str, allowed_path) + cached = self._ref_audio_resolve_cache.get(cache_key) + if cached is not None: + self._ref_audio_resolve_cache.move_to_end(cache_key) + wav_list, sr, _, _ = cached + logger.debug( + "Resolved ref_audio from cache: samples=%d sr=%d duration_s=%.3f", + len(wav_list), + sr, + len(wav_list) / sr if sr > 0 else 0.0, + ) + return wav_list, sr, cache_key + + if connector is None: + # In diffusion mode, model_config may not be available + if self._diffusion_mode: + connector = MediaConnector() + else: + model_config = self.model_config + connector = MediaConnector( + allowed_local_media_path=model_config.allowed_local_media_path, + allowed_media_domains=model_config.allowed_media_domains, + ) + + fetch_start_s = time.perf_counter() + wav_np, fetched_sr = await connector.fetch_audio_async(ref_audio_str) + fetch_decode_ms = (time.perf_counter() - fetch_start_s) * 1000.0 + tolist_start_s = time.perf_counter() + wav_list, sr, artifact_key, duration = self._finalize_fetched_ref_audio(wav_np, fetched_sr) + tolist_ms = (time.perf_counter() - tolist_start_s) * 1000.0 + logger.debug( + "Resolved ref_audio: fetch_decode_ms=%.3f tolist_ms=%.3f samples=%d sr=%d duration_s=%.3f", + fetch_decode_ms, + tolist_ms, + len(wav_list), + sr, + duration, + ) + post_key = await self._ref_audio_cache_key(ref_audio_str, allowed_path) + if post_key == cache_key: + self._put_resolved_ref_audio(cache_key, wav_list, sr, artifact_key) + return wav_list, sr, cache_key + logger.debug( + "ref_audio metadata changed during fetch (attempt %d/%d); retrying", + attempt + 1, + _REF_AUDIO_METADATA_FETCH_ATTEMPTS, + ) + + logger.warning( + "ref_audio file changed during fetch after %d attempts; skipping resolve cache", + _REF_AUDIO_METADATA_FETCH_ATTEMPTS, ) - self._put_resolved_ref_audio(cache_key, wav_list, sr, artifact_key) - return wav_list, sr + assert wav_list is not None and post_key + return wav_list, sr, post_key @staticmethod def _make_ref_audio_artifact_cache_key(wav: np.ndarray, sr: int) -> str: @@ -1761,12 +1934,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): h.update(wav_f32.tobytes(order="C")) return h.hexdigest() - def _get_resolved_ref_audio_artifact_key(self, ref_audio_str: str) -> str | None: - source_key = hashlib.sha1(ref_audio_str.encode("utf-8")).hexdigest() - cached = self._ref_audio_resolve_cache.get(source_key) + def _get_resolved_ref_audio_artifact_key(self, cache_key: str) -> str | None: + """Look up the artifact key for a previously resolved ref_audio. + + *cache_key* must be the exact key returned by the preceding + ``_resolve_ref_audio`` call. Passing the key explicitly avoids a + second ``os.stat`` syscall and eliminates the TOCTOU window that + would arise from recomputing the key independently. + """ + cached = self._ref_audio_resolve_cache.get(cache_key) if cached is None: return None - self._ref_audio_resolve_cache.move_to_end(source_key) + self._ref_audio_resolve_cache.move_to_end(cache_key) return cached[3] def _put_resolved_ref_audio(self, cache_key: str, wav_list: list[float], sr: int, artifact_key: str) -> None: @@ -1831,7 +2010,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): async def _resolve_ref_audio_many(self, ref_audio_list: list[str]) -> list[tuple[list[float], int]]: resolved = [] for ref_audio in ref_audio_list: - resolved.append(await self._resolve_ref_audio(ref_audio)) + wav_list, sr, _ = await self._resolve_ref_audio(ref_audio) + resolved.append((wav_list, sr)) return resolved # ---- Ming TTS helpers ---- @@ -2502,7 +2682,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): + mm_processor_kwargs with prompt_text. """ # Resolve reference audio - wav_samples, sr = await self._resolve_ref_audio(request.ref_audio) + wav_samples, sr, _ = await self._resolve_ref_audio(request.ref_audio) audio_data = (np.asarray(wav_samples, dtype=np.float32), sr) # Wrap the reference transcript in the CosyVoice3 instruction template @@ -2616,7 +2796,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): """ # Voice cloning requires ref_audio + ref_text if request.ref_audio is not None and request.ref_text: - wav_samples, sr = await self._resolve_ref_audio(request.ref_audio) + wav_samples, sr, _ = await self._resolve_ref_audio(request.ref_audio) audio_data = (np.asarray(wav_samples, dtype=np.float32), int(sr)) mm_kwargs: dict[str, Any] = { @@ -2748,8 +2928,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Uploaded voice: ref_audio was auto-set as [base64_data_url] ref_audio_source = tts_params["ref_audio"][0] if ref_audio_source is not None and isinstance(ref_audio_source, str): - wav_list, sr = await self._resolve_ref_audio(ref_audio_source) - artifact_key = self._get_resolved_ref_audio_artifact_key(ref_audio_source) + wav_list, sr, cache_key = await self._resolve_ref_audio(ref_audio_source) + tts_params["ref_audio_cache_key"] = cache_key + artifact_key = self._get_resolved_ref_audio_artifact_key(cache_key) if artifact_key: tts_params[_QWEN3_TTS_REF_AUDIO_CACHE_KEY] = [artifact_key] ref_code_length = self._estimate_ref_code_len([wav_list, sr]) @@ -3142,7 +3323,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): request_id = f"speech-{random_uuid()}" prompt: dict[str, Any] = {"input": request.input} if request.ref_audio: - wav, sr = await self._resolve_ref_audio(request.ref_audio) + wav, sr, _ = await self._resolve_ref_audio(request.ref_audio) prompt["ref_audio"] = (np.asarray(wav, dtype=np.float32), sr) if request.ref_text: prompt["ref_text"] = request.ref_text diff --git a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py index 48d5886c4..a6426a13d 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/fish_speech.py @@ -49,13 +49,14 @@ class FishSpeechAdapter(ARTTSAdapter): ) -> PreparedRequest: server = self.ctx.server ref_audio_data = None + tts_params: dict = {} if request.ref_audio is not None: - wav_list, sr = await server._resolve_ref_audio(request.ref_audio) + wav_list, sr, cache_key = await server._resolve_ref_audio(request.ref_audio) ref_audio_data = (wav_list, sr) + tts_params["ref_audio_cache_key"] = cache_key prompt = await server._build_fish_speech_prompt_async( request, ref_audio_data=ref_audio_data, has_inline_ref_audio=has_inline_ref_audio ) - tts_params = {} prompt["cache_salt"] = conditioning_cache_salt(request, tts_params) return PreparedRequest(prompt=prompt, tts_params=tts_params, model_type="fish_speech") diff --git a/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py b/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py index de70cd5b8..500287e1c 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/indextts2.py @@ -108,6 +108,8 @@ def indextts2_conditioning_cache_salt( "use_random", "lang", "text_normalization", + "ref_audio_cache_key", + "emo_audio_cache_key", ): h.update(b"\x00") h.update(key.encode("utf-8")) @@ -260,8 +262,9 @@ class IndexTTS2Adapter(ARTTSAdapter): ref_audio_source = server._get_uploaded_audio_data(voice_lower) using_uploaded_voice = ref_audio_source is not None if ref_audio_source is not None and isinstance(ref_audio_source, str): - wav_list, sr = await server._resolve_ref_audio(ref_audio_source) + wav_list, sr, cache_key = await server._resolve_ref_audio(ref_audio_source) params["voice"] = [[wav_list, sr]] + params["ref_audio_cache_key"] = [cache_key] if using_uploaded_voice and voice_lower: params["voice_name"] = [voice_lower] params["voice_created_at"] = [server._voice_created_at(voice_lower)] @@ -271,8 +274,9 @@ class IndexTTS2Adapter(ARTTSAdapter): if key not in extras: continue if key == "emo_audio": - wav_list, sr = await server._resolve_ref_audio(extras[key]) + wav_list, sr, emo_key = await server._resolve_ref_audio(extras[key]) params[key] = [[wav_list, sr]] + params["emo_audio_cache_key"] = [emo_key] else: params[key] = [extras[key]] return params diff --git a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py index eab4acd74..613968b8c 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/ming_tts.py @@ -64,7 +64,7 @@ class MingTTSAdapter(ARTTSAdapter): if isinstance(ref_audio_source, list): ref_audio_data = await server._resolve_ref_audio_many(ref_audio_source) elif ref_audio_source is not None and isinstance(ref_audio_source, str): - wav_list, sr = await server._resolve_ref_audio(ref_audio_source) + wav_list, sr, _ = await server._resolve_ref_audio(ref_audio_source) ref_audio_data = (wav_list, sr) prompt = server._build_ming_dense_prompt( request, diff --git a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py index 81bda2078..e51b14c67 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/moss_tts.py @@ -91,7 +91,7 @@ class _MossTTSAdapterBase(ARTTSAdapter): self, request: "OpenAICreateSpeechRequest", sampling_params_list: list, has_inline_ref_audio: bool ) -> PreparedRequest: server = self.ctx.server - tts_params = await server._build_moss_tts_params(request) + tts_params = await server._build_moss_tts_params(request, has_inline_ref_audio=has_inline_ref_audio) if request.voice: voice_lower = request.voice.lower() if voice_lower in server.uploaded_speakers and not has_inline_ref_audio: diff --git a/vllm_omni/entrypoints/openai/tts_adapters/omnivoice.py b/vllm_omni/entrypoints/openai/tts_adapters/omnivoice.py index aa3ddd3b4..deafab685 100644 --- a/vllm_omni/entrypoints/openai/tts_adapters/omnivoice.py +++ b/vllm_omni/entrypoints/openai/tts_adapters/omnivoice.py @@ -33,7 +33,7 @@ class OmniVoiceAdapter(ARTTSAdapter): server = self.ctx.server prompt: dict = {"input": request.input} if request.ref_audio: - wav, sr = await server._resolve_ref_audio(request.ref_audio) + wav, sr, _ = await server._resolve_ref_audio(request.ref_audio) prompt["ref_audio"] = (np.asarray(wav, dtype=np.float32), sr) if request.ref_text: prompt["ref_text"] = request.ref_text diff --git a/vllm_omni/model_executor/models/moss_tts/reference_encoder.py b/vllm_omni/model_executor/models/moss_tts/reference_encoder.py index 701f673a5..ef1abfa86 100644 --- a/vllm_omni/model_executor/models/moss_tts/reference_encoder.py +++ b/vllm_omni/model_executor/models/moss_tts/reference_encoder.py @@ -8,14 +8,13 @@ in ``serving_speech.py``. The serving layer just calls :func:`encode_reference_codes` with its generic helpers (the audio resolver and the process-wide speaker cache). -Kept import-light (only ``asyncio`` / ``hashlib`` / ``torch`` plus the logger) +Kept import-light (only ``asyncio`` / ``torch`` plus the logger) so importing it from the API-server process does not pull the talker/codec. """ from __future__ import annotations import asyncio -import hashlib from collections.abc import Awaitable, Callable from typing import Any @@ -43,7 +42,7 @@ async def encode_reference_codes( ref_str: str, *, processor: Any, - resolve_ref_audio: Callable[[str], Awaitable[tuple[list, int]]], + resolve_ref_audio: Callable[[str], Awaitable[tuple[list, int, str]]], speaker_cache: Any, variant: str, n_vq: int, @@ -62,10 +61,15 @@ async def encode_reference_codes( ``asyncio.to_thread`` so cold/anonymous encodes from concurrent requests overlap instead of serializing on the event loop. + For named voices the speaker cache is checked *before* resolving the audio, + avoiding the decode cost when the cache is warm. For anonymous references + the audio is resolved first so the cache key incorporates mtime/size from + a single stat (no TOCTOU window). + Args: ref_str: The raw reference audio (URL / path / data URL) as received. processor: Upstream MOSS processor exposing ``encode_audios_from_wav``. - resolve_ref_audio: Async callable mapping ``ref_str`` to ``(wav_list, sr)``. + resolve_ref_audio: Async callable mapping ``ref_str`` to ``(wav_list, sr, cache_key)``. speaker_cache: Process-wide ``SpeakerEmbeddingCache``. variant: MOSS sub-variant (``tts`` / ``ttsd`` / ...), namespaces the cache key. n_vq: Number of RVQ codebooks (also namespaces the cache key). @@ -76,24 +80,45 @@ async def encode_reference_codes( Returns: The reference RVQ codes tensor (CPU), ready to pass to the processor. """ + model_type = f"moss_tts_{variant}_nq{n_vq}" + + # ---- Named-voice branch: check cache *before* resolving ---- + # When voice_name is set, the cache key is (voice_name, created_at) which + # does not depend on the resolved audio content. The serving layer only + # passes voice_name for uploaded speakers without an inline ref_audio; + # re-upload bumps created_at and clears the speaker-cache slot. if voice_name: speaker_name = voice_name created_at = int(voice_created_at) + cache_key = speaker_cache.make_cache_key( + speaker_name, + model_type=model_type, + created_at=created_at, + ) + cached = speaker_cache.get(cache_key) + if cached is not None: + logger.debug("Speaker cache HIT for MOSS-TTS reference '%s'", speaker_name) + return cached["codes"].clone() + # Cache miss — resolve the audio for encoding. + wav_list, sr, _ = await resolve_ref_audio(ref_str) else: - speaker_name = "ref:" + hashlib.sha1((ref_str or "").encode("utf-8")).hexdigest() + # ---- Anonymous branch: resolve first for content-addressed key ---- + # The resolve incorporates mtime/size from a single stat so the key + # and the waveform always come from the same filesystem snapshot. + wav_list, sr, resolve_cache_key = await resolve_ref_audio(ref_str) + speaker_name = "ref:" + resolve_cache_key created_at = 0 + cache_key = speaker_cache.make_cache_key( + speaker_name, + model_type=model_type, + created_at=created_at, + ) + cached = speaker_cache.get(cache_key) + if cached is not None: + logger.debug("Speaker cache HIT for MOSS-TTS reference '%s'", speaker_name) + return cached["codes"].clone() - cache_key = speaker_cache.make_cache_key( - speaker_name, - model_type=f"moss_tts_{variant}_nq{n_vq}", - created_at=created_at, - ) - cached = speaker_cache.get(cache_key) - if cached is not None: - logger.debug("Speaker cache HIT for MOSS-TTS reference '%s'", speaker_name) - return cached["codes"].clone() - - wav_list, sr = await resolve_ref_audio(ref_str) + # ---- Cold miss: encode in a worker thread and store ---- codes = await asyncio.to_thread(_encode_wav_sync, processor, wav_list, sr, sr_target, n_vq) speaker_cache.put(cache_key, {"codes": codes.detach().cpu()}) logger.debug("Speaker cache STORE for MOSS-TTS reference '%s'", speaker_name)