[Frontend] Tune reference cache limits and registered MOSS voice requests (#7883)

Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
Canlin Guo authored and GitHub committed 2026-09-29 04:10:17 +00:00
1 parent 1c1fdf719c
commit 235c430326
22 files changed
+752 -70

No files matched your search

@@ -366,6 +366,8 @@ class _FakeSocket:
class _FakeEngineClient:
config_path: str | None = None
def __init__(self, *, stage_configs=None, endpoint_restrictions=None, vllm_config=None) -> None:
self.stage_configs = stage_configs if stage_configs is not None else []
self.endpoint_restrictions = endpoint_restrictions if endpoint_restrictions is not None else {}
@@ -999,9 +1001,14 @@ async def test_pure_diffusion_app_state_key_snapshot(monkeypatch) -> None:
@pytest.mark.asyncio
async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) -> None:
async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch, tmp_path) -> None:
stage = SimpleNamespace(stage_type="diffusion", engine_args={})
engine = _FakeEngineClient(stage_configs=[stage])
base = tmp_path / "base.yaml"
base.write_text("speech_cache:\n resolve_max_bytes: 1234\n speaker_max_bytes: 0\n")
deploy = tmp_path / "deploy.yaml"
deploy.write_text("base_config: base.yaml\nspeech_cache:\n resolve_max_entries: 17\n")
engine.config_path = str(deploy)
speech_kwargs = {}
def _for_diffusion_factory(label: str):
@@ -1042,10 +1049,14 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
assert speech_kwargs["allowed_local_media_path"] == "/allowed/media"
assert speech_kwargs["allowed_media_domains"] == ["media.example.com"]
config = speech_kwargs["speech_cache_config"]
assert config.resolve_max_bytes == 1234
assert config.resolve_max_entries == 17
assert config.speaker_max_bytes == 0
@pytest.mark.asyncio
async def test_multistage_app_state_key_snapshot(monkeypatch) -> None:
async def test_multistage_app_state_key_snapshot(monkeypatch, tmp_path) -> None:
"""Lock multi-stage ``app.state`` keys after init, including live vs None.
Fails if chat/speech/video/realtime/tokenization keys disappear or are
@@ -1075,7 +1086,15 @@ async def test_multistage_app_state_key_snapshot(monkeypatch) -> None:
def warmup(self):
return None
deploy = tmp_path / "deploy.yaml"
deploy.write_text("speech_cache:\n resolve_max_bytes: 1024\n speaker_max_bytes: 8\n")
engine.config_path = str(deploy)
speech_kwargs = {}
class _FakeSpeech(_FakeCtor):
def __init__(self, *args, **kwargs):
speech_kwargs.update(kwargs)
async def warmup(self):
return None
@@ -1104,6 +1123,9 @@ async def test_multistage_app_state_key_snapshot(monkeypatch) -> None:
state = State()
await api_server.omni_init_app_state(engine, state, _minimal_args())
assert speech_kwargs["speech_cache_config"].resolve_max_bytes == 1024
assert speech_kwargs["speech_cache_config"].resolve_max_entries == 2048
assert speech_kwargs["speech_cache_config"].speaker_max_bytes == 8
_assert_app_state_snapshot(
state,
@@ -0,0 +1,200 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import asyncio
from unittest.mock import Mock
import numpy as np
import pytest
import torch
from safetensors.torch import save_file
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext, conditioning_cache_salt
from vllm_omni.entrypoints.openai.tts_adapters.moss_tts import MossTTSAdapter
from vllm_omni.model_executor.models.moss_tts.reference_encoder import MossReferenceEncoder
from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def setup(tmp_path):
server = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
server.uploaded_speakers_dir = tmp_path
server.uploaded_speakers = {
"alice": dict(
embedding_source="audio", created_at=123, file_path=str(tmp_path / "old.safetensors"), ref_text="Reference."
)
}
server._speaker_cache = SpeakerEmbeddingCache()
server._ref_audio_data_url_cache = {}
server._ref_audio_resolve_cache = {}
server._voice_created_at = Mock(side_effect=[123, 124])
adapter = MossTTSAdapter(SpeechServingContext(server=server))
adapter._moss_variant = "local"
encoder = MossReferenceEncoder(None, variant="local", n_vq=12, sr_target=24000, speaker_cache=server._speaker_cache)
adapter._get_moss_ref_encoder = lambda: encoder
class Processor:
def build_user_message(self, **kwargs):
return kwargs
def build_assistant_message(self, **kwargs):
return kwargs
def __call__(self, *, conversations, mode):
message = conversations[0][-1]
codes = message.get("audio_codes_list", message.get("reference"))[0]
text = torch.zeros((len(codes), 1), dtype=torch.int64)
return {"input_ids": [torch.cat([text, codes], dim=1)]}
adapter._get_moss_processor = lambda: Processor()
return server, adapter, encoder
@pytest.mark.asyncio
async def test_hot_registered_voice_skips_uri_and_resolve(setup):
server, adapter, encoder = setup
codes = torch.arange(36, dtype=torch.int32).reshape(3, 12)
server._speaker_cache.put(encoder._make_cache_key("alice", 123), {"codes": codes})
for name in [
"_get_uploaded_audio_data",
"_validate_ref_audio_format",
"_load_registered_reference",
"_resolve_ref_audio_array",
]:
setattr(server, name, Mock(side_effect=AssertionError(name)))
request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
assert adapter.validate(request) is None
result = await adapter.build(request, [], False)
assert torch.equal(result.tts_params["codes"]["ref"], codes)
assert request.ref_audio is None
assert not server._ref_audio_data_url_cache
server._voice_created_at.assert_not_called()
@pytest.mark.asyncio
async def test_reupload_during_encoding_keeps_codes_and_salt_generation(setup):
server, adapter, encoder = setup
entered, release = asyncio.Event(), asyncio.Event()
server._load_registered_reference = Mock(return_value=(np.zeros(24000, dtype=np.float32), 24000))
async def submit(waveform, sr):
entered.set()
await release.wait()
return torch.full((3, 12), 7, dtype=torch.int64)
encoder._batcher.submit = submit
request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
assert adapter.validate(request) is None
task = asyncio.create_task(adapter.build(request, [], False))
await entered.wait()
server.uploaded_speakers["alice"] = dict(server.uploaded_speakers["alice"], created_at=124, ref_text="New.")
release.set()
result = await task
assert torch.equal(result.tts_params["codes"]["ref"], torch.full((3, 12), 7))
assert result.tts_params["voice_created_at"] == [123]
assert request.ref_text == "Reference."
assert result.prompt["cache_salt"] == conditioning_cache_salt(
request, result.tts_params, registered_voice=("alice", 123)
)
assert server._speaker_cache.get(encoder._make_cache_key("alice", 124)) is None
server._voice_created_at.assert_not_called()
def test_inline_override_is_not_bound(setup):
server, adapter, _ = setup
request = OpenAICreateSpeechRequest(input="Hello", voice="alice", ref_audio="file:///reference.wav")
assert adapter.validate(request) is None
assert request._registered_voice_reference is None
def test_private_snapshot_cannot_be_supplied_as_json(setup):
request = OpenAICreateSpeechRequest.model_validate(
dict(input="Hello", voice="alice", _registered_voice_reference={"name": "alice", "created_at": 999})
)
assert request._registered_voice_reference is None
@pytest.mark.asyncio
@pytest.mark.parametrize("channels", [1, 2])
async def test_loaded_waveform_matches_legacy_data_uri(setup, channels):
server, adapter, _ = setup
samples = np.random.default_rng(1).uniform(-1.1, 1.1, (24000, channels)).astype(np.float32).squeeze()
info = server.uploaded_speakers["alice"]
save_file(
{"audio": torch.from_numpy(samples)}, info["file_path"], metadata={"created_at": "123", "sample_rate": "24000"}
)
request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
assert adapter.validate(request) is None
waveform, sr = server._load_registered_reference(request._registered_voice_reference)
from vllm.multimodal.media import MediaConnector
legacy, legacy_sr = await MediaConnector().fetch_audio_async(server._get_uploaded_audio_data("alice"))
expected, _, _, _ = server._finalize_fetched_ref_audio(legacy, legacy_sr)
np.testing.assert_array_equal(waveform, expected)
assert sr == legacy_sr
# A deleted captured generation must not fall back to the new upload.
from pathlib import Path
Path(info["file_path"]).unlink()
with pytest.raises((FileNotFoundError, OSError)):
server._load_registered_reference(request._registered_voice_reference)
@pytest.mark.asyncio
async def test_registered_cold_and_hot_salt_match(setup):
server, adapter, encoder = setup
server._load_registered_reference = Mock(return_value=(np.zeros(24000, dtype=np.float32), 24000))
async def submit(waveform, sr):
return torch.full((3, 12), 7, dtype=torch.int64)
encoder._batcher.submit = submit
results = []
for _ in range(2):
request = OpenAICreateSpeechRequest(input="Hello", voice="alice", ref_text="Override.")
assert adapter.validate(request) is None
assert request.ref_text == "Override."
results.append(await adapter.build(request, [], False))
assert results[0].prompt["cache_salt"] == results[1].prompt["cache_salt"]
server._load_registered_reference.assert_called_once()
@pytest.mark.parametrize("generation", [0, -1])
def test_invalid_registered_generation_is_rejected(setup, generation):
server, adapter, _ = setup
server.uploaded_speakers["alice"]["created_at"] = generation
request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
assert "no valid generation" in adapter.validate(request)
assert request._registered_voice_reference is None
@pytest.mark.asyncio
async def test_new_request_uses_reuploaded_generation(setup):
server, adapter, encoder = setup
results = []
for generation in [123, 124]:
server.uploaded_speakers["alice"]["created_at"] = generation
codes = torch.full((3, 12), generation, dtype=torch.int32)
server._speaker_cache.put(encoder._make_cache_key("alice", generation), {"codes": codes})
request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
assert adapter.validate(request) is None
result = await adapter.build(request, [], False)
assert torch.equal(result.tts_params["codes"]["ref"], codes)
results.append(result)
assert results[0].prompt["cache_salt"] != results[1].prompt["cache_salt"]
server.uploaded_speakers.clear()
assert "requires 'ref_audio'" in adapter.validate(OpenAICreateSpeechRequest(input="Hello", voice="alice"))
def test_cold_load_rejects_mismatched_generation(setup):
server, adapter, _ = setup
info = server.uploaded_speakers["alice"]
save_file({"audio": torch.zeros(24000)}, info["file_path"], metadata={"created_at": "124", "sample_rate": "24000"})
request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
assert adapter.validate(request) is None
with pytest.raises(ValueError, match="generation changed"):
server._load_registered_reference(request._registered_voice_reference)
@@ -275,6 +275,7 @@ def test_sleep_then_wakeup_roundtrip(sleep_capable_engine):
@pytest.fixture
def pure_diffusion_engine(mocker):
engine = mocker.MagicMock()
engine.config_path = None
engine.stage_configs = [{"stage_type": "diffusion"}]
engine.sleep = mocker.AsyncMock(return_value=[FakeAck(stage_id=0)])
engine.wake_up = mocker.AsyncMock(return_value=[FakeAck(stage_id=0)])
@@ -8,11 +8,8 @@ from collections import OrderedDict
import numpy as np
import pytest
from vllm_omni.entrypoints.openai.serving_speech import (
_REF_AUDIO_RESOLVE_CACHE_MAX_BYTES,
_REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES,
OmniOpenAIServingSpeech,
)
from vllm_omni.config.speech_cache import SpeechCacheConfig
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -24,8 +21,9 @@ def cache():
server.model_config = None
server._ref_audio_resolve_cache = OrderedDict()
server._ref_audio_resolve_cache_bytes = 0
server._ref_audio_resolve_cache_max_entries = _REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES
server._ref_audio_resolve_cache_max_bytes = _REF_AUDIO_RESOLVE_CACHE_MAX_BYTES
config = SpeechCacheConfig()
server._ref_audio_resolve_cache_max_entries = config.resolve_max_entries
server._ref_audio_resolve_cache_max_bytes = config.resolve_max_bytes
server._ref_audio_model_artifact_ready = set()
server._request_ref_audio_artifact_keys = {}
return server
@@ -36,8 +34,8 @@ def _put(cache, key, samples, artifact=None):
def test_compact_storage_and_default_capacity(cache):
assert cache._ref_audio_resolve_cache_max_entries == 1024
assert cache._ref_audio_resolve_cache_max_bytes == 512 * 1024 * 1024
assert cache._ref_audio_resolve_cache_max_entries == 2048
assert cache._ref_audio_resolve_cache_max_bytes == 4 * 1024**3
_put(cache, "a", [0.0, 0.25, -0.5, 1.0])
value = cache._ref_audio_resolve_cache["a"]
assert value[0].dtype == np.float32
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import numpy as np
import pytest
from vllm_omni.config.speech_cache import SpeechCacheConfig
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def server(monkeypatch, tmp_path):
import vllm_omni.utils.speaker_cache as module
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
monkeypatch.setattr(module, "_SINGLETON", None)
instance = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
instance.speech_cache_config = SpeechCacheConfig()
return instance
def test_defaults(server):
server._init_speaker_storage()
assert server._ref_audio_resolve_cache_max_bytes == 4 * 1024**3
assert server._ref_audio_resolve_cache_max_entries == 2048
assert server._speaker_cache.stats()["max_bytes"] == 512 * 1024**2
@pytest.mark.parametrize("entries,budget", [(1, 1000), (10, 8)])
def test_reference_cache_enforces_configured_limits(server, entries, budget):
server.speech_cache_config = SpeechCacheConfig(
resolve_max_entries=entries, resolve_max_bytes=budget, speaker_max_bytes=16
)
server._init_speaker_storage()
server._put_resolved_ref_audio("a", np.asarray([0.0, 0.1], dtype=np.float32), 24000, "artifact-a")
server._put_resolved_ref_audio("b", np.asarray([0.2, 0.3], dtype=np.float32), 24000, "artifact-b")
assert list(server._ref_audio_resolve_cache) == ["b"]
assert server._ref_audio_resolve_cache_bytes == 8
assert server._speaker_cache.stats()["max_bytes"] == 16
@pytest.mark.parametrize("field", ["resolve_max_entries", "resolve_max_bytes"])
def test_zero_disables_storage(server, field):
server.speech_cache_config = SpeechCacheConfig(**{field: 0})
server._init_speaker_storage()
server._put_resolved_ref_audio("a", np.asarray([0.0], dtype=np.float32), 24000, "artifact-a")
assert not server._ref_audio_resolve_cache
assert server._ref_audio_resolve_cache_bytes == 0
def test_diffusion_receives_config(server):
config = SpeechCacheConfig(resolve_max_bytes=80, resolve_max_entries=1, speaker_max_bytes=0)
speech = OmniOpenAIServingSpeech.for_diffusion(object(), "test", speech_cache_config=config)
assert speech.speech_cache_config is config
assert speech._ref_audio_resolve_cache_max_bytes == 80
assert speech._speaker_cache.stats()["max_bytes"] == 0
@@ -4,6 +4,7 @@
import numpy as np
import pytest
from vllm_omni.config.speech_cache import SpeechCacheConfig
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -12,7 +13,9 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def server(monkeypatch, tmp_path):
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
return OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
instance = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
instance.speech_cache_config = SpeechCacheConfig()
return instance
@pytest.mark.parametrize("stereo", [False, True])
@@ -2144,7 +2144,9 @@ class TestTTSMethods:
speech_server._tts_model_type = "moss_tts"
speech_server._adapter = speech_server._get_tts_adapter()
speech_server._adapter._moss_variant = "ttsd"
speech_server.uploaded_speakers = {"alice": {}}
speech_server.uploaded_speakers = {
"alice": {"embedding_source": "audio", "created_at": 42, "file_path": "/test.safetensors"}
}
mocker.patch.object(speech_server, "_voice_created_at", return_value=42)
proc = mocker.MagicMock()
@@ -2166,16 +2168,16 @@ class TestTTSMethods:
req = OpenAICreateSpeechRequest(
input="hello",
voice="alice",
ref_audio="data:audio/wav;base64,aaa",
ref_audio_2="data:audio/wav;base64,bbb",
)
assert speech_server._adapter._bind_registered_reference(req) is None
params = await speech_server._adapter._build_moss_tts_params(req, has_inline_ref_audio=False)
assert sorted(seen) == [
("data:audio/wav;base64,aaa", "alice"),
("data:audio/wav;base64,bbb", None),
("registered:alice:42", "alice"),
]
# Per-slot salt keys survive the concurrent (gather) encode order.
assert params["ref_audio_cache_key"] == "rk:data:audio/wav;base64,aaa"
assert "ref_audio_cache_key" not in params
assert params["ref_audio_2_cache_key"] == "rk:data:audio/wav;base64,bbb"
def test_precomputed_qwen3_voice_infers_base_without_ref_audio(self, speech_server):
@@ -219,7 +219,12 @@ def test_moss_reference_transcript_mode(variant, ref_text, mode, mocker):
reference = [torch.ones((3, 12), dtype=torch.int64)]
unified = torch.arange(52, dtype=torch.int64).reshape(1, 4, 13)
processor = mocker.Mock(return_value={"input_ids": unified})
server = mocker.Mock(_moss_variant="local", uploaded_speakers={"speaker": {}})
server = mocker.Mock(
_moss_variant="local",
uploaded_speakers={
"speaker": {"embedding_source": "audio", "created_at": 123, "file_path": "/test.safetensors"}
},
)
server._voice_created_at.return_value = 123
request = OpenAICreateSpeechRequest(
input="Target.", ref_text=ref_text, language="English", voice="Speaker", seed=0, max_new_tokens=2048
@@ -780,3 +785,42 @@ def test_qwen3_validate_rejects_no_voice_no_default_then_accepts():
if __name__ == "__main__":
pytest.main([__file__, "-v"])
def test_registered_voice_salt_does_not_materialize_audio():
from vllm_omni.entrypoints.openai.tts_adapters.base import conditioning_cache_salt
class NoRepr(str):
def __repr__(self):
raise AssertionError("Uploaded audio must not be serialized on the hot path")
request = OpenAICreateSpeechRequest(input="Target.", voice="speaker").model_copy(
update={"ref_audio": NoRepr("data:audio/wav;base64,AAAA")}
)
salt = conditioning_cache_salt(request, registered_voice=("speaker", 123))
assert salt
with pytest.raises(AssertionError, match="must not be serialized"):
conditioning_cache_salt(request)
@pytest.mark.parametrize(
"field, value",
[("input", "Different target."), ("ref_text", "Different reference."), ("language", "Chinese")],
)
def test_registered_voice_salt_preserves_request_conditioning(field, value):
from vllm_omni.entrypoints.openai.tts_adapters.base import conditioning_cache_salt
request = OpenAICreateSpeechRequest(input="Target.", voice="speaker")
salt = conditioning_cache_salt(request, registered_voice=("speaker", 123))
changed = request.model_copy(update={field: value})
assert conditioning_cache_salt(changed, registered_voice=("speaker", 123)) != salt
@pytest.mark.parametrize("key", ["ref_audio_2_cache_key", "task_type", "ref_text"])
def test_registered_voice_salt_preserves_resolved_conditioning(key):
from vllm_omni.entrypoints.openai.tts_adapters.base import conditioning_cache_salt
request = OpenAICreateSpeechRequest(input="Target.", voice="speaker")
first = conditioning_cache_salt(request, {key: "a"}, registered_voice=("speaker", 123))
second = conditioning_cache_salt(request, {key: "b"}, registered_voice=("speaker", 123))
assert first != second