vllm-omni/tests/entrypoints/openai_api/test_reference_waveform_cache.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

import numpy as np
import pytest

from vllm_omni.config.speech_cache import SpeechCacheConfig
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech

pytestmark = [pytest.mark.core_model, pytest.mark.cpu]


@pytest.fixture
def server(monkeypatch, tmp_path):
    monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
    instance = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
    instance.speech_cache_config = SpeechCacheConfig()
    return instance


@pytest.mark.parametrize("stereo", [False, True])
def test_finalized_waveform_owns_contiguous_numeric_storage(server, stereo):
    import gc

    source = np.linspace(-1, 1, 96000, dtype=np.float64).reshape(48000, 2)
    decoded = source if stereo else source[:, 0]
    expected = np.asarray(decoded, dtype=np.float32)
    if stereo:
        expected = expected.mean(axis=-1)
    waveform, sr, artifact, duration = server._finalize_fetched_ref_audio(decoded, 24000)
    assert waveform.dtype == np.float32
    assert waveform.flags.c_contiguous and waveform.flags.owndata
    assert not np.shares_memory(waveform, decoded)
    assert not gc.is_tracked(waveform)
    np.testing.assert_array_equal(waveform, expected)
    assert artifact == server._make_ref_audio_artifact_cache_key(expected, sr)
    assert duration == 2.0


@pytest.mark.asyncio
async def test_array_resolver_reuses_buffer_and_legacy_lists_are_temporary(server, mocker):
    server._init_speaker_storage()
    server._diffusion_mode = True
    server._allowed_local_media_path = ""
    server._ref_audio_cache_key = mocker.AsyncMock(return_value="locator")
    server._media_connector = mocker.Mock()
    server._media_connector.fetch_audio_async = mocker.AsyncMock(
        return_value=(np.linspace(-1, 1, 24000, dtype=np.float32), 24000)
    )
    first, sr, key = await server._resolve_ref_audio_array("reference")
    second, _, _ = await server._resolve_ref_audio_array("reference")
    legacy, _, _ = await server._resolve_ref_audio("reference")
    assert first is second
    assert isinstance(legacy, list)
    np.testing.assert_array_equal(first, legacy)
    legacy[0] = 100
    assert first[0] == -1
    assert server._ref_audio_resolve_cache[key][0] is first
    assert server._ref_audio_resolve_cache_bytes == first.nbytes == 96000
    server._media_connector.fetch_audio_async.assert_awaited_once()


@pytest.mark.parametrize("variant", [None, "local", "delay", "realtime"])
@pytest.mark.asyncio
async def test_moss_array_resolution_keeps_nano_list_transport(mocker, variant):
    from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
    from vllm_omni.entrypoints.openai.tts_adapters.moss_tts import _MossTTSAdapterBase

    server = mocker.Mock()
    server._resolve_ref_audio = mocker.AsyncMock(return_value=([0.0], 24000, "key"))
    waveform = np.zeros(1, dtype=np.float32)
    server._resolve_ref_audio_array = mocker.AsyncMock(return_value=(waveform, 24000, "key"))
    adapter = _MossTTSAdapterBase.__new__(_MossTTSAdapterBase)
    adapter.ctx = SpeechServingContext(server=server)
    adapter._moss_variant = variant
    result, _, _ = await adapter._resolve_ref_audio("reference")
    if variant is None:
        assert isinstance(result, list)
        server._resolve_ref_audio_array.assert_not_awaited()
    else:
        assert result is waveform
        server._resolve_ref_audio.assert_not_awaited()


def test_waveform_cache_eviction_releases_owned_buffer(server):
    import weakref

    server._init_speaker_storage()
    server._ref_audio_resolve_cache_max_entries = 1
    waveform = np.zeros(24000, dtype=np.float32)
    ref = weakref.ref(waveform)
    server._put_resolved_ref_audio("first", waveform, 24000, "artifact-first")
    del waveform
    assert ref() is not None
    server._put_resolved_ref_audio("second", np.zeros(24000, dtype=np.float32), 24000, "artifact-second")
    assert ref() is None
    assert server._ref_audio_resolve_cache_bytes == 96000


@pytest.mark.parametrize("entries, budget", [(1, 1000), (10, 8)])
def test_numeric_waveform_cache_limits(server, entries, budget):
    server._init_speaker_storage()
    server._ref_audio_resolve_cache_max_entries = entries
    server._ref_audio_resolve_cache_max_bytes = budget
    server._put_resolved_ref_audio("a", np.zeros(2, dtype=np.float32), 24000, "artifact-a")
    server._put_resolved_ref_audio("b", np.ones(2, dtype=np.float32), 24000, "artifact-b")
    assert list(server._ref_audio_resolve_cache) == ["b"]
    assert server._ref_audio_resolve_cache_bytes == 8


@pytest.mark.asyncio
async def test_cosyvoice_array_prompt_owns_buffer_without_list_roundtrip(mocker, monkeypatch):
    from types import SimpleNamespace

    from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
    from vllm_omni.entrypoints.openai.tts_adapters.cosyvoice3 import CosyVoice3Adapter

    monkeypatch.setenv("COSYVOICE3_REFERENCE_PREFETCH", "0")
    waveform = np.linspace(-1, 1, 24000, dtype=np.float32)
    server = mocker.Mock()
    server._resolve_ref_audio_array = mocker.AsyncMock(return_value=(waveform, 24000, "key"))
    server._resolve_ref_audio = mocker.AsyncMock(side_effect=AssertionError("unnecessary list conversion"))
    adapter = CosyVoice3Adapter(SpeechServingContext(server=server))
    request = SimpleNamespace(input="Target text.", ref_audio="reference", ref_text="Reference text.", voice=None)
    prompt = await adapter._build_prompt(request)
    actual, rate = prompt["multi_modal_data"]["audio"]
    np.testing.assert_array_equal(actual, waveform)
    assert rate == 24000 and actual.dtype == np.float32
    assert not np.shares_memory(actual, waveform)
    actual[0] = 100
    assert waveform[0] == -1
    assert prompt["prompt"] == request.input
    assert prompt["mm_processor_kwargs"]["prompt_text"].endswith("<|endofprompt|>Reference text.")
    server._resolve_ref_audio.assert_not_awaited()
    server._resolve_ref_audio_array.assert_awaited_once_with("reference")


@pytest.mark.asyncio
async def test_cosyvoice_reference_prefetch_resamples_and_warms_conditioning(mocker):
    from types import SimpleNamespace

    import vllm_omni.model_executor.models.cosyvoice3.cosyvoice3 as cosyvoice3
    from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
    from vllm_omni.entrypoints.openai.tts_adapters.cosyvoice3 import CosyVoice3Adapter

    waveform = np.linspace(-1, 1, 24000, dtype=np.float32)
    server = mocker.Mock()
    server._resolve_ref_audio_array = mocker.AsyncMock(return_value=(waveform, 24000, "key"))
    server.model_config.hf_config = SimpleNamespace(target_sr=16000)
    server.model_config.model = "model-dir"
    warm = mocker.patch.object(cosyvoice3, "prefetch_reference_conditioning")
    adapter = CosyVoice3Adapter(SpeechServingContext(server=server))
    request = SimpleNamespace(input="Target text.", ref_audio="reference", ref_text="Reference text.", voice=None)
    prompt = await adapter._build_prompt(request)
    audio, rate = prompt["multi_modal_data"]["audio"]
    # The processor receives samples at its own rate, so its resampling is a
    # no-op and its content-addressed cache lookup hits the warmed entry.
    assert rate == 16000 and prompt["mm_processor_kwargs"]["sample_rate"] == 16000
    assert audio.shape == (16000,) and audio.dtype == np.float32
    warm.assert_called_once()
    model_dir, config, (warmed, warmed_rate) = warm.call_args.args
    assert model_dir == "model-dir" and config is server.model_config.hf_config
    assert warmed is audio and warmed_rate == 16000
    assert waveform[0] == -1


@pytest.mark.parametrize("mode", ["ras", "standard"])
def test_cosyvoice_sampling_mode_sets_control_stops_without_mutating_defaults(mocker, mode):
    from vllm.sampling_params import SamplingParams

    from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
    from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
    from vllm_omni.entrypoints.openai.tts_adapters.cosyvoice3 import CosyVoice3Adapter
    from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config

    config = CosyVoice3Config()
    config.cosyvoice3_sampling_mode = mode
    serving = mocker.Mock(spec=OmniOpenAIServingSpeech)
    serving.model_config = mocker.Mock(hf_config=config)
    adapter = CosyVoice3Adapter(SpeechServingContext(server=serving))
    adapter._tokenizer = mocker.Mock()
    mocker.patch("vllm_omni.model_executor.models.cosyvoice3.utils.extract_text_token", return_value=(None, 9))
    defaults = [SamplingParams(stop_token_ids=[6562])]
    request = OpenAICreateSpeechRequest(input="Example text.", max_new_tokens=2048)
    result = adapter.apply_sampling_overrides(defaults, request)
    assert defaults[0].stop_token_ids == [6562]
    assert result[0].min_tokens == 18
    assert result[0].max_tokens == 2048
    expected = set(range(6561, 6761)) if mode == "standard" else {6562}
    assert set(result[0].stop_token_ids) == expected
    assert expected <= result[0].all_stop_token_ids