vllm-omni/tests/entrypoints/openai_api/test_moss_registered_reference.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

import asyncio
from unittest.mock import Mock

import numpy as np
import pytest
import torch
from safetensors.torch import save_file

from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext, conditioning_cache_salt
from vllm_omni.entrypoints.openai.tts_adapters.moss_tts import MossTTSAdapter
from vllm_omni.model_executor.models.moss_tts.reference_encoder import MossReferenceEncoder
from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache

pytestmark = [pytest.mark.core_model, pytest.mark.cpu]


@pytest.fixture
def setup(tmp_path):
    server = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
    server.uploaded_speakers_dir = tmp_path
    server.uploaded_speakers = {
        "alice": dict(
            embedding_source="audio", created_at=123, file_path=str(tmp_path / "old.safetensors"), ref_text="Reference."
        )
    }
    server._speaker_cache = SpeakerEmbeddingCache()
    server._ref_audio_data_url_cache = {}
    server._ref_audio_resolve_cache = {}
    server._voice_created_at = Mock(side_effect=[123, 124])
    adapter = MossTTSAdapter(SpeechServingContext(server=server))
    adapter._moss_variant = "local"
    encoder = MossReferenceEncoder(None, variant="local", n_vq=12, sr_target=24000, speaker_cache=server._speaker_cache)
    adapter._get_moss_ref_encoder = lambda: encoder

    class Processor:
        def build_user_message(self, **kwargs):
            return kwargs

        def build_assistant_message(self, **kwargs):
            return kwargs

        def __call__(self, *, conversations, mode):
            message = conversations[0][-1]
            codes = message.get("audio_codes_list", message.get("reference"))[0]
            text = torch.zeros((len(codes), 1), dtype=torch.int64)
            return {"input_ids": [torch.cat([text, codes], dim=1)]}

    adapter._get_moss_processor = lambda: Processor()
    return server, adapter, encoder


@pytest.mark.asyncio
async def test_hot_registered_voice_skips_uri_and_resolve(setup):
    server, adapter, encoder = setup
    codes = torch.arange(36, dtype=torch.int32).reshape(3, 12)
    server._speaker_cache.put(encoder._make_cache_key("alice", 123), {"codes": codes})
    for name in [
        "_get_uploaded_audio_data",
        "_validate_ref_audio_format",
        "_load_registered_reference",
        "_resolve_ref_audio_array",
    ]:
        setattr(server, name, Mock(side_effect=AssertionError(name)))
    request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
    assert adapter.validate(request) is None
    result = await adapter.build(request, [], False)
    assert torch.equal(result.tts_params["codes"]["ref"], codes)
    assert request.ref_audio is None
    assert not server._ref_audio_data_url_cache
    server._voice_created_at.assert_not_called()


@pytest.mark.asyncio
async def test_reupload_during_encoding_keeps_codes_and_salt_generation(setup):
    server, adapter, encoder = setup
    entered, release = asyncio.Event(), asyncio.Event()
    server._load_registered_reference = Mock(return_value=(np.zeros(24000, dtype=np.float32), 24000))

    async def submit(waveform, sr):
        entered.set()
        await release.wait()
        return torch.full((3, 12), 7, dtype=torch.int64)

    encoder._batcher.submit = submit
    request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
    assert adapter.validate(request) is None
    task = asyncio.create_task(adapter.build(request, [], False))
    await entered.wait()
    server.uploaded_speakers["alice"] = dict(server.uploaded_speakers["alice"], created_at=124, ref_text="New.")
    release.set()
    result = await task
    assert torch.equal(result.tts_params["codes"]["ref"], torch.full((3, 12), 7))
    assert result.tts_params["voice_created_at"] == [123]
    assert request.ref_text == "Reference."
    assert result.prompt["cache_salt"] == conditioning_cache_salt(
        request, result.tts_params, registered_voice=("alice", 123)
    )
    assert server._speaker_cache.get(encoder._make_cache_key("alice", 124)) is None
    server._voice_created_at.assert_not_called()


def test_inline_override_is_not_bound(setup):
    server, adapter, _ = setup
    request = OpenAICreateSpeechRequest(input="Hello", voice="alice", ref_audio="file:///reference.wav")
    assert adapter.validate(request) is None
    assert request._registered_voice_reference is None


def test_private_snapshot_cannot_be_supplied_as_json(setup):
    request = OpenAICreateSpeechRequest.model_validate(
        dict(input="Hello", voice="alice", _registered_voice_reference={"name": "alice", "created_at": 999})
    )
    assert request._registered_voice_reference is None


@pytest.mark.asyncio
@pytest.mark.parametrize("channels", [1, 2])
async def test_loaded_waveform_matches_legacy_data_uri(setup, channels):
    server, adapter, _ = setup
    samples = np.random.default_rng(1).uniform(-1.1, 1.1, (24000, channels)).astype(np.float32).squeeze()
    info = server.uploaded_speakers["alice"]
    save_file(
        {"audio": torch.from_numpy(samples)}, info["file_path"], metadata={"created_at": "123", "sample_rate": "24000"}
    )
    request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
    assert adapter.validate(request) is None
    waveform, sr = server._load_registered_reference(request._registered_voice_reference)
    from vllm.multimodal.media import MediaConnector

    legacy, legacy_sr = await MediaConnector().fetch_audio_async(server._get_uploaded_audio_data("alice"))
    expected, _, _, _ = server._finalize_fetched_ref_audio(legacy, legacy_sr)
    np.testing.assert_array_equal(waveform, expected)
    assert sr == legacy_sr
    # A deleted captured generation must not fall back to the new upload.
    from pathlib import Path

    Path(info["file_path"]).unlink()
    with pytest.raises((FileNotFoundError, OSError)):
        server._load_registered_reference(request._registered_voice_reference)


@pytest.mark.asyncio
async def test_registered_cold_and_hot_salt_match(setup):
    server, adapter, encoder = setup
    server._load_registered_reference = Mock(return_value=(np.zeros(24000, dtype=np.float32), 24000))

    async def submit(waveform, sr):
        return torch.full((3, 12), 7, dtype=torch.int64)

    encoder._batcher.submit = submit
    results = []
    for _ in range(2):
        request = OpenAICreateSpeechRequest(input="Hello", voice="alice", ref_text="Override.")
        assert adapter.validate(request) is None
        assert request.ref_text == "Override."
        results.append(await adapter.build(request, [], False))
    assert results[0].prompt["cache_salt"] == results[1].prompt["cache_salt"]
    server._load_registered_reference.assert_called_once()


@pytest.mark.parametrize("generation", [0, -1])
def test_invalid_registered_generation_is_rejected(setup, generation):
    server, adapter, _ = setup
    server.uploaded_speakers["alice"]["created_at"] = generation
    request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
    assert "no valid generation" in adapter.validate(request)
    assert request._registered_voice_reference is None


@pytest.mark.asyncio
async def test_new_request_uses_reuploaded_generation(setup):
    server, adapter, encoder = setup
    results = []
    for generation in [123, 124]:
        server.uploaded_speakers["alice"]["created_at"] = generation
        codes = torch.full((3, 12), generation, dtype=torch.int32)
        server._speaker_cache.put(encoder._make_cache_key("alice", generation), {"codes": codes})
        request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
        assert adapter.validate(request) is None
        result = await adapter.build(request, [], False)
        assert torch.equal(result.tts_params["codes"]["ref"], codes)
        results.append(result)
    assert results[0].prompt["cache_salt"] != results[1].prompt["cache_salt"]
    server.uploaded_speakers.clear()
    assert "requires 'ref_audio'" in adapter.validate(OpenAICreateSpeechRequest(input="Hello", voice="alice"))


def test_cold_load_rejects_mismatched_generation(setup):
    server, adapter, _ = setup
    info = server.uploaded_speakers["alice"]
    save_file({"audio": torch.zeros(24000)}, info["file_path"], metadata={"created_at": "124", "sample_rate": "24000"})
    request = OpenAICreateSpeechRequest(input="Hello", voice="alice")
    assert adapter.validate(request) is None
    with pytest.raises(ValueError, match="generation changed"):
        server._load_registered_reference(request._registered_voice_reference)