vllm-omni/tests/benchmarks/test_realtime_tts_probe.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

"""The Seed-TTS Realtime probe queues for a duplex session slot instead of failing."""

from __future__ import annotations

import pytest

from vllm_omni.benchmarks.patch import patch as bench_patch
from vllm_omni.clients import duplex as duplex_client

pytestmark = [pytest.mark.core_model, pytest.mark.cpu]


class _AdmissionGate:
    """Stands in for ``DuplexClient``: refuses admission ``refusals`` times, then admits."""

    instances: list[_AdmissionGate] = []
    refusals = 0
    refusal_code = "resource_exhausted"

    def __init__(self, url: str, **kwargs: object) -> None:
        del url, kwargs
        self.entered = False
        type(self).instances.append(self)

    async def __aenter__(self) -> _AdmissionGate:
        cls = type(self)
        if cls.refusals > 0:
            cls.refusals -= 1
            raise duplex_client.DuplexProtocolError("duplex_session_capacity_exhausted: limit=4", code=cls.refusal_code)
        self.entered = True
        return self

    async def __aexit__(self, exc_type, exc, tb) -> None:
        del exc_type, exc, tb

    async def events(self):  # the collector's consume loop ends at once
        return
        yield  # pragma: no cover


@pytest.fixture
def gate(monkeypatch: pytest.MonkeyPatch) -> type[_AdmissionGate]:
    _AdmissionGate.instances = []
    _AdmissionGate.refusals = 0
    _AdmissionGate.refusal_code = "resource_exhausted"
    monkeypatch.setattr(duplex_client, "DuplexClient", _AdmissionGate)
    return _AdmissionGate


async def _configure(probe: bench_patch._RealtimeTTSProbe) -> None:
    try:
        await probe.configure("test-model", auto_response=True)
    finally:
        await probe.__aexit__(None, None, None)


@pytest.mark.asyncio
async def test_configure_waits_for_a_free_session_slot(gate) -> None:
    """Two refusals, then a slot: the request is served, not counted as failed."""
    gate.refusals = 2
    probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
    await _configure(probe)
    assert len(gate.instances) == 3
    assert gate.instances[-1].entered is True
    assert probe._client is gate.instances[-1]


@pytest.mark.asyncio
async def test_configure_raises_other_rejections_at_once(gate) -> None:
    gate.refusals = 1
    gate.refusal_code = "unsupported_turn_detection"
    probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
    with pytest.raises(duplex_client.DuplexProtocolError):
        await _configure(probe)
    assert len(gate.instances) == 1


@pytest.mark.asyncio
async def test_configure_gives_up_when_the_slot_wait_budget_is_spent(gate, monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(bench_patch._RealtimeTTSProbe, "_SESSION_SLOT_WAIT_S", 0.0)
    gate.refusals = 5
    probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
    with pytest.raises(duplex_client.DuplexProtocolError) as excinfo:
        await _configure(probe)
    assert excinfo.value.code == "resource_exhausted"
    assert len(gate.instances) == 1


class _SilenceSink:
    """Stands in for a configured ``DuplexClient``: records every appended chunk."""

    def __init__(self) -> None:
        self.config = duplex_client.SessionConfig()
        self.chunks: list[bytes] = []

    async def append_audio(self, pcm: bytes, *, is_speech: bool | None = None, video_frames=None) -> None:
        del video_frames
        assert is_speech is False
        self.chunks.append(pcm)


@pytest.mark.asyncio
async def test_stream_silence_stops_at_the_turn_end(monkeypatch: pytest.MonkeyPatch) -> None:
    """Silence is fed until the turn settles, not for the whole cap."""
    probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
    sink = _SilenceSink()
    probe._client = sink

    async def no_wait(_: float) -> None:
        return

    monkeypatch.setattr(bench_patch.asyncio, "sleep", no_wait)
    streamed = await probe.stream_silence(seconds=12.0, chunk_ms=200, until=lambda: len(sink.chunks) >= 3)
    assert len(sink.chunks) == 3
    assert streamed == pytest.approx(0.6)
    assert all(len(chunk) == sink.config.input_audio.byte_count(200) for chunk in sink.chunks)


@pytest.mark.asyncio
async def test_stream_silence_runs_to_the_cap_when_the_turn_never_settles(monkeypatch: pytest.MonkeyPatch) -> None:
    probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
    sink = _SilenceSink()
    probe._client = sink

    async def no_wait(_: float) -> None:
        return

    monkeypatch.setattr(bench_patch.asyncio, "sleep", no_wait)
    streamed = await probe.stream_silence(seconds=1.0, chunk_ms=200, until=lambda: False)
    assert len(sink.chunks) == 5
    assert streamed == pytest.approx(1.0)


class _TurnEvents:
    def __init__(self, response_ids: list[str], audio: dict[str, bytes], text: dict[str, str] | None = None) -> None:
        self.response_ids = response_ids
        self._audio = audio
        self._text = text or {}

    def audio_bytes(self, response_id: str) -> bytes:
        return self._audio.get(response_id, b"")

    def response_text(self, response_id: str) -> str:
        return self._text.get(response_id, "")


def test_turn_response_is_the_first_one_with_audio_even_if_the_model_spoke_again(caplog) -> None:
    events = _TurnEvents(["r0", "r1", "r2"], {"r1": b"\x01\x00", "r2": b"\x02\x00"})
    with caplog.at_level("WARNING", logger=bench_patch.logger.name):
        assert bench_patch._seed_tts_turn_response_id(events, 0, 0) == "r1"
    assert "spoke again" in caplog.text


def test_turn_response_ignores_responses_from_before_the_turn() -> None:
    events = _TurnEvents(["r0", "r1"], {"r0": b"\x01\x00", "r1": b"\x02\x00"})
    assert bench_patch._seed_tts_turn_response_id(events, 1, 0) == "r1"


def test_turn_without_audio_is_an_error() -> None:
    events = _TurnEvents(["r0"], {})
    with pytest.raises(RuntimeError, match="no audio response"):
        bench_patch._seed_tts_turn_response_id(events, 0, 0)


def test_stall_report_says_the_model_never_answered() -> None:
    events = _TurnEvents(["r0"], {"r0": b"\x01\x00"})
    report = bench_patch._seed_tts_turn_stall_report(events, 1, 3, 30.0)
    assert "turn 3 never started a response" in report
    assert "30.0s of silence" in report


def test_stall_report_lists_a_response_that_never_finished() -> None:
    events = _TurnEvents(["r1"], {"r1": b"\x01\x00\x02\x00"}, {"r1": "hello"})
    report = bench_patch._seed_tts_turn_stall_report(events, 0, 0, 4.2)
    assert "started 1 response(s) after 4.2s of silence" in report
    assert "r1 (4 audio bytes, text 'hello')" in report