# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """OpenAI Realtime client events -> typed ``DuplexCommand`` objects and their mailbox payloads.""" from __future__ import annotations import base64 import numpy as np import pytest from vllm_omni.engine.duplex.commands import ( REALTIME_COMMAND_TYPES, AckPlayback, AppendAudio, AppendText, BargeIn, CancelInput, CancelResponse, ClearInput, ClearOutputAudio, CloseSession, Commit, CreateItem, CreateResponse, DeleteItem, DuplexCommand, DuplexCommandError, Heartbeat, SignalTurn, TruncateItem, UpdateSession, command_from_realtime, ) from vllm_omni.engine.duplex.realtime_commands import ( RealtimeInputDefaults, translate_realtime_command, ) pytestmark = [pytest.mark.core_model, pytest.mark.cpu] _LOUD_PCM16 = base64.b64encode(b"\x00\x10" * 8).decode("ascii") _SILENT_PCM16 = base64.b64encode(b"\x00\x00" * 8).decode("ascii") _JPEG_FRAME = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 12).decode("ascii") _MINIMAL_PAYLOADS: dict[str, tuple[dict[str, object], type[DuplexCommand]]] = { "input_audio_buffer.append": ({"audio": _LOUD_PCM16}, AppendAudio), "input_audio_buffer.commit": ({}, Commit), "input_audio_buffer.clear": ({}, ClearInput), "output_audio_buffer.clear": ({}, ClearOutputAudio), "response.create": ({}, CreateResponse), "response.cancel": ({}, CancelResponse), "conversation.item.create": ({"item": {"type": "message", "role": "user", "content": []}}, CreateItem), "conversation.item.delete": ({"item_id": "item_1"}, DeleteItem), "conversation.item.truncate": ({"item_id": "item_1", "audio_end_ms": 100}, TruncateItem), "session.update": ({"session": {"instructions": "hi"}}, UpdateSession), "playback.ack": ({"played_ms": 10}, AckPlayback), "session.heartbeat": ({}, Heartbeat), "session.close": ({}, CloseSession), "turn.signal": ({"event": "user_started"}, SignalTurn), "input.text.append": ({"text": "hello"}, AppendText), "input.cancel": ({}, CancelInput), "barge_in": ({}, BargeIn), } def test_every_realtime_command_type_has_a_mapping_case(): assert set(_MINIMAL_PAYLOADS) == set(REALTIME_COMMAND_TYPES) @pytest.mark.parametrize("event_type", sorted(REALTIME_COMMAND_TYPES)) def test_command_from_realtime_maps_each_type_to_its_dataclass_and_propagates_event_id(event_type: str): body, expected_cls = _MINIMAL_PAYLOADS[event_type] command = command_from_realtime({"type": event_type, "event_id": f"evt-{event_type}", **body}) assert type(command) is expected_cls assert command.event_id == f"evt-{event_type}" payload = command.payload() assert payload["type"] == expected_cls.type assert payload["realtime_event_id"] == f"evt-{event_type}" assert "event_id" not in payload anonymous = command_from_realtime({"type": event_type, **body}) assert anonymous.event_id is None assert "realtime_event_id" not in anonymous.payload() @pytest.mark.parametrize( ("alias", "expected_cls", "body"), [ ("push_text", AppendText, {"text": "hello"}), ("input_text.append", AppendText, {"text": "hello"}), ("signal_turn", SignalTurn, {"event": "user_started"}), ("close_session", CloseSession, {}), ("close", CloseSession, {}), ("audio.playback_ack", AckPlayback, {"played_ms": 10}), ("input.commit", Commit, {}), ], ) def test_pre_realtime_client_event_aliases_map_to_their_canonical_command( alias: str, expected_cls: type[DuplexCommand], body: dict[str, object] ): assert type(command_from_realtime({"type": alias, **body})) is expected_cls def test_append_audio_decodes_and_converts_pcm16_to_16k_float32(): command = command_from_realtime( {"type": "input_audio_buffer.append", "audio": _LOUD_PCM16, "format": "pcm16", "sample_rate_hz": 16000} ) assert isinstance(command, AppendAudio) assert command.format == "pcm_f32le" assert command.sample_rate_hz == 16000 samples = np.frombuffer(command.audio, dtype=" 8 * 4 passthrough = translate_realtime_command( {"type": "input_audio_buffer.append", "audio": base64.b64encode(np.zeros(4, dtype="