383 lines
16 KiB
Python
383 lines
16 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
"""OpenAI Realtime client events -> :class:`DuplexCommand` (stateless).
|
|
|
|
This module is the *duplex binding* of the shared Realtime codec. Parsing,
|
|
validating and decoding a client event is not duplex-specific and lives in
|
|
``vllm_omni.protocol.realtime``, reached through ``vllm_omni.protocol.duplex``;
|
|
what lives here is the part that is: which
|
|
:class:`~vllm_omni.engine.duplex.commands.DuplexCommand` a decoded event
|
|
becomes, what the duplex engine can serve
|
|
(:data:`DUPLEX_REALTIME_CAPABILITIES`), and the mapping between a Realtime
|
|
output format and the duplex ``response_format`` vocabulary.
|
|
|
|
Everything that needs per-session state (input-buffer emptiness for commits,
|
|
response-id fallbacks for cancels, conversation-item lookups, VAD) is resolved
|
|
by the session runner through the helpers on
|
|
:class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
|
|
the commands produced here carry the raw client intent only.
|
|
|
|
The shared names re-exported at the bottom are a compatibility surface for
|
|
existing importers. The canonical definitions are in
|
|
``vllm_omni.protocol.realtime`` and there is exactly one of each.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping
|
|
from typing import cast
|
|
|
|
from vllm_omni.engine.duplex.commands import (
|
|
AckPlayback,
|
|
AppendAudio,
|
|
AppendText,
|
|
BargeIn,
|
|
CancelInput,
|
|
CancelResponse,
|
|
ClearInput,
|
|
ClearOutputAudio,
|
|
CloseSession,
|
|
Commit,
|
|
CreateItem,
|
|
CreateResponse,
|
|
DeleteItem,
|
|
DuplexCommand,
|
|
DuplexCommandError,
|
|
Heartbeat,
|
|
SignalTurn,
|
|
TruncateItem,
|
|
UpdateSession,
|
|
)
|
|
from vllm_omni.protocol.duplex import (
|
|
REALTIME_INPUT_AUDIO_FORMATS,
|
|
REALTIME_INPUT_HINT_KEYS,
|
|
REALTIME_OUTPUT_AUDIO_FORMATS,
|
|
RealtimeInputDefaults,
|
|
RealtimeProtocolCapabilities,
|
|
RealtimeProtocolError,
|
|
apply_realtime_session_defaults,
|
|
copy_realtime_input_hints,
|
|
decode_audio_append,
|
|
input_audio_transcription_config,
|
|
input_explicitly_non_speech,
|
|
input_looks_like_speech,
|
|
input_transcript_from_item,
|
|
is_supported_realtime_input_format,
|
|
json_safe_realtime_payload,
|
|
normalize_conversation_item,
|
|
parse_realtime_audio_format,
|
|
realtime_audio_format_object,
|
|
realtime_max_output_tokens,
|
|
realtime_output_format,
|
|
realtime_overlap_fields,
|
|
text_chars_for_audio_ms_from_marks,
|
|
truncate_realtime_item_content,
|
|
validate_conversation_item_audio_formats,
|
|
validate_realtime_item_truncate,
|
|
validate_realtime_response_audio_formats,
|
|
validate_realtime_session_audio_formats,
|
|
validate_realtime_video_frames,
|
|
validate_session_payload,
|
|
)
|
|
|
|
|
|
def _validate_duplex_turn_detection(session_payload: Mapping[str, object]) -> str | None:
|
|
"""The duplex engine's answer for ``turn_detection`` on a session object.
|
|
|
|
Imported lazily: the validator lives with the Silero VAD backend, and a
|
|
connection that never sends a session object should not pay for loading it.
|
|
"""
|
|
from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
|
|
|
|
return validate_realtime_turn_detection(session_payload)
|
|
|
|
|
|
#: What the duplex engine can serve, for the shared session-object validator.
|
|
DUPLEX_REALTIME_CAPABILITIES = RealtimeProtocolCapabilities(
|
|
validate_turn_detection=_validate_duplex_turn_detection,
|
|
)
|
|
|
|
|
|
def duplex_response_format(realtime_format: str) -> str:
|
|
"""A Realtime output format -> the duplex ``response_format`` vocabulary."""
|
|
normalized = realtime_format.lower()
|
|
if normalized in {"pcm16", "pcm_s16le", "s16le"}:
|
|
return "pcm"
|
|
if normalized in {"g711_ulaw", "g711_alaw"}:
|
|
return "pcm"
|
|
if normalized in {"wav", "pcm"}:
|
|
return normalized
|
|
return "wav"
|
|
|
|
|
|
# ---- append / item audio conversion ----
|
|
|
|
|
|
def build_append_audio(
|
|
event: Mapping[str, object],
|
|
*,
|
|
defaults: RealtimeInputDefaults,
|
|
hints_source: Mapping[str, object] | None = None,
|
|
) -> AppendAudio:
|
|
"""Decode one audio append (shared codec) and pack it as the duplex command."""
|
|
try:
|
|
decoded = decode_audio_append(event, defaults=defaults, hints_source=hints_source)
|
|
except RealtimeProtocolError as exc:
|
|
raise DuplexCommandError(str(exc), code=exc.code, event_id=exc.event_id) from exc
|
|
return AppendAudio(
|
|
event_id=decoded.event_id,
|
|
audio=decoded.audio,
|
|
format=decoded.format,
|
|
sample_rate_hz=decoded.sample_rate_hz,
|
|
is_speech=decoded.is_speech,
|
|
video_frames=decoded.video_frames,
|
|
duration_ms=decoded.duration_ms,
|
|
audio_end_ms=decoded.audio_end_ms,
|
|
hints=decoded.hints,
|
|
)
|
|
|
|
|
|
# ---- translation ----
|
|
|
|
|
|
def translate_realtime_command(
|
|
payload: Mapping[str, object],
|
|
*,
|
|
defaults: RealtimeInputDefaults | None = None,
|
|
) -> DuplexCommand:
|
|
"""Map one OpenAI Realtime client event onto a :class:`DuplexCommand`.
|
|
|
|
Raises :class:`DuplexCommandError` for malformed or unsupported payloads.
|
|
``session.resume`` and ``session.event_ack`` are transport concerns and are
|
|
rejected with ``code="unknown_event"``.
|
|
"""
|
|
defaults = defaults or RealtimeInputDefaults()
|
|
event_type = payload.get("type")
|
|
event_id = cast("str", payload.get("event_id")) if isinstance(payload.get("event_id"), str) else None
|
|
if not isinstance(event_type, str):
|
|
raise DuplexCommandError("Duplex event missing string type", code="bad_event", event_id=event_id)
|
|
|
|
if event_type == "session.update":
|
|
session = payload.get("session")
|
|
session_payload: Mapping[str, object] = session if isinstance(session, dict) else payload
|
|
rejection = validate_session_payload(session_payload, capabilities=DUPLEX_REALTIME_CAPABILITIES)
|
|
if rejection is not None:
|
|
raise DuplexCommandError(rejection.message, code=rejection.code, event_id=event_id)
|
|
return UpdateSession(event_id=event_id, patch=dict(session_payload))
|
|
|
|
if event_type == "conversation.item.create":
|
|
item = payload.get("item")
|
|
format_error = validate_conversation_item_audio_formats(item)
|
|
if format_error is not None:
|
|
raise DuplexCommandError(format_error, code="unsupported_audio_format", event_id=event_id)
|
|
if not isinstance(item, dict):
|
|
raise DuplexCommandError("conversation.item.create requires item", code="bad_event", event_id=event_id)
|
|
previous_item_id = payload.get("previous_item_id")
|
|
return CreateItem(
|
|
event_id=event_id,
|
|
item=normalize_conversation_item(item),
|
|
previous_item_id=previous_item_id if isinstance(previous_item_id, str) else None,
|
|
)
|
|
|
|
if event_type == "conversation.item.delete":
|
|
item_id = payload.get("item_id")
|
|
if not isinstance(item_id, str) or not item_id:
|
|
raise DuplexCommandError(
|
|
"conversation.item.delete requires item_id", code="missing_item_id", event_id=event_id
|
|
)
|
|
return DeleteItem(event_id=event_id, item_id=item_id)
|
|
|
|
if event_type == "conversation.item.truncate":
|
|
item_id = payload.get("item_id")
|
|
audio_end_ms = payload.get("audio_end_ms")
|
|
content_index = payload.get("content_index", 0)
|
|
if not isinstance(item_id, str) or not item_id:
|
|
raise DuplexCommandError(
|
|
"conversation.item.truncate requires item_id", code="missing_item_id", event_id=event_id
|
|
)
|
|
if not isinstance(audio_end_ms, int | float):
|
|
raise DuplexCommandError(
|
|
"conversation.item.truncate requires numeric audio_end_ms", code="bad_event", event_id=event_id
|
|
)
|
|
return TruncateItem(
|
|
event_id=event_id,
|
|
item_id=item_id,
|
|
audio_end_ms=int(audio_end_ms),
|
|
content_index=int(content_index) if isinstance(content_index, int | float) else 0,
|
|
)
|
|
|
|
if event_type == "input_audio_buffer.append":
|
|
return build_append_audio(dict(payload), defaults=defaults)
|
|
|
|
if event_type in {"input_audio_buffer.commit", "input.commit"}:
|
|
final = payload.get("final", True)
|
|
create_response = payload.get("response_create", payload.get("create_response"))
|
|
is_speech = payload.get("is_speech")
|
|
return Commit(
|
|
event_id=event_id,
|
|
final=bool(final) if isinstance(final, bool) else True,
|
|
create_response=(
|
|
bool(create_response)
|
|
if isinstance(create_response, bool)
|
|
else (True if event_type == "input.commit" and create_response is None else None)
|
|
),
|
|
is_speech=is_speech if isinstance(is_speech, bool) else None,
|
|
)
|
|
|
|
if event_type == "input_audio_buffer.clear":
|
|
return ClearInput(event_id=event_id)
|
|
|
|
if event_type == "output_audio_buffer.clear":
|
|
response_id = payload.get("response_id")
|
|
return ClearOutputAudio(
|
|
event_id=event_id,
|
|
response_id=response_id if isinstance(response_id, str) and response_id else None,
|
|
)
|
|
|
|
if event_type == "response.cancel":
|
|
response_id = payload.get("response_id")
|
|
return CancelResponse(
|
|
event_id=event_id,
|
|
response_id=response_id if isinstance(response_id, str) and response_id else None,
|
|
)
|
|
|
|
if event_type == "response.create":
|
|
response_payload = payload.get("response")
|
|
if isinstance(response_payload, dict):
|
|
format_error = validate_realtime_response_audio_formats(response_payload)
|
|
if format_error is not None:
|
|
raise DuplexCommandError(format_error, code="unsupported_audio_format", event_id=event_id)
|
|
return CreateResponse(
|
|
event_id=event_id,
|
|
options=dict(response_payload) if isinstance(response_payload, dict) else {},
|
|
)
|
|
|
|
if event_type in {"playback.ack", "audio.playback_ack"}:
|
|
played_ms = payload.get("played_ms")
|
|
if not isinstance(played_ms, int | float):
|
|
raise DuplexCommandError("playback.ack requires numeric played_ms", code="bad_event", event_id=event_id)
|
|
committed_ms = payload.get("committed_ms")
|
|
response_id = payload.get("response_id")
|
|
item_id = payload.get("item_id")
|
|
return AckPlayback(
|
|
event_id=event_id,
|
|
played_ms=int(played_ms),
|
|
committed_ms=int(committed_ms) if isinstance(committed_ms, int | float) else None,
|
|
response_id=response_id if isinstance(response_id, str) and response_id else None,
|
|
item_id=item_id if isinstance(item_id, str) and item_id else None,
|
|
)
|
|
|
|
if event_type == "session.heartbeat":
|
|
return Heartbeat(event_id=event_id)
|
|
|
|
if event_type == "conversation.item.retrieve":
|
|
# The projected conversation items live engine-side; the runner answers.
|
|
return SignalTurn(event_id=event_id, event="conversation.item.retrieve", signal_payload=dict(payload))
|
|
|
|
if event_type in {"session.close", "close", "close_session"}:
|
|
reason = payload.get("reason")
|
|
return CloseSession(event_id=event_id, reason=reason if isinstance(reason, str) and reason else "client_close")
|
|
|
|
if event_type in {"input.text.append", "input_text.append", "push_text"}:
|
|
text = payload.get("text")
|
|
if not isinstance(text, str):
|
|
raise DuplexCommandError("input.text.append requires text", code="bad_event", event_id=event_id)
|
|
return AppendText(event_id=event_id, text=text)
|
|
|
|
if event_type == "input.cancel":
|
|
return CancelInput(event_id=event_id)
|
|
|
|
if event_type == "barge_in":
|
|
return BargeIn(event_id=event_id)
|
|
|
|
if event_type in {"turn.signal", "signal_turn"}:
|
|
signal_event = payload.get("event")
|
|
if not isinstance(signal_event, str) or not signal_event:
|
|
raise DuplexCommandError("turn.signal requires event", code="bad_event", event_id=event_id)
|
|
signal_payload = payload.get("payload")
|
|
signal_payload = dict(signal_payload) if isinstance(signal_payload, dict) else {}
|
|
if signal_event == "input.cancel":
|
|
return CancelInput(event_id=event_id)
|
|
if signal_event == "barge_in":
|
|
return BargeIn(event_id=event_id)
|
|
if signal_event == "response.cancel":
|
|
response_id = signal_payload.get("response_id", payload.get("response_id"))
|
|
return CancelResponse(
|
|
event_id=event_id,
|
|
response_id=response_id if isinstance(response_id, str) and response_id else None,
|
|
)
|
|
if signal_event == "session.update":
|
|
return UpdateSession(event_id=event_id, patch=signal_payload)
|
|
if signal_event == "conversation.item.create":
|
|
item = signal_payload.get("item")
|
|
if not isinstance(item, dict):
|
|
raise DuplexCommandError("conversation.item.create requires item", code="bad_event", event_id=event_id)
|
|
previous_item_id = signal_payload.get("previous_item_id")
|
|
return CreateItem(
|
|
event_id=event_id,
|
|
item=normalize_conversation_item(item),
|
|
previous_item_id=previous_item_id if isinstance(previous_item_id, str) else None,
|
|
)
|
|
if signal_event == "conversation.item.delete":
|
|
item_id = signal_payload.get("item_id")
|
|
if not isinstance(item_id, str) or not item_id:
|
|
raise DuplexCommandError(
|
|
"conversation.item.delete requires item_id", code="missing_item_id", event_id=event_id
|
|
)
|
|
return DeleteItem(event_id=event_id, item_id=item_id)
|
|
if signal_event == "conversation.item.truncate":
|
|
item_id = signal_payload.get("item_id")
|
|
audio_end_ms = signal_payload.get("audio_end_ms")
|
|
content_index = signal_payload.get("content_index", 0)
|
|
if not isinstance(item_id, str) or not item_id or not isinstance(audio_end_ms, int | float):
|
|
raise DuplexCommandError(
|
|
"conversation.item.truncate requires item_id and numeric audio_end_ms",
|
|
code="bad_event",
|
|
event_id=event_id,
|
|
)
|
|
return TruncateItem(
|
|
event_id=event_id,
|
|
item_id=item_id,
|
|
audio_end_ms=int(audio_end_ms),
|
|
content_index=int(content_index) if isinstance(content_index, int | float) else 0,
|
|
)
|
|
return SignalTurn(event_id=event_id, event=signal_event, signal_payload=signal_payload)
|
|
|
|
raise DuplexCommandError(f"Unknown duplex event type: {event_type}", code="unknown_event", event_id=event_id)
|
|
|
|
|
|
#: Names this module used to define, re-exported so existing importers keep
|
|
#: working. The canonical definitions live in ``vllm_omni.protocol.realtime``.
|
|
__all__ = [
|
|
"DUPLEX_REALTIME_CAPABILITIES",
|
|
"REALTIME_INPUT_AUDIO_FORMATS",
|
|
"REALTIME_INPUT_HINT_KEYS",
|
|
"REALTIME_OUTPUT_AUDIO_FORMATS",
|
|
"RealtimeInputDefaults",
|
|
"apply_realtime_session_defaults",
|
|
"build_append_audio",
|
|
"copy_realtime_input_hints",
|
|
"duplex_response_format",
|
|
"input_audio_transcription_config",
|
|
"input_explicitly_non_speech",
|
|
"input_looks_like_speech",
|
|
"input_transcript_from_item",
|
|
"is_supported_realtime_input_format",
|
|
"json_safe_realtime_payload",
|
|
"normalize_conversation_item",
|
|
"parse_realtime_audio_format",
|
|
"realtime_audio_format_object",
|
|
"realtime_max_output_tokens",
|
|
"realtime_output_format",
|
|
"realtime_overlap_fields",
|
|
"text_chars_for_audio_ms_from_marks",
|
|
"translate_realtime_command",
|
|
"truncate_realtime_item_content",
|
|
"validate_conversation_item_audio_formats",
|
|
"validate_realtime_item_truncate",
|
|
"validate_realtime_response_audio_formats",
|
|
"validate_realtime_session_audio_formats",
|
|
"validate_realtime_video_frames",
|
|
]
|