[Core] Give the OpenAI Realtime wire codec a reusable home (RFC #6592 P0a) (#7640)

Signed-off-by: psv666 <2693925048@qq.com>

vllm_omni/protocol/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Wire protocols vLLM-Omni speaks, independent of who serves or runs them.
+
+A module under ``vllm_omni.protocol`` describes *what is on the wire*: how a
+client payload is parsed and validated, and what shape a server payload has.
+It never decides how a request is executed, which model answers it, or where
+the session state lives. That keeps one protocol implementation usable by more
+than one runtime (see ``vllm_omni.protocol.realtime``).
+
+Dependency rule, asserted by ``tests/protocol/realtime/test_protocol_import_boundary.py``:
+nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``,
+``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``.
+The arrows point inwards --- those packages import the protocol, never the
+other way round.
+"""

vllm_omni/protocol/duplex/__init__.py

@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""vLLM-Omni's full-duplex extension of the OpenAI Realtime wire protocol.
+
+OpenAI's Realtime API assumes one turn at a time: the client commits input, the
+server answers. A full-duplex session does not --- the model listens while it
+speaks, it decides when to take a turn, the client reports playback progress,
+and a dropped socket re-attaches to a session that kept running. Those need
+events and commands the GA vocabulary does not have.
+
+This package is that delta. It builds on ``vllm_omni.protocol.realtime`` (same
+base classes, same pure wire rendering, no duplication) and, like it, owns no
+session and imports no runtime: the duplex engine depends on this package, not
+the other way round.
+
+The one door
+------------
+It is also the *only* door a duplex consumer uses. ``protocol.duplex.commands``
+and ``protocol.duplex.events`` carry the complete vocabulary --- Tier 1 classes
+re-exported unchanged alongside the Tier 2 and Tier 3 ones --- and this module
+re-exports the Tier 1 helper functions. So the layering is a chain, not a mesh::
+
+    protocol.realtime   (OpenAI's surface)
+        ^
+        | imports
+        |
+    protocol.duplex     (this package: re-exports Tier 1, adds Tier 2 and 3)
+        ^
+        | imports
+        |
+    duplex engine / entrypoints / clients
+
+``tests/protocol/duplex/test_duplex_protocol_facade.py`` asserts that the last
+arrow is the only one: no duplex consumer imports ``protocol.realtime``
+directly. The payoff is that a helper which later needs a duplex-specific
+version --- ``convert_input_audio_with_rate`` is the standing example, it
+resamples to MiniCPM-o's 16 kHz rather than the client's rate --- is overridden
+in one file instead of at every call site.
+
+What stays outside
+------------------
+The *mailbox* half of a command does not live here. A duplex command carries
+two different things: the client event it decodes from (wire, this package) and
+the dictionary the session runner consumes (engine). They genuinely differ ---
+``session.update``, ``conversation.item.create`` / ``.delete`` / ``.truncate``
+all travel on the runner's ``turn.signal`` channel --- so ``payload()`` and its
+``type`` stay in ``vllm_omni.engine.duplex.commands``. Events have no such
+half, which is why :data:`~vllm_omni.protocol.duplex.events.DuplexEvent` can be
+a plain alias of ``RealtimeEvent`` while ``DuplexCommand`` cannot.
+"""
+
+from vllm_omni.protocol.duplex.errors import (
+    REALTIME_ERROR_TYPES_BY_CODE,
+    RealtimeProtocolError,
+    realtime_error_type,
+)
+from vllm_omni.protocol.realtime.audio import (
+    MAX_INPUT_SAMPLE_RATE_HZ,
+    MIN_INPUT_SAMPLE_RATE_HZ,
+    convert_input_audio_with_rate,
+    convert_output_audio,
+    decode_g711_alaw,
+    decode_g711_ulaw,
+    encode_float32_mono_wav_base64,
+    encode_g711_alaw,
+    encode_g711_ulaw,
+    resample_pcm16_mono,
+    validate_input_sample_rate_hz,
+    wav_payload_to_pcm16,
+)
+from vllm_omni.protocol.realtime.audio_input import (
+    REALTIME_INPUT_HINT_KEYS,
+    RealtimeAudioAppend,
+    copy_realtime_input_hints,
+    decode_audio_append,
+    input_explicitly_non_speech,
+    input_looks_like_speech,
+)
+from vllm_omni.protocol.realtime.capabilities import (
+    RealtimeProtocolCapabilities,
+    RealtimeSessionRejection,
+    validate_session_payload,
+)
+from vllm_omni.protocol.realtime.formats import (
+    REALTIME_INPUT_AUDIO_FORMATS,
+    REALTIME_OUTPUT_AUDIO_FORMATS,
+    is_supported_realtime_input_format,
+    parse_realtime_audio_format,
+    realtime_audio_format_object,
+    realtime_output_format,
+    validate_conversation_item_audio_formats,
+    validate_realtime_response_audio_formats,
+    validate_realtime_session_audio_formats,
+)
+from vllm_omni.protocol.realtime.items import (
+    input_transcript_from_item,
+    normalize_conversation_item,
+    text_chars_for_audio_ms_from_marks,
+    truncate_realtime_item_content,
+    validate_realtime_item_truncate,
+    validate_realtime_video_frames,
+)
+from vllm_omni.protocol.realtime.session import (
+    RealtimeInputDefaults,
+    apply_realtime_session_defaults,
+    input_audio_transcription_config,
+    json_safe_realtime_payload,
+    realtime_max_output_tokens,
+    realtime_overlap_fields,
+)
+
+__all__ = [
+    "MAX_INPUT_SAMPLE_RATE_HZ",
+    "MIN_INPUT_SAMPLE_RATE_HZ",
+    "REALTIME_ERROR_TYPES_BY_CODE",
+    "REALTIME_INPUT_AUDIO_FORMATS",
+    "REALTIME_INPUT_HINT_KEYS",
+    "REALTIME_OUTPUT_AUDIO_FORMATS",
+    "RealtimeAudioAppend",
+    "RealtimeInputDefaults",
+    "RealtimeProtocolCapabilities",
+    "RealtimeProtocolError",
+    "RealtimeSessionRejection",
+    "apply_realtime_session_defaults",
+    "convert_input_audio_with_rate",
+    "convert_output_audio",
+    "copy_realtime_input_hints",
+    "decode_audio_append",
+    "decode_g711_alaw",
+    "decode_g711_ulaw",
+    "encode_float32_mono_wav_base64",
+    "encode_g711_alaw",
+    "encode_g711_ulaw",
+    "input_audio_transcription_config",
+    "input_explicitly_non_speech",
+    "input_looks_like_speech",
+    "input_transcript_from_item",
+    "is_supported_realtime_input_format",
+    "json_safe_realtime_payload",
+    "normalize_conversation_item",
+    "parse_realtime_audio_format",
+    "realtime_audio_format_object",
+    "realtime_error_type",
+    "realtime_max_output_tokens",
+    "realtime_output_format",
+    "realtime_overlap_fields",
+    "resample_pcm16_mono",
+    "text_chars_for_audio_ms_from_marks",
+    "truncate_realtime_item_content",
+    "validate_conversation_item_audio_formats",
+    "validate_input_sample_rate_hz",
+    "validate_realtime_item_truncate",
+    "validate_realtime_response_audio_formats",
+    "validate_realtime_session_audio_formats",
+    "validate_realtime_video_frames",
+    "validate_session_payload",
+    "wav_payload_to_pcm16",
+]

vllm_omni/protocol/duplex/commands.py

@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""The client-event vocabulary a full-duplex session accepts.
+
+Seven of the seventeen are events OpenAI's Realtime API has no equivalent for,
+because they only make sense when the model and the user can talk at the same
+time: the user cuts in (``barge_in``), the client reports how much audio it
+played (``playback.ack``), a turn transition is signalled locally
+(``turn.signal``), input is abandoned without committing (``input.cancel``),
+text is pushed into a live turn (``input.text.append``), and the session is
+kept alive or closed explicitly (``session.heartbeat`` / ``session.close``).
+Two more are OpenAI commands carrying duplex-only fields (Tier 2).
+
+The remaining eight are OpenAI's, unchanged, and re-exported from
+``vllm_omni.protocol.realtime.commands`` rather than redeclared --- they are
+the same objects. They are re-exported so this module is the *whole*
+vocabulary and a duplex consumer never has to import the Tier 1 package
+directly; keeping them declared there means a GA-only consumer is still not
+handed events its clients never send.
+
+None of these carries the engine's mailbox rendering: ``payload()`` and its
+channel live in ``vllm_omni.engine.duplex.commands``, because for four of the
+seventeen the runner's channel is not the client event type at all.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import ClassVar
+
+from vllm_omni.protocol.realtime import commands as realtime_commands
+from vllm_omni.protocol.realtime.commands import (
+    CancelResponse,
+    ClearInput,
+    ClearOutputAudio,
+    CreateItem,
+    CreateResponse,
+    DeleteItem,
+    RealtimeCommand,
+    TruncateItem,
+    UpdateSession,
+)
+
+# ---- Tier 2: OpenAI names carrying vLLM-Omni extensions ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AppendAudio(realtime_commands.AppendAudio):
+    """Tier 2: OpenAI's ``input_audio_buffer.append`` plus the duplex hints.
+
+    OpenAI's append is just base64 audio. The duplex lane also lets a client
+    declare this chunk's own format and rate, whether it believes the chunk is
+    speech, camera frames captured alongside it, and its timing --- all
+    additive, all ignored by a stock client.
+    """
+
+    #: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
+    format: str = "pcm16"
+    sample_rate_hz: int | None = None
+    is_speech: bool | None = None
+    video_frames: tuple[str, ...] = ()
+    duration_ms: int | None = None
+    audio_end_ms: int | None = None
+    hints: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Commit(realtime_commands.Commit):
+    """Tier 2: OpenAI's ``input_audio_buffer.commit`` plus duplex turn control.
+
+    OpenAI's commit carries nothing and always produces an item. Here a commit
+    may be non-final, may decline to start a response, and may declare itself
+    silence --- the semantic divergence documented in
+    ``docs/serving/realtime_duplex_api.md`` ("Commit != response").
+    """
+
+    #: Realtime conversation item created for this commit (wire correlation only).
+    #: ``None`` means "no explicit request": the runner decides on commit (auto-response
+    #: sessions answer on their own); ``True`` / ``False`` force it.
+    create_response: bool | None = None
+    is_speech: bool | None = None
+    realtime_item_id: str | None = None
+
+
+# ---- Tier 3: vLLM-Omni only ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AppendText(RealtimeCommand):
+    wire_type: ClassVar[str] = "input.text.append"
+    text: str
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CancelInput(RealtimeCommand):
+    wire_type: ClassVar[str] = "input.cancel"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class BargeIn(RealtimeCommand):
+    wire_type: ClassVar[str] = "barge_in"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SignalTurn(RealtimeCommand):
+    """Generic ``turn.signal`` (local turn transitions such as ``user_started``)."""
+
+    wire_type: ClassVar[str] = "turn.signal"
+    event: str
+    signal_payload: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AckPlayback(RealtimeCommand):
+    wire_type: ClassVar[str] = "playback.ack"
+    played_ms: int
+    committed_ms: int | None = None
+    response_id: str | None = None
+    item_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Heartbeat(RealtimeCommand):
+    wire_type: ClassVar[str] = "session.heartbeat"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CloseSession(RealtimeCommand):
+    """Graceful close requested through the command stream (``session.close``)."""
+
+    wire_type: ClassVar[str] = "session.close"
+    reason: str = "client_close"
+
+
+#: The complete command vocabulary a duplex client may send: the eight Tier 1
+#: classes re-exported unchanged, the two Tier 2 classes defined above, and the
+#: seven Tier 3 ones. A duplex consumer imports from here and never reaches
+#: past this module into ``vllm_omni.protocol.realtime``, so a command that
+#: later grows a duplex extension changes only this file.
+__all__ = [
+    # Tier 1 --- re-exported from ``vllm_omni.protocol.realtime.commands``.
+    "CancelResponse",
+    "ClearInput",
+    "ClearOutputAudio",
+    "CreateItem",
+    "CreateResponse",
+    "DeleteItem",
+    "RealtimeCommand",
+    "TruncateItem",
+    "UpdateSession",
+    # Tier 2 --- an OpenAI command plus duplex-only fields.
+    "AppendAudio",
+    "Commit",
+    # Tier 3 --- vLLM-Omni only.
+    "AckPlayback",
+    "AppendText",
+    "BargeIn",
+    "CancelInput",
+    "CloseSession",
+    "Heartbeat",
+    "SignalTurn",
+]

vllm_omni/protocol/duplex/errors.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""vLLM-Omni's Realtime error-code vocabulary.
+
+OpenAI's Realtime API standardises the ``error.type`` classes
+(``invalid_request_error`` / ``server_error`` / ``rate_limit_error``) but not
+the ``code`` values inside them. Ours are listed here --- Tier 3 in
+``docs/serving/realtime_duplex_api.md``, "the full error-code vocabulary" ---
+so ``protocol/realtime`` stays the OpenAI surface and nothing more.
+"""
+
+from __future__ import annotations
+
+from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
+
+__all__ = ["REALTIME_ERROR_TYPES_BY_CODE", "RealtimeProtocolError", "realtime_error_type"]
+
+
+#: OpenAI Realtime ``error.type`` for each internal error code.
+REALTIME_ERROR_TYPES_BY_CODE: dict[str, str] = {
+    "bad_event": "invalid_request_error",
+    "bad_audio": "invalid_request_error",
+    "config_timeout": "invalid_request_error",
+    "invalid_json": "invalid_request_error",
+    "event_too_large": "invalid_request_error",
+    "unknown_event": "invalid_request_error",
+    "internal_error": "server_error",
+    "runtime_append_failed": "server_error",
+    "runtime_append_task_failed": "server_error",
+    "runtime_signal_failed": "server_error",
+    "runtime_abort_failed": "server_error",
+    "runtime_data_plane_stream_failed": "server_error",
+    "runtime_data_plane_text_without_audio": "server_error",
+    "resource_exhausted": "rate_limit_error",
+    "session_exists": "invalid_request_error",
+    "session_closed": "invalid_request_error",
+    "unknown_session": "invalid_request_error",
+    "invalid_duplex_runtime_config": "invalid_request_error",
+    "instructions_update_unsupported": "invalid_request_error",
+    "persona_update_unsupported": "invalid_request_error",
+    "voice_update_unsupported": "invalid_request_error",
+    "unsupported_nemotron_duplex_mode": "invalid_request_error",
+    "unsupported_native_response_options": "invalid_request_error",
+    "runtime_touch_failed": "server_error",
+    "engine_error": "server_error",
+    "input_backpressure": "rate_limit_error",
+    "response_already_active": "invalid_request_error",
+    "response_not_active": "invalid_request_error",
+    "response_create_without_input": "invalid_request_error",
+    "text_only_turn_unsupported": "invalid_request_error",
+    "commit_aborted": "server_error",
+    "input_audio_buffer_empty": "invalid_request_error",
+    "missing_item_id": "invalid_request_error",
+    "item_not_found": "invalid_request_error",
+    "playback_item_mismatch": "invalid_request_error",
+    "playback_item_not_found": "invalid_request_error",
+    "playback_ack_too_late": "invalid_request_error",
+    "unsupported_audio_format": "invalid_request_error",
+    "unsupported_turn_detection": "invalid_request_error",
+    "unsupported_ref_audio_path": "invalid_request_error",
+    "ref_audio_required": "invalid_request_error",
+    "model_update_unsupported": "invalid_request_error",
+    "voice_update_after_audio_unsupported": "invalid_request_error",
+    "ref_audio_update_unsupported": "invalid_request_error",
+    "native_text_append_unsupported": "invalid_request_error",
+    "invalid_video_frames": "invalid_request_error",
+    "invalid_function_call_output": "invalid_request_error",
+    "server_vad_unavailable": "server_error",
+}
+
+
+def realtime_error_type(code: str) -> str:
+    """The OpenAI ``error.type`` bucket for an internal error code."""
+    return REALTIME_ERROR_TYPES_BY_CODE.get(code, "invalid_request_error")

vllm_omni/protocol/duplex/events.py

@@ -0,0 +1,419 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""vLLM-Omni's half of the Realtime server-event vocabulary.
+
+``docs/serving/realtime_duplex_api.md`` sorts every message into three tiers.
+Tier 1 --- identical to OpenAI --- is ``vllm_omni.protocol.realtime.events``.
+The other two are here:
+
+**Tier 2, OpenAI names carrying vLLM-Omni extensions.** Same wire type, same
+OpenAI fields, plus additive keys a stock client ignores. Each one subclasses
+its Tier 1 twin and declares only what it adds, so the OpenAI surface stays
+honest: a consumer that wants pure GA takes ``protocol/realtime`` and gets
+exactly OpenAI's fields.
+
+**Tier 3, vLLM-Omni only.** Events OpenAI has no equivalent for, because they
+only make sense when the model and the user can talk at the same time: the
+model decides when to listen and when to speak, the client reports how much
+audio it played, overlapping speech needs a recorded decision, and a dropped
+socket can re-attach to a live session.
+
+Same-named classes are deliberate --- ``duplex.events.SessionCreated`` *is*
+``realtime.events.SessionCreated`` with our extensions. Import whichever
+surface you mean.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import cast
+
+from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
+from vllm_omni.protocol.realtime import events as realtime_events
+from vllm_omni.protocol.realtime.events import (
+    AudioDone,
+    ContentPartAdded,
+    ContentPartDone,
+    FunctionCallArgumentsDelta,
+    FunctionCallArgumentsDone,
+    InputCleared,
+    InputTranscriptionCompleted,
+    ItemAdded,
+    ItemCreated,
+    ItemDone,
+    ItemRetrieved,
+    OutputAudioCleared,
+    OutputItemAdded,
+    OutputItemDone,
+    RateLimitsUpdated,
+    RealtimeEvent,
+    ResponseScopedEvent,
+    SessionUpdated,
+    SpeechStarted,
+    SpeechStopped,
+    TextDelta,
+    TextDone,
+    TranscriptDelta,
+    TranscriptDone,
+    new_event_id,
+    wire_value,
+)
+
+#: The duplex event base class *is* the Realtime one: unlike a command, an
+#: event has no engine-internal half to carry --- what the session emits is
+#: exactly what the socket sends. Named here rather than in the engine so the
+#: duplex vocabulary is described in one place, and kept as an alias rather
+#: than an empty subclass so ``isinstance`` and the class identity tests in
+#: ``tests/protocol/`` stay meaningful.
+DuplexEvent = RealtimeEvent
+
+# ---- Tier 2: OpenAI names carrying vLLM-Omni extensions ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionCreated(realtime_events.SessionCreated):
+    """Tier 2: OpenAI's ``SessionCreated`` plus the resume credentials a reconnecting client needs."""
+
+    #: Transport resume credentials (set by the websocket handler when resume is supported).
+    optional_wire_fields = frozenset({"attachment_generation", "resume_token"})
+    attachment_generation: int | None = None
+    resume_token: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseCreated(realtime_events.ResponseCreated):
+    """Tier 2: OpenAI's ``ResponseCreated`` plus a top-level ``response_id``.
+
+    OpenAI carries the id nested as ``response.id``; the duplex lane also
+    reports it at the top level, where every other response-scoped event
+    already has one.
+    """
+
+    response_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseDone(realtime_events.ResponseDone):
+    """Tier 2: OpenAI's ``ResponseDone`` plus a top-level ``response_id``.
+
+    OpenAI carries the id nested as ``response.id``; the duplex lane also
+    reports it at the top level, where every other response-scoped event
+    already has one.
+    """
+
+    response_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AudioDelta(realtime_events.AudioDelta):
+    """Tier 2: OpenAI's ``AudioDelta`` plus the chunk's own format/rate and the duplex playback metadata."""
+
+    optional_wire_fields = frozenset({"sample_rate_hz", "metadata"})
+    format: str = "pcm16"
+    sample_rate_hz: int | None = None
+    metadata: Mapping[str, object] | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class InputCommitted(realtime_events.InputCommitted):
+    """Tier 2: OpenAI's ``InputCommitted`` plus the native ``input.committed`` event echoed under ``event``."""
+
+    #: The session-internal ``input.committed`` event.
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        # Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
+        # rebuilds the class, so the zero-argument form's ``__class__`` cell
+        # points at the pre-slots class and is no longer in the MRO.
+        base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
+        return {**base, "event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemDeleted(realtime_events.ItemDeleted):
+    """Tier 2: OpenAI's ``ItemDeleted`` plus the originating event echoed under ``event``."""
+
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        # Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
+        # rebuilds the class, so the zero-argument form's ``__class__`` cell
+        # points at the pre-slots class and is no longer in the MRO.
+        base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
+        return {**base, "event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemTruncated(realtime_events.ItemTruncated):
+    """Tier 2: OpenAI's ``ItemTruncated`` plus the originating event echoed under ``event``."""
+
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        # Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
+        # rebuilds the class, so the zero-argument form's ``__class__`` cell
+        # points at the pre-slots class and is no longer in the MRO.
+        base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
+        return {**base, "event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ErrorEvent(realtime_events.ErrorEvent):
+    """Tier 2: OpenAI's ``ErrorEvent`` plus vLLM-Omni's error codes and any extra keys merged into the envelope."""
+
+    #: Extra keys merged into the wire ``error`` object (``retryable`` ...).
+    extra: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def error_type(self) -> str:
+        return REALTIME_ERROR_TYPES_BY_CODE.get(self.code, "invalid_request_error")
+
+    @property
+    def error(self) -> dict[str, object]:
+        # Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
+        # rebuilds the class, so the zero-argument form's ``__class__`` cell
+        # points at the pre-slots class and is no longer in the MRO. Reached
+        # through ``__dict__`` because attribute access on the class gives the
+        # getter function, not the ``property`` that owns it.
+        error = dict(realtime_events.ErrorEvent.__dict__["error"].fget(self))
+        error.update(cast("Mapping[str, object]", wire_value(self.extra)))
+        return error
+
+
+# ---- Tier 3: vLLM-Omni only ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionResumed(SessionCreated):
+    wire_type = "session.resumed"
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionReplaced(RealtimeEvent):
+    """A newer connection took the session over (sent to the replaced socket)."""
+
+    wire_type = "session.replaced"
+
+    attachment_generation: int = 0
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionResyncRequired(RealtimeEvent):
+    """The replay journal cannot bridge the gap; the client must start over."""
+
+    wire_type = "session.resync_required"
+
+    reason: str = "journal_gap"
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionClosed(RealtimeEvent):
+    wire_type = "session.closed"
+
+    reason: str = "closed"
+    #: The session-internal close event (kept for clients that inspect it).
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def is_terminal(self) -> bool:
+        return True
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, "reason": self.reason, "event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionExpired(SessionClosed):
+    wire_type = "session.expired"
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, "reason": self.reason}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionHeartbeatAck(RealtimeEvent):
+    wire_type = "session.heartbeat_ack"
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TurnEvent(RealtimeEvent):
+    """Turn-state transition (``turn.event``)."""
+
+    wire_type = "turn.event"
+
+    event: str = ""
+    turn_state: str = ""
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, "event": self.event, "turn_state": self.turn_state, "epoch": self.epoch}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Listen(RealtimeEvent):
+    """The model decided to keep listening (no spoken response for this turn)."""
+
+    wire_type = "response.listen"
+    optional_wire_fields = frozenset({"response_id"})
+
+    response_id: str | None = None  # type: ignore[assignment]
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        response: dict[str, object] = {
+            "object": "realtime.response",
+            "status": "listening",
+            "metadata": wire_value(self.details),
+        }
+        data: dict[str, object] = {"session_id": self.session_id, "epoch": self.epoch, "response": response}
+        if self.response_id:
+            response["id"] = self.response_id
+            data["response_id"] = self.response_id
+        return data
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Speak(ResponseScopedEvent):
+    wire_type = "response.speak"
+
+    metadata: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class OverlapDecision(RealtimeEvent):
+    wire_type = "overlap.decision"
+
+    policy: str | None = None
+    action: str | None = None
+    reason: str | None = None
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {
+            "session_id": self.session_id,
+            "epoch": self.epoch,
+            "policy": self.policy,
+            "action": self.action,
+            "reason": self.reason,
+            "metadata": wire_value(self.details),
+        }
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class PlaybackAcknowledged(RealtimeEvent):
+    wire_type = "playback.acknowledged"
+
+    #: The session-internal ``playback.acknowledged`` event (cursor, committed_ms ...).
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class DuplexRawEvent(RealtimeEvent):
+    """A session-internal event with no dedicated Realtime type (``duplex.<internal type>``)."""
+
+    wire_type = "duplex.raw"
+
+    internal_type: str = ""
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def type(self) -> str:
+        return f"duplex.{self.internal_type}"
+
+    def to_realtime(self) -> dict[str, object]:
+        return {"type": self.type, "event_id": self.event_id, "event": wire_value(self.details)}
+
+
+def error_event(
+    code: str,
+    message: str,
+    *,
+    event_id: object | None = None,
+    param: object | None = None,
+    extra: Mapping[str, object] | None = None,
+) -> ErrorEvent:
+    """Build the error event for an internal code; ``event_id`` is the *client* event id."""
+    return ErrorEvent(
+        code=code,
+        message=message,
+        related_event_id=event_id if isinstance(event_id, str) and event_id else None,
+        param=param if isinstance(param, str) and param else None,
+        extra=dict(extra or {}),
+    )
+
+
+#: The complete event vocabulary a duplex client may receive: 22 Tier 1
+#: classes re-exported unchanged, the 8 Tier 2 ones defined above, and 12
+#: Tier 3 ones. A duplex consumer imports from here and never reaches past
+#: this module into ``vllm_omni.protocol.realtime``.
+__all__ = [
+    # Tier 1 --- re-exported from ``vllm_omni.protocol.realtime.events``.
+    "AudioDone",
+    "ContentPartAdded",
+    "ContentPartDone",
+    "FunctionCallArgumentsDelta",
+    "FunctionCallArgumentsDone",
+    "InputCleared",
+    "InputTranscriptionCompleted",
+    "ItemAdded",
+    "ItemCreated",
+    "ItemDone",
+    "ItemRetrieved",
+    "OutputAudioCleared",
+    "OutputItemAdded",
+    "OutputItemDone",
+    "RateLimitsUpdated",
+    "SessionUpdated",
+    "SpeechStarted",
+    "SpeechStopped",
+    "TextDelta",
+    "TextDone",
+    "TranscriptDelta",
+    "TranscriptDone",
+    # Tier 2 --- an OpenAI event plus duplex-only fields.
+    "AudioDelta",
+    "ErrorEvent",
+    "InputCommitted",
+    "ItemDeleted",
+    "ItemTruncated",
+    "ResponseCreated",
+    "ResponseDone",
+    "SessionCreated",
+    # Tier 3 --- vLLM-Omni only.
+    "DuplexRawEvent",
+    "Listen",
+    "OverlapDecision",
+    "PlaybackAcknowledged",
+    "SessionClosed",
+    "SessionExpired",
+    "SessionHeartbeatAck",
+    "SessionReplaced",
+    "SessionResumed",
+    "SessionResyncRequired",
+    "Speak",
+    "TurnEvent",
+    # Bases and helpers shared with the Tier 1 surface.
+    "DuplexEvent",
+    "REALTIME_ERROR_TYPES_BY_CODE",
+    "RealtimeEvent",
+    "ResponseScopedEvent",
+    "error_event",
+    "new_event_id",
+    "wire_value",
+]

vllm_omni/protocol/realtime/__init__.py

@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""The OpenAI Realtime wire codec, shared by every vLLM-Omni Realtime surface.
+
+What this is
+------------
+The model-agnostic half of ``WS /v1/realtime``: how a client event is parsed,
+which audio formats and session fields are accepted, what a conversation item
+means, how audio is decoded and resampled, and the shape of the error envelope
+(the *codes* inside it are a consumer's own --- ours are in
+``vllm_omni.protocol.duplex.errors``). It is a library of pure functions and value objects.
+
+What it is not
+--------------
+It owns no session. It does not decide when a response starts, what a model is
+prompted with, whether a turn is over, or where committed audio goes. Those are
+runtime decisions, and after PR #7413 a duplex session's runtime decisions all
+belong to ``vllm_omni.engine.duplex`` --- the session runner, the typed
+``DuplexCommand`` / ``DuplexEvent`` boundary and the ``DuplexModelPlugin`` seam.
+This package sits *under* that: the duplex engine binds the codec to its own
+command and event vocabulary in
+``vllm_omni.engine.duplex.realtime_commands`` and
+``vllm_omni.engine.duplex.realtime_events``.
+
+Why it is separate (RFC #6592, P0a)
+-----------------------------------
+So that a second Realtime surface --- a Qwen3-Omni GA profile, say --- can reuse
+the parsing, validation, format negotiation and error envelope without adopting
+the duplex session control plane (lease manager, commit policy, model channel),
+and without a second copy of the codec drifting away from this one. A consumer
+states what it can serve through
+:class:`~vllm_omni.protocol.realtime.capabilities.RealtimeProtocolCapabilities`
+and keeps its own session state.
+
+Known duplex-shaped defaults
+----------------------------
+This package is model-agnostic in its *dependencies*, not yet in every
+*default*. Two constants still carry the assumptions of its first consumer and
+a second one will have to parameterize them:
+
+* :func:`~vllm_omni.protocol.realtime.audio.convert_input_audio_with_rate`
+  resamples to 16 kHz, which is MiniCPM-o's feature-extractor rate. A GA client
+  sending 24 kHz is converted down rather than served at its own rate.
+* :func:`~vllm_omni.protocol.realtime.items.validate_realtime_video_frames`
+  encodes the MiniCPM-o duplex camera contract (at most two frames per append,
+  ``max_slice_nums`` not selectable on the wire).
+
+Both are behaviour this extraction deliberately did **not** change (RFC #6592
+P0a is behaviour-preserving); making them per-consumer is P0b/P1 work.
+
+Dependency rule
+---------------
+Nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``,
+``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``;
+``tests/protocol/realtime/test_protocol_import_boundary.py`` asserts it.
+"""
+
+from vllm_omni.protocol.realtime.audio import (
+    MAX_INPUT_SAMPLE_RATE_HZ,
+    MIN_INPUT_SAMPLE_RATE_HZ,
+    convert_input_audio_with_rate,
+    convert_output_audio,
+    encode_float32_mono_wav_base64,
+    resample_pcm16_mono,
+    validate_input_sample_rate_hz,
+    wav_payload_to_pcm16,
+)
+from vllm_omni.protocol.realtime.audio_input import (
+    REALTIME_INPUT_HINT_KEYS,
+    RealtimeAudioAppend,
+    copy_realtime_input_hints,
+    decode_audio_append,
+    input_explicitly_non_speech,
+    input_looks_like_speech,
+)
+from vllm_omni.protocol.realtime.capabilities import (
+    RealtimeProtocolCapabilities,
+    RealtimeSessionRejection,
+    validate_session_payload,
+)
+from vllm_omni.protocol.realtime.commands import RealtimeCommand
+from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
+from vllm_omni.protocol.realtime.events import (
+    RealtimeEvent,
+    ResponseScopedEvent,
+    new_event_id,
+    wire_value,
+)
+from vllm_omni.protocol.realtime.formats import (
+    REALTIME_INPUT_AUDIO_FORMATS,
+    REALTIME_OUTPUT_AUDIO_FORMATS,
+    is_supported_realtime_input_format,
+    parse_realtime_audio_format,
+    realtime_audio_format_object,
+    realtime_output_format,
+    validate_conversation_item_audio_formats,
+    validate_realtime_response_audio_formats,
+    validate_realtime_session_audio_formats,
+)
+from vllm_omni.protocol.realtime.items import (
+    input_transcript_from_item,
+    normalize_conversation_item,
+    text_chars_for_audio_ms_from_marks,
+    truncate_realtime_item_content,
+    validate_realtime_item_truncate,
+    validate_realtime_video_frames,
+)
+from vllm_omni.protocol.realtime.session import (
+    RealtimeInputDefaults,
+    apply_realtime_session_defaults,
+    input_audio_transcription_config,
+    json_safe_realtime_payload,
+    realtime_max_output_tokens,
+    realtime_overlap_fields,
+)
+
+__all__ = [
+    "MAX_INPUT_SAMPLE_RATE_HZ",
+    "MIN_INPUT_SAMPLE_RATE_HZ",
+    "REALTIME_INPUT_AUDIO_FORMATS",
+    "REALTIME_INPUT_HINT_KEYS",
+    "REALTIME_OUTPUT_AUDIO_FORMATS",
+    "RealtimeAudioAppend",
+    "RealtimeCommand",
+    "RealtimeEvent",
+    "RealtimeInputDefaults",
+    "RealtimeProtocolCapabilities",
+    "RealtimeProtocolError",
+    "RealtimeSessionRejection",
+    "ResponseScopedEvent",
+    "apply_realtime_session_defaults",
+    "convert_input_audio_with_rate",
+    "convert_output_audio",
+    "copy_realtime_input_hints",
+    "decode_audio_append",
+    "encode_float32_mono_wav_base64",
+    "input_audio_transcription_config",
+    "input_explicitly_non_speech",
+    "input_looks_like_speech",
+    "input_transcript_from_item",
+    "is_supported_realtime_input_format",
+    "json_safe_realtime_payload",
+    "new_event_id",
+    "normalize_conversation_item",
+    "parse_realtime_audio_format",
+    "realtime_audio_format_object",
+    "realtime_max_output_tokens",
+    "realtime_output_format",
+    "realtime_overlap_fields",
+    "resample_pcm16_mono",
+    "text_chars_for_audio_ms_from_marks",
+    "truncate_realtime_item_content",
+    "validate_conversation_item_audio_formats",
+    "validate_input_sample_rate_hz",
+    "validate_realtime_item_truncate",
+    "validate_realtime_response_audio_formats",
+    "validate_realtime_session_audio_formats",
+    "validate_realtime_video_frames",
+    "validate_session_payload",
+    "wav_payload_to_pcm16",
+    "wire_value",
+]

vllm_omni/protocol/realtime/audio.py

@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from __future__ import annotations
+
+import base64
+import binascii
+import io
+import math
+import wave
+
+import numpy as np
+
+from vllm_omni.utils.audio_resample import StreamingAudioResampler
+
+try:
+    from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin
+except ImportError:  # pragma: no cover - audioop is removed in newer Python.
+    alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None
+
+
+MIN_INPUT_SAMPLE_RATE_HZ = 8_000
+MAX_INPUT_SAMPLE_RATE_HZ = 192_000
+
+
+def validate_input_sample_rate_hz(sample_rate_hz: object) -> int:
+    if (
+        isinstance(sample_rate_hz, bool)
+        or not isinstance(sample_rate_hz, int | float)
+        or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz))
+    ):
+        raise ValueError("sample_rate_hz must be a finite integer")
+    rate = int(sample_rate_hz)
+    if rate != sample_rate_hz:
+        raise ValueError("sample_rate_hz must be an integer")
+    if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ:
+        raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}")
+    return rate
+
+
+def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes:
+    if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz:
+        return raw
+    samples = np.frombuffer(raw, dtype="<i2")
+    rate_gcd = math.gcd(source_rate_hz, target_rate_hz)
+    reduced_factor = max(source_rate_hz // rate_gcd, target_rate_hz // rate_gcd)
+    if reduced_factor > StreamingAudioResampler.max_polyphase_factor:
+        # The FIR size scales with the reduced ratio. Keep unusual client
+        # rates bounded while common audio rates use the high-quality path.
+        if samples.size <= 1:
+            return raw
+        target_size = max(1, int(round(samples.size * target_rate_hz / source_rate_hz)))
+        source_x = np.linspace(0.0, 1.0, num=samples.size, endpoint=True)
+        target_x = np.linspace(0.0, 1.0, num=target_size, endpoint=True)
+        resampled = np.interp(target_x, source_x, samples)
+        return np.clip(resampled, -32_768, 32_767).astype("<i2").tobytes()
+    resampler = StreamingAudioResampler(source_rate_hz, target_rate_hz)
+    normalized = samples.astype(np.float32) * np.float32(1.0 / 32768.0)
+    resampled = resampler.process(normalized, final=True)
+    return np.clip(np.rint(resampled * 32768.0), -32768, 32767).astype("<i2").tobytes()
+
+
+def decode_g711_ulaw(raw: bytes) -> bytes:
+    if ulaw2lin is not None:
+        return ulaw2lin(raw, 2)
+    data = np.frombuffer(raw, dtype=np.uint8)
+    value = np.bitwise_not(data).astype(np.int16)
+    sign = value & 0x80
+    exponent = (value >> 4) & 0x07
+    mantissa = value & 0x0F
+    sample = ((mantissa << 3) + 0x84) << exponent
+    return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype("<i2").tobytes()
+
+
+def decode_g711_alaw(raw: bytes) -> bytes:
+    if alaw2lin is not None:
+        return alaw2lin(raw, 2)
+    data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16)
+    sign = data & 0x80
+    exponent = (data >> 4) & 0x07
+    mantissa = data & 0x0F
+    sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1))
+    return np.where(sign != 0, sample, -sample).astype("<i2").tobytes()
+
+
+def encode_g711_ulaw(raw: bytes) -> bytes:
+    if lin2ulaw is not None:
+        return lin2ulaw(raw, 2)
+    pcm = np.clip(np.frombuffer(raw, dtype="<i2").astype(np.int32), -32635, 32635)
+    sign = np.where(pcm < 0, 0x80, 0)
+    magnitude = np.abs(pcm) + 0x84
+    exponent = np.zeros_like(magnitude)
+    for exp in range(7):
+        exponent = np.where(magnitude > (0xFF << exp), exp + 1, exponent)
+    mantissa = (magnitude >> (exponent + 3)) & 0x0F
+    return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes()
+
+
+def encode_g711_alaw(raw: bytes) -> bytes:
+    if lin2alaw is not None:
+        return lin2alaw(raw, 2)
+    pcm = np.frombuffer(raw, dtype="<i2").astype(np.int32)
+    sign = np.where(pcm >= 0, 0x80, 0x00)
+    magnitude = np.abs(pcm)
+    exponent = np.zeros_like(magnitude)
+    for exp in range(1, 8):
+        exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent)
+    mantissa = np.where(
+        exponent == 0,
+        (magnitude >> 4) & 0x0F,
+        (magnitude >> (exponent + 3)) & 0x0F,
+    )
+    return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes()
+
+
+def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]:
+    try:
+        with wave.open(io.BytesIO(raw), "rb") as wav_file:
+            channels = wav_file.getnchannels()
+            sample_width = wav_file.getsampwidth()
+            sample_rate_hz = wav_file.getframerate()
+            frames = wav_file.readframes(wav_file.getnframes())
+        if sample_width != 2:
+            return None, sample_rate_hz
+        if channels <= 1:
+            return frames, sample_rate_hz
+        pcm = np.frombuffer(frames, dtype="<i2").reshape(-1, channels)
+        mono = np.mean(pcm.astype(np.float32), axis=1)
+        return np.clip(mono, -32768, 32767).astype("<i2").tobytes(), sample_rate_hz
+    except (EOFError, ValueError, wave.Error):
+        return None, None
+
+
+def encode_float32_mono_wav_base64(
+    samples: np.ndarray,
+    *,
+    sample_rate_hz: int,
+) -> str:
+    """Package normalized mono float32 samples as a PCM16 WAV data payload."""
+    rate = validate_input_sample_rate_hz(sample_rate_hz)
+    normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1)
+    pcm16 = np.clip(np.rint(normalized * 32768.0), -32768, 32767).astype("<i2")
+    with io.BytesIO() as buffer:
+        with wave.open(buffer, "wb") as wav_file:
+            wav_file.setnchannels(1)
+            wav_file.setsampwidth(2)
+            wav_file.setframerate(rate)
+            wav_file.writeframes(pcm16.tobytes())
+        return base64.b64encode(buffer.getvalue()).decode("ascii")
+
+
+def convert_input_audio_with_rate(
+    audio: object,
+    fmt: object,
+    *,
+    sample_rate_hz: int | float | None = None,
+    target_sample_rate_hz: int = 16_000,
+) -> tuple[object, object, int | float | None]:
+    if not isinstance(audio, str) or not isinstance(fmt, str):
+        return audio, fmt, sample_rate_hz
+    normalized = fmt.lower()
+    if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}:
+        return audio, fmt, sample_rate_hz
+    if isinstance(sample_rate_hz, int | float):
+        sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz)
+    try:
+        raw = base64.b64decode(audio.strip(), validate=False)
+    except (binascii.Error, ValueError):
+        return audio, fmt, sample_rate_hz
+    if normalized == "g711_ulaw":
+        raw = decode_g711_ulaw(raw)
+        sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
+    elif normalized == "g711_alaw":
+        raw = decode_g711_alaw(raw)
+        sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
+    elif len(raw) % 2:
+        return audio, fmt, sample_rate_hz
+    if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz:
+        raw = resample_pcm16_mono(
+            raw,
+            source_rate_hz=int(sample_rate_hz),
+            target_rate_hz=target_sample_rate_hz,
+        )
+        sample_rate_hz = target_sample_rate_hz
+    pcm = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
+    encoded = base64.b64encode(np.ascontiguousarray(pcm, dtype="<f4").tobytes()).decode("ascii")
+    return encoded, "pcm_f32le", sample_rate_hz
+
+
+def convert_output_audio(
+    audio: str,
+    *,
+    source_fmt: str,
+    target_fmt: str,
+    source_sample_rate_hz: int | None = None,
+    target_sample_rate_hz: int | None = None,
+) -> tuple[str, str, int | None]:
+    target = target_fmt.lower()
+    if target not in {"g711_ulaw", "g711_alaw"}:
+        return audio, source_fmt, source_sample_rate_hz
+    try:
+        raw = base64.b64decode(audio, validate=False)
+    except (binascii.Error, ValueError):
+        return audio, source_fmt, source_sample_rate_hz
+    source = source_fmt.lower()
+    if source == "wav":
+        pcm_raw, wav_rate = wav_payload_to_pcm16(raw)
+        if pcm_raw is None:
+            return audio, source_fmt, source_sample_rate_hz
+        raw = pcm_raw
+        source_sample_rate_hz = source_sample_rate_hz or wav_rate
+    elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2:
+        return audio, source_fmt, source_sample_rate_hz
+    target_rate = target_sample_rate_hz or 8_000
+    if source_sample_rate_hz is not None:
+        raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate)
+    encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw)
+    return base64.b64encode(encoded).decode("ascii"), target, target_rate

vllm_omni/protocol/realtime/audio_input.py

@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Decoding one ``input_audio_buffer.append``.
+
+An append is the hottest client event on a Realtime connection and the one with
+the most wire surface: base64 audio in any accepted format, an optional rate,
+optional camera frames, and a set of optional hints (``duration_ms``,
+``is_speech``, a VAD probability, a transcript) that clients attach and servers
+are free to use or ignore.
+
+:func:`decode_audio_append` turns that into :class:`RealtimeAudioAppend`, a
+value object: bytes at 16 kHz ``pcm_f32le``, the surviving hints, and a speech
+verdict. It is a pure function of the event plus the session's
+:class:`~vllm_omni.protocol.realtime.session.RealtimeInputDefaults`, so any
+consumer decodes an append the same way. Building a consumer's own command
+object out of it is one constructor call --- see
+``vllm_omni.engine.duplex.realtime_commands.build_append_audio``, which wraps it
+for the duplex ``AppendAudio``.
+
+The speech verdict here is the *client-declared* one (explicit flags, a VAD
+probability the client sent, or an RMS floor as a last resort). It is not
+server-side VAD; that is a runtime concern and lives with whoever owns the
+session.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import cast
+
+import numpy as np
+
+from vllm_omni.protocol.realtime.audio import convert_input_audio_with_rate
+from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
+from vllm_omni.protocol.realtime.formats import (
+    is_supported_realtime_input_format,
+    parse_realtime_audio_format,
+)
+from vllm_omni.protocol.realtime.items import validate_realtime_video_frames
+from vllm_omni.protocol.realtime.session import RealtimeInputDefaults
+
+__all__ = [
+    "REALTIME_INPUT_HINT_KEYS",
+    "RealtimeAudioAppend",
+    "copy_realtime_input_hints",
+    "decode_audio_append",
+    "input_explicitly_non_speech",
+    "input_looks_like_speech",
+]
+
+#: Wire hint keys copied verbatim from a client append onto the internal payload.
+REALTIME_INPUT_HINT_KEYS = (
+    "duration_ms",
+    "audio_duration_ms",
+    "audio_start_ms",
+    "audio_end_ms",
+    "is_speech",
+    "speech",
+    "speech_probability",
+    "vad",
+    "overlap_action",
+    "overlap",
+    "force_barge_in",
+    "force_listen",
+    "text",
+    "transcript",
+)
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeAudioAppend:
+    """One decoded ``input_audio_buffer.append``.
+
+    ``audio`` is raw bytes in ``format`` at ``sample_rate_hz`` --- after
+    :func:`decode_audio_append` that is 16 kHz ``pcm_f32le`` for every input
+    format the codec converts.
+    """
+
+    audio: bytes
+    format: str
+    sample_rate_hz: int | None = None
+    is_speech: bool | None = None
+    video_frames: tuple[str, ...] = ()
+    duration_ms: int | None = None
+    audio_end_ms: int | None = None
+    #: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
+    hints: dict[str, object] = field(default_factory=dict)
+    #: Client correlation id (OpenAI ``event_id``), echoed on error events.
+    event_id: str | None = None
+
+
+# ---- speech hints ----
+
+
+def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
+    for key in ("is_speech", "speech"):
+        value = event.get(key)
+        if isinstance(value, bool):
+            return not value
+    vad = event.get("vad")
+    if isinstance(vad, dict):
+        value = vad.get("is_speech")
+        if isinstance(value, bool):
+            return not value
+        probability = vad.get("speech_probability", vad.get("probability"))
+        if isinstance(probability, int | float):
+            return float(probability) < 0.5
+    probability = event.get("speech_probability")
+    return isinstance(probability, int | float) and float(probability) < 0.5
+
+
+def input_looks_like_speech(
+    event: Mapping[str, object],
+    *,
+    audio: object,
+    fmt: object,
+    overlap_silence_rms: float,
+) -> bool:
+    if input_explicitly_non_speech(event):
+        return False
+    for key in ("is_speech", "speech"):
+        value = event.get(key)
+        if isinstance(value, bool):
+            return value
+    vad = event.get("vad")
+    if isinstance(vad, dict):
+        probability = vad.get("speech_probability", vad.get("probability"))
+        if isinstance(probability, int | float):
+            return float(probability) >= 0.5
+    probability = event.get("speech_probability")
+    if isinstance(probability, int | float):
+        return float(probability) >= 0.5
+    if fmt != "pcm_f32le":
+        return True
+    if isinstance(audio, bytes | bytearray):
+        raw = bytes(audio)
+    elif isinstance(audio, str):
+        try:
+            raw = base64.b64decode(audio, validate=True)
+        except (binascii.Error, ValueError):
+            return True
+    else:
+        return True
+    if len(raw) < 4 or len(raw) % 4 != 0:
+        return True
+    samples = np.frombuffer(raw, dtype=np.float32)
+    if samples.size == 0:
+        return False
+    rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
+    threshold = event.get("overlap_silence_rms")
+    if not isinstance(threshold, int | float):
+        vad = event.get("vad")
+        if isinstance(vad, dict):
+            threshold = vad.get("silence_rms")
+    silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
+    return rms >= max(0.0, silence_rms)
+
+
+def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
+    for key in REALTIME_INPUT_HINT_KEYS:
+        if key in source:
+            target[key] = source[key]
+
+
+# ---- append decoding ----
+
+
+def decode_audio_append(
+    event: Mapping[str, object],
+    *,
+    defaults: RealtimeInputDefaults,
+    hints_source: Mapping[str, object] | None = None,
+) -> RealtimeAudioAppend:
+    """Validate, convert (to 16 kHz ``pcm_f32le``) and pack one audio append.
+
+    ``hints_source`` is the enclosing payload when the audio arrives inside
+    something larger than a bare append --- a ``conversation.item.create``
+    audio part, say --- so its hints apply unless the part overrides them.
+
+    Raises :class:`RealtimeProtocolError` for an unsupported format, undecodable
+    audio or invalid camera frames.
+    """
+    event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
+    audio = event.get("audio") or event.get("delta")
+    fmt, format_rate = parse_realtime_audio_format(
+        event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
+    )
+    sample_rate_hz = (
+        event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
+    )
+    if not is_supported_realtime_input_format(fmt):
+        raise RealtimeProtocolError(
+            f"Unsupported input_audio_format: {fmt}",
+            code="unsupported_audio_format",
+            event_id=event_id,
+        )
+    try:
+        audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
+            audio,
+            fmt,
+            sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
+        )
+    except ValueError as exc:
+        raise RealtimeProtocolError(str(exc), code="bad_event", event_id=event_id) from exc
+    hints: dict[str, object] = {}
+    if hints_source is not None:
+        copy_realtime_input_hints(hints_source, hints)
+    copy_realtime_input_hints(event, hints)
+    looks_like_speech = input_looks_like_speech(
+        {**(hints_source or {}), **event},
+        audio=audio,
+        fmt=fmt,
+        overlap_silence_rms=defaults.overlap_silence_rms,
+    )
+    video_frames: tuple[str, ...] = ()
+    raw_frames = event.get("video_frames")
+    if raw_frames is not None:
+        frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
+        if frames_error is not None:
+            raise RealtimeProtocolError(frames_error, code="invalid_video_frames", event_id=event_id)
+        if isinstance(raw_frames, list):
+            video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
+    duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
+    audio_end_ms = hints.get("audio_end_ms")
+    try:
+        audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
+    except (binascii.Error, ValueError) as exc:
+        raise RealtimeProtocolError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
+    return RealtimeAudioAppend(
+        event_id=event_id,
+        audio=audio_bytes,
+        format=str(fmt),
+        sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
+        is_speech=looks_like_speech,
+        video_frames=video_frames,
+        duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
+        audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
+        hints=hints,
+    )

vllm_omni/protocol/realtime/capabilities.py

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""What a Realtime consumer can serve, and the session check that reads it.
+
+``session.update`` is the one client event whose acceptance depends on who is
+behind the socket. The audio formats a server can decode, and whether it can do
+server-side turn detection at all, are properties of the *consumer* --- the
+duplex engine, or some other runtime --- not of the protocol. The codec still
+has to do the checking, because it owns the error envelope and the order the
+checks run in.
+
+:class:`RealtimeProtocolCapabilities` is how a consumer states those answers.
+It is the post-#7413 form of the ``supports(turn_detection, formats)`` half of
+RFC #6592's proposed ``RealtimeModelAdapter``. The other three halves of that
+proposal --- building a prompt, starting a response, cancelling one --- are not
+here and must not come here: for a duplex model they are already owned by
+``DuplexModelPlugin`` together with ``DuplexSessionRunner``, and a second
+interface over the same responsibility is exactly the fork this layer exists to
+prevent.
+
+The duplex binding of this object is
+``vllm_omni.engine.duplex.realtime_commands.DUPLEX_REALTIME_CAPABILITIES``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+
+from vllm_omni.protocol.realtime.formats import (
+    REALTIME_INPUT_AUDIO_FORMATS,
+    REALTIME_OUTPUT_AUDIO_FORMATS,
+    validate_realtime_session_audio_formats,
+)
+
+__all__ = [
+    "RealtimeProtocolCapabilities",
+    "RealtimeSessionRejection",
+    "validate_session_payload",
+]
+
+#: A consumer's answer to "can you do this turn detection?": an error message, or None.
+TurnDetectionValidator = Callable[[Mapping[str, object]], str | None]
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeProtocolCapabilities:
+    """One consumer's answer to what a ``session`` object may ask for.
+
+    The defaults are permissive, not restrictive: every format the codec can
+    decode, and **no turn-detection validation at all**. A consumer that leaves
+    ``validate_turn_detection`` at ``None`` therefore accepts whatever
+    ``turn_detection`` the session object carries --- including values it cannot
+    actually serve, such as ``semantic_vad``. A consumer that supports only some
+    turn-detection modes (or none) must supply its own validator; see
+    ``vllm_omni.engine.duplex.realtime_commands.DUPLEX_REALTIME_CAPABILITIES``.
+    """
+
+    input_audio_formats: frozenset[str] = field(default_factory=lambda: frozenset(REALTIME_INPUT_AUDIO_FORMATS))
+    output_audio_formats: frozenset[str] = field(default_factory=lambda: frozenset(REALTIME_OUTPUT_AUDIO_FORMATS))
+    #: Consumer-specific ``turn_detection`` validation. ``None`` means the
+    #: consumer takes whatever the session object says without a check of its own.
+    validate_turn_detection: TurnDetectionValidator | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeSessionRejection:
+    """Why a session object was refused: the internal error code and the message.
+
+    ``param`` names the offending session field where there is one, for the
+    ``error.param`` slot of the OpenAI error envelope.
+    """
+
+    code: str
+    message: str
+    param: str | None = None
+
+
+def validate_session_payload(
+    session_payload: Mapping[str, object],
+    *,
+    capabilities: RealtimeProtocolCapabilities,
+) -> RealtimeSessionRejection | None:
+    """Check a ``session`` object against one consumer; ``None`` when it is acceptable.
+
+    Audio formats are checked before turn detection, so a session object that
+    is wrong in both ways reports the format problem --- keep that order, it is
+    what clients already see.
+    """
+    format_error = validate_realtime_session_audio_formats(
+        session_payload,
+        input_audio_formats=capabilities.input_audio_formats,
+        output_audio_formats=capabilities.output_audio_formats,
+    )
+    if format_error is not None:
+        return RealtimeSessionRejection(code="unsupported_audio_format", message=format_error)
+    validate_turn_detection = capabilities.validate_turn_detection
+    if validate_turn_detection is not None:
+        turn_detection_error = validate_turn_detection(session_payload)
+        if turn_detection_error is not None:
+            return RealtimeSessionRejection(
+                code="unsupported_turn_detection", message=turn_detection_error, param="turn_detection"
+            )
+    return None

vllm_omni/protocol/realtime/commands.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""OpenAI Realtime client events as typed commands.
+
+A Realtime server's first job is turning a client JSON event into something
+typed. These are the 10 client events OpenAI's Realtime API defines, as frozen
+dataclasses carrying the fields that survive decoding.
+
+vLLM-Omni's own client events (barge-in, playback acknowledgement, text append,
+heartbeat, explicit close, generic turn signals) are a separate vocabulary in
+``vllm_omni.protocol.duplex.commands``.
+
+Decoding a payload *into* these stays with the consumer, because which command
+a given event becomes can depend on what that consumer supports --- see
+``vllm_omni.engine.duplex.realtime_commands.translate_realtime_command``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import ClassVar
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class RealtimeCommand:
+    """One decoded OpenAI Realtime client event.
+
+    ``wire_type`` is the event type the client sent; the typed fields are what
+    survived decoding, so a consumer reads them instead of re-parsing JSON.
+
+    There is deliberately no rendering method here. A server receives commands,
+    it does not emit them, and how a runtime represents one internally is its
+    own business --- the duplex engine renders its mailbox dictionary in
+    ``vllm_omni.engine.duplex.commands``.
+    """
+
+    #: OpenAI Realtime client event this command decodes from.
+    wire_type: ClassVar[str] = ""
+    #: Client correlation id (OpenAI ``event_id``), echoed on error events.
+    event_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class UpdateSession(RealtimeCommand):
+    """``session.update``: a Realtime ``session`` object patch."""
+
+    wire_type: ClassVar[str] = "session.update"
+    patch: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AppendAudio(RealtimeCommand):
+    wire_type: ClassVar[str] = "input_audio_buffer.append"
+    #: Raw audio bytes in ``format`` at ``sample_rate_hz`` (base64 only on the wire).
+    audio: bytes
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Commit(RealtimeCommand):
+    wire_type: ClassVar[str] = "input_audio_buffer.commit"
+    final: bool = True
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ClearInput(RealtimeCommand):
+    wire_type: ClassVar[str] = "input_audio_buffer.clear"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ClearOutputAudio(RealtimeCommand):
+    wire_type: ClassVar[str] = "output_audio_buffer.clear"
+    #: Explicit response to clear; ``None`` targets the active/last response.
+    response_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CreateResponse(RealtimeCommand):
+    wire_type: ClassVar[str] = "response.create"
+    #: Raw Realtime ``response`` object (instructions, voice, modalities, ...).
+    options: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CancelResponse(RealtimeCommand):
+    wire_type: ClassVar[str] = "response.cancel"
+    response_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CreateItem(RealtimeCommand):
+    """``conversation.item.create`` (history injection or function-call output)."""
+
+    wire_type: ClassVar[str] = "conversation.item.create"
+    item: Mapping[str, object]
+    previous_item_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class DeleteItem(RealtimeCommand):
+    wire_type: ClassVar[str] = "conversation.item.delete"
+    item_id: str
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TruncateItem(RealtimeCommand):
+    wire_type: ClassVar[str] = "conversation.item.truncate"
+    item_id: str
+    audio_end_ms: int
+    content_index: int = 0
+
+
+__all__ = [
+    "AppendAudio",
+    "CancelResponse",
+    "ClearInput",
+    "ClearOutputAudio",
+    "Commit",
+    "CreateItem",
+    "CreateResponse",
+    "DeleteItem",
+    "RealtimeCommand",
+    "TruncateItem",
+    "UpdateSession",
+]

vllm_omni/protocol/realtime/errors.py

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""The OpenAI Realtime error envelope.
+
+A Realtime server reports a rejected client event as one ``error`` event whose
+``error`` object carries ``type`` / ``code`` / ``message`` (plus the client's
+``event_id`` and an optional ``param``). ``type`` is not free-form: it is one
+of OpenAI's four buckets, derived from our internal code by
+:data:`REALTIME_ERROR_TYPES_BY_CODE`.
+
+OpenAI standardises the three ``type`` classes but not the *codes*, so the code
+vocabulary is a consumer's own. vLLM-Omni's lives in
+``vllm_omni.protocol.duplex.errors`` (Tier 3 in
+``docs/serving/realtime_duplex_api.md``); what stays here is the exception type
+every consumer raises, so the envelope is rendered identically whoever built it.
+"""
+
+from __future__ import annotations
+
+__all__ = ["RealtimeProtocolError"]
+
+
+class RealtimeProtocolError(ValueError):
+    """A client payload could not be decoded into a valid Realtime intent.
+
+    ``code`` is the internal error code that :data:`REALTIME_ERROR_TYPES_BY_CODE`
+    maps to an OpenAI ``error.type``; ``event_id`` is the *client* event id the
+    error answers. ``vllm_omni.engine.duplex.commands.DuplexCommandError`` is
+    the duplex specialization, so a consumer catching either sees the same
+    three attributes.
+    """
+
+    def __init__(self, message: str, *, code: str = "bad_event", event_id: str | None = None) -> None:
+        super().__init__(message)
+        self.code = code
+        self.event_id = event_id

vllm_omni/protocol/realtime/events.py

@@ -0,0 +1,467 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""OpenAI Realtime server events: typed classes and their wire rendering.
+
+Every event is a frozen dataclass with explicit fields, and the Realtime wire
+JSON is *derived* from those fields by :meth:`RealtimeEvent.to_realtime`. That
+rendering is pure --- it never consults session state --- which is why it can
+live here, away from whoever owns the session. The stateful part (response and
+item ids, content-part bookkeeping) is consumed when the events are
+*constructed*, by the consumer.
+
+This module holds the 30 events OpenAI's Realtime API defines. vLLM-Omni's own
+additions to the wire (listen/speak turn-taking, playback acknowledgement,
+session resume) are a separate vocabulary in
+``vllm_omni.protocol.duplex.events``, so a consumer that only speaks GA does
+not inherit them.
+
+``session_id`` / ``epoch`` are bound by the consumer when an event leaves it;
+producers build events with the defaults.
+"""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Mapping
+from dataclasses import dataclass, field, fields
+from typing import ClassVar
+from uuid import uuid4
+
+
+def new_event_id() -> str:
+    return f"event_{uuid4().hex}"
+
+
+_IDENTITY_FIELDS = frozenset({"session_id", "epoch", "event_id"})
+
+
+def wire_value(value: object) -> object:
+    if isinstance(value, Mapping):
+        return {str(k): wire_value(v) for k, v in value.items()}
+    if isinstance(value, tuple | list):
+        return [wire_value(v) for v in value]
+    return value
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class RealtimeEvent:
+    """Base of every public session event."""
+
+    #: Realtime ``type`` this class renders to.
+    wire_type: ClassVar[str] = ""
+    #: Fields that are omitted from the wire object when ``None``.
+    optional_wire_fields: ClassVar[frozenset[str]] = frozenset()
+
+    session_id: str = ""
+    epoch: int | None = None
+    #: Server event id (OpenAI ``event_id``); generated at construction.
+    event_id: str = field(default_factory=new_event_id)
+
+    # ---- identity ----
+
+    @property
+    def type(self) -> str:
+        return self.wire_type
+
+    @property
+    def is_terminal(self) -> bool:
+        return False
+
+    # ---- generic accessors (None when the class has no such field) ----
+
+    @property
+    def response_id(self) -> str | None:
+        return None
+
+    @property
+    def item_id(self) -> str | None:
+        return None
+
+    @property
+    def text(self) -> str | None:
+        return None
+
+    @property
+    def audio(self) -> bytes | None:
+        return None
+
+    # ---- wire rendering (pure) ----
+
+    def _wire_fields(self) -> dict[str, object]:
+        data: dict[str, object] = {}
+        for f in fields(self):
+            if f.name in _IDENTITY_FIELDS:
+                continue
+            value = getattr(self, f.name)
+            if value is None and f.name in self.optional_wire_fields:
+                continue
+            data[f.name] = wire_value(value)
+        return data
+
+    def to_realtime(self) -> dict[str, object]:
+        """The Realtime wire JSON object for this event (derived from the fields)."""
+        return {"type": self.wire_type, "event_id": self.event_id, **self._wire_fields()}
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging aid
+        return f"{type(self).__name__}(type={self.wire_type!r}, session_id={self.session_id!r}, epoch={self.epoch!r})"
+
+
+# ---- shared bases ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseScopedEvent(RealtimeEvent):
+    """Events addressed to one response (and usually one output item / content part)."""
+
+    response_id: str | None = None  # type: ignore[assignment]
+    item_id: str | None = None  # type: ignore[assignment]
+    output_index: int = 0
+    content_index: int = 0
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class _ItemEvent(RealtimeEvent):
+    previous_item_id: str | None = None
+    item: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def item_id(self) -> str | None:
+        value = self.item.get("id")
+        return value if isinstance(value, str) else None
+
+
+# ---- GA server events ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionCreated(RealtimeEvent):
+    wire_type = "session.created"
+
+    session: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionUpdated(RealtimeEvent):
+    wire_type = "session.updated"
+
+    session: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseCreated(RealtimeEvent):
+    wire_type = "response.created"
+
+    response: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class OutputItemAdded(RealtimeEvent):
+    wire_type = "response.output_item.added"
+
+    response_id: str | None = None  # type: ignore[assignment]
+    output_index: int = 0
+    item: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def item_id(self) -> str | None:
+        value = self.item.get("id")
+        return value if isinstance(value, str) else None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class OutputItemDone(OutputItemAdded):
+    wire_type = "response.output_item.done"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ContentPartAdded(ResponseScopedEvent):
+    wire_type = "response.content_part.added"
+
+    part: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ContentPartDone(ContentPartAdded):
+    wire_type = "response.content_part.done"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AudioDelta(ResponseScopedEvent):
+    wire_type = "response.output_audio.delta"
+
+    #: Base64 audio in ``format``.
+    delta: str = ""
+
+    @property
+    def audio(self) -> bytes | None:
+        try:
+            return base64.b64decode(self.delta)
+        except (ValueError, TypeError):
+            return None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AudioDone(ResponseScopedEvent):
+    wire_type = "response.output_audio.done"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TranscriptDelta(ResponseScopedEvent):
+    wire_type = "response.output_audio_transcript.delta"
+
+    delta: str = ""
+
+    @property
+    def text(self) -> str | None:
+        return self.delta
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TranscriptDone(ResponseScopedEvent):
+    wire_type = "response.output_audio_transcript.done"
+
+    transcript: str = ""
+
+    @property
+    def text(self) -> str | None:
+        return self.transcript
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TextDelta(ResponseScopedEvent):
+    wire_type = "response.output_text.delta"
+
+    delta: str = ""
+
+    @property
+    def text(self) -> str | None:
+        return self.delta
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TextDone(ResponseScopedEvent):
+    wire_type = "response.output_text.done"
+
+    text: str = ""  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class FunctionCallArgumentsDelta(RealtimeEvent):
+    wire_type = "response.function_call_arguments.delta"
+
+    response_id: str | None = None  # type: ignore[assignment]
+    item_id: str | None = None  # type: ignore[assignment]
+    output_index: int = 0
+    call_id: str = ""
+    delta: str = ""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class FunctionCallArgumentsDone(RealtimeEvent):
+    wire_type = "response.function_call_arguments.done"
+
+    response_id: str | None = None  # type: ignore[assignment]
+    item_id: str | None = None  # type: ignore[assignment]
+    output_index: int = 0
+    call_id: str = ""
+    arguments: str = ""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseDone(RealtimeEvent):
+    wire_type = "response.done"
+
+    response: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def status(self) -> str | None:
+        value = self.response.get("status")
+        return value if isinstance(value, str) else None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class RateLimitsUpdated(RealtimeEvent):
+    wire_type = "rate_limits.updated"
+
+    rate_limits: tuple[Mapping[str, object], ...] = ()
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class InputCommitted(RealtimeEvent):
+    wire_type = "input_audio_buffer.committed"
+
+    previous_item_id: str | None = None
+    item_id: str | None = None  # type: ignore[assignment]
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"previous_item_id": self.previous_item_id, "item_id": self.item_id}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class InputCleared(RealtimeEvent):
+    wire_type = "input_audio_buffer.cleared"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SpeechStarted(RealtimeEvent):
+    wire_type = "input_audio_buffer.speech_started"
+
+    audio_start_ms: int = 0
+    item_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SpeechStopped(RealtimeEvent):
+    wire_type = "input_audio_buffer.speech_stopped"
+
+    audio_end_ms: int = 0
+    item_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemAdded(_ItemEvent):
+    wire_type = "conversation.item.added"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemCreated(_ItemEvent):
+    wire_type = "conversation.item.created"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemDone(_ItemEvent):
+    wire_type = "conversation.item.done"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemRetrieved(RealtimeEvent):
+    wire_type = "conversation.item.retrieved"
+
+    item: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def item_id(self) -> str | None:
+        value = self.item.get("id")
+        return value if isinstance(value, str) else None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemDeleted(RealtimeEvent):
+    wire_type = "conversation.item.deleted"
+
+    item_id: str | None = None  # type: ignore[assignment]
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"item_id": self.item_id}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemTruncated(RealtimeEvent):
+    wire_type = "conversation.item.truncated"
+
+    item_id: str | None = None  # type: ignore[assignment]
+    content_index: int = 0
+    audio_end_ms: int = 0
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {
+            "item_id": self.item_id,
+            "content_index": self.content_index,
+            "audio_end_ms": self.audio_end_ms,
+        }
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class InputTranscriptionCompleted(RealtimeEvent):
+    wire_type = "conversation.item.input_audio_transcription.completed"
+
+    item_id: str | None = None  # type: ignore[assignment]
+    content_index: int = 0
+    transcript: str = ""
+
+    @property
+    def text(self) -> str | None:
+        return self.transcript
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class OutputAudioCleared(RealtimeEvent):
+    wire_type = "output_audio_buffer.cleared"
+
+    response_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ErrorEvent(RealtimeEvent):
+    wire_type = "error"
+
+    code: str = "internal_error"
+    message: str = ""
+    #: Client ``event_id`` this error answers (OpenAI ``error.event_id``).
+    related_event_id: str | None = None
+    param: str | None = None
+
+    @property
+    def error_type(self) -> str:
+        """OpenAI's error class for this error.
+
+        OpenAI standardises only the three classes, not the codes, so the base
+        reports the default. A consumer with its own code vocabulary overrides
+        this --- see ``vllm_omni.protocol.duplex.events.ErrorEvent``.
+        """
+        return "invalid_request_error"
+
+    @property
+    def error(self) -> dict[str, object]:
+        error: dict[str, object] = {"type": self.error_type, "code": self.code, "message": self.message}
+        if self.related_event_id:
+            error["event_id"] = self.related_event_id
+        if self.param:
+            error["param"] = self.param
+        return error
+
+    @property
+    def text(self) -> str | None:
+        return self.message
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"error": self.error}
+
+
+__all__ = [
+    "AudioDelta",
+    "AudioDone",
+    "ContentPartAdded",
+    "ContentPartDone",
+    "ErrorEvent",
+    "FunctionCallArgumentsDelta",
+    "FunctionCallArgumentsDone",
+    "InputCleared",
+    "InputCommitted",
+    "InputTranscriptionCompleted",
+    "ItemAdded",
+    "ItemCreated",
+    "ItemDeleted",
+    "ItemDone",
+    "ItemRetrieved",
+    "ItemTruncated",
+    "OutputAudioCleared",
+    "OutputItemAdded",
+    "OutputItemDone",
+    "RateLimitsUpdated",
+    "RealtimeEvent",
+    "ResponseCreated",
+    "ResponseDone",
+    "ResponseScopedEvent",
+    "SessionCreated",
+    "SessionUpdated",
+    "SpeechStarted",
+    "SpeechStopped",
+    "TextDelta",
+    "TextDone",
+    "TranscriptDelta",
+    "TranscriptDone",
+    "new_event_id",
+    "wire_value",
+]

vllm_omni/protocol/realtime/formats.py

@@ -0,0 +1,177 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Realtime audio-format negotiation: what a client may declare and how it is read.
+
+A Realtime client declares audio formats in three places --- the session object
+(``session.audio.input.format`` and the older flat ``input_audio_format``), a
+``response.create`` override, and a ``conversation.item.create`` audio part ---
+and may spell one format several ways (``audio/pcm``, ``pcm16``, ``s16le``).
+This module is the single place that normalizes those spellings and says which
+are supported. Folding a session's declaration into the per-connection append
+defaults is the job of ``session.py``, which builds on this module.
+
+Everything here is a pure function of wire payloads: no session state, no model,
+no engine.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Mapping
+
+__all__ = [
+    "REALTIME_INPUT_AUDIO_FORMATS",
+    "REALTIME_OUTPUT_AUDIO_FORMATS",
+    "is_supported_realtime_input_format",
+    "parse_realtime_audio_format",
+    "realtime_audio_format_object",
+    "realtime_output_format",
+    "validate_conversation_item_audio_formats",
+    "validate_realtime_response_audio_formats",
+    "validate_realtime_session_audio_formats",
+]
+
+REALTIME_INPUT_AUDIO_FORMATS = {
+    "pcm16",
+    "pcm_s16le",
+    "s16le",
+    "pcm_f32le",
+    "g711_ulaw",
+    "g711_alaw",
+}
+REALTIME_OUTPUT_AUDIO_FORMATS = {
+    "pcm16",
+    "pcm_s16le",
+    "s16le",
+    "wav",
+    "pcm",
+    "g711_ulaw",
+    "g711_alaw",
+}
+
+
+def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]:
+    def normalize_format(fmt: str) -> str:
+        normalized = fmt.lower()
+        if normalized in {"audio/pcm", "pcm"}:
+            return "pcm16"
+        if normalized in {"audio/wav", "wav"}:
+            return "wav"
+        if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}:
+            return "pcm16"
+        if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}:
+            return "pcm_f32le"
+        if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}:
+            return "g711_ulaw"
+        if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}:
+            return "g711_alaw"
+        return fmt
+
+    if isinstance(raw_format, str):
+        return normalize_format(raw_format), None
+    if not isinstance(raw_format, dict):
+        return raw_format, None
+    rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate")
+    sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None
+    fmt = raw_format.get("type") or raw_format.get("format")
+    if not isinstance(fmt, str):
+        return raw_format, sample_rate_hz
+    return normalize_format(fmt), sample_rate_hz
+
+
+def realtime_output_format(duplex_format: object) -> str:
+    if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}:
+        return duplex_format.lower()
+    if isinstance(duplex_format, str) and duplex_format.lower() == "pcm":
+        return "pcm16"
+    return str(duplex_format or "wav")
+
+
+def is_supported_realtime_input_format(fmt: object) -> bool:
+    return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS
+
+
+def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]:
+    if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}:
+        payload: dict[str, object] = {"type": "audio/pcm"}
+    elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le":
+        payload = {"type": "audio/pcm_f32le"}
+    elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw":
+        payload = {"type": "audio/g711_ulaw"}
+    elif isinstance(fmt, str) and fmt.lower() == "g711_alaw":
+        payload = {"type": "audio/g711_alaw"}
+    else:
+        payload = {"type": "audio/wav"}
+    if sample_rate_hz is not None:
+        payload["rate"] = int(sample_rate_hz)
+    return payload
+
+
+def validate_realtime_session_audio_formats(
+    session_payload: Mapping[str, object],
+    *,
+    input_audio_formats: Collection[str] | None = None,
+    output_audio_formats: Collection[str] | None = None,
+) -> str | None:
+    """Reject a session object that declares an audio format we cannot serve.
+
+    The format sets default to everything the codec can decode; a consumer that
+    serves a narrower set passes its own (see
+    ``vllm_omni.protocol.realtime.capabilities``).
+    """
+    supported_input = REALTIME_INPUT_AUDIO_FORMATS if input_audio_formats is None else input_audio_formats
+    supported_output = REALTIME_OUTPUT_AUDIO_FORMATS if output_audio_formats is None else output_audio_formats
+    audio_config = session_payload.get("audio")
+    input_format: object = session_payload.get("input_audio_format")
+    if input_format is None and isinstance(audio_config, dict):
+        audio_input = audio_config.get("input")
+        if isinstance(audio_input, dict):
+            input_format = audio_input.get("format")
+    parsed_input, _ = parse_realtime_audio_format(input_format)
+    if input_format is not None and not (isinstance(parsed_input, str) and parsed_input.lower() in supported_input):
+        return f"Unsupported input_audio_format: {input_format}"
+
+    output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
+    if output_format is None and isinstance(audio_config, dict):
+        audio_output = audio_config.get("output")
+        if isinstance(audio_output, dict):
+            output_format = audio_output.get("format")
+    parsed_output, _ = parse_realtime_audio_format(output_format)
+    if output_format is not None and not (isinstance(parsed_output, str) and parsed_output.lower() in supported_output):
+        return f"Unsupported output_audio_format: {output_format}"
+    return None
+
+
+def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None:
+    output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format")
+    audio_config = response_payload.get("audio")
+    if output_format is None and isinstance(audio_config, dict):
+        audio_output = audio_config.get("output")
+        if isinstance(audio_output, dict):
+            output_format = audio_output.get("format")
+    parsed_output, _ = parse_realtime_audio_format(output_format)
+    if output_format is not None and not (
+        isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
+    ):
+        return f"Unsupported output_audio_format: {output_format}"
+    return None
+
+
+def validate_conversation_item_audio_formats(item: object) -> str | None:
+    if not isinstance(item, dict):
+        return None
+    content = item.get("content")
+    if not isinstance(content, list):
+        return None
+    for part in content:
+        if not isinstance(part, dict):
+            continue
+        if part.get("type") not in {"input_audio", "audio"}:
+            continue
+        raw_format = part.get("format")
+        parsed_format, _ = parse_realtime_audio_format(raw_format)
+        if raw_format is not None and not (
+            isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS
+        ):
+            return f"Unsupported input_audio format in conversation.item.create: {raw_format}"
+    return None

vllm_omni/protocol/realtime/items.py

@@ -0,0 +1,197 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Realtime conversation items: shape, truncation and the transcript inside one.
+
+A Realtime conversation is a list of *items* --- a user message, an assistant
+message, a function call --- each with a ``content`` list of typed parts. The
+client creates them (``conversation.item.create``), truncates an assistant
+item's audio after a barge-in (``conversation.item.truncate``), and the server
+echoes them back inside ``conversation.item.*`` and ``response.done`` events.
+
+The rules in this module are the ones both sides have to agree on: which
+defaults a bare item gets, how far a transcript is cut when the client says it
+only heard N milliseconds of audio, and what makes a truncate request invalid.
+Item *storage* --- which items a session currently holds, in what order --- is
+the consumer's state, not this module's.
+
+``validate_realtime_video_frames`` sits here too: camera frames ride along an
+``input_audio_buffer.append`` in the vLLM-Omni extension of the protocol, and
+validating them is the same kind of wire-shape check.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+from collections.abc import Mapping
+from uuid import uuid4
+
+__all__ = [
+    "input_transcript_from_item",
+    "normalize_conversation_item",
+    "text_chars_for_audio_ms_from_marks",
+    "truncate_realtime_item_content",
+    "validate_realtime_item_truncate",
+    "validate_realtime_video_frames",
+]
+
+
+def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
+    """Validate omni-duplex camera frames on input_audio_buffer.append.
+
+    Wire contract matches the official MiniCPM-o duplex loop: one base
+    base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
+    stacked composite tiling the sub-frames captured inside it (at most 2
+    images either way). A caller-supplied ``max_slice_nums`` is rejected rather
+    than silently ignored: slicing is Stage 0's decision here, and Stage 0
+    already applies the official HD suggestion for a stacked unit
+    (``max_slice_nums=[2, 1]``). The wire simply does not let the client choose
+    it.
+    """
+    if max_slice_nums not in (None, 1):
+        return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself"
+    if not isinstance(video_frames, list):
+        return "video_frames must be a list of base64-encoded images"
+    frames = [frame for frame in video_frames if frame is not None]
+    if len(frames) > 2:
+        return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
+    for frame in frames:
+        if not isinstance(frame, str) or not frame:
+            return "video_frames entries must be non-empty base64 strings"
+        if len(frame) > 4_000_000:
+            return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
+        try:
+            header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
+        except (binascii.Error, ValueError):
+            return "video_frames entries must be valid base64"
+        if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
+            return "video_frames entries must be JPEG or PNG images"
+    return None
+
+
+def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]:
+    normalized = dict(item)
+    normalized.setdefault("id", f"item_{uuid4().hex}")
+    normalized.setdefault("object", "realtime.item")
+    normalized.setdefault("type", "message")
+    normalized.setdefault("status", "completed")
+    if "role" not in normalized and normalized.get("type") == "message":
+        normalized["role"] = "user"
+    if not isinstance(normalized.get("content"), list):
+        normalized["content"] = []
+    return normalized
+
+
+def text_chars_for_audio_ms_from_marks(
+    audio_end_ms: int,
+    text_len: int,
+    marks: list[object],
+    *,
+    final_ms: object | None = None,
+) -> int:
+    if text_len <= 0:
+        return 0
+    clean_marks: list[tuple[int, int]] = []
+    for mark in marks:
+        if not isinstance(mark, dict):
+            continue
+        raw_text_chars = mark.get("text_chars")
+        raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
+        if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
+            continue
+        clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
+    if not clean_marks:
+        return 0 if audio_end_ms <= 0 else text_len
+    clean_marks.sort(key=lambda item: item[0])
+    audio_end_ms = max(0, int(audio_end_ms))
+    if audio_end_ms <= 0:
+        return 0
+    previous_ms = 0
+    previous_chars = 0
+    for mark_ms, mark_chars in clean_marks:
+        mark_ms = max(previous_ms, mark_ms)
+        mark_chars = max(previous_chars, min(text_len, mark_chars))
+        if audio_end_ms <= mark_ms:
+            if mark_ms <= previous_ms:
+                return mark_chars
+            ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
+            return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
+        previous_ms = mark_ms
+        previous_chars = mark_chars
+    if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
+        if audio_end_ms >= int(final_ms):
+            return text_len
+        ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
+        return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
+    return text_len if audio_end_ms >= previous_ms else previous_chars
+
+
+def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None:
+    content = item.get("content")
+    if not isinstance(content, list) or not content:
+        return
+    index = max(0, int(content_index))
+    if index >= len(content):
+        return
+    part = content[index]
+    if not isinstance(part, dict):
+        return
+    transcript = part.get("transcript")
+    if not isinstance(transcript, str) or not transcript:
+        return
+    marks = part.get("audio_text_marks")
+    if isinstance(marks, list):
+        keep_chars = text_chars_for_audio_ms_from_marks(
+            audio_end_ms,
+            len(transcript),
+            marks,
+            final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
+        )
+        part["transcript"] = transcript[:keep_chars].rstrip()
+        return
+    duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
+    if isinstance(duration_ms, int | float) and duration_ms > 0:
+        keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
+        part["transcript"] = transcript[:keep_chars].rstrip()
+    elif audio_end_ms <= 0:
+        part["transcript"] = ""
+
+
+def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None:
+    if item.get("type") != "message" or item.get("role") != "assistant":
+        return "conversation.item.truncate only supports assistant message items"
+    if audio_end_ms < 0:
+        return "conversation.item.truncate requires non-negative audio_end_ms"
+    content = item.get("content")
+    if not isinstance(content, list) or not content:
+        return None
+    index = max(0, int(content_index))
+    if index >= len(content):
+        return f"conversation.item.truncate content_index out of range: {content_index}"
+    part = content[index]
+    if not isinstance(part, dict):
+        return "conversation.item.truncate target content part is invalid"
+    if part.get("type") not in {"audio", "output_audio"}:
+        return "conversation.item.truncate target content part must be audio"
+    duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
+    if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
+        return "conversation.item.truncate audio_end_ms exceeds item audio duration"
+    return None
+
+
+def input_transcript_from_item(item: Mapping[str, object]) -> str:
+    content = item.get("content")
+    if not isinstance(content, list):
+        return ""
+    parts: list[str] = []
+    for part in content:
+        if not isinstance(part, dict):
+            continue
+        for key in ("transcript", "text"):
+            value = part.get(key)
+            if isinstance(value, str) and value:
+                if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
+                    parts.append(value)
+                    break
+    return "".join(parts).strip()

vllm_omni/protocol/realtime/session.py

@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Reading the Realtime ``session`` object.
+
+``session.update`` carries one JSON object that a client may spell in either of
+two shapes: the GA nest (``session.audio.input.format``,
+``session.audio.output.sample_rate_hz``) or the older flat beta keys
+(``input_audio_format``, ``sample_rate_hz``). Everything downstream --- the
+append decoder, the projection that renders ``session.created`` back out, a
+model's runtime configuration --- has to agree on what a given session object
+means, so the readers live here once.
+
+:class:`RealtimeInputDefaults` is the resolved answer for the input path: the
+format, sample rate and silence threshold an ``input_audio_buffer.append``
+inherits when it omits them. A connection keeps one of these and re-derives it
+on every ``session.update``.
+
+``overlap_policy`` / ``playback_commit_policy`` are vLLM-Omni extension fields
+on the session object rather than GA OpenAI fields. They are read here because
+this module owns what the wire may say; what a server *does* with them is the
+consumer's business.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from typing import Any
+
+from vllm_omni.protocol.realtime.formats import (
+    REALTIME_INPUT_AUDIO_FORMATS,
+    REALTIME_OUTPUT_AUDIO_FORMATS,
+    parse_realtime_audio_format,
+    realtime_output_format,
+)
+
+__all__ = [
+    "RealtimeInputDefaults",
+    "apply_realtime_session_defaults",
+    "input_audio_transcription_config",
+    "json_safe_realtime_payload",
+    "realtime_max_output_tokens",
+    "realtime_overlap_fields",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeInputDefaults:
+    """Session-level wire defaults an append may omit (derived from the session object)."""
+
+    input_audio_format: str = "pcm16"
+    input_sample_rate_hz: int = 16000
+    output_audio_format: str = "pcm16"
+    output_sample_rate_hz: int | None = None
+    overlap_silence_rms: float = 0.003
+
+    def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults:
+        """Return defaults updated from a Realtime ``session`` object (session.update)."""
+        values = apply_realtime_session_defaults(self, session_payload)
+        return values
+
+
+def apply_realtime_session_defaults(
+    defaults: RealtimeInputDefaults,
+    session_payload: Mapping[str, object],
+) -> RealtimeInputDefaults:
+    """Derive the wire defaults a Realtime ``session`` object declares."""
+    input_format: object = session_payload.get("input_audio_format")
+    audio_config = session_payload.get("audio")
+    if input_format is None and isinstance(audio_config, dict):
+        audio_input = audio_config.get("input")
+        if isinstance(audio_input, dict):
+            input_format = audio_input.get("format")
+    input_format, input_rate = parse_realtime_audio_format(input_format)
+    # Any: heterogeneous field values splatted into ``dataclasses.replace`` below.
+    updates: dict[str, Any] = {}
+    if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS:
+        updates["input_audio_format"] = input_format
+    output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
+    output_rate_raw: object | None = None
+    if output_format is None and isinstance(audio_config, dict):
+        audio_output = audio_config.get("output")
+        if isinstance(audio_output, dict):
+            output_format = audio_output.get("format")
+            output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate")
+    output_format, output_rate = parse_realtime_audio_format(output_format)
+    if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0:
+        output_rate = int(output_rate_raw)
+    if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
+        updates["output_audio_format"] = realtime_output_format(output_format)
+    sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate")
+    if sample_rate is None and isinstance(audio_config, dict):
+        audio_input = audio_config.get("input")
+        if isinstance(audio_input, dict):
+            sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate")
+    if sample_rate is None:
+        sample_rate = input_rate
+    if isinstance(sample_rate, int | float) and sample_rate > 0:
+        updates["input_sample_rate_hz"] = int(sample_rate)
+    if isinstance(output_rate, int | float) and output_rate > 0:
+        updates["output_sample_rate_hz"] = int(output_rate)
+    overlap_fields = realtime_overlap_fields(session_payload)
+    overlap_silence_rms = overlap_fields.get("overlap_silence_rms")
+    if isinstance(overlap_silence_rms, int | float):
+        updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms))
+    return replace(defaults, **updates) if updates else defaults
+
+
+def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]:
+    fields: dict[str, object] = {}
+    if isinstance(session_payload.get("overlap_policy"), str):
+        fields["overlap_policy"] = session_payload["overlap_policy"]
+    for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"):
+        value = session_payload.get(key)
+        if isinstance(value, int | float):
+            fields[key] = value
+    if isinstance(session_payload.get("playback_commit_policy"), str):
+        fields["playback_commit_policy"] = session_payload["playback_commit_policy"]
+    return fields
+
+
+def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]:
+    clean: dict[str, object] = {}
+    for key, value in payload.items():
+        if key == "extra_body":
+            continue
+        if isinstance(value, str | int | float | bool) or value is None:
+            clean[key] = value
+        elif isinstance(value, dict):
+            clean[key] = json_safe_realtime_payload(value)
+        elif isinstance(value, list):
+            clean[key] = [
+                (json_safe_realtime_payload(item) if isinstance(item, dict) else item)
+                for item in value
+                if isinstance(item, str | int | float | bool | dict) or item is None
+            ]
+    return clean
+
+
+def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
+    transcription = session_payload.get("input_audio_transcription")
+    if isinstance(transcription, dict):
+        return transcription
+    audio_config = session_payload.get("audio")
+    if not isinstance(audio_config, dict):
+        return None
+    audio_input = audio_config.get("input")
+    if not isinstance(audio_input, dict):
+        return None
+    transcription = audio_input.get("transcription")
+    return transcription if isinstance(transcription, dict) else None
+
+
+def realtime_max_output_tokens(value: object) -> int | None:
+    """Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
+    if value is None:
+        return None
+    if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}:
+        return None
+    if isinstance(value, int) and value > 0:
+        return int(value)
+    return None