[Core] Give the OpenAI Realtime wire codec a reusable home (RFC #6592 P0a) (#7640)

Signed-off-by: psv666 <2693925048@qq.com>

.buildkite/common/ci_source_file_dependencies.yml

@@ -44,6 +44,7 @@ x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
   - vllm_omni/entrypoints/duplex/
   - vllm_omni/entrypoints/duplex_omni.py
   - vllm_omni/entrypoints/async_omni_base.py
+  - vllm_omni/protocol/
   - vllm_omni/clients/
   - vllm_omni/outputs/duplex.py
   - vllm_omni/model_executor/duplex_sampling.py
@@ -259,6 +260,7 @@ source_file_dependencies:
     - tests/entrypoints/test_duplex_omni.py
     - tests/entrypoints/openai_api/test_duplex_api_server.py
     - tests/model_executor/models/minicpmo_4_5/duplex/
+    - tests/protocol/
 
   omni_minicpmo_4_5_duplex_perf:
     - *minicpmo_4_5

.github/CODEOWNERS

@@ -67,6 +67,11 @@
 # File-level overrides within entrypoints/
 /vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
 /vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
+# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex
+# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the
+# engine depends on it — so it is owned by the serving-boundary document rather
+# than by any one consumer. — docs/design/module/entrypoints.md
+/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao
 
 # Error contracts — docs/design/module/error_contracts.md
 /vllm_omni/errors.py @alex-jw-brooks @NickCao

docs/design/fullduplex.md

@@ -9,6 +9,12 @@ the owner document for `vllm_omni/engine/duplex/**`,
 `vllm_omni/entrypoints/duplex_omni.py`, `vllm_omni/entrypoints/duplex/**`,
 `vllm_omni/clients/**` and the per-model `duplex/` packages.
 
+`vllm_omni/protocol/realtime/**` (the OpenAI Realtime API) and
+`vllm_omni/protocol/duplex/**` (vLLM-Omni's extension of it) are the shared
+wire codec duplex is built on. Duplex is its first consumer but not its owner: the codec
+is deliberately runtime-agnostic so a second Realtime surface can reuse it
+(RFC #6592 P0a).
+
 The public wire contract lives in
 [`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md);
 this page describes how it is produced.
@@ -124,6 +130,40 @@ plugin and the session runtime config to `DuplexOrchestrator` directly.
    attachment/resume/replay bookkeeping; it holds no session state.
 6. **No `typing.Protocol`** in the duplex surfaces: the plugin, data plane,
    session state, PCM buffer, stage port and client transport seams are ABCs.
+7. **The wire codec is not duplex, and a wire type is not a mailbox message.**
+   The event and command *vocabulary* plus its wire rendering, the audio format
+   negotiation, the conversation-item rules and the error envelope are model-
+   and runtime-agnostic, and split along the three tiers
+   [`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md)
+   already defines: **Tier 1** (identical to OpenAI) is
+   `vllm_omni/protocol/realtime/**`; **Tier 2** (OpenAI names carrying our
+   extensions) and **Tier 3** (ours alone) are `vllm_omni/protocol/duplex/**`.
+   A Tier 2 class *subclasses* its Tier 1 twin and declares only what it adds,
+   so the OpenAI surface stays honestly pure --- `session.created` without
+   `resume_token`, `input_audio_buffer.append` without `video_frames`. Our
+   error **codes** are Tier 3 (`protocol/duplex/errors.py`); only the envelope
+   shape and OpenAI's three `error.type` classes are Tier 1.
+   What stays engine-side is everything that is *internal representation*
+   rather than contract: `DuplexCommand.payload()` renders the session runner's
+   mailbox dictionary, whose channel genuinely differs from the client event
+   (`session.update` and the three `conversation.item.*` commands all travel on
+   `turn.signal`); `engine/duplex/realtime_commands.py` decides which command a
+   decoded event becomes; `engine/duplex/realtime_events.py` holds the
+   session's projection state. The codec may not import the engine, the
+   entrypoints, `model_executor` or the clients
+   (`tests/protocol/realtime/test_protocol_import_boundary.py`), and there is
+   exactly one implementation of each codec behaviour
+   (`tests/protocol/realtime/test_realtime_codec_single_source.py`).
+   The dependency chain is `protocol/realtime` <- `protocol/duplex` <- engine /
+   entrypoints / clients, and a duplex consumer uses **only** the middle link:
+   `protocol/duplex/**` re-exports the Tier 1 names it does not extend, so a
+   helper that later needs a duplex-specific version (`convert_input_audio_with_rate`
+   resamples to MiniCPM-o's 16 kHz rather than the client's rate) is overridden
+   in one file instead of at every call site
+   (`tests/protocol/duplex/test_duplex_protocol_facade.py`).
+   `DuplexEvent` is therefore a plain alias of `RealtimeEvent` --- an event has
+   no engine-internal half --- while `DuplexCommand` is a real class, because a
+   command does.
 
 ## Package layout
 
@@ -142,6 +182,31 @@ vllm_omni/
 │   │   ├── chat_completions.py      DuplexChatCompletionsAdapter (/v1/chat/completions on a session per request)
 │   │   └── websocket.py             websocket send/close/receive helpers
 │   └── openai/api_server.py         builds DuplexOmni for duplex models; session-backed app state
+├── protocol/                        SHARED WIRE CODEC (no engine / no model / no transport)
+│   ├── realtime/                    the OpenAI Realtime API itself
+│   │   ├── events.py                RealtimeEvent + OpenAI's 30 server events (22 ship as Tier 1;
+│   │   │                            8 have a Tier 2 twin in protocol/duplex)
+│   │   ├── commands.py              RealtimeCommand + OpenAI's 10 client commands (8 Tier 1,
+│   │   │                            2 with a Tier 2 twin) --- wire_type + fields, no payload()
+│   │   ├── formats.py               audio format spellings, what is supported, per-session validation
+│   │   ├── session.py               session-object readers; RealtimeInputDefaults (append defaults)
+│   │   ├── items.py                 conversation item shape, truncation, transcripts, camera frames
+│   │   ├── audio_input.py           decode_audio_append -> RealtimeAudioAppend; client speech hints
+│   │   ├── audio.py                 PCM / G.711 / WAV decode, resample, re-encode
+│   │   ├── errors.py                RealtimeProtocolError (the envelope's exception type)
+│   │   └── capabilities.py          RealtimeProtocolCapabilities: what one consumer can serve
+│   └── duplex/                      Tier 2 + Tier 3, and the one door a duplex consumer uses
+│       ├── __init__.py              re-exports the Tier 1 helper functions (audio, formats,
+│       │                            items, session, capabilities) so the engine never
+│       │                            imports protocol/realtime directly
+│       ├── events.py                the whole 42-event vocabulary: 22 Tier-1 re-exports,
+│       │                            8 Tier-2 subclasses (session.created + resume_token, ...),
+│       │                            12 Tier-3 events (listen/speak, playback, resume, ...);
+│       │                            DuplexEvent = RealtimeEvent
+│       ├── commands.py              the whole 17-command vocabulary: 8 Tier-1 re-exports,
+│       │                            2 Tier-2 subclasses (append + hints, commit + final),
+│       │                            7 Tier-3 commands (barge_in, playback.ack, ...)
+│       └── errors.py                our error-code vocabulary (Tier 3) + RealtimeProtocolError
 ├── engine/
 │   ├── omni_engine_base.py          OmniEngineBase (stage processes, orchestrator thread, queues, RPC)
 │   ├── async_omni_engine.py         AsyncOmniEngine (turn-based request building)
@@ -149,16 +214,20 @@ vllm_omni/
 │   ├── orchestrator.py              OrchestratorBase + Orchestrator
 │   ├── duplex_orchestrator.py       DuplexOrchestrator (+ DuplexOrchestratorRequestState; implements DuplexStagePort)
 │   └── duplex/
-│       ├── commands.py              DuplexCommand dataclasses, command_from_realtime
+│       ├── commands.py              mailbox half: DuplexCommand.payload() + the mailbox `type`,
-│       ├── realtime_commands.py     Realtime client event -> DuplexCommand translation
+│       │                            paired with each protocol command; command_from_realtime
-│       ├── events.py                DuplexEvent dataclasses (+ to_realtime), REALTIME_ERROR_TYPES_BY_CODE
+│       ├── realtime_commands.py     duplex binding of the codec: decoded event -> DuplexCommand,
+│       │                            DUPLEX_REALTIME_CAPABILITIES, duplex_response_format
+│       ├── events.py                re-export shim (DuplexEvent = RealtimeEvent) + the runner's
+│       │                            epoch-filter sets DOMAIN_TERMINAL_EVENTS / MODEL_OUTPUT_EVENTS
 │       ├── realtime_events.py       RealtimeProjectionState: internal event -> typed events
 │       ├── messages.py              queue envelopes (Open/Close/Resume/Touch/Command/Result/Event), DuplexSessionError
 │       ├── config.py                DuplexSessionConfig, DuplexCapabilities, ResponseCreateOptions
 │       ├── contracts.py             DuplexFence (session_id, epoch, turn_id), stage request records, DuplexStagePort
 │       ├── plugin.py                DuplexModelPlugin, DuplexModelSessionState, DuplexDataPlane, PcmAppendBuffer ABCs
 │       ├── turn_detection.py        server-side VAD turn detector used by the session
-│       ├── audio.py / vad.py / intermediate.py
+│       ├── audio.py                 compatibility re-export, via protocol/duplex
+│       ├── vad.py / intermediate.py
 │       └── session/                 one engine-resident session and everything that runs it
 │           ├── engine_session.py    DuplexEngineSession: ledgers, lease, fence, stage resources, append sequencing
 │           ├── runner.py            DuplexSessionRunner (per-session mailbox on the orchestrator loop)

docs/design/module/entrypoints.md

@@ -30,6 +30,7 @@ primary_code_paths:
   - vllm_omni/entrypoints/utils.py
   - vllm_omni/entrypoints/duplex/**
   - vllm_omni/clients/**
+  - vllm_omni/protocol/**
 primary_path_exceptions:
   - path: vllm_omni/entrypoints/openai/errors.py
     owner: error_contracts.md
@@ -53,6 +54,7 @@ depends_on:
   - error_contracts.md
   - engine_orchestration.md
 validation_paths:
+  - tests/protocol/**
   - tests/entrypoints/test_omni_entrypoints.py
   - tests/entrypoints/test_async_omni.py
   - tests/entrypoints/test_async_omni_pause_sleep_routing.py
@@ -106,6 +108,31 @@ placement, payload implementation, or semantic error classification.
 `entrypoints/openai/errors.py` is an explicit primary-path exception owned by
 `error_contracts.md`.
 
+It also owns `vllm_omni/protocol/**`, the shared wire codec promoted by RFC
+[#6592](https://github.com/vllm-project/vllm-omni/issues/6592) P0a. It follows the three tiers
+`docs/serving/realtime_duplex_api.md` already defines. `protocol/realtime/**`
+is Tier 1, identical to OpenAI: 30 server events, 10 client commands, their
+typed fields and pure wire rendering, audio-format negotiation,
+conversation-item rules and the error-envelope shape. `protocol/duplex/**` is
+Tier 2 (OpenAI names carrying vLLM-Omni extensions, each a subclass of its
+Tier 1 twin declaring only what it adds) and Tier 3 (ours alone, including our
+error-code vocabulary), kept separate so a GA-only consumer is not handed a
+vocabulary its clients never send. That package sits
+outside `entrypoints/` on purpose --- the engine depends on it, so it belongs
+to neither layer --- but deciding what a Realtime client may put on the wire is
+this document's subject, and the same reasoning already places
+`vllm_omni/errors.py`, `inputs/` and `outputs/` under contract documents rather
+than under a consumer.
+
+It is deliberately **not** owned by `../fullduplex.md`, even though duplex is
+the codec's only consumer today. RFC #6592 asks for a named owner precisely so
+a second Realtime consumer does not need duplex review to change the codec
+(#6592 open question 8). Ownership here is review scope, not a dependency: the
+codec may not import the engine, the entrypoints or the model code, which
+`tests/protocol/realtime/test_protocol_import_boundary.py` enforces. When a
+second consumer lands (#6592 P0b) the package is expected to graduate to its
+own module document with owners drawn from both consumers.
+
 ## Candidate invariants
 
 These identifiers are proposals while the document is `draft`.
@@ -115,6 +142,22 @@ These identifiers are proposals while the document is `draft`.
 **Rule:** Entrypoints MUST NOT implement cross-stage routing or stage lifecycle
 policy.
 
+### ENTRY-INV-002: The shared Realtime codec does not depend on its consumers
+
+**Rule:** `vllm_omni/protocol/**` MUST NOT import `vllm_omni.engine`,
+`vllm_omni.entrypoints`, `vllm_omni.model_executor`, `vllm_omni.worker` or
+`vllm_omni.clients`; `protocol/duplex/**` MAY depend on `protocol/realtime/**`
+but never the reverse, and a Tier 1 class MUST NOT carry a vLLM-Omni extension
+field; and each protocol/codec behavior MUST have exactly one
+implementation that every consumer uses. A wire type MUST NOT carry a
+consumer's internal representation: the duplex mailbox rendering
+(`DuplexCommand.payload()`, whose channel differs from the client event for
+`session.update` and the `conversation.item.*` commands) stays engine-side.
+A duplex consumer (`engine/duplex/**`, `entrypoints/duplex/**`, the duplex
+clients) MUST import `vllm_omni.protocol.duplex` and MUST NOT import
+`vllm_omni.protocol.realtime` directly, so the tier boundary has exactly one
+extension point.
+
 ### ENTRY-INV-100: Public requests are normalized once
 
 **Rule:** Public protocol values MUST be validated and converted to internal

tests/protocol/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

tests/protocol/duplex/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

tests/protocol/duplex/test_duplex_protocol_facade.py

@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""``protocol.duplex`` is the only door a duplex consumer uses.
+
+The layering is a chain::
+
+    protocol.realtime  <-  protocol.duplex  <-  engine / entrypoints / clients
+
+The middle link only earns its keep if the last arrow is the *only* one. If the
+engine may also reach past it straight into ``protocol.realtime``, then giving a
+Tier 1 helper a duplex-specific version later means touching every call site
+instead of one file --- which is exactly the change
+``convert_input_audio_with_rate`` is expected to need (it resamples to
+MiniCPM-o's 16 kHz, not the client's rate).
+
+So this asserts two things: nobody skips the middle link, and the middle link is
+complete enough that nobody has to.
+"""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+
+#: Every module that speaks the duplex protocol. These import
+#: ``vllm_omni.protocol.duplex``; reaching into ``vllm_omni.protocol.realtime``
+#: from here skips the extension point.
+CONSUMER_PATHS = (
+    "vllm_omni/engine/duplex",
+    "vllm_omni/engine/duplex_omni_engine.py",
+    "vllm_omni/entrypoints/duplex",
+    "vllm_omni/entrypoints/duplex_omni.py",
+    "vllm_omni/clients/duplex.py",
+    "vllm_omni/clients/inline_duplex.py",
+)
+
+
+def _consumer_modules() -> list[Path]:
+    found: list[Path] = []
+    for entry in CONSUMER_PATHS:
+        path = REPO_ROOT / entry
+        found.extend(sorted(path.rglob("*.py")) if path.is_dir() else [path])
+    assert found, "consumer list is stale"
+    return found
+
+
+@pytest.mark.parametrize("module", _consumer_modules(), ids=lambda p: p.name)
+def test_a_duplex_consumer_does_not_skip_the_duplex_protocol_package(module: Path) -> None:
+    # Source-level, so a lazy import inside a function body is caught too.
+    tree = ast.parse(module.read_text())
+    offenders = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ImportFrom) and (node.module or "").startswith("vllm_omni.protocol.realtime"):
+            offenders.append(f"line {node.lineno}: from {node.module} import ...")
+        elif isinstance(node, ast.Import):
+            offenders.extend(
+                f"line {node.lineno}: import {alias.name}"
+                for alias in node.names
+                if alias.name.startswith("vllm_omni.protocol.realtime")
+            )
+    assert not offenders, (
+        f"{module.relative_to(REPO_ROOT)} imports the Tier 1 package directly; "
+        f"import from vllm_omni.protocol.duplex instead:\n  " + "\n  ".join(offenders)
+    )
+
+
+def test_the_duplex_package_carries_the_whole_command_vocabulary() -> None:
+    from vllm_omni.engine.duplex import commands as engine_commands
+    from vllm_omni.protocol.duplex import commands as duplex_wire
+
+    # Every command the engine handles is nameable through the one door.
+    engine_names = {
+        name
+        for name in engine_commands.__all__
+        if isinstance(getattr(engine_commands, name), type)
+        and issubclass(getattr(engine_commands, name), engine_commands.DuplexCommand)
+        and getattr(engine_commands, name) is not engine_commands.DuplexCommand
+    }
+    assert engine_names <= set(duplex_wire.__all__)
+
+
+def test_the_duplex_package_carries_the_whole_event_vocabulary() -> None:
+    from vllm_omni.engine.duplex import events as engine_events
+    from vllm_omni.protocol.duplex import events as duplex_wire
+
+    engine_names = {
+        name
+        for name in engine_events.__all__
+        if isinstance(getattr(engine_events, name), type)
+        and issubclass(getattr(engine_events, name), engine_events.DuplexEvent)
+    }
+    assert engine_names <= set(duplex_wire.__all__)
+
+
+@pytest.mark.parametrize(
+    "module_pair",
+    [("events", "events"), ("commands", "commands")],
+    ids=["events", "commands"],
+)
+def test_a_reexported_tier1_name_is_the_tier1_object(module_pair: tuple[str, str]) -> None:
+    """Re-export, not re-declaration: the Tier 1 class is one object."""
+    import importlib
+
+    duplex_mod = importlib.import_module(f"vllm_omni.protocol.duplex.{module_pair[0]}")
+    realtime_mod = importlib.import_module(f"vllm_omni.protocol.realtime.{module_pair[1]}")
+
+    shared = [n for n in duplex_mod.__all__ if hasattr(realtime_mod, n)]
+    assert shared, "nothing re-exported --- the facade is not doing its job"
+    for name in shared:
+        duplex_obj = getattr(duplex_mod, name)
+        realtime_obj = getattr(realtime_mod, name)
+        if duplex_obj is realtime_obj:
+            continue
+        # The only permitted divergence is a Tier 2 class extending its twin.
+        assert isinstance(duplex_obj, type) and issubclass(duplex_obj, realtime_obj), (
+            f"{name} in protocol.duplex is neither the Tier 1 object nor a subclass of it"
+        )
+
+
+def test_duplex_event_is_an_alias_but_duplex_command_is_not() -> None:
+    """The asymmetry is deliberate, and a later reader should not 'fix' it.
+
+    An event has no engine-internal half --- what the session emits is what the
+    socket sends --- so ``DuplexEvent`` is simply ``RealtimeEvent``. A command
+    does: ``payload()`` renders the session runner's mailbox dictionary, and for
+    four of the seventeen the mailbox channel is not even the client event type.
+    """
+    from vllm_omni.engine.duplex.commands import DuplexCommand
+    from vllm_omni.protocol.duplex.events import DuplexEvent
+    from vllm_omni.protocol.realtime.commands import RealtimeCommand
+    from vllm_omni.protocol.realtime.events import RealtimeEvent
+
+    assert DuplexEvent is RealtimeEvent
+
+    assert DuplexCommand is not RealtimeCommand
+    assert issubclass(DuplexCommand, RealtimeCommand)
+    assert hasattr(DuplexCommand, "payload") and not hasattr(RealtimeCommand, "payload")

tests/protocol/realtime/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

tests/protocol/realtime/test_protocol_import_boundary.py

@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""The shared Realtime codec must not depend on any runtime that uses it.
+
+``vllm_omni.protocol.realtime`` exists so a second Realtime surface can reuse
+the wire codec without adopting the duplex session control plane. That only
+holds while the arrows point one way: the engine, the entrypoints and the model
+code import the protocol, never the reverse. Nothing in the package's own code
+enforces that, so it is asserted here --- once on the source (which catches a
+lazy import inside a function body) and once on a real import (which catches a
+dependency arriving through a third module).
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+PROTOCOL_DIR = REPO_ROOT / "vllm_omni" / "protocol"
+
+#: Packages that consume the protocol. A dependency on any of them would make
+#: the codec unusable outside the runtime it accidentally reached into.
+#: ``vllm_omni.utils`` is deliberately absent: it is leaf helpers (the audio
+#: resampler) that consume nothing, so depending on it costs a reuser nothing.
+FORBIDDEN_PREFIXES = (
+    "vllm_omni.engine",
+    "vllm_omni.entrypoints",
+    "vllm_omni.model_executor",
+    "vllm_omni.worker",
+    "vllm_omni.clients",
+)
+
+
+def _protocol_sources() -> list[Path]:
+    sources = sorted(PROTOCOL_DIR.rglob("*.py"))
+    assert sources, f"no python sources under {PROTOCOL_DIR}"
+    return sources
+
+
+def _imported_modules(source: Path) -> set[str]:
+    """Every module name imported anywhere in the file, lazy imports included."""
+    names: set[str] = set()
+    for node in ast.walk(ast.parse(source.read_text())):
+        if isinstance(node, ast.Import):
+            names.update(alias.name for alias in node.names)
+        elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
+            names.add(node.module)
+    return names
+
+
+@pytest.mark.parametrize("source", _protocol_sources(), ids=lambda path: path.name)
+def test_protocol_sources_do_not_import_a_consumer(source: Path) -> None:
+    offenders = sorted(
+        name
+        for name in _imported_modules(source)
+        if any(name == prefix or name.startswith(prefix + ".") for prefix in FORBIDDEN_PREFIXES)
+    )
+    assert not offenders, f"{source.relative_to(REPO_ROOT)} imports {', '.join(offenders)}"
+
+
+def test_importing_the_protocol_package_loads_no_consumer() -> None:
+    # A child interpreter, because this test session has already imported the
+    # duplex engine: only a fresh process can tell what the package pulls in.
+    # ``vllm_omni`` itself is imported first, because its ``__init__`` loads a
+    # baseline graph (patches, the model registry) that has nothing to do with
+    # this package; only what the protocol import *adds* is measured.
+    env = os.environ.copy()
+    env.pop("CUDA_VISIBLE_DEVICES", None)
+    env.pop("HIP_VISIBLE_DEVICES", None)
+    script = f"""
+import sys
+
+import vllm_omni  # noqa: F401
+
+baseline = set(sys.modules)
+
+import vllm_omni.protocol.realtime  # noqa: F401
+
+forbidden_prefixes = {FORBIDDEN_PREFIXES!r}
+added = sorted(
+    name
+    for name in set(sys.modules) - baseline
+    if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes)
+)
+if added:
+    raise SystemExit("the realtime protocol package loaded consumers: " + ", ".join(added))
+"""
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        cwd=REPO_ROOT,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=180,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr

tests/protocol/realtime/test_realtime_capabilities.py

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""``RealtimeProtocolCapabilities``: one consumer's answers, the codec's checking.
+
+This is the seam that lets a non-duplex Realtime surface validate a
+``session.update`` without a ``DuplexModelPlugin``. What it has to get right is
+narrow: the order the checks run in (formats before turn detection, because
+that is the order clients already see), the error codes, and that a consumer
+which declares a narrower format set is actually narrowed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+import pytest
+
+from vllm_omni.protocol.realtime.capabilities import (
+    RealtimeProtocolCapabilities,
+    RealtimeSessionRejection,
+    validate_session_payload,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+_DEFAULT = RealtimeProtocolCapabilities()
+
+
+def test_a_consumer_without_a_turn_detection_check_accepts_a_plain_session():
+    assert (
+        validate_session_payload({"model": "m", "audio": {"input": {"format": "pcm16"}}}, capabilities=_DEFAULT) is None
+    )
+
+
+def test_a_consumer_without_a_turn_detection_check_ignores_turn_detection():
+    payload = {"turn_detection": {"type": "semantic_vad"}}
+
+    assert validate_session_payload(payload, capabilities=_DEFAULT) is None
+
+
+def test_an_unsupported_input_format_is_rejected_with_the_realtime_code():
+    rejection = validate_session_payload({"input_audio_format": "opus"}, capabilities=_DEFAULT)
+
+    assert rejection == RealtimeSessionRejection(
+        code="unsupported_audio_format", message="Unsupported input_audio_format: opus"
+    )
+
+
+def test_a_consumer_may_declare_a_narrower_format_set():
+    pcm_only = RealtimeProtocolCapabilities(input_audio_formats=frozenset({"pcm16"}))
+
+    assert validate_session_payload({"input_audio_format": "pcm16"}, capabilities=pcm_only) is None
+    rejection = validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=pcm_only)
+    assert rejection is not None
+    assert rejection.code == "unsupported_audio_format"
+    # The default capabilities still accept it: the narrowing is per consumer.
+    assert validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=_DEFAULT) is None
+
+
+def test_the_turn_detection_check_runs_and_reports_its_own_code():
+    def reject_everything(session_payload: Mapping[str, object]) -> str | None:
+        del session_payload
+        return "server_vad is not available"
+
+    capabilities = RealtimeProtocolCapabilities(validate_turn_detection=reject_everything)
+
+    rejection = validate_session_payload({"turn_detection": {"type": "server_vad"}}, capabilities=capabilities)
+
+    assert rejection == RealtimeSessionRejection(
+        code="unsupported_turn_detection", message="server_vad is not available", param="turn_detection"
+    )
+
+
+def test_a_bad_format_is_reported_before_a_bad_turn_detection():
+    calls: list[Mapping[str, object]] = []
+
+    def record(session_payload: Mapping[str, object]) -> str | None:
+        calls.append(session_payload)
+        return "turn detection is wrong too"
+
+    capabilities = RealtimeProtocolCapabilities(validate_turn_detection=record)
+
+    rejection = validate_session_payload(
+        {"input_audio_format": "opus", "turn_detection": {"type": "semantic_vad"}},
+        capabilities=capabilities,
+    )
+
+    assert rejection is not None
+    assert rejection.code == "unsupported_audio_format"
+    assert calls == []

tests/protocol/realtime/test_realtime_codec_single_source.py

@@ -0,0 +1,256 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""There is one Realtime codec, and the duplex engine uses it.
+
+RFC #6592 P0a rejects the "copy the codec and fix it up" option because the two
+copies drift. The extraction is only worth anything while the duplex names are
+*the same objects* as the protocol ones, not lookalikes --- so assert identity,
+which a re-implementation cannot satisfy.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+import pytest
+
+from vllm_omni.engine.duplex import audio as duplex_audio
+from vllm_omni.engine.duplex import events as duplex_events
+from vllm_omni.engine.duplex import realtime_commands as duplex_codec
+from vllm_omni.engine.duplex.commands import DuplexCommandError
+from vllm_omni.protocol import realtime as protocol
+from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+#: Names ``engine.duplex.realtime_commands`` re-exports for its existing importers.
+_REEXPORTED_FROM_PROTOCOL = (
+    "REALTIME_INPUT_AUDIO_FORMATS",
+    "REALTIME_INPUT_HINT_KEYS",
+    "REALTIME_OUTPUT_AUDIO_FORMATS",
+    "RealtimeInputDefaults",
+    "apply_realtime_session_defaults",
+    "copy_realtime_input_hints",
+    "input_audio_transcription_config",
+    "input_explicitly_non_speech",
+    "input_looks_like_speech",
+    "input_transcript_from_item",
+    "is_supported_realtime_input_format",
+    "json_safe_realtime_payload",
+    "normalize_conversation_item",
+    "parse_realtime_audio_format",
+    "realtime_audio_format_object",
+    "realtime_max_output_tokens",
+    "realtime_output_format",
+    "realtime_overlap_fields",
+    "text_chars_for_audio_ms_from_marks",
+    "truncate_realtime_item_content",
+    "validate_conversation_item_audio_formats",
+    "validate_realtime_item_truncate",
+    "validate_realtime_response_audio_formats",
+    "validate_realtime_session_audio_formats",
+    "validate_realtime_video_frames",
+)
+
+
+@pytest.mark.parametrize("name", _REEXPORTED_FROM_PROTOCOL)
+def test_the_duplex_codec_name_is_the_protocol_object(name: str) -> None:
+    assert getattr(duplex_codec, name) is getattr(protocol, name)
+
+
+@pytest.mark.parametrize(
+    "name",
+    ("convert_input_audio_with_rate", "convert_output_audio", "resample_pcm16_mono", "wav_payload_to_pcm16"),
+)
+def test_the_duplex_audio_shim_is_the_protocol_object(name: str) -> None:
+    assert getattr(duplex_audio, name) is getattr(protocol, name)
+
+
+def test_a_duplex_command_error_is_a_realtime_protocol_error() -> None:
+    # The duplex binding converts the codec's error into its own type; both
+    # carry the same code/event_id, which is what the error envelope renders.
+    error = DuplexCommandError("bad", code="bad_audio", event_id="event_1")
+
+    assert isinstance(error, RealtimeProtocolError)
+    assert (error.code, error.event_id) == ("bad_audio", "event_1")
+
+
+def test_the_duplex_append_command_is_built_from_the_shared_decoder() -> None:
+    defaults = protocol.RealtimeInputDefaults()
+    event = {"event_id": "event_1", "audio": "", "format": "pcm16", "duration_ms": 40}
+
+    decoded = protocol.decode_audio_append(event, defaults=defaults)
+    command = duplex_codec.build_append_audio(event, defaults=defaults)
+
+    assert (command.audio, command.format, command.sample_rate_hz) == (
+        decoded.audio,
+        decoded.format,
+        decoded.sample_rate_hz,
+    )
+    assert (command.is_speech, command.video_frames, command.duration_ms) == (
+        decoded.is_speech,
+        decoded.video_frames,
+        decoded.duration_ms,
+    )
+    assert (command.audio_end_ms, dict(command.hints), command.event_id) == (
+        decoded.audio_end_ms,
+        decoded.hints,
+        decoded.event_id,
+    )
+
+
+def test_the_shared_decoder_error_reaches_the_client_as_a_duplex_command_error() -> None:
+    defaults = protocol.RealtimeInputDefaults()
+    event = {"event_id": "event_1", "audio": "AAAA", "format": "opus"}
+
+    with pytest.raises(RealtimeProtocolError) as shared:
+        protocol.decode_audio_append(event, defaults=defaults)
+    with pytest.raises(DuplexCommandError) as duplex:
+        duplex_codec.build_append_audio(event, defaults=defaults)
+
+    assert (duplex.value.code, str(duplex.value)) == (shared.value.code, str(shared.value))
+    assert duplex.value.event_id == shared.value.event_id == "event_1"
+
+
+def test_the_duplex_capabilities_reject_unimplemented_turn_detection() -> None:
+    # The duplex answer comes from engine.duplex.turn_detection, reached through
+    # the capability object rather than imported by the codec.
+    rejection = protocol.validate_session_payload(
+        {"turn_detection": {"type": "semantic_vad"}},
+        capabilities=duplex_codec.DUPLEX_REALTIME_CAPABILITIES,
+    )
+
+    assert rejection is not None
+    assert rejection.code == "unsupported_turn_detection"
+
+
+# ---- the class split (RFC #6592 P0a, second half) ----
+#
+# The event and command classes moved out of the engine too. The same identity
+# rule applies: the duplex names must BE the protocol objects, and the mailbox
+# half must have stayed behind.
+
+# `docs/serving/realtime_duplex_api.md` sorts every message into three tiers.
+# Tier 1 is pure OpenAI and lives in protocol/realtime; Tier 2 (OpenAI names
+# carrying our extensions) and Tier 3 (ours alone) live in protocol/duplex.
+_TIER1_EVENTS = ("SessionUpdated", "ItemAdded", "ItemCreated", "TextDelta", "OutputItemAdded")
+_TIER2_EVENTS = (
+    "SessionCreated",
+    "ResponseCreated",
+    "ResponseDone",
+    "AudioDelta",
+    "InputCommitted",
+    "ItemDeleted",
+    "ItemTruncated",
+    "ErrorEvent",
+)
+_TIER3_EVENTS = ("Listen", "Speak", "OverlapDecision", "PlaybackAcknowledged", "SessionResumed")
+_TIER1_COMMANDS = ("UpdateSession", "CreateItem", "CancelResponse", "ClearInput")
+_TIER2_COMMANDS = ("AppendAudio", "Commit")
+_TIER3_COMMANDS = ("BargeIn", "AckPlayback", "Heartbeat", "SignalTurn", "CloseSession")
+
+
+@pytest.mark.parametrize("name", _TIER1_EVENTS)
+def test_tier1_events_are_the_pure_openai_objects(name: str) -> None:
+    from vllm_omni.protocol.realtime import events as realtime_events
+
+    assert duplex_events.__dict__[name] is getattr(realtime_events, name)
+
+
+@pytest.mark.parametrize("name", _TIER2_EVENTS + _TIER3_EVENTS)
+def test_tier2_and_tier3_events_come_from_protocol_duplex(name: str) -> None:
+    from vllm_omni.protocol.duplex import events as duplex_wire_events
+
+    assert duplex_events.__dict__[name] is getattr(duplex_wire_events, name)
+
+
+@pytest.mark.parametrize("name", _TIER2_EVENTS)
+def test_tier2_extends_its_tier1_twin_rather_than_replacing_it(name: str) -> None:
+    """A Tier 2 class must be its Tier 1 class plus fields, never a fork of it."""
+    from vllm_omni.protocol.duplex import events as duplex_wire_events
+    from vllm_omni.protocol.realtime import events as realtime_events
+
+    tier1 = getattr(realtime_events, name)
+    tier2 = getattr(duplex_wire_events, name)
+
+    assert issubclass(tier2, tier1)
+    assert tier2.wire_type == tier1.wire_type
+    tier1_fields = {f.name for f in dataclasses.fields(tier1)}
+    tier2_fields = {f.name for f in dataclasses.fields(tier2)}
+    # Purely additive: Tier 2 adds, never drops or renames.
+    assert tier1_fields < tier2_fields
+
+
+@pytest.mark.parametrize("name", _TIER1_EVENTS + _TIER1_COMMANDS)
+def test_tier1_carries_no_vllm_omni_extension_fields(name: str) -> None:
+    """The point of the split: protocol/realtime must be honestly pure OpenAI."""
+    from vllm_omni.protocol.realtime import commands as realtime_commands
+    from vllm_omni.protocol.realtime import events as realtime_events
+
+    cls = getattr(realtime_events, name, None) or getattr(realtime_commands, name)
+    fields = {f.name for f in dataclasses.fields(cls)}
+    # Extension fields catalogued as Tier 2 in the serving doc.
+    assert not (
+        fields
+        & {
+            "attachment_generation",
+            "resume_token",
+            "details",
+            "extra",
+            "is_speech",
+            "video_frames",
+            "hints",
+            "realtime_item_id",
+        }
+    )
+
+
+def test_the_error_code_vocabulary_is_tier3() -> None:
+    """OpenAI standardises the error classes, not our codes (doc: Tier 3)."""
+    from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
+    from vllm_omni.protocol.realtime import errors as realtime_errors
+
+    assert duplex_events.REALTIME_ERROR_TYPES_BY_CODE is REALTIME_ERROR_TYPES_BY_CODE
+    assert not hasattr(realtime_errors, "REALTIME_ERROR_TYPES_BY_CODE")
+
+
+def test_the_tier1_error_event_reports_only_openai_classes() -> None:
+    from vllm_omni.protocol.duplex import events as duplex_wire_events
+    from vllm_omni.protocol.realtime import events as realtime_events
+
+    # Tier 1 knows the envelope shape but not our code vocabulary.
+    assert realtime_events.ErrorEvent(code="resource_exhausted").error_type == "invalid_request_error"
+    # Tier 2 resolves it through our table.
+    assert duplex_wire_events.ErrorEvent(code="resource_exhausted").error_type == "rate_limit_error"
+
+
+@pytest.mark.parametrize("name", _TIER1_COMMANDS + _TIER2_COMMANDS + _TIER3_COMMANDS)
+def test_engine_commands_subclass_their_protocol_wire_type(name: str) -> None:
+    from vllm_omni.engine.duplex import commands as engine_commands
+    from vllm_omni.protocol.duplex import commands as duplex_wire_commands
+    from vllm_omni.protocol.realtime import commands as realtime_commands
+
+    engine_cls = getattr(engine_commands, name)
+    wire_cls = getattr(duplex_wire_commands, name, None) or getattr(realtime_commands, name)
+
+    assert issubclass(engine_cls, wire_cls)
+    assert engine_cls.wire_type == wire_cls.wire_type
+
+
+def test_the_mailbox_channel_stayed_engine_side() -> None:
+    """``payload()``'s ``type`` is the runner's channel, not the client event.
+
+    These four are the proof that the two halves are genuinely different, and
+    so the reason the classes were split rather than relocated wholesale.
+    """
+    from vllm_omni.engine.duplex import commands as engine_commands
+    from vllm_omni.protocol.realtime import commands as realtime_commands
+
+    for name in ("UpdateSession", "CreateItem", "DeleteItem", "TruncateItem"):
+        engine_cls = getattr(engine_commands, name)
+        wire_cls = getattr(realtime_commands, name)
+        assert engine_cls.type == "turn.signal"
+        assert wire_cls.wire_type != "turn.signal"
+        # The wire class carries no mailbox vocabulary at all.
+        assert "type" not in vars(wire_cls)

tools/pre_commit/check_forbidden_imports.py

@@ -79,11 +79,12 @@ CHECK_IMPORTS = {
             "vllm_omni/entrypoints/openai/video_stream_base.py",
             "vllm_omni/entrypoints/openai/video_stream_session.py",
             "vllm_omni/experimental/fullduplex/client.py",
-            "vllm_omni/engine/duplex/audio.py",
             "vllm_omni/engine/duplex/commands.py",
             "vllm_omni/engine/duplex/contracts.py",
-            "vllm_omni/engine/duplex/events.py",
+            "vllm_omni/protocol/realtime/audio.py",
-            "vllm_omni/engine/duplex/realtime_commands.py",
+            "vllm_omni/protocol/realtime/audio_input.py",
+            "vllm_omni/protocol/realtime/events.py",
+            "vllm_omni/protocol/realtime/items.py",
             "vllm_omni/experimental/fullduplex/joyvl/memory/memory.py",
             "vllm_omni/model_executor/models/minicpmo_4_5/duplex/input.py",
             "vllm_omni/model_executor/models/minicpmo_4_5/duplex/plugin.py",

vllm_omni/clients/inline_duplex.py

@@ -32,8 +32,8 @@ from typing import TYPE_CHECKING
 from vllm_omni.clients.duplex import DuplexClientBase, DuplexConnectionError, SessionConfig
 
 if TYPE_CHECKING:
-    from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
     from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle
+    from vllm_omni.protocol.duplex import RealtimeInputDefaults
 
 __all__ = ["InlineDuplexClient"]
 
@@ -71,7 +71,7 @@ class InlineDuplexClient(DuplexClientBase):
 
     def _input_defaults(self) -> RealtimeInputDefaults:
         """Audio format / sample-rate / VAD defaults declared by this session's object."""
-        from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
+        from vllm_omni.protocol.duplex import RealtimeInputDefaults
 
         session_payload = self.session_info or self.config.to_session_payload(model=self.model)
         return RealtimeInputDefaults().with_session_payload(session_payload)

vllm_omni/engine/duplex/audio.py

@@ -1,218 +1,44 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
+"""Compatibility re-export: the Realtime audio codec moved out of the duplex engine.
+
+Decoding PCM / G.711 / WAV, resampling and re-encoding are wire-format work with
+no duplex session in them, so the implementation now lives in
+``vllm_omni.protocol.realtime.audio`` where a non-duplex Realtime surface can
+use it too. Duplex code reaches it through ``vllm_omni.protocol.duplex``, the
+one door the engine uses --- which is also where a duplex-specific conversion
+would be introduced. Import from there in new code.
+"""
+
 from __future__ import annotations
 
-import base64
+from vllm_omni.protocol.duplex import (
-import binascii
+    MAX_INPUT_SAMPLE_RATE_HZ,
-import io
+    MIN_INPUT_SAMPLE_RATE_HZ,
-import math
+    convert_input_audio_with_rate,
-import wave
+    convert_output_audio,
+    decode_g711_alaw,
+    decode_g711_ulaw,
+    encode_float32_mono_wav_base64,
+    encode_g711_alaw,
+    encode_g711_ulaw,
+    resample_pcm16_mono,
+    validate_input_sample_rate_hz,
+    wav_payload_to_pcm16,
+)
 
-import numpy as np
+__all__ = [
-
+    "MAX_INPUT_SAMPLE_RATE_HZ",
-from vllm_omni.utils.audio_resample import StreamingAudioResampler
+    "MIN_INPUT_SAMPLE_RATE_HZ",
-
+    "convert_input_audio_with_rate",
-try:
+    "convert_output_audio",
-    from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin
+    "decode_g711_alaw",
-except ImportError:  # pragma: no cover - audioop is removed in newer Python.
+    "decode_g711_ulaw",
-    alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None
+    "encode_float32_mono_wav_base64",
-
+    "encode_g711_alaw",
-
+    "encode_g711_ulaw",
-MIN_INPUT_SAMPLE_RATE_HZ = 8_000
+    "resample_pcm16_mono",
-MAX_INPUT_SAMPLE_RATE_HZ = 192_000
+    "validate_input_sample_rate_hz",
-
+    "wav_payload_to_pcm16",
-
+]
-def validate_input_sample_rate_hz(sample_rate_hz: object) -> int:
-    if (
-        isinstance(sample_rate_hz, bool)
-        or not isinstance(sample_rate_hz, int | float)
-        or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz))
-    ):
-        raise ValueError("sample_rate_hz must be a finite integer")
-    rate = int(sample_rate_hz)
-    if rate != sample_rate_hz:
-        raise ValueError("sample_rate_hz must be an integer")
-    if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ:
-        raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}")
-    return rate
-
-
-def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes:
-    if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz:
-        return raw
-    samples = np.frombuffer(raw, dtype="<i2")
-    rate_gcd = math.gcd(source_rate_hz, target_rate_hz)
-    reduced_factor = max(source_rate_hz // rate_gcd, target_rate_hz // rate_gcd)
-    if reduced_factor > StreamingAudioResampler.max_polyphase_factor:
-        # The FIR size scales with the reduced ratio. Keep unusual client
-        # rates bounded while common audio rates use the high-quality path.
-        if samples.size <= 1:
-            return raw
-        target_size = max(1, int(round(samples.size * target_rate_hz / source_rate_hz)))
-        source_x = np.linspace(0.0, 1.0, num=samples.size, endpoint=True)
-        target_x = np.linspace(0.0, 1.0, num=target_size, endpoint=True)
-        resampled = np.interp(target_x, source_x, samples)
-        return np.clip(resampled, -32_768, 32_767).astype("<i2").tobytes()
-    resampler = StreamingAudioResampler(source_rate_hz, target_rate_hz)
-    normalized = samples.astype(np.float32) * np.float32(1.0 / 32768.0)
-    resampled = resampler.process(normalized, final=True)
-    return np.clip(np.rint(resampled * 32768.0), -32768, 32767).astype("<i2").tobytes()
-
-
-def decode_g711_ulaw(raw: bytes) -> bytes:
-    if ulaw2lin is not None:
-        return ulaw2lin(raw, 2)
-    data = np.frombuffer(raw, dtype=np.uint8)
-    value = np.bitwise_not(data).astype(np.int16)
-    sign = value & 0x80
-    exponent = (value >> 4) & 0x07
-    mantissa = value & 0x0F
-    sample = ((mantissa << 3) + 0x84) << exponent
-    return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype("<i2").tobytes()
-
-
-def decode_g711_alaw(raw: bytes) -> bytes:
-    if alaw2lin is not None:
-        return alaw2lin(raw, 2)
-    data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16)
-    sign = data & 0x80
-    exponent = (data >> 4) & 0x07
-    mantissa = data & 0x0F
-    sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1))
-    return np.where(sign != 0, sample, -sample).astype("<i2").tobytes()
-
-
-def encode_g711_ulaw(raw: bytes) -> bytes:
-    if lin2ulaw is not None:
-        return lin2ulaw(raw, 2)
-    pcm = np.clip(np.frombuffer(raw, dtype="<i2").astype(np.int32), -32635, 32635)
-    sign = np.where(pcm < 0, 0x80, 0)
-    magnitude = np.abs(pcm) + 0x84
-    exponent = np.zeros_like(magnitude)
-    for exp in range(7):
-        exponent = np.where(magnitude > (0xFF << exp), exp + 1, exponent)
-    mantissa = (magnitude >> (exponent + 3)) & 0x0F
-    return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes()
-
-
-def encode_g711_alaw(raw: bytes) -> bytes:
-    if lin2alaw is not None:
-        return lin2alaw(raw, 2)
-    pcm = np.frombuffer(raw, dtype="<i2").astype(np.int32)
-    sign = np.where(pcm >= 0, 0x80, 0x00)
-    magnitude = np.abs(pcm)
-    exponent = np.zeros_like(magnitude)
-    for exp in range(1, 8):
-        exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent)
-    mantissa = np.where(
-        exponent == 0,
-        (magnitude >> 4) & 0x0F,
-        (magnitude >> (exponent + 3)) & 0x0F,
-    )
-    return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes()
-
-
-def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]:
-    try:
-        with wave.open(io.BytesIO(raw), "rb") as wav_file:
-            channels = wav_file.getnchannels()
-            sample_width = wav_file.getsampwidth()
-            sample_rate_hz = wav_file.getframerate()
-            frames = wav_file.readframes(wav_file.getnframes())
-        if sample_width != 2:
-            return None, sample_rate_hz
-        if channels <= 1:
-            return frames, sample_rate_hz
-        pcm = np.frombuffer(frames, dtype="<i2").reshape(-1, channels)
-        mono = np.mean(pcm.astype(np.float32), axis=1)
-        return np.clip(mono, -32768, 32767).astype("<i2").tobytes(), sample_rate_hz
-    except (EOFError, ValueError, wave.Error):
-        return None, None
-
-
-def encode_float32_mono_wav_base64(
-    samples: np.ndarray,
-    *,
-    sample_rate_hz: int,
-) -> str:
-    """Package normalized mono float32 samples as a PCM16 WAV data payload."""
-    rate = validate_input_sample_rate_hz(sample_rate_hz)
-    normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1)
-    pcm16 = np.clip(np.rint(normalized * 32768.0), -32768, 32767).astype("<i2")
-    with io.BytesIO() as buffer:
-        with wave.open(buffer, "wb") as wav_file:
-            wav_file.setnchannels(1)
-            wav_file.setsampwidth(2)
-            wav_file.setframerate(rate)
-            wav_file.writeframes(pcm16.tobytes())
-        return base64.b64encode(buffer.getvalue()).decode("ascii")
-
-
-def convert_input_audio_with_rate(
-    audio: object,
-    fmt: object,
-    *,
-    sample_rate_hz: int | float | None = None,
-    target_sample_rate_hz: int = 16_000,
-) -> tuple[object, object, int | float | None]:
-    if not isinstance(audio, str) or not isinstance(fmt, str):
-        return audio, fmt, sample_rate_hz
-    normalized = fmt.lower()
-    if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}:
-        return audio, fmt, sample_rate_hz
-    if isinstance(sample_rate_hz, int | float):
-        sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz)
-    try:
-        raw = base64.b64decode(audio.strip(), validate=False)
-    except (binascii.Error, ValueError):
-        return audio, fmt, sample_rate_hz
-    if normalized == "g711_ulaw":
-        raw = decode_g711_ulaw(raw)
-        sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
-    elif normalized == "g711_alaw":
-        raw = decode_g711_alaw(raw)
-        sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
-    elif len(raw) % 2:
-        return audio, fmt, sample_rate_hz
-    if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz:
-        raw = resample_pcm16_mono(
-            raw,
-            source_rate_hz=int(sample_rate_hz),
-            target_rate_hz=target_sample_rate_hz,
-        )
-        sample_rate_hz = target_sample_rate_hz
-    pcm = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
-    encoded = base64.b64encode(np.ascontiguousarray(pcm, dtype="<f4").tobytes()).decode("ascii")
-    return encoded, "pcm_f32le", sample_rate_hz
-
-
-def convert_output_audio(
-    audio: str,
-    *,
-    source_fmt: str,
-    target_fmt: str,
-    source_sample_rate_hz: int | None = None,
-    target_sample_rate_hz: int | None = None,
-) -> tuple[str, str, int | None]:
-    target = target_fmt.lower()
-    if target not in {"g711_ulaw", "g711_alaw"}:
-        return audio, source_fmt, source_sample_rate_hz
-    try:
-        raw = base64.b64decode(audio, validate=False)
-    except (binascii.Error, ValueError):
-        return audio, source_fmt, source_sample_rate_hz
-    source = source_fmt.lower()
-    if source == "wav":
-        pcm_raw, wav_rate = wav_payload_to_pcm16(raw)
-        if pcm_raw is None:
-            return audio, source_fmt, source_sample_rate_hz
-        raw = pcm_raw
-        source_sample_rate_hz = source_sample_rate_hz or wav_rate
-    elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2:
-        return audio, source_fmt, source_sample_rate_hz
-    target_rate = target_sample_rate_hz or 8_000
-    if source_sample_rate_hz is not None:
-        raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate)
-    encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw)
-    return base64.b64encode(encoded).decode("ascii"), target, target_rate

vllm_omni/engine/duplex/commands.py

@@ -1,41 +1,62 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
-"""Typed commands accepted by a duplex session.
+"""The command vocabulary one duplex session accepts, plus its mailbox rendering.
 
 ``DuplexSessionHandle.submit()`` takes one of these; the websocket handler and
-``InlineDuplexClient`` build them with :func:`command_from_realtime` from the
+``InlineDuplexClient`` build them with :func:`command_from_realtime`.
-OpenAI Realtime client event vocabulary. Every command can also render the
+
-session-internal mailbox payload (``payload()``), which is the dictionary
+The *wire* half of each command --- which client event it decodes from and what
-vocabulary the session runner bodies were written against.
+fields survive decoding --- now lives in ``vllm_omni.protocol.duplex.commands``,
+which carries the whole vocabulary: the eight Tier 1 classes re-exported from
+``vllm_omni.protocol.realtime.commands``, the two Tier 2 ones and the seven
+Tier 3 ones. The engine imports that module and never the Tier 1 one directly,
+so a command that later grows a duplex extension changes one file
+(RFC #6592 P0a).
+
+The *engine* half stays here, because it is not wire contract at all:
+``payload()`` renders the session-internal mailbox dictionary the runner bodies
+were written against, and its ``type`` is the mailbox channel rather than the
+client event. Those two genuinely differ --- ``session.update`` and
+``conversation.item.create`` / ``.delete`` / ``.truncate`` all travel on the
+``turn.signal`` channel --- which is why the halves are separated instead of
+the classes being relocated wholesale.
 """
 
 from __future__ import annotations
 
 import base64
 from collections.abc import Mapping
-from dataclasses import dataclass, field, fields
+from dataclasses import dataclass, fields
 from typing import TYPE_CHECKING, ClassVar
 
+from vllm_omni.protocol.duplex import RealtimeProtocolError
+from vllm_omni.protocol.duplex import commands as _duplex_wire
+from vllm_omni.protocol.duplex.commands import RealtimeCommand
+
 if TYPE_CHECKING:
-    from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
+    from vllm_omni.protocol.duplex import RealtimeInputDefaults
 
 
-class DuplexCommandError(ValueError):
+class DuplexCommandError(RealtimeProtocolError):
-    """A client payload could not be turned into a command."""
+    """A client payload could not be turned into a duplex command.
 
-    def __init__(self, message: str, *, code: str = "bad_event", event_id: str | None = None) -> None:
+    The duplex name for a Realtime protocol error: same ``code`` /
-        super().__init__(message)
+    ``event_id`` contract, so the error envelope is rendered identically
-        self.code = code
+    whichever consumer raised it.
-        self.event_id = event_id
+    """
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
-class DuplexCommand:
+class DuplexCommand(RealtimeCommand):
+    """A Realtime command as the duplex engine handles it.
+
+    Adds the mailbox channel (``type``) and its rendering on top of the wire
+    command; every concrete class below pairs this with its protocol twin.
+    """
+
     #: Mailbox event type this command renders to (see ``payload()``).
     type: ClassVar[str] = ""
-    #: Client correlation id (OpenAI ``event_id``), echoed on error events.
-    event_id: str | None = None
 
     def payload(self) -> dict[str, object]:
         """Render the session-internal mailbox dictionary."""
@@ -56,19 +77,23 @@ class DuplexCommand:
         return data
 
 
+# ---- GA commands ----
+
+
 @dataclass(frozen=True, slots=True, kw_only=True)
-class AppendAudio(DuplexCommand):
+class UpdateSession(DuplexCommand, _duplex_wire.UpdateSession):
+    type: ClassVar[str] = "turn.signal"
+
+    def payload(self) -> dict[str, object]:
+        data = DuplexCommand.payload(self)
+        data["event"] = "session.update"
+        data["payload"] = dict(data.pop("patch", {}) or {})
+        return data
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AppendAudio(DuplexCommand, _duplex_wire.AppendAudio):
     type: ClassVar[str] = "input_audio_buffer.append"
-    #: Raw audio bytes in ``format`` at ``sample_rate_hz`` (base64 only on the wire).
-    audio: bytes
-    format: str = "pcm16"
-    sample_rate_hz: int | None = None
-    is_speech: bool | None = None
-    video_frames: tuple[str, ...] = ()
-    duration_ms: int | None = None
-    audio_end_ms: int | None = None
-    #: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
-    hints: Mapping[str, object] = field(default_factory=dict)
 
     def payload(self) -> dict[str, object]:
         data = DuplexCommand.payload(self)
@@ -90,21 +115,8 @@ class AppendAudio(DuplexCommand):
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
-class AppendText(DuplexCommand):
+class Commit(DuplexCommand, _duplex_wire.Commit):
-    type: ClassVar[str] = "input.text.append"
-    text: str
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class Commit(DuplexCommand):
     type: ClassVar[str] = "input_audio_buffer.commit"
-    final: bool = True
-    #: ``None`` means "no explicit request": the runner decides on commit (auto-response
-    #: sessions answer on their own); ``True`` / ``False`` force it.
-    create_response: bool | None = None
-    is_speech: bool | None = None
-    #: Realtime conversation item created for this commit (wire correlation only).
-    realtime_item_id: str | None = None
 
     def payload(self) -> dict[str, object]:
         data = DuplexCommand.payload(self)
@@ -114,10 +126,18 @@ class Commit(DuplexCommand):
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
-class CreateResponse(DuplexCommand):
+class ClearInput(DuplexCommand, _duplex_wire.ClearInput):
+    type: ClassVar[str] = "input_audio_buffer.clear"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ClearOutputAudio(DuplexCommand, _duplex_wire.ClearOutputAudio):
+    type: ClassVar[str] = "output_audio_buffer.clear"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CreateResponse(DuplexCommand, _duplex_wire.CreateResponse):
     type: ClassVar[str] = "response.create"
-    #: Raw Realtime ``response`` object (instructions, voice, modalities, ...).
-    options: Mapping[str, object] = field(default_factory=dict)
 
     def payload(self) -> dict[str, object]:
         data = DuplexCommand.payload(self)
@@ -128,84 +148,13 @@ class CreateResponse(DuplexCommand):
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
-class ClearInput(DuplexCommand):
+class CancelResponse(DuplexCommand, _duplex_wire.CancelResponse):
-    type: ClassVar[str] = "input_audio_buffer.clear"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class CancelInput(DuplexCommand):
-    type: ClassVar[str] = "input.cancel"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class CancelResponse(DuplexCommand):
     type: ClassVar[str] = "response.cancel"
-    response_id: str | None = None
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
-class BargeIn(DuplexCommand):
+class CreateItem(DuplexCommand, _duplex_wire.CreateItem):
-    type: ClassVar[str] = "barge_in"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ClearOutputAudio(DuplexCommand):
-    type: ClassVar[str] = "output_audio_buffer.clear"
-    #: Explicit response to clear; ``None`` targets the active/last response.
-    response_id: str | None = None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SignalTurn(DuplexCommand):
-    """Generic ``turn.signal`` (local turn transitions such as ``user_started``)."""
-
     type: ClassVar[str] = "turn.signal"
-    event: str
-    signal_payload: Mapping[str, object] = field(default_factory=dict)
-
-    def payload(self) -> dict[str, object]:
-        data = DuplexCommand.payload(self)
-        signal_payload = data.pop("signal_payload", None)
-        if isinstance(signal_payload, Mapping) and signal_payload:
-            data["payload"] = dict(signal_payload)
-        return data
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class UpdateSession(DuplexCommand):
-    """``session.update``: a Realtime ``session`` object patch."""
-
-    type: ClassVar[str] = "turn.signal"
-    patch: Mapping[str, object] = field(default_factory=dict)
-
-    def payload(self) -> dict[str, object]:
-        data = DuplexCommand.payload(self)
-        data["event"] = "session.update"
-        data["payload"] = dict(data.pop("patch", {}) or {})
-        return data
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class AckPlayback(DuplexCommand):
-    type: ClassVar[str] = "playback.ack"
-    played_ms: int
-    committed_ms: int | None = None
-    response_id: str | None = None
-    item_id: str | None = None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class Heartbeat(DuplexCommand):
-    type: ClassVar[str] = "session.heartbeat"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class CreateItem(DuplexCommand):
-    """``conversation.item.create`` (history injection or function-call output)."""
-
-    type: ClassVar[str] = "turn.signal"
-    item: Mapping[str, object]
-    previous_item_id: str | None = None
 
     def payload(self) -> dict[str, object]:
         data = DuplexCommand.payload(self)
@@ -219,9 +168,8 @@ class CreateItem(DuplexCommand):
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
-class DeleteItem(DuplexCommand):
+class DeleteItem(DuplexCommand, _duplex_wire.DeleteItem):
     type: ClassVar[str] = "turn.signal"
-    item_id: str
 
     def payload(self) -> dict[str, object]:
         data = DuplexCommand.payload(self)
@@ -231,11 +179,8 @@ class DeleteItem(DuplexCommand):
 
 
 @dataclass(frozen=True, slots=True, kw_only=True)
-class TruncateItem(DuplexCommand):
+class TruncateItem(DuplexCommand, _duplex_wire.TruncateItem):
     type: ClassVar[str] = "turn.signal"
-    item_id: str
-    audio_end_ms: int
-    content_index: int = 0
 
     def payload(self) -> dict[str, object]:
         data = DuplexCommand.payload(self)
@@ -248,15 +193,51 @@ class TruncateItem(DuplexCommand):
         return data
 
 
+# ---- duplex extension commands ----
+
+
 @dataclass(frozen=True, slots=True, kw_only=True)
-class CloseSession(DuplexCommand):
+class AppendText(DuplexCommand, _duplex_wire.AppendText):
-    """Graceful close requested through the command stream (``session.close``)."""
+    type: ClassVar[str] = "input.text.append"
 
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CancelInput(DuplexCommand, _duplex_wire.CancelInput):
+    type: ClassVar[str] = "input.cancel"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class BargeIn(DuplexCommand, _duplex_wire.BargeIn):
+    type: ClassVar[str] = "barge_in"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SignalTurn(DuplexCommand, _duplex_wire.SignalTurn):
+    type: ClassVar[str] = "turn.signal"
+
+    def payload(self) -> dict[str, object]:
+        data = DuplexCommand.payload(self)
+        signal_payload = data.pop("signal_payload", None)
+        if isinstance(signal_payload, Mapping) and signal_payload:
+            data["payload"] = dict(signal_payload)
+        return data
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AckPlayback(DuplexCommand, _duplex_wire.AckPlayback):
+    type: ClassVar[str] = "playback.ack"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Heartbeat(DuplexCommand, _duplex_wire.Heartbeat):
+    type: ClassVar[str] = "session.heartbeat"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CloseSession(DuplexCommand, _duplex_wire.CloseSession):
     type: ClassVar[str] = "session.close"
-    reason: str = "client_close"
 
 
-#: Realtime client event types that map onto commands (wire vocabulary).
 REALTIME_COMMAND_TYPES: frozenset[str] = frozenset(
     {
         "input_audio_buffer.append",
@@ -300,7 +281,6 @@ def command_from_realtime(
 
 
 __all__ = [
-    "REALTIME_COMMAND_TYPES",
     "AckPlayback",
     "AppendAudio",
     "AppendText",
@@ -317,6 +297,8 @@ __all__ = [
     "DuplexCommand",
     "DuplexCommandError",
     "Heartbeat",
+    "REALTIME_COMMAND_TYPES",
+    "RealtimeCommand",
     "SignalTurn",
     "TruncateItem",
     "UpdateSession",

vllm_omni/engine/duplex/config.py

@@ -339,26 +339,26 @@ class DuplexSessionConfig:
         ``_session_create_from_realtime``.
         """
         from vllm_omni.engine.duplex.realtime_commands import (
-            RealtimeInputDefaults,
+            DUPLEX_REALTIME_CAPABILITIES,
             duplex_response_format,
+        )
+        from vllm_omni.engine.duplex.turn_detection import normalize_turn_detection_session_payload
+        from vllm_omni.protocol.duplex import (
+            RealtimeInputDefaults,
             input_audio_transcription_config,
             json_safe_realtime_payload,
             realtime_max_output_tokens,
             realtime_overlap_fields,
-            validate_realtime_session_audio_formats,
+            validate_session_payload,
-        )
-        from vllm_omni.engine.duplex.turn_detection import (
-            normalize_turn_detection_session_payload,
-            validate_realtime_turn_detection,
         )
 
         payload: dict[str, object] = dict(session_payload)
-        format_error = validate_realtime_session_audio_formats(payload)
+        # One session check for every consumer (ENTRY-INV-002): the same
-        if format_error is not None:
+        # capability object ``translate_realtime_command`` uses, so a session
-            raise DuplexConfigError(format_error, code="unsupported_audio_format")
+        # object is accepted or refused identically whichever door it came in.
-        turn_detection_error = validate_realtime_turn_detection(payload)
+        rejection = validate_session_payload(payload, capabilities=DUPLEX_REALTIME_CAPABILITIES)
-        if turn_detection_error is not None:
+        if rejection is not None:
-            raise DuplexConfigError(turn_detection_error, code="unsupported_turn_detection", param="turn_detection")
+            raise DuplexConfigError(rejection.message, code=rejection.code, param=rejection.param)
         normalize_turn_detection_session_payload(payload)
         defaults = RealtimeInputDefaults().with_session_payload(payload)
         payload.update(realtime_overlap_fields(payload))
@@ -446,9 +446,9 @@ class DuplexSessionConfig:
         when the patch changes something a live session cannot change.
         ``audio_started`` is ``playback.generated_ms > 0 or playback.sent_ms > 0``.
         """
-        from vllm_omni.engine.duplex.realtime_commands import (
+        from vllm_omni.engine.duplex.realtime_commands import duplex_response_format
+        from vllm_omni.protocol.duplex import (
             REALTIME_OUTPUT_AUDIO_FORMATS,
-            duplex_response_format,
             input_audio_transcription_config,
             json_safe_realtime_payload,
             parse_realtime_audio_format,
@@ -629,9 +629,9 @@ class ResponseCreateOptions:
         for options a model-native duplex session cannot apply per response.
         Private runtime keys in ``extra_body`` are dropped.
         """
-        from vllm_omni.engine.duplex.realtime_commands import (
+        from vllm_omni.engine.duplex.realtime_commands import duplex_response_format
+        from vllm_omni.protocol.duplex import (
             REALTIME_OUTPUT_AUDIO_FORMATS,
-            duplex_response_format,
             parse_realtime_audio_format,
             realtime_max_output_tokens,
         )
@@ -782,14 +782,14 @@ def realtime_item_to_history_message(item: object) -> dict[str, object] | None:
 
 def realtime_max_output_tokens(value: object) -> int | None:
     """Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
-    from vllm_omni.engine.duplex.realtime_commands import realtime_max_output_tokens as _impl
+    from vllm_omni.protocol.duplex import realtime_max_output_tokens as _impl
 
     return _impl(value)
 
 
 def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
     """Return the ``input_audio_transcription`` object of a Realtime session payload."""
-    from vllm_omni.engine.duplex.realtime_commands import input_audio_transcription_config as _impl
+    from vllm_omni.protocol.duplex import input_audio_transcription_config as _impl
 
     return _impl(session_payload)
 

vllm_omni/engine/duplex/events.py

@@ -1,677 +1,79 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
-"""Typed events emitted by a duplex session.
+"""The event vocabulary one duplex session speaks, assembled from two layers.
 
-Every public event is a frozen dataclass with explicit fields. The Realtime
+The classes themselves no longer live here. ``vllm_omni.protocol.duplex.events``
-wire JSON is *derived* from those fields by :meth:`DuplexEvent.to_realtime`,
+carries the whole vocabulary --- 22 Tier 1 classes re-exported from
-which is pure: it never consults session state. The stateful part (response /
+``vllm_omni.protocol.realtime.events``, 8 Tier 2 ones and 12 Tier 3 ones --- so
-item ids, content-part bookkeeping) lives in
+a non-duplex Realtime consumer can take the Tier 1 half and its wire rendering
-``vllm_omni.engine.duplex.realtime_events`` on the session runner and is
+without the duplex session control plane (RFC #6592 P0a). The engine imports
-consumed when the events are *constructed*.
+the duplex module and never the Tier 1 one directly.
 
-``session_id`` / ``epoch`` are bound by
+What is genuinely engine-side stays: :data:`DOMAIN_TERMINAL_EVENTS` and
-``DuplexSessionManager.emit`` when an event leaves the runner; producers
+:data:`MODEL_OUTPUT_EVENTS` are the session runner's epoch-filter policy, not
-(projector, serving handshake helpers) build events with the defaults.
+wire contract --- they name *internal* event types, and which of them may never
+be dropped as stale is a runtime decision.
+
+``DuplexEvent`` is re-exported as the name the engine and the clients already
+use. It is an alias of
+:class:`~vllm_omni.protocol.realtime.events.RealtimeEvent`, not a subclass,
+because an event has no engine-internal half --- what the session emits is
+exactly what the socket sends. A *command* does have one (``payload()`` renders
+the runner's mailbox), which is why ``DuplexCommand`` is a real class.
 """
 
 from __future__ import annotations
 
-import base64
+from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
-from collections.abc import Mapping
+from vllm_omni.protocol.duplex.events import (
-from dataclasses import dataclass, field, fields
+    AudioDelta,
-from typing import ClassVar, cast
+    AudioDone,
-from uuid import uuid4
+    ContentPartAdded,
+    ContentPartDone,
+    DuplexEvent,
+    DuplexRawEvent,
+    ErrorEvent,
+    FunctionCallArgumentsDelta,
+    FunctionCallArgumentsDone,
+    InputCleared,
+    InputCommitted,
+    InputTranscriptionCompleted,
+    ItemAdded,
+    ItemCreated,
+    ItemDeleted,
+    ItemDone,
+    ItemRetrieved,
+    ItemTruncated,
+    Listen,
+    OutputAudioCleared,
+    OutputItemAdded,
+    OutputItemDone,
+    OverlapDecision,
+    PlaybackAcknowledged,
+    RateLimitsUpdated,
+    ResponseCreated,
+    ResponseDone,
+    SessionClosed,
+    SessionCreated,
+    SessionExpired,
+    SessionHeartbeatAck,
+    SessionReplaced,
+    SessionResumed,
+    SessionResyncRequired,
+    SessionUpdated,
+    Speak,
+    SpeechStarted,
+    SpeechStopped,
+    TextDelta,
+    TextDone,
+    TranscriptDelta,
+    TranscriptDone,
+    TurnEvent,
+    error_event,
+    new_event_id,
+)
 
-
-def new_event_id() -> str:
-    return f"event_{uuid4().hex}"
-
-
-#: OpenAI Realtime ``error.type`` for each internal error code.
-REALTIME_ERROR_TYPES_BY_CODE: dict[str, str] = {
-    "bad_event": "invalid_request_error",
-    "bad_audio": "invalid_request_error",
-    "config_timeout": "invalid_request_error",
-    "invalid_json": "invalid_request_error",
-    "event_too_large": "invalid_request_error",
-    "unknown_event": "invalid_request_error",
-    "internal_error": "server_error",
-    "runtime_append_failed": "server_error",
-    "runtime_append_task_failed": "server_error",
-    "runtime_signal_failed": "server_error",
-    "runtime_abort_failed": "server_error",
-    "runtime_data_plane_stream_failed": "server_error",
-    "runtime_data_plane_text_without_audio": "server_error",
-    "resource_exhausted": "rate_limit_error",
-    "session_exists": "invalid_request_error",
-    "session_closed": "invalid_request_error",
-    "unknown_session": "invalid_request_error",
-    "invalid_duplex_runtime_config": "invalid_request_error",
-    "instructions_update_unsupported": "invalid_request_error",
-    "persona_update_unsupported": "invalid_request_error",
-    "voice_update_unsupported": "invalid_request_error",
-    "unsupported_nemotron_duplex_mode": "invalid_request_error",
-    "unsupported_native_response_options": "invalid_request_error",
-    "runtime_touch_failed": "server_error",
-    "engine_error": "server_error",
-    "input_backpressure": "rate_limit_error",
-    "response_already_active": "invalid_request_error",
-    "response_not_active": "invalid_request_error",
-    "response_create_without_input": "invalid_request_error",
-    "text_only_turn_unsupported": "invalid_request_error",
-    "commit_aborted": "server_error",
-    "input_audio_buffer_empty": "invalid_request_error",
-    "missing_item_id": "invalid_request_error",
-    "item_not_found": "invalid_request_error",
-    "playback_item_mismatch": "invalid_request_error",
-    "playback_item_not_found": "invalid_request_error",
-    "playback_ack_too_late": "invalid_request_error",
-    "unsupported_audio_format": "invalid_request_error",
-    "unsupported_turn_detection": "invalid_request_error",
-    "unsupported_ref_audio_path": "invalid_request_error",
-    "ref_audio_required": "invalid_request_error",
-    "model_update_unsupported": "invalid_request_error",
-    "voice_update_after_audio_unsupported": "invalid_request_error",
-    "ref_audio_update_unsupported": "invalid_request_error",
-    "native_text_append_unsupported": "invalid_request_error",
-    "invalid_video_frames": "invalid_request_error",
-    "invalid_function_call_output": "invalid_request_error",
-    "server_vad_unavailable": "server_error",
-}
-
-_IDENTITY_FIELDS = frozenset({"session_id", "epoch", "event_id"})
-
-
-def _wire_value(value: object) -> object:
-    if isinstance(value, Mapping):
-        return {str(k): _wire_value(v) for k, v in value.items()}
-    if isinstance(value, tuple | list):
-        return [_wire_value(v) for v in value]
-    return value
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class DuplexEvent:
-    """Base of every public session event."""
-
-    #: Realtime ``type`` this class renders to.
-    wire_type: ClassVar[str] = ""
-    #: Fields that are omitted from the wire object when ``None``.
-    optional_wire_fields: ClassVar[frozenset[str]] = frozenset()
-
-    session_id: str = ""
-    epoch: int | None = None
-    #: Server event id (OpenAI ``event_id``); generated at construction.
-    event_id: str = field(default_factory=new_event_id)
-
-    # ---- identity ----
-
-    @property
-    def type(self) -> str:
-        return self.wire_type
-
-    @property
-    def is_terminal(self) -> bool:
-        return False
-
-    # ---- generic accessors (None when the class has no such field) ----
-
-    @property
-    def response_id(self) -> str | None:
-        return None
-
-    @property
-    def item_id(self) -> str | None:
-        return None
-
-    @property
-    def text(self) -> str | None:
-        return None
-
-    @property
-    def audio(self) -> bytes | None:
-        return None
-
-    # ---- wire rendering (pure) ----
-
-    def _wire_fields(self) -> dict[str, object]:
-        data: dict[str, object] = {}
-        for f in fields(self):
-            if f.name in _IDENTITY_FIELDS:
-                continue
-            value = getattr(self, f.name)
-            if value is None and f.name in self.optional_wire_fields:
-                continue
-            data[f.name] = _wire_value(value)
-        return data
-
-    def to_realtime(self) -> dict[str, object]:
-        """The Realtime wire JSON object for this event (derived from the fields)."""
-        return {"type": self.wire_type, "event_id": self.event_id, **self._wire_fields()}
-
-    def __repr__(self) -> str:  # pragma: no cover - debugging aid
-        return f"{type(self).__name__}(type={self.wire_type!r}, session_id={self.session_id!r}, epoch={self.epoch!r})"
-
-
-# ---- session lifecycle ----
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SessionCreated(DuplexEvent):
-    wire_type = "session.created"
-    optional_wire_fields = frozenset({"attachment_generation", "resume_token"})
-
-    session: Mapping[str, object] = field(default_factory=dict)
-    #: Transport resume credentials (set by the websocket handler when resume is supported).
-    attachment_generation: int | None = None
-    resume_token: str | None = None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SessionUpdated(DuplexEvent):
-    wire_type = "session.updated"
-
-    session: Mapping[str, object] = field(default_factory=dict)
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SessionResumed(SessionCreated):
-    wire_type = "session.resumed"
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SessionReplaced(DuplexEvent):
-    """A newer connection took the session over (sent to the replaced socket)."""
-
-    wire_type = "session.replaced"
-
-    attachment_generation: int = 0
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SessionResyncRequired(DuplexEvent):
-    """The replay journal cannot bridge the gap; the client must start over."""
-
-    wire_type = "session.resync_required"
-
-    reason: str = "journal_gap"
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SessionClosed(DuplexEvent):
-    wire_type = "session.closed"
-
-    reason: str = "closed"
-    #: The session-internal close event (kept for clients that inspect it).
-    details: Mapping[str, object] = field(default_factory=dict)
-
-    @property
-    def is_terminal(self) -> bool:
-        return True
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"session_id": self.session_id, "reason": self.reason, "event": _wire_value(self.details)}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SessionExpired(SessionClosed):
-    wire_type = "session.expired"
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"session_id": self.session_id, "reason": self.reason}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SessionHeartbeatAck(DuplexEvent):
-    wire_type = "session.heartbeat_ack"
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"session_id": self.session_id}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class TurnEvent(DuplexEvent):
-    """Turn-state transition (``turn.event``)."""
-
-    wire_type = "turn.event"
-
-    event: str = ""
-    turn_state: str = ""
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"session_id": self.session_id, "event": self.event, "turn_state": self.turn_state, "epoch": self.epoch}
-
-
-# ---- response lifecycle ----
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class _ResponseEvent(DuplexEvent):
-    """Events addressed to one response (and usually one output item / content part)."""
-
-    response_id: str | None = None  # type: ignore[assignment]
-    item_id: str | None = None  # type: ignore[assignment]
-    output_index: int = 0
-    content_index: int = 0
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ResponseCreated(DuplexEvent):
-    wire_type = "response.created"
-
-    response_id: str | None = None  # type: ignore[assignment]
-    response: Mapping[str, object] = field(default_factory=dict)
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class Listen(DuplexEvent):
-    """The model decided to keep listening (no spoken response for this turn)."""
-
-    wire_type = "response.listen"
-    optional_wire_fields = frozenset({"response_id"})
-
-    response_id: str | None = None  # type: ignore[assignment]
-    details: Mapping[str, object] = field(default_factory=dict)
-
-    def _wire_fields(self) -> dict[str, object]:
-        response: dict[str, object] = {
-            "object": "realtime.response",
-            "status": "listening",
-            "metadata": _wire_value(self.details),
-        }
-        data: dict[str, object] = {"session_id": self.session_id, "epoch": self.epoch, "response": response}
-        if self.response_id:
-            response["id"] = self.response_id
-            data["response_id"] = self.response_id
-        return data
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class Speak(_ResponseEvent):
-    wire_type = "response.speak"
-
-    metadata: Mapping[str, object] = field(default_factory=dict)
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class OverlapDecision(DuplexEvent):
-    wire_type = "overlap.decision"
-
-    policy: str | None = None
-    action: str | None = None
-    reason: str | None = None
-    details: Mapping[str, object] = field(default_factory=dict)
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {
-            "session_id": self.session_id,
-            "epoch": self.epoch,
-            "policy": self.policy,
-            "action": self.action,
-            "reason": self.reason,
-            "metadata": _wire_value(self.details),
-        }
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class OutputItemAdded(DuplexEvent):
-    wire_type = "response.output_item.added"
-
-    response_id: str | None = None  # type: ignore[assignment]
-    output_index: int = 0
-    item: Mapping[str, object] = field(default_factory=dict)
-
-    @property
-    def item_id(self) -> str | None:
-        value = self.item.get("id")
-        return value if isinstance(value, str) else None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class OutputItemDone(OutputItemAdded):
-    wire_type = "response.output_item.done"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ContentPartAdded(_ResponseEvent):
-    wire_type = "response.content_part.added"
-
-    part: Mapping[str, object] = field(default_factory=dict)
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ContentPartDone(ContentPartAdded):
-    wire_type = "response.content_part.done"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class AudioDelta(_ResponseEvent):
-    wire_type = "response.output_audio.delta"
-    optional_wire_fields = frozenset({"sample_rate_hz", "metadata"})
-
-    #: Base64 audio in ``format``.
-    delta: str = ""
-    format: str = "pcm16"
-    sample_rate_hz: int | None = None
-    metadata: Mapping[str, object] | None = None
-
-    @property
-    def audio(self) -> bytes | None:
-        try:
-            return base64.b64decode(self.delta)
-        except (ValueError, TypeError):
-            return None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class AudioDone(_ResponseEvent):
-    wire_type = "response.output_audio.done"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class TranscriptDelta(_ResponseEvent):
-    wire_type = "response.output_audio_transcript.delta"
-
-    delta: str = ""
-
-    @property
-    def text(self) -> str | None:
-        return self.delta
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class TranscriptDone(_ResponseEvent):
-    wire_type = "response.output_audio_transcript.done"
-
-    transcript: str = ""
-
-    @property
-    def text(self) -> str | None:
-        return self.transcript
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class TextDelta(_ResponseEvent):
-    wire_type = "response.output_text.delta"
-
-    delta: str = ""
-
-    @property
-    def text(self) -> str | None:
-        return self.delta
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class TextDone(_ResponseEvent):
-    wire_type = "response.output_text.done"
-
-    text: str = ""  # type: ignore[assignment]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class FunctionCallArgumentsDelta(DuplexEvent):
-    wire_type = "response.function_call_arguments.delta"
-
-    response_id: str | None = None  # type: ignore[assignment]
-    item_id: str | None = None  # type: ignore[assignment]
-    output_index: int = 0
-    call_id: str = ""
-    delta: str = ""
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class FunctionCallArgumentsDone(DuplexEvent):
-    wire_type = "response.function_call_arguments.done"
-
-    response_id: str | None = None  # type: ignore[assignment]
-    item_id: str | None = None  # type: ignore[assignment]
-    output_index: int = 0
-    call_id: str = ""
-    arguments: str = ""
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ResponseDone(DuplexEvent):
-    wire_type = "response.done"
-
-    response_id: str | None = None  # type: ignore[assignment]
-    response: Mapping[str, object] = field(default_factory=dict)
-
-    @property
-    def status(self) -> str | None:
-        value = self.response.get("status")
-        return value if isinstance(value, str) else None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class RateLimitsUpdated(DuplexEvent):
-    wire_type = "rate_limits.updated"
-
-    rate_limits: tuple[Mapping[str, object], ...] = ()
-
-
-# ---- input buffer / conversation ----
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class InputCommitted(DuplexEvent):
-    wire_type = "input_audio_buffer.committed"
-
-    previous_item_id: str | None = None
-    item_id: str | None = None  # type: ignore[assignment]
-    #: The session-internal ``input.committed`` event.
-    details: Mapping[str, object] = field(default_factory=dict)
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {
-            "previous_item_id": self.previous_item_id,
-            "item_id": self.item_id,
-            "event": _wire_value(self.details),
-        }
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class InputCleared(DuplexEvent):
-    wire_type = "input_audio_buffer.cleared"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SpeechStarted(DuplexEvent):
-    wire_type = "input_audio_buffer.speech_started"
-
-    audio_start_ms: int = 0
-    item_id: str | None = None  # type: ignore[assignment]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class SpeechStopped(DuplexEvent):
-    wire_type = "input_audio_buffer.speech_stopped"
-
-    audio_end_ms: int = 0
-    item_id: str | None = None  # type: ignore[assignment]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class _ItemEvent(DuplexEvent):
-    previous_item_id: str | None = None
-    item: Mapping[str, object] = field(default_factory=dict)
-
-    @property
-    def item_id(self) -> str | None:
-        value = self.item.get("id")
-        return value if isinstance(value, str) else None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ItemAdded(_ItemEvent):
-    wire_type = "conversation.item.added"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ItemCreated(_ItemEvent):
-    wire_type = "conversation.item.created"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ItemDone(_ItemEvent):
-    wire_type = "conversation.item.done"
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ItemRetrieved(DuplexEvent):
-    wire_type = "conversation.item.retrieved"
-
-    item: Mapping[str, object] = field(default_factory=dict)
-
-    @property
-    def item_id(self) -> str | None:
-        value = self.item.get("id")
-        return value if isinstance(value, str) else None
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ItemDeleted(DuplexEvent):
-    wire_type = "conversation.item.deleted"
-
-    item_id: str | None = None  # type: ignore[assignment]
-    details: Mapping[str, object] = field(default_factory=dict)
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"item_id": self.item_id, "event": _wire_value(self.details)}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ItemTruncated(DuplexEvent):
-    wire_type = "conversation.item.truncated"
-
-    item_id: str | None = None  # type: ignore[assignment]
-    content_index: int = 0
-    audio_end_ms: int = 0
-    details: Mapping[str, object] = field(default_factory=dict)
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {
-            "item_id": self.item_id,
-            "content_index": self.content_index,
-            "audio_end_ms": self.audio_end_ms,
-            "event": _wire_value(self.details),
-        }
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class InputTranscriptionCompleted(DuplexEvent):
-    wire_type = "conversation.item.input_audio_transcription.completed"
-
-    item_id: str | None = None  # type: ignore[assignment]
-    content_index: int = 0
-    transcript: str = ""
-
-    @property
-    def text(self) -> str | None:
-        return self.transcript
-
-
-# ---- playback / control ----
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class OutputAudioCleared(DuplexEvent):
-    wire_type = "output_audio_buffer.cleared"
-
-    response_id: str | None = None  # type: ignore[assignment]
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class PlaybackAcknowledged(DuplexEvent):
-    wire_type = "playback.acknowledged"
-
-    #: The session-internal ``playback.acknowledged`` event (cursor, committed_ms ...).
-    details: Mapping[str, object] = field(default_factory=dict)
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"event": _wire_value(self.details)}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class ErrorEvent(DuplexEvent):
-    wire_type = "error"
-
-    code: str = "internal_error"
-    message: str = ""
-    #: Client ``event_id`` this error answers (OpenAI ``error.event_id``).
-    related_event_id: str | None = None
-    param: str | None = None
-    #: Extra keys merged into the wire ``error`` object (``retryable`` ...).
-    extra: Mapping[str, object] = field(default_factory=dict)
-
-    @property
-    def error_type(self) -> str:
-        return REALTIME_ERROR_TYPES_BY_CODE.get(self.code, "invalid_request_error")
-
-    @property
-    def error(self) -> dict[str, object]:
-        error: dict[str, object] = {"type": self.error_type, "code": self.code, "message": self.message}
-        if self.related_event_id:
-            error["event_id"] = self.related_event_id
-        if self.param:
-            error["param"] = self.param
-        error.update(cast("Mapping[str, object]", _wire_value(self.extra)))
-        return error
-
-    @property
-    def text(self) -> str | None:
-        return self.message
-
-    def _wire_fields(self) -> dict[str, object]:
-        return {"error": self.error}
-
-
-@dataclass(frozen=True, slots=True, kw_only=True)
-class DuplexRawEvent(DuplexEvent):
-    """A session-internal event with no dedicated Realtime type (``duplex.<internal type>``)."""
-
-    wire_type = "duplex.raw"
-
-    internal_type: str = ""
-    details: Mapping[str, object] = field(default_factory=dict)
-
-    @property
-    def type(self) -> str:
-        return f"duplex.{self.internal_type}"
-
-    def to_realtime(self) -> dict[str, object]:
-        return {"type": self.type, "event_id": self.event_id, "event": _wire_value(self.details)}
-
-
-def error_event(
-    code: str,
-    message: str,
-    *,
-    event_id: object | None = None,
-    param: object | None = None,
-    extra: Mapping[str, object] | None = None,
-) -> ErrorEvent:
-    """Build the error event for an internal code; ``event_id`` is the *client* event id."""
-    return ErrorEvent(
-        code=code,
-        message=message,
-        related_event_id=event_id if isinstance(event_id, str) and event_id else None,
-        param=param if isinstance(param, str) and param else None,
-        extra=dict(extra or {}),
-    )
-
-
-#: Internal events that terminate a response/session and must never be dropped as stale.
 DOMAIN_TERMINAL_EVENTS = frozenset(
     {
         "response.done",
@@ -682,7 +84,6 @@ DOMAIN_TERMINAL_EVENTS = frozenset(
     }
 )
 
-#: Internal streaming model-output events subject to epoch/close stale filtering.
 MODEL_OUTPUT_EVENTS = frozenset(
     {
         "response.created",
@@ -705,13 +106,11 @@ MODEL_OUTPUT_EVENTS = frozenset(
 
 
 __all__ = [
-    "DOMAIN_TERMINAL_EVENTS",
-    "MODEL_OUTPUT_EVENTS",
-    "REALTIME_ERROR_TYPES_BY_CODE",
     "AudioDelta",
     "AudioDone",
     "ContentPartAdded",
     "ContentPartDone",
+    "DOMAIN_TERMINAL_EVENTS",
     "DuplexEvent",
     "DuplexRawEvent",
     "ErrorEvent",
@@ -727,11 +126,13 @@ __all__ = [
     "ItemRetrieved",
     "ItemTruncated",
     "Listen",
+    "MODEL_OUTPUT_EVENTS",
     "OutputAudioCleared",
     "OutputItemAdded",
     "OutputItemDone",
     "OverlapDecision",
     "PlaybackAcknowledged",
+    "REALTIME_ERROR_TYPES_BY_CODE",
     "RateLimitsUpdated",
     "ResponseCreated",
     "ResponseDone",

vllm_omni/engine/duplex/realtime_commands.py

@@ -3,26 +3,31 @@
 
 """OpenAI Realtime client events -> :class:`DuplexCommand` (stateless).
 
-This is the mapping half of the former ``entrypoints/duplex/realtime_input.py``
+This module is the *duplex binding* of the shared Realtime codec. Parsing,
-translator. Everything that needed per-session state in the old translator
+validating and decoding a client event is not duplex-specific and lives in
-(input-buffer emptiness for commits, response-id fallbacks for cancels,
+``vllm_omni.protocol.realtime``, reached through ``vllm_omni.protocol.duplex``;
-conversation-item lookups, VAD) is resolved by the session runner through the
+what lives here is the part that is: which
-helpers on :class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
+:class:`~vllm_omni.engine.duplex.commands.DuplexCommand` a decoded event
+becomes, what the duplex engine can serve
+(:data:`DUPLEX_REALTIME_CAPABILITIES`), and the mapping between a Realtime
+output format and the duplex ``response_format`` vocabulary.
+
+Everything that needs per-session state (input-buffer emptiness for commits,
+response-id fallbacks for cancels, conversation-item lookups, VAD) is resolved
+by the session runner through the helpers on
+:class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
 the commands produced here carry the raw client intent only.
+
+The shared names re-exported at the bottom are a compatibility surface for
+existing importers. The canonical definitions are in
+``vllm_omni.protocol.realtime`` and there is exactly one of each.
 """
 
 from __future__ import annotations
 
-import base64
-import binascii
 from collections.abc import Mapping
-from dataclasses import dataclass, replace
+from typing import cast
-from typing import Any, cast
-from uuid import uuid4
 
-import numpy as np
-
-from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate
 from vllm_omni.engine.duplex.commands import (
     AckPlayback,
     AppendAudio,
@@ -44,93 +49,58 @@ from vllm_omni.engine.duplex.commands import (
     TruncateItem,
     UpdateSession,
 )
-
+from vllm_omni.protocol.duplex import (
-REALTIME_INPUT_AUDIO_FORMATS = {
+    REALTIME_INPUT_AUDIO_FORMATS,
-    "pcm16",
+    REALTIME_INPUT_HINT_KEYS,
-    "pcm_s16le",
+    REALTIME_OUTPUT_AUDIO_FORMATS,
-    "s16le",
+    RealtimeInputDefaults,
-    "pcm_f32le",
+    RealtimeProtocolCapabilities,
-    "g711_ulaw",
+    RealtimeProtocolError,
-    "g711_alaw",
+    apply_realtime_session_defaults,
-}
+    copy_realtime_input_hints,
-REALTIME_OUTPUT_AUDIO_FORMATS = {
+    decode_audio_append,
-    "pcm16",
+    input_audio_transcription_config,
-    "pcm_s16le",
+    input_explicitly_non_speech,
-    "s16le",
+    input_looks_like_speech,
-    "wav",
+    input_transcript_from_item,
-    "pcm",
+    is_supported_realtime_input_format,
-    "g711_ulaw",
+    json_safe_realtime_payload,
-    "g711_alaw",
+    normalize_conversation_item,
-}
+    parse_realtime_audio_format,
-
+    realtime_audio_format_object,
-#: Wire hint keys copied verbatim from a client append onto the internal payload.
+    realtime_max_output_tokens,
-REALTIME_INPUT_HINT_KEYS = (
+    realtime_output_format,
-    "duration_ms",
+    realtime_overlap_fields,
-    "audio_duration_ms",
+    text_chars_for_audio_ms_from_marks,
-    "audio_start_ms",
+    truncate_realtime_item_content,
-    "audio_end_ms",
+    validate_conversation_item_audio_formats,
-    "is_speech",
+    validate_realtime_item_truncate,
-    "speech",
+    validate_realtime_response_audio_formats,
-    "speech_probability",
+    validate_realtime_session_audio_formats,
-    "vad",
+    validate_realtime_video_frames,
-    "overlap_action",
+    validate_session_payload,
-    "overlap",
-    "force_barge_in",
-    "force_listen",
-    "text",
-    "transcript",
 )
 
 
-@dataclass(frozen=True, slots=True)
+def _validate_duplex_turn_detection(session_payload: Mapping[str, object]) -> str | None:
-class RealtimeInputDefaults:
+    """The duplex engine's answer for ``turn_detection`` on a session object.
-    """Session-level wire defaults an append may omit (derived from the session object)."""
 
-    input_audio_format: str = "pcm16"
+    Imported lazily: the validator lives with the Silero VAD backend, and a
-    input_sample_rate_hz: int = 16000
+    connection that never sends a session object should not pay for loading it.
-    output_audio_format: str = "pcm16"
+    """
-    output_sample_rate_hz: int | None = None
+    from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
-    overlap_silence_rms: float = 0.003
 
-    def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults:
+    return validate_realtime_turn_detection(session_payload)
-        """Return defaults updated from a Realtime ``session`` object (session.update)."""
-        values = apply_realtime_session_defaults(self, session_payload)
-        return values
 
 
-# ---- audio format helpers ----
+#: What the duplex engine can serve, for the shared session-object validator.
-
+DUPLEX_REALTIME_CAPABILITIES = RealtimeProtocolCapabilities(
-
+    validate_turn_detection=_validate_duplex_turn_detection,
-def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]:
+)
-    def normalize_format(fmt: str) -> str:
-        normalized = fmt.lower()
-        if normalized in {"audio/pcm", "pcm"}:
-            return "pcm16"
-        if normalized in {"audio/wav", "wav"}:
-            return "wav"
-        if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}:
-            return "pcm16"
-        if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}:
-            return "pcm_f32le"
-        if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}:
-            return "g711_ulaw"
-        if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}:
-            return "g711_alaw"
-        return fmt
-
-    if isinstance(raw_format, str):
-        return normalize_format(raw_format), None
-    if not isinstance(raw_format, dict):
-        return raw_format, None
-    rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate")
-    sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None
-    fmt = raw_format.get("type") or raw_format.get("format")
-    if not isinstance(fmt, str):
-        return raw_format, sample_rate_hz
-    return normalize_format(fmt), sample_rate_hz
 
 
 def duplex_response_format(realtime_format: str) -> str:
+    """A Realtime output format -> the duplex ``response_format`` vocabulary."""
     normalized = realtime_format.lower()
     if normalized in {"pcm16", "pcm_s16le", "s16le"}:
         return "pcm"
@@ -141,433 +111,6 @@ def duplex_response_format(realtime_format: str) -> str:
     return "wav"
 
 
-def realtime_output_format(duplex_format: object) -> str:
-    if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}:
-        return duplex_format.lower()
-    if isinstance(duplex_format, str) and duplex_format.lower() == "pcm":
-        return "pcm16"
-    return str(duplex_format or "wav")
-
-
-def is_supported_realtime_input_format(fmt: object) -> bool:
-    return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS
-
-
-def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]:
-    if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}:
-        payload: dict[str, object] = {"type": "audio/pcm"}
-    elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le":
-        payload = {"type": "audio/pcm_f32le"}
-    elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw":
-        payload = {"type": "audio/g711_ulaw"}
-    elif isinstance(fmt, str) and fmt.lower() == "g711_alaw":
-        payload = {"type": "audio/g711_alaw"}
-    else:
-        payload = {"type": "audio/wav"}
-    if sample_rate_hz is not None:
-        payload["rate"] = int(sample_rate_hz)
-    return payload
-
-
-def validate_realtime_session_audio_formats(session_payload: Mapping[str, object]) -> str | None:
-    audio_config = session_payload.get("audio")
-    input_format: object = session_payload.get("input_audio_format")
-    if input_format is None and isinstance(audio_config, dict):
-        audio_input = audio_config.get("input")
-        if isinstance(audio_input, dict):
-            input_format = audio_input.get("format")
-    parsed_input, _ = parse_realtime_audio_format(input_format)
-    if input_format is not None and not (
-        isinstance(parsed_input, str) and parsed_input.lower() in REALTIME_INPUT_AUDIO_FORMATS
-    ):
-        return f"Unsupported input_audio_format: {input_format}"
-
-    output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
-    if output_format is None and isinstance(audio_config, dict):
-        audio_output = audio_config.get("output")
-        if isinstance(audio_output, dict):
-            output_format = audio_output.get("format")
-    parsed_output, _ = parse_realtime_audio_format(output_format)
-    if output_format is not None and not (
-        isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
-    ):
-        return f"Unsupported output_audio_format: {output_format}"
-    return None
-
-
-def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None:
-    output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format")
-    audio_config = response_payload.get("audio")
-    if output_format is None and isinstance(audio_config, dict):
-        audio_output = audio_config.get("output")
-        if isinstance(audio_output, dict):
-            output_format = audio_output.get("format")
-    parsed_output, _ = parse_realtime_audio_format(output_format)
-    if output_format is not None and not (
-        isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
-    ):
-        return f"Unsupported output_audio_format: {output_format}"
-    return None
-
-
-def validate_conversation_item_audio_formats(item: object) -> str | None:
-    if not isinstance(item, dict):
-        return None
-    content = item.get("content")
-    if not isinstance(content, list):
-        return None
-    for part in content:
-        if not isinstance(part, dict):
-            continue
-        if part.get("type") not in {"input_audio", "audio"}:
-            continue
-        raw_format = part.get("format")
-        parsed_format, _ = parse_realtime_audio_format(raw_format)
-        if raw_format is not None and not (
-            isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS
-        ):
-            return f"Unsupported input_audio format in conversation.item.create: {raw_format}"
-    return None
-
-
-def apply_realtime_session_defaults(
-    defaults: RealtimeInputDefaults,
-    session_payload: Mapping[str, object],
-) -> RealtimeInputDefaults:
-    """Derive the wire defaults a Realtime ``session`` object declares."""
-    input_format: object = session_payload.get("input_audio_format")
-    audio_config = session_payload.get("audio")
-    if input_format is None and isinstance(audio_config, dict):
-        audio_input = audio_config.get("input")
-        if isinstance(audio_input, dict):
-            input_format = audio_input.get("format")
-    input_format, input_rate = parse_realtime_audio_format(input_format)
-    # Any: heterogeneous field values splatted into ``dataclasses.replace`` below.
-    updates: dict[str, Any] = {}
-    if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS:
-        updates["input_audio_format"] = input_format
-    output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
-    output_rate_raw: object | None = None
-    if output_format is None and isinstance(audio_config, dict):
-        audio_output = audio_config.get("output")
-        if isinstance(audio_output, dict):
-            output_format = audio_output.get("format")
-            output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate")
-    output_format, output_rate = parse_realtime_audio_format(output_format)
-    if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0:
-        output_rate = int(output_rate_raw)
-    if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
-        updates["output_audio_format"] = realtime_output_format(output_format)
-    sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate")
-    if sample_rate is None and isinstance(audio_config, dict):
-        audio_input = audio_config.get("input")
-        if isinstance(audio_input, dict):
-            sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate")
-    if sample_rate is None:
-        sample_rate = input_rate
-    if isinstance(sample_rate, int | float) and sample_rate > 0:
-        updates["input_sample_rate_hz"] = int(sample_rate)
-    if isinstance(output_rate, int | float) and output_rate > 0:
-        updates["output_sample_rate_hz"] = int(output_rate)
-    overlap_fields = realtime_overlap_fields(session_payload)
-    overlap_silence_rms = overlap_fields.get("overlap_silence_rms")
-    if isinstance(overlap_silence_rms, int | float):
-        updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms))
-    return replace(defaults, **updates) if updates else defaults
-
-
-def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]:
-    fields: dict[str, object] = {}
-    if isinstance(session_payload.get("overlap_policy"), str):
-        fields["overlap_policy"] = session_payload["overlap_policy"]
-    for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"):
-        value = session_payload.get(key)
-        if isinstance(value, int | float):
-            fields[key] = value
-    if isinstance(session_payload.get("playback_commit_policy"), str):
-        fields["playback_commit_policy"] = session_payload["playback_commit_policy"]
-    return fields
-
-
-def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]:
-    clean: dict[str, object] = {}
-    for key, value in payload.items():
-        if key == "extra_body":
-            continue
-        if isinstance(value, str | int | float | bool) or value is None:
-            clean[key] = value
-        elif isinstance(value, dict):
-            clean[key] = json_safe_realtime_payload(value)
-        elif isinstance(value, list):
-            clean[key] = [
-                (json_safe_realtime_payload(item) if isinstance(item, dict) else item)
-                for item in value
-                if isinstance(item, str | int | float | bool | dict) or item is None
-            ]
-    return clean
-
-
-def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
-    transcription = session_payload.get("input_audio_transcription")
-    if isinstance(transcription, dict):
-        return transcription
-    audio_config = session_payload.get("audio")
-    if not isinstance(audio_config, dict):
-        return None
-    audio_input = audio_config.get("input")
-    if not isinstance(audio_input, dict):
-        return None
-    transcription = audio_input.get("transcription")
-    return transcription if isinstance(transcription, dict) else None
-
-
-def realtime_max_output_tokens(value: object) -> int | None:
-    """Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
-    if value is None:
-        return None
-    if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}:
-        return None
-    if isinstance(value, int) and value > 0:
-        return int(value)
-    return None
-
-
-# ---- speech hints ----
-
-
-def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
-    for key in ("is_speech", "speech"):
-        value = event.get(key)
-        if isinstance(value, bool):
-            return not value
-    vad = event.get("vad")
-    if isinstance(vad, dict):
-        value = vad.get("is_speech")
-        if isinstance(value, bool):
-            return not value
-        probability = vad.get("speech_probability", vad.get("probability"))
-        if isinstance(probability, int | float):
-            return float(probability) < 0.5
-    probability = event.get("speech_probability")
-    return isinstance(probability, int | float) and float(probability) < 0.5
-
-
-def input_looks_like_speech(
-    event: Mapping[str, object],
-    *,
-    audio: object,
-    fmt: object,
-    overlap_silence_rms: float,
-) -> bool:
-    if input_explicitly_non_speech(event):
-        return False
-    for key in ("is_speech", "speech"):
-        value = event.get(key)
-        if isinstance(value, bool):
-            return value
-    vad = event.get("vad")
-    if isinstance(vad, dict):
-        probability = vad.get("speech_probability", vad.get("probability"))
-        if isinstance(probability, int | float):
-            return float(probability) >= 0.5
-    probability = event.get("speech_probability")
-    if isinstance(probability, int | float):
-        return float(probability) >= 0.5
-    if fmt != "pcm_f32le":
-        return True
-    if isinstance(audio, bytes | bytearray):
-        raw = bytes(audio)
-    elif isinstance(audio, str):
-        try:
-            raw = base64.b64decode(audio, validate=True)
-        except (binascii.Error, ValueError):
-            return True
-    else:
-        return True
-    if len(raw) < 4 or len(raw) % 4 != 0:
-        return True
-    samples = np.frombuffer(raw, dtype=np.float32)
-    if samples.size == 0:
-        return False
-    rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
-    threshold = event.get("overlap_silence_rms")
-    if not isinstance(threshold, int | float):
-        vad = event.get("vad")
-        if isinstance(vad, dict):
-            threshold = vad.get("silence_rms")
-    silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
-    return rms >= max(0.0, silence_rms)
-
-
-def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
-    for key in REALTIME_INPUT_HINT_KEYS:
-        if key in source:
-            target[key] = source[key]
-
-
-def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
-    """Validate omni-duplex camera frames on input_audio_buffer.append.
-
-    Wire contract matches the official MiniCPM-o duplex loop: one base
-    base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
-    stacked composite tiling the sub-frames captured inside it (at most 2
-    images either way). A caller-supplied ``max_slice_nums`` is rejected rather
-    than silently ignored: slicing is Stage 0's decision here, and Stage 0
-    already applies the official HD suggestion for a stacked unit
-    (``max_slice_nums=[2, 1]``). The wire simply does not let the client choose
-    it.
-    """
-    if max_slice_nums not in (None, 1):
-        return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself"
-    if not isinstance(video_frames, list):
-        return "video_frames must be a list of base64-encoded images"
-    frames = [frame for frame in video_frames if frame is not None]
-    if len(frames) > 2:
-        return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
-    for frame in frames:
-        if not isinstance(frame, str) or not frame:
-            return "video_frames entries must be non-empty base64 strings"
-        if len(frame) > 4_000_000:
-            return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
-        try:
-            header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
-        except (binascii.Error, ValueError):
-            return "video_frames entries must be valid base64"
-        if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
-            return "video_frames entries must be JPEG or PNG images"
-    return None
-
-
-# ---- conversation items ----
-
-
-def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]:
-    normalized = dict(item)
-    normalized.setdefault("id", f"item_{uuid4().hex}")
-    normalized.setdefault("object", "realtime.item")
-    normalized.setdefault("type", "message")
-    normalized.setdefault("status", "completed")
-    if "role" not in normalized and normalized.get("type") == "message":
-        normalized["role"] = "user"
-    if not isinstance(normalized.get("content"), list):
-        normalized["content"] = []
-    return normalized
-
-
-def text_chars_for_audio_ms_from_marks(
-    audio_end_ms: int,
-    text_len: int,
-    marks: list[object],
-    *,
-    final_ms: object | None = None,
-) -> int:
-    if text_len <= 0:
-        return 0
-    clean_marks: list[tuple[int, int]] = []
-    for mark in marks:
-        if not isinstance(mark, dict):
-            continue
-        raw_text_chars = mark.get("text_chars")
-        raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
-        if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
-            continue
-        clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
-    if not clean_marks:
-        return 0 if audio_end_ms <= 0 else text_len
-    clean_marks.sort(key=lambda item: item[0])
-    audio_end_ms = max(0, int(audio_end_ms))
-    if audio_end_ms <= 0:
-        return 0
-    previous_ms = 0
-    previous_chars = 0
-    for mark_ms, mark_chars in clean_marks:
-        mark_ms = max(previous_ms, mark_ms)
-        mark_chars = max(previous_chars, min(text_len, mark_chars))
-        if audio_end_ms <= mark_ms:
-            if mark_ms <= previous_ms:
-                return mark_chars
-            ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
-            return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
-        previous_ms = mark_ms
-        previous_chars = mark_chars
-    if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
-        if audio_end_ms >= int(final_ms):
-            return text_len
-        ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
-        return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
-    return text_len if audio_end_ms >= previous_ms else previous_chars
-
-
-def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None:
-    content = item.get("content")
-    if not isinstance(content, list) or not content:
-        return
-    index = max(0, int(content_index))
-    if index >= len(content):
-        return
-    part = content[index]
-    if not isinstance(part, dict):
-        return
-    transcript = part.get("transcript")
-    if not isinstance(transcript, str) or not transcript:
-        return
-    marks = part.get("audio_text_marks")
-    if isinstance(marks, list):
-        keep_chars = text_chars_for_audio_ms_from_marks(
-            audio_end_ms,
-            len(transcript),
-            marks,
-            final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
-        )
-        part["transcript"] = transcript[:keep_chars].rstrip()
-        return
-    duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
-    if isinstance(duration_ms, int | float) and duration_ms > 0:
-        keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
-        part["transcript"] = transcript[:keep_chars].rstrip()
-    elif audio_end_ms <= 0:
-        part["transcript"] = ""
-
-
-def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None:
-    if item.get("type") != "message" or item.get("role") != "assistant":
-        return "conversation.item.truncate only supports assistant message items"
-    if audio_end_ms < 0:
-        return "conversation.item.truncate requires non-negative audio_end_ms"
-    content = item.get("content")
-    if not isinstance(content, list) or not content:
-        return None
-    index = max(0, int(content_index))
-    if index >= len(content):
-        return f"conversation.item.truncate content_index out of range: {content_index}"
-    part = content[index]
-    if not isinstance(part, dict):
-        return "conversation.item.truncate target content part is invalid"
-    if part.get("type") not in {"audio", "output_audio"}:
-        return "conversation.item.truncate target content part must be audio"
-    duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
-    if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
-        return "conversation.item.truncate audio_end_ms exceeds item audio duration"
-    return None
-
-
-def input_transcript_from_item(item: Mapping[str, object]) -> str:
-    content = item.get("content")
-    if not isinstance(content, list):
-        return ""
-    parts: list[str] = []
-    for part in content:
-        if not isinstance(part, dict):
-            continue
-        for key in ("transcript", "text"):
-            value = part.get(key)
-            if isinstance(value, str) and value:
-                if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
-                    parts.append(value)
-                    break
-    return "".join(parts).strip()
-
-
 # ---- append / item audio conversion ----
 
 
@@ -577,67 +120,21 @@ def build_append_audio(
     defaults: RealtimeInputDefaults,
     hints_source: Mapping[str, object] | None = None,
 ) -> AppendAudio:
-    """Validate, convert (to 16 kHz ``pcm_f32le`` base64) and pack one audio append.
+    """Decode one audio append (shared codec) and pack it as the duplex command."""
-
-    Speech classification without VAD (``is_speech`` on the command) uses the
-    explicit wire hints and the RMS fallback exactly like the old translator.
-    """
-    event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
-    audio = event.get("audio") or event.get("delta")
-    fmt, format_rate = parse_realtime_audio_format(
-        event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
-    )
-    sample_rate_hz = (
-        event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
-    )
-    if not is_supported_realtime_input_format(fmt):
-        raise DuplexCommandError(
-            f"Unsupported input_audio_format: {fmt}",
-            code="unsupported_audio_format",
-            event_id=event_id,
-        )
     try:
-        audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
+        decoded = decode_audio_append(event, defaults=defaults, hints_source=hints_source)
-            audio,
+    except RealtimeProtocolError as exc:
-            fmt,
+        raise DuplexCommandError(str(exc), code=exc.code, event_id=exc.event_id) from exc
-            sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
-        )
-    except ValueError as exc:
-        raise DuplexCommandError(str(exc), code="bad_event", event_id=event_id) from exc
-    hints: dict[str, object] = {}
-    if hints_source is not None:
-        copy_realtime_input_hints(hints_source, hints)
-    copy_realtime_input_hints(event, hints)
-    looks_like_speech = input_looks_like_speech(
-        {**(hints_source or {}), **event},
-        audio=audio,
-        fmt=fmt,
-        overlap_silence_rms=defaults.overlap_silence_rms,
-    )
-    video_frames: tuple[str, ...] = ()
-    raw_frames = event.get("video_frames")
-    if raw_frames is not None:
-        frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
-        if frames_error is not None:
-            raise DuplexCommandError(frames_error, code="invalid_video_frames", event_id=event_id)
-        if isinstance(raw_frames, list):
-            video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
-    duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
-    audio_end_ms = hints.get("audio_end_ms")
-    try:
-        audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
-    except (binascii.Error, ValueError) as exc:
-        raise DuplexCommandError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
     return AppendAudio(
-        event_id=event_id,
+        event_id=decoded.event_id,
-        audio=audio_bytes,
+        audio=decoded.audio,
-        format=str(fmt),
+        format=decoded.format,
-        sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
+        sample_rate_hz=decoded.sample_rate_hz,
-        is_speech=looks_like_speech,
+        is_speech=decoded.is_speech,
-        video_frames=video_frames,
+        video_frames=decoded.video_frames,
-        duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
+        duration_ms=decoded.duration_ms,
-        audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
+        audio_end_ms=decoded.audio_end_ms,
-        hints=hints,
+        hints=decoded.hints,
     )
 
 
@@ -664,14 +161,9 @@ def translate_realtime_command(
     if event_type == "session.update":
         session = payload.get("session")
         session_payload: Mapping[str, object] = session if isinstance(session, dict) else payload
-        format_error = validate_realtime_session_audio_formats(session_payload)
+        rejection = validate_session_payload(session_payload, capabilities=DUPLEX_REALTIME_CAPABILITIES)
-        if format_error is not None:
+        if rejection is not None:
-            raise DuplexCommandError(format_error, code="unsupported_audio_format", event_id=event_id)
+            raise DuplexCommandError(rejection.message, code=rejection.code, event_id=event_id)
-        from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
-
-        turn_detection_error = validate_realtime_turn_detection(session_payload)
-        if turn_detection_error is not None:
-            raise DuplexCommandError(turn_detection_error, code="unsupported_turn_detection", event_id=event_id)
         return UpdateSession(event_id=event_id, patch=dict(session_payload))
 
     if event_type == "conversation.item.create":
@@ -855,7 +347,10 @@ def translate_realtime_command(
     raise DuplexCommandError(f"Unknown duplex event type: {event_type}", code="unknown_event", event_id=event_id)
 
 
+#: Names this module used to define, re-exported so existing importers keep
+#: working. The canonical definitions live in ``vllm_omni.protocol.realtime``.
 __all__ = [
+    "DUPLEX_REALTIME_CAPABILITIES",
     "REALTIME_INPUT_AUDIO_FORMATS",
     "REALTIME_INPUT_HINT_KEYS",
     "REALTIME_OUTPUT_AUDIO_FORMATS",

vllm_omni/engine/duplex/realtime_events.py

@@ -15,6 +15,12 @@ carries the input-side bookkeeping the old input translator kept (input buffer
 flags, conversation items, response-id fallbacks); the ``resolve_*`` /
 ``note_*`` helpers give the runner the same behaviour for the corresponding
 commands.
+
+What is *not* here is the model- and runtime-agnostic half of the codec ---
+audio format negotiation, conversation-item shape and truncation, transcript
+extraction, audio conversion. That lives in ``vllm_omni.protocol.realtime`` so
+a non-duplex Realtime surface can use it without the duplex session; this
+module is the duplex consumer of it (RFC #6592 P0a).
 """
 
 from __future__ import annotations
@@ -24,7 +30,6 @@ from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, cast
 from uuid import uuid4
 
-from vllm_omni.engine.duplex.audio import convert_output_audio
 from vllm_omni.engine.duplex.commands import (
     AppendAudio,
     CancelResponse,
@@ -74,10 +79,11 @@ from vllm_omni.engine.duplex.events import (
     TranscriptDone,
     error_event,
 )
-from vllm_omni.engine.duplex.realtime_commands import (
+from vllm_omni.engine.duplex.realtime_commands import build_append_audio
+from vllm_omni.protocol.duplex import (
     RealtimeInputDefaults,
     apply_realtime_session_defaults,
-    build_append_audio,
+    convert_output_audio,
     copy_realtime_input_hints,
     input_looks_like_speech,
     input_transcript_from_item,

vllm_omni/engine/duplex/session/runner.py

@@ -28,7 +28,6 @@ from typing import TYPE_CHECKING, TypeVar
 
 from vllm.logger import init_logger
 
-from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate
 from vllm_omni.engine.duplex.commands import (
     AckPlayback,
     AppendAudio,
@@ -103,6 +102,7 @@ from vllm_omni.engine.duplex.turn_detection import (
     TurnDetectionResult,
 )
 from vllm_omni.metrics.stats import StageRequestStats
+from vllm_omni.protocol.duplex import convert_input_audio_with_rate
 
 if TYPE_CHECKING:
     from vllm.config import ModelConfig

vllm_omni/entrypoints/duplex/realtime_input.py

@@ -5,7 +5,9 @@
 
 Everything that needed session state now lives engine-side
 (``vllm_omni.engine.duplex.realtime_commands`` for the command mapping,
-``vllm_omni.engine.duplex.realtime_events`` for the projection state). What is
+``vllm_omni.engine.duplex.realtime_events`` for the projection state), and the
+model-agnostic parsing both of those build on lives in
+``vllm_omni.protocol.realtime``. What is
 left here is the per-connection handshake policy (query-param defaults,
 autostart / resume-only rules, ``session.resume`` parsing), the wire defaults
 used to translate appends, and error rendering through the typed
@@ -20,10 +22,8 @@ from typing import TYPE_CHECKING
 
 from vllm_omni.engine.duplex.commands import DuplexCommand, DuplexCommandError
 from vllm_omni.engine.duplex.events import error_event
-from vllm_omni.engine.duplex.realtime_commands import (
+from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
-    RealtimeInputDefaults,
+from vllm_omni.protocol.duplex import RealtimeInputDefaults
-    translate_realtime_command,
-)
 
 if TYPE_CHECKING:
     from starlette.websockets import WebSocket

vllm_omni/entrypoints/duplex/serving.py

@@ -36,7 +36,6 @@ from vllm_omni.engine.duplex.events import (
     SessionResyncRequired,
 )
 from vllm_omni.engine.duplex.messages import DuplexSessionError
-from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
 from vllm_omni.entrypoints.duplex.realtime_input import RealtimeEnvelope, parse_resume_request
 from vllm_omni.entrypoints.duplex.session_attachment import (
     DuplexJournalGapError,
@@ -52,6 +51,7 @@ from vllm_omni.entrypoints.duplex.websocket import (
     receive_text_with_timeout,
 )
 from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle
+from vllm_omni.protocol.duplex import RealtimeInputDefaults
 
 logger = init_logger(__name__)
 

vllm_omni/protocol/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Wire protocols vLLM-Omni speaks, independent of who serves or runs them.
+
+A module under ``vllm_omni.protocol`` describes *what is on the wire*: how a
+client payload is parsed and validated, and what shape a server payload has.
+It never decides how a request is executed, which model answers it, or where
+the session state lives. That keeps one protocol implementation usable by more
+than one runtime (see ``vllm_omni.protocol.realtime``).
+
+Dependency rule, asserted by ``tests/protocol/realtime/test_protocol_import_boundary.py``:
+nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``,
+``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``.
+The arrows point inwards --- those packages import the protocol, never the
+other way round.
+"""

vllm_omni/protocol/duplex/__init__.py

@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""vLLM-Omni's full-duplex extension of the OpenAI Realtime wire protocol.
+
+OpenAI's Realtime API assumes one turn at a time: the client commits input, the
+server answers. A full-duplex session does not --- the model listens while it
+speaks, it decides when to take a turn, the client reports playback progress,
+and a dropped socket re-attaches to a session that kept running. Those need
+events and commands the GA vocabulary does not have.
+
+This package is that delta. It builds on ``vllm_omni.protocol.realtime`` (same
+base classes, same pure wire rendering, no duplication) and, like it, owns no
+session and imports no runtime: the duplex engine depends on this package, not
+the other way round.
+
+The one door
+------------
+It is also the *only* door a duplex consumer uses. ``protocol.duplex.commands``
+and ``protocol.duplex.events`` carry the complete vocabulary --- Tier 1 classes
+re-exported unchanged alongside the Tier 2 and Tier 3 ones --- and this module
+re-exports the Tier 1 helper functions. So the layering is a chain, not a mesh::
+
+    protocol.realtime   (OpenAI's surface)
+        ^
+        | imports
+        |
+    protocol.duplex     (this package: re-exports Tier 1, adds Tier 2 and 3)
+        ^
+        | imports
+        |
+    duplex engine / entrypoints / clients
+
+``tests/protocol/duplex/test_duplex_protocol_facade.py`` asserts that the last
+arrow is the only one: no duplex consumer imports ``protocol.realtime``
+directly. The payoff is that a helper which later needs a duplex-specific
+version --- ``convert_input_audio_with_rate`` is the standing example, it
+resamples to MiniCPM-o's 16 kHz rather than the client's rate --- is overridden
+in one file instead of at every call site.
+
+What stays outside
+------------------
+The *mailbox* half of a command does not live here. A duplex command carries
+two different things: the client event it decodes from (wire, this package) and
+the dictionary the session runner consumes (engine). They genuinely differ ---
+``session.update``, ``conversation.item.create`` / ``.delete`` / ``.truncate``
+all travel on the runner's ``turn.signal`` channel --- so ``payload()`` and its
+``type`` stay in ``vllm_omni.engine.duplex.commands``. Events have no such
+half, which is why :data:`~vllm_omni.protocol.duplex.events.DuplexEvent` can be
+a plain alias of ``RealtimeEvent`` while ``DuplexCommand`` cannot.
+"""
+
+from vllm_omni.protocol.duplex.errors import (
+    REALTIME_ERROR_TYPES_BY_CODE,
+    RealtimeProtocolError,
+    realtime_error_type,
+)
+from vllm_omni.protocol.realtime.audio import (
+    MAX_INPUT_SAMPLE_RATE_HZ,
+    MIN_INPUT_SAMPLE_RATE_HZ,
+    convert_input_audio_with_rate,
+    convert_output_audio,
+    decode_g711_alaw,
+    decode_g711_ulaw,
+    encode_float32_mono_wav_base64,
+    encode_g711_alaw,
+    encode_g711_ulaw,
+    resample_pcm16_mono,
+    validate_input_sample_rate_hz,
+    wav_payload_to_pcm16,
+)
+from vllm_omni.protocol.realtime.audio_input import (
+    REALTIME_INPUT_HINT_KEYS,
+    RealtimeAudioAppend,
+    copy_realtime_input_hints,
+    decode_audio_append,
+    input_explicitly_non_speech,
+    input_looks_like_speech,
+)
+from vllm_omni.protocol.realtime.capabilities import (
+    RealtimeProtocolCapabilities,
+    RealtimeSessionRejection,
+    validate_session_payload,
+)
+from vllm_omni.protocol.realtime.formats import (
+    REALTIME_INPUT_AUDIO_FORMATS,
+    REALTIME_OUTPUT_AUDIO_FORMATS,
+    is_supported_realtime_input_format,
+    parse_realtime_audio_format,
+    realtime_audio_format_object,
+    realtime_output_format,
+    validate_conversation_item_audio_formats,
+    validate_realtime_response_audio_formats,
+    validate_realtime_session_audio_formats,
+)
+from vllm_omni.protocol.realtime.items import (
+    input_transcript_from_item,
+    normalize_conversation_item,
+    text_chars_for_audio_ms_from_marks,
+    truncate_realtime_item_content,
+    validate_realtime_item_truncate,
+    validate_realtime_video_frames,
+)
+from vllm_omni.protocol.realtime.session import (
+    RealtimeInputDefaults,
+    apply_realtime_session_defaults,
+    input_audio_transcription_config,
+    json_safe_realtime_payload,
+    realtime_max_output_tokens,
+    realtime_overlap_fields,
+)
+
+__all__ = [
+    "MAX_INPUT_SAMPLE_RATE_HZ",
+    "MIN_INPUT_SAMPLE_RATE_HZ",
+    "REALTIME_ERROR_TYPES_BY_CODE",
+    "REALTIME_INPUT_AUDIO_FORMATS",
+    "REALTIME_INPUT_HINT_KEYS",
+    "REALTIME_OUTPUT_AUDIO_FORMATS",
+    "RealtimeAudioAppend",
+    "RealtimeInputDefaults",
+    "RealtimeProtocolCapabilities",
+    "RealtimeProtocolError",
+    "RealtimeSessionRejection",
+    "apply_realtime_session_defaults",
+    "convert_input_audio_with_rate",
+    "convert_output_audio",
+    "copy_realtime_input_hints",
+    "decode_audio_append",
+    "decode_g711_alaw",
+    "decode_g711_ulaw",
+    "encode_float32_mono_wav_base64",
+    "encode_g711_alaw",
+    "encode_g711_ulaw",
+    "input_audio_transcription_config",
+    "input_explicitly_non_speech",
+    "input_looks_like_speech",
+    "input_transcript_from_item",
+    "is_supported_realtime_input_format",
+    "json_safe_realtime_payload",
+    "normalize_conversation_item",
+    "parse_realtime_audio_format",
+    "realtime_audio_format_object",
+    "realtime_error_type",
+    "realtime_max_output_tokens",
+    "realtime_output_format",
+    "realtime_overlap_fields",
+    "resample_pcm16_mono",
+    "text_chars_for_audio_ms_from_marks",
+    "truncate_realtime_item_content",
+    "validate_conversation_item_audio_formats",
+    "validate_input_sample_rate_hz",
+    "validate_realtime_item_truncate",
+    "validate_realtime_response_audio_formats",
+    "validate_realtime_session_audio_formats",
+    "validate_realtime_video_frames",
+    "validate_session_payload",
+    "wav_payload_to_pcm16",
+]

vllm_omni/protocol/duplex/commands.py

@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""The client-event vocabulary a full-duplex session accepts.
+
+Seven of the seventeen are events OpenAI's Realtime API has no equivalent for,
+because they only make sense when the model and the user can talk at the same
+time: the user cuts in (``barge_in``), the client reports how much audio it
+played (``playback.ack``), a turn transition is signalled locally
+(``turn.signal``), input is abandoned without committing (``input.cancel``),
+text is pushed into a live turn (``input.text.append``), and the session is
+kept alive or closed explicitly (``session.heartbeat`` / ``session.close``).
+Two more are OpenAI commands carrying duplex-only fields (Tier 2).
+
+The remaining eight are OpenAI's, unchanged, and re-exported from
+``vllm_omni.protocol.realtime.commands`` rather than redeclared --- they are
+the same objects. They are re-exported so this module is the *whole*
+vocabulary and a duplex consumer never has to import the Tier 1 package
+directly; keeping them declared there means a GA-only consumer is still not
+handed events its clients never send.
+
+None of these carries the engine's mailbox rendering: ``payload()`` and its
+channel live in ``vllm_omni.engine.duplex.commands``, because for four of the
+seventeen the runner's channel is not the client event type at all.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import ClassVar
+
+from vllm_omni.protocol.realtime import commands as realtime_commands
+from vllm_omni.protocol.realtime.commands import (
+    CancelResponse,
+    ClearInput,
+    ClearOutputAudio,
+    CreateItem,
+    CreateResponse,
+    DeleteItem,
+    RealtimeCommand,
+    TruncateItem,
+    UpdateSession,
+)
+
+# ---- Tier 2: OpenAI names carrying vLLM-Omni extensions ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AppendAudio(realtime_commands.AppendAudio):
+    """Tier 2: OpenAI's ``input_audio_buffer.append`` plus the duplex hints.
+
+    OpenAI's append is just base64 audio. The duplex lane also lets a client
+    declare this chunk's own format and rate, whether it believes the chunk is
+    speech, camera frames captured alongside it, and its timing --- all
+    additive, all ignored by a stock client.
+    """
+
+    #: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
+    format: str = "pcm16"
+    sample_rate_hz: int | None = None
+    is_speech: bool | None = None
+    video_frames: tuple[str, ...] = ()
+    duration_ms: int | None = None
+    audio_end_ms: int | None = None
+    hints: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Commit(realtime_commands.Commit):
+    """Tier 2: OpenAI's ``input_audio_buffer.commit`` plus duplex turn control.
+
+    OpenAI's commit carries nothing and always produces an item. Here a commit
+    may be non-final, may decline to start a response, and may declare itself
+    silence --- the semantic divergence documented in
+    ``docs/serving/realtime_duplex_api.md`` ("Commit != response").
+    """
+
+    #: Realtime conversation item created for this commit (wire correlation only).
+    #: ``None`` means "no explicit request": the runner decides on commit (auto-response
+    #: sessions answer on their own); ``True`` / ``False`` force it.
+    create_response: bool | None = None
+    is_speech: bool | None = None
+    realtime_item_id: str | None = None
+
+
+# ---- Tier 3: vLLM-Omni only ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AppendText(RealtimeCommand):
+    wire_type: ClassVar[str] = "input.text.append"
+    text: str
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CancelInput(RealtimeCommand):
+    wire_type: ClassVar[str] = "input.cancel"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class BargeIn(RealtimeCommand):
+    wire_type: ClassVar[str] = "barge_in"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SignalTurn(RealtimeCommand):
+    """Generic ``turn.signal`` (local turn transitions such as ``user_started``)."""
+
+    wire_type: ClassVar[str] = "turn.signal"
+    event: str
+    signal_payload: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AckPlayback(RealtimeCommand):
+    wire_type: ClassVar[str] = "playback.ack"
+    played_ms: int
+    committed_ms: int | None = None
+    response_id: str | None = None
+    item_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Heartbeat(RealtimeCommand):
+    wire_type: ClassVar[str] = "session.heartbeat"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CloseSession(RealtimeCommand):
+    """Graceful close requested through the command stream (``session.close``)."""
+
+    wire_type: ClassVar[str] = "session.close"
+    reason: str = "client_close"
+
+
+#: The complete command vocabulary a duplex client may send: the eight Tier 1
+#: classes re-exported unchanged, the two Tier 2 classes defined above, and the
+#: seven Tier 3 ones. A duplex consumer imports from here and never reaches
+#: past this module into ``vllm_omni.protocol.realtime``, so a command that
+#: later grows a duplex extension changes only this file.
+__all__ = [
+    # Tier 1 --- re-exported from ``vllm_omni.protocol.realtime.commands``.
+    "CancelResponse",
+    "ClearInput",
+    "ClearOutputAudio",
+    "CreateItem",
+    "CreateResponse",
+    "DeleteItem",
+    "RealtimeCommand",
+    "TruncateItem",
+    "UpdateSession",
+    # Tier 2 --- an OpenAI command plus duplex-only fields.
+    "AppendAudio",
+    "Commit",
+    # Tier 3 --- vLLM-Omni only.
+    "AckPlayback",
+    "AppendText",
+    "BargeIn",
+    "CancelInput",
+    "CloseSession",
+    "Heartbeat",
+    "SignalTurn",
+]

vllm_omni/protocol/duplex/errors.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""vLLM-Omni's Realtime error-code vocabulary.
+
+OpenAI's Realtime API standardises the ``error.type`` classes
+(``invalid_request_error`` / ``server_error`` / ``rate_limit_error``) but not
+the ``code`` values inside them. Ours are listed here --- Tier 3 in
+``docs/serving/realtime_duplex_api.md``, "the full error-code vocabulary" ---
+so ``protocol/realtime`` stays the OpenAI surface and nothing more.
+"""
+
+from __future__ import annotations
+
+from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
+
+__all__ = ["REALTIME_ERROR_TYPES_BY_CODE", "RealtimeProtocolError", "realtime_error_type"]
+
+
+#: OpenAI Realtime ``error.type`` for each internal error code.
+REALTIME_ERROR_TYPES_BY_CODE: dict[str, str] = {
+    "bad_event": "invalid_request_error",
+    "bad_audio": "invalid_request_error",
+    "config_timeout": "invalid_request_error",
+    "invalid_json": "invalid_request_error",
+    "event_too_large": "invalid_request_error",
+    "unknown_event": "invalid_request_error",
+    "internal_error": "server_error",
+    "runtime_append_failed": "server_error",
+    "runtime_append_task_failed": "server_error",
+    "runtime_signal_failed": "server_error",
+    "runtime_abort_failed": "server_error",
+    "runtime_data_plane_stream_failed": "server_error",
+    "runtime_data_plane_text_without_audio": "server_error",
+    "resource_exhausted": "rate_limit_error",
+    "session_exists": "invalid_request_error",
+    "session_closed": "invalid_request_error",
+    "unknown_session": "invalid_request_error",
+    "invalid_duplex_runtime_config": "invalid_request_error",
+    "instructions_update_unsupported": "invalid_request_error",
+    "persona_update_unsupported": "invalid_request_error",
+    "voice_update_unsupported": "invalid_request_error",
+    "unsupported_nemotron_duplex_mode": "invalid_request_error",
+    "unsupported_native_response_options": "invalid_request_error",
+    "runtime_touch_failed": "server_error",
+    "engine_error": "server_error",
+    "input_backpressure": "rate_limit_error",
+    "response_already_active": "invalid_request_error",
+    "response_not_active": "invalid_request_error",
+    "response_create_without_input": "invalid_request_error",
+    "text_only_turn_unsupported": "invalid_request_error",
+    "commit_aborted": "server_error",
+    "input_audio_buffer_empty": "invalid_request_error",
+    "missing_item_id": "invalid_request_error",
+    "item_not_found": "invalid_request_error",
+    "playback_item_mismatch": "invalid_request_error",
+    "playback_item_not_found": "invalid_request_error",
+    "playback_ack_too_late": "invalid_request_error",
+    "unsupported_audio_format": "invalid_request_error",
+    "unsupported_turn_detection": "invalid_request_error",
+    "unsupported_ref_audio_path": "invalid_request_error",
+    "ref_audio_required": "invalid_request_error",
+    "model_update_unsupported": "invalid_request_error",
+    "voice_update_after_audio_unsupported": "invalid_request_error",
+    "ref_audio_update_unsupported": "invalid_request_error",
+    "native_text_append_unsupported": "invalid_request_error",
+    "invalid_video_frames": "invalid_request_error",
+    "invalid_function_call_output": "invalid_request_error",
+    "server_vad_unavailable": "server_error",
+}
+
+
+def realtime_error_type(code: str) -> str:
+    """The OpenAI ``error.type`` bucket for an internal error code."""
+    return REALTIME_ERROR_TYPES_BY_CODE.get(code, "invalid_request_error")

vllm_omni/protocol/duplex/events.py

@@ -0,0 +1,419 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""vLLM-Omni's half of the Realtime server-event vocabulary.
+
+``docs/serving/realtime_duplex_api.md`` sorts every message into three tiers.
+Tier 1 --- identical to OpenAI --- is ``vllm_omni.protocol.realtime.events``.
+The other two are here:
+
+**Tier 2, OpenAI names carrying vLLM-Omni extensions.** Same wire type, same
+OpenAI fields, plus additive keys a stock client ignores. Each one subclasses
+its Tier 1 twin and declares only what it adds, so the OpenAI surface stays
+honest: a consumer that wants pure GA takes ``protocol/realtime`` and gets
+exactly OpenAI's fields.
+
+**Tier 3, vLLM-Omni only.** Events OpenAI has no equivalent for, because they
+only make sense when the model and the user can talk at the same time: the
+model decides when to listen and when to speak, the client reports how much
+audio it played, overlapping speech needs a recorded decision, and a dropped
+socket can re-attach to a live session.
+
+Same-named classes are deliberate --- ``duplex.events.SessionCreated`` *is*
+``realtime.events.SessionCreated`` with our extensions. Import whichever
+surface you mean.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import cast
+
+from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
+from vllm_omni.protocol.realtime import events as realtime_events
+from vllm_omni.protocol.realtime.events import (
+    AudioDone,
+    ContentPartAdded,
+    ContentPartDone,
+    FunctionCallArgumentsDelta,
+    FunctionCallArgumentsDone,
+    InputCleared,
+    InputTranscriptionCompleted,
+    ItemAdded,
+    ItemCreated,
+    ItemDone,
+    ItemRetrieved,
+    OutputAudioCleared,
+    OutputItemAdded,
+    OutputItemDone,
+    RateLimitsUpdated,
+    RealtimeEvent,
+    ResponseScopedEvent,
+    SessionUpdated,
+    SpeechStarted,
+    SpeechStopped,
+    TextDelta,
+    TextDone,
+    TranscriptDelta,
+    TranscriptDone,
+    new_event_id,
+    wire_value,
+)
+
+#: The duplex event base class *is* the Realtime one: unlike a command, an
+#: event has no engine-internal half to carry --- what the session emits is
+#: exactly what the socket sends. Named here rather than in the engine so the
+#: duplex vocabulary is described in one place, and kept as an alias rather
+#: than an empty subclass so ``isinstance`` and the class identity tests in
+#: ``tests/protocol/`` stay meaningful.
+DuplexEvent = RealtimeEvent
+
+# ---- Tier 2: OpenAI names carrying vLLM-Omni extensions ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionCreated(realtime_events.SessionCreated):
+    """Tier 2: OpenAI's ``SessionCreated`` plus the resume credentials a reconnecting client needs."""
+
+    #: Transport resume credentials (set by the websocket handler when resume is supported).
+    optional_wire_fields = frozenset({"attachment_generation", "resume_token"})
+    attachment_generation: int | None = None
+    resume_token: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseCreated(realtime_events.ResponseCreated):
+    """Tier 2: OpenAI's ``ResponseCreated`` plus a top-level ``response_id``.
+
+    OpenAI carries the id nested as ``response.id``; the duplex lane also
+    reports it at the top level, where every other response-scoped event
+    already has one.
+    """
+
+    response_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseDone(realtime_events.ResponseDone):
+    """Tier 2: OpenAI's ``ResponseDone`` plus a top-level ``response_id``.
+
+    OpenAI carries the id nested as ``response.id``; the duplex lane also
+    reports it at the top level, where every other response-scoped event
+    already has one.
+    """
+
+    response_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AudioDelta(realtime_events.AudioDelta):
+    """Tier 2: OpenAI's ``AudioDelta`` plus the chunk's own format/rate and the duplex playback metadata."""
+
+    optional_wire_fields = frozenset({"sample_rate_hz", "metadata"})
+    format: str = "pcm16"
+    sample_rate_hz: int | None = None
+    metadata: Mapping[str, object] | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class InputCommitted(realtime_events.InputCommitted):
+    """Tier 2: OpenAI's ``InputCommitted`` plus the native ``input.committed`` event echoed under ``event``."""
+
+    #: The session-internal ``input.committed`` event.
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        # Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
+        # rebuilds the class, so the zero-argument form's ``__class__`` cell
+        # points at the pre-slots class and is no longer in the MRO.
+        base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
+        return {**base, "event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemDeleted(realtime_events.ItemDeleted):
+    """Tier 2: OpenAI's ``ItemDeleted`` plus the originating event echoed under ``event``."""
+
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        # Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
+        # rebuilds the class, so the zero-argument form's ``__class__`` cell
+        # points at the pre-slots class and is no longer in the MRO.
+        base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
+        return {**base, "event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemTruncated(realtime_events.ItemTruncated):
+    """Tier 2: OpenAI's ``ItemTruncated`` plus the originating event echoed under ``event``."""
+
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        # Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
+        # rebuilds the class, so the zero-argument form's ``__class__`` cell
+        # points at the pre-slots class and is no longer in the MRO.
+        base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
+        return {**base, "event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ErrorEvent(realtime_events.ErrorEvent):
+    """Tier 2: OpenAI's ``ErrorEvent`` plus vLLM-Omni's error codes and any extra keys merged into the envelope."""
+
+    #: Extra keys merged into the wire ``error`` object (``retryable`` ...).
+    extra: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def error_type(self) -> str:
+        return REALTIME_ERROR_TYPES_BY_CODE.get(self.code, "invalid_request_error")
+
+    @property
+    def error(self) -> dict[str, object]:
+        # Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
+        # rebuilds the class, so the zero-argument form's ``__class__`` cell
+        # points at the pre-slots class and is no longer in the MRO. Reached
+        # through ``__dict__`` because attribute access on the class gives the
+        # getter function, not the ``property`` that owns it.
+        error = dict(realtime_events.ErrorEvent.__dict__["error"].fget(self))
+        error.update(cast("Mapping[str, object]", wire_value(self.extra)))
+        return error
+
+
+# ---- Tier 3: vLLM-Omni only ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionResumed(SessionCreated):
+    wire_type = "session.resumed"
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionReplaced(RealtimeEvent):
+    """A newer connection took the session over (sent to the replaced socket)."""
+
+    wire_type = "session.replaced"
+
+    attachment_generation: int = 0
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionResyncRequired(RealtimeEvent):
+    """The replay journal cannot bridge the gap; the client must start over."""
+
+    wire_type = "session.resync_required"
+
+    reason: str = "journal_gap"
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionClosed(RealtimeEvent):
+    wire_type = "session.closed"
+
+    reason: str = "closed"
+    #: The session-internal close event (kept for clients that inspect it).
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def is_terminal(self) -> bool:
+        return True
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, "reason": self.reason, "event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionExpired(SessionClosed):
+    wire_type = "session.expired"
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, "reason": self.reason}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionHeartbeatAck(RealtimeEvent):
+    wire_type = "session.heartbeat_ack"
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TurnEvent(RealtimeEvent):
+    """Turn-state transition (``turn.event``)."""
+
+    wire_type = "turn.event"
+
+    event: str = ""
+    turn_state: str = ""
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"session_id": self.session_id, "event": self.event, "turn_state": self.turn_state, "epoch": self.epoch}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Listen(RealtimeEvent):
+    """The model decided to keep listening (no spoken response for this turn)."""
+
+    wire_type = "response.listen"
+    optional_wire_fields = frozenset({"response_id"})
+
+    response_id: str | None = None  # type: ignore[assignment]
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        response: dict[str, object] = {
+            "object": "realtime.response",
+            "status": "listening",
+            "metadata": wire_value(self.details),
+        }
+        data: dict[str, object] = {"session_id": self.session_id, "epoch": self.epoch, "response": response}
+        if self.response_id:
+            response["id"] = self.response_id
+            data["response_id"] = self.response_id
+        return data
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Speak(ResponseScopedEvent):
+    wire_type = "response.speak"
+
+    metadata: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class OverlapDecision(RealtimeEvent):
+    wire_type = "overlap.decision"
+
+    policy: str | None = None
+    action: str | None = None
+    reason: str | None = None
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {
+            "session_id": self.session_id,
+            "epoch": self.epoch,
+            "policy": self.policy,
+            "action": self.action,
+            "reason": self.reason,
+            "metadata": wire_value(self.details),
+        }
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class PlaybackAcknowledged(RealtimeEvent):
+    wire_type = "playback.acknowledged"
+
+    #: The session-internal ``playback.acknowledged`` event (cursor, committed_ms ...).
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"event": wire_value(self.details)}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class DuplexRawEvent(RealtimeEvent):
+    """A session-internal event with no dedicated Realtime type (``duplex.<internal type>``)."""
+
+    wire_type = "duplex.raw"
+
+    internal_type: str = ""
+    details: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def type(self) -> str:
+        return f"duplex.{self.internal_type}"
+
+    def to_realtime(self) -> dict[str, object]:
+        return {"type": self.type, "event_id": self.event_id, "event": wire_value(self.details)}
+
+
+def error_event(
+    code: str,
+    message: str,
+    *,
+    event_id: object | None = None,
+    param: object | None = None,
+    extra: Mapping[str, object] | None = None,
+) -> ErrorEvent:
+    """Build the error event for an internal code; ``event_id`` is the *client* event id."""
+    return ErrorEvent(
+        code=code,
+        message=message,
+        related_event_id=event_id if isinstance(event_id, str) and event_id else None,
+        param=param if isinstance(param, str) and param else None,
+        extra=dict(extra or {}),
+    )
+
+
+#: The complete event vocabulary a duplex client may receive: 22 Tier 1
+#: classes re-exported unchanged, the 8 Tier 2 ones defined above, and 12
+#: Tier 3 ones. A duplex consumer imports from here and never reaches past
+#: this module into ``vllm_omni.protocol.realtime``.
+__all__ = [
+    # Tier 1 --- re-exported from ``vllm_omni.protocol.realtime.events``.
+    "AudioDone",
+    "ContentPartAdded",
+    "ContentPartDone",
+    "FunctionCallArgumentsDelta",
+    "FunctionCallArgumentsDone",
+    "InputCleared",
+    "InputTranscriptionCompleted",
+    "ItemAdded",
+    "ItemCreated",
+    "ItemDone",
+    "ItemRetrieved",
+    "OutputAudioCleared",
+    "OutputItemAdded",
+    "OutputItemDone",
+    "RateLimitsUpdated",
+    "SessionUpdated",
+    "SpeechStarted",
+    "SpeechStopped",
+    "TextDelta",
+    "TextDone",
+    "TranscriptDelta",
+    "TranscriptDone",
+    # Tier 2 --- an OpenAI event plus duplex-only fields.
+    "AudioDelta",
+    "ErrorEvent",
+    "InputCommitted",
+    "ItemDeleted",
+    "ItemTruncated",
+    "ResponseCreated",
+    "ResponseDone",
+    "SessionCreated",
+    # Tier 3 --- vLLM-Omni only.
+    "DuplexRawEvent",
+    "Listen",
+    "OverlapDecision",
+    "PlaybackAcknowledged",
+    "SessionClosed",
+    "SessionExpired",
+    "SessionHeartbeatAck",
+    "SessionReplaced",
+    "SessionResumed",
+    "SessionResyncRequired",
+    "Speak",
+    "TurnEvent",
+    # Bases and helpers shared with the Tier 1 surface.
+    "DuplexEvent",
+    "REALTIME_ERROR_TYPES_BY_CODE",
+    "RealtimeEvent",
+    "ResponseScopedEvent",
+    "error_event",
+    "new_event_id",
+    "wire_value",
+]

vllm_omni/protocol/realtime/__init__.py

@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""The OpenAI Realtime wire codec, shared by every vLLM-Omni Realtime surface.
+
+What this is
+------------
+The model-agnostic half of ``WS /v1/realtime``: how a client event is parsed,
+which audio formats and session fields are accepted, what a conversation item
+means, how audio is decoded and resampled, and the shape of the error envelope
+(the *codes* inside it are a consumer's own --- ours are in
+``vllm_omni.protocol.duplex.errors``). It is a library of pure functions and value objects.
+
+What it is not
+--------------
+It owns no session. It does not decide when a response starts, what a model is
+prompted with, whether a turn is over, or where committed audio goes. Those are
+runtime decisions, and after PR #7413 a duplex session's runtime decisions all
+belong to ``vllm_omni.engine.duplex`` --- the session runner, the typed
+``DuplexCommand`` / ``DuplexEvent`` boundary and the ``DuplexModelPlugin`` seam.
+This package sits *under* that: the duplex engine binds the codec to its own
+command and event vocabulary in
+``vllm_omni.engine.duplex.realtime_commands`` and
+``vllm_omni.engine.duplex.realtime_events``.
+
+Why it is separate (RFC #6592, P0a)
+-----------------------------------
+So that a second Realtime surface --- a Qwen3-Omni GA profile, say --- can reuse
+the parsing, validation, format negotiation and error envelope without adopting
+the duplex session control plane (lease manager, commit policy, model channel),
+and without a second copy of the codec drifting away from this one. A consumer
+states what it can serve through
+:class:`~vllm_omni.protocol.realtime.capabilities.RealtimeProtocolCapabilities`
+and keeps its own session state.
+
+Known duplex-shaped defaults
+----------------------------
+This package is model-agnostic in its *dependencies*, not yet in every
+*default*. Two constants still carry the assumptions of its first consumer and
+a second one will have to parameterize them:
+
+* :func:`~vllm_omni.protocol.realtime.audio.convert_input_audio_with_rate`
+  resamples to 16 kHz, which is MiniCPM-o's feature-extractor rate. A GA client
+  sending 24 kHz is converted down rather than served at its own rate.
+* :func:`~vllm_omni.protocol.realtime.items.validate_realtime_video_frames`
+  encodes the MiniCPM-o duplex camera contract (at most two frames per append,
+  ``max_slice_nums`` not selectable on the wire).
+
+Both are behaviour this extraction deliberately did **not** change (RFC #6592
+P0a is behaviour-preserving); making them per-consumer is P0b/P1 work.
+
+Dependency rule
+---------------
+Nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``,
+``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``;
+``tests/protocol/realtime/test_protocol_import_boundary.py`` asserts it.
+"""
+
+from vllm_omni.protocol.realtime.audio import (
+    MAX_INPUT_SAMPLE_RATE_HZ,
+    MIN_INPUT_SAMPLE_RATE_HZ,
+    convert_input_audio_with_rate,
+    convert_output_audio,
+    encode_float32_mono_wav_base64,
+    resample_pcm16_mono,
+    validate_input_sample_rate_hz,
+    wav_payload_to_pcm16,
+)
+from vllm_omni.protocol.realtime.audio_input import (
+    REALTIME_INPUT_HINT_KEYS,
+    RealtimeAudioAppend,
+    copy_realtime_input_hints,
+    decode_audio_append,
+    input_explicitly_non_speech,
+    input_looks_like_speech,
+)
+from vllm_omni.protocol.realtime.capabilities import (
+    RealtimeProtocolCapabilities,
+    RealtimeSessionRejection,
+    validate_session_payload,
+)
+from vllm_omni.protocol.realtime.commands import RealtimeCommand
+from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
+from vllm_omni.protocol.realtime.events import (
+    RealtimeEvent,
+    ResponseScopedEvent,
+    new_event_id,
+    wire_value,
+)
+from vllm_omni.protocol.realtime.formats import (
+    REALTIME_INPUT_AUDIO_FORMATS,
+    REALTIME_OUTPUT_AUDIO_FORMATS,
+    is_supported_realtime_input_format,
+    parse_realtime_audio_format,
+    realtime_audio_format_object,
+    realtime_output_format,
+    validate_conversation_item_audio_formats,
+    validate_realtime_response_audio_formats,
+    validate_realtime_session_audio_formats,
+)
+from vllm_omni.protocol.realtime.items import (
+    input_transcript_from_item,
+    normalize_conversation_item,
+    text_chars_for_audio_ms_from_marks,
+    truncate_realtime_item_content,
+    validate_realtime_item_truncate,
+    validate_realtime_video_frames,
+)
+from vllm_omni.protocol.realtime.session import (
+    RealtimeInputDefaults,
+    apply_realtime_session_defaults,
+    input_audio_transcription_config,
+    json_safe_realtime_payload,
+    realtime_max_output_tokens,
+    realtime_overlap_fields,
+)
+
+__all__ = [
+    "MAX_INPUT_SAMPLE_RATE_HZ",
+    "MIN_INPUT_SAMPLE_RATE_HZ",
+    "REALTIME_INPUT_AUDIO_FORMATS",
+    "REALTIME_INPUT_HINT_KEYS",
+    "REALTIME_OUTPUT_AUDIO_FORMATS",
+    "RealtimeAudioAppend",
+    "RealtimeCommand",
+    "RealtimeEvent",
+    "RealtimeInputDefaults",
+    "RealtimeProtocolCapabilities",
+    "RealtimeProtocolError",
+    "RealtimeSessionRejection",
+    "ResponseScopedEvent",
+    "apply_realtime_session_defaults",
+    "convert_input_audio_with_rate",
+    "convert_output_audio",
+    "copy_realtime_input_hints",
+    "decode_audio_append",
+    "encode_float32_mono_wav_base64",
+    "input_audio_transcription_config",
+    "input_explicitly_non_speech",
+    "input_looks_like_speech",
+    "input_transcript_from_item",
+    "is_supported_realtime_input_format",
+    "json_safe_realtime_payload",
+    "new_event_id",
+    "normalize_conversation_item",
+    "parse_realtime_audio_format",
+    "realtime_audio_format_object",
+    "realtime_max_output_tokens",
+    "realtime_output_format",
+    "realtime_overlap_fields",
+    "resample_pcm16_mono",
+    "text_chars_for_audio_ms_from_marks",
+    "truncate_realtime_item_content",
+    "validate_conversation_item_audio_formats",
+    "validate_input_sample_rate_hz",
+    "validate_realtime_item_truncate",
+    "validate_realtime_response_audio_formats",
+    "validate_realtime_session_audio_formats",
+    "validate_realtime_video_frames",
+    "validate_session_payload",
+    "wav_payload_to_pcm16",
+    "wire_value",
+]

vllm_omni/protocol/realtime/audio.py

@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from __future__ import annotations
+
+import base64
+import binascii
+import io
+import math
+import wave
+
+import numpy as np
+
+from vllm_omni.utils.audio_resample import StreamingAudioResampler
+
+try:
+    from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin
+except ImportError:  # pragma: no cover - audioop is removed in newer Python.
+    alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None
+
+
+MIN_INPUT_SAMPLE_RATE_HZ = 8_000
+MAX_INPUT_SAMPLE_RATE_HZ = 192_000
+
+
+def validate_input_sample_rate_hz(sample_rate_hz: object) -> int:
+    if (
+        isinstance(sample_rate_hz, bool)
+        or not isinstance(sample_rate_hz, int | float)
+        or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz))
+    ):
+        raise ValueError("sample_rate_hz must be a finite integer")
+    rate = int(sample_rate_hz)
+    if rate != sample_rate_hz:
+        raise ValueError("sample_rate_hz must be an integer")
+    if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ:
+        raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}")
+    return rate
+
+
+def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes:
+    if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz:
+        return raw
+    samples = np.frombuffer(raw, dtype="<i2")
+    rate_gcd = math.gcd(source_rate_hz, target_rate_hz)
+    reduced_factor = max(source_rate_hz // rate_gcd, target_rate_hz // rate_gcd)
+    if reduced_factor > StreamingAudioResampler.max_polyphase_factor:
+        # The FIR size scales with the reduced ratio. Keep unusual client
+        # rates bounded while common audio rates use the high-quality path.
+        if samples.size <= 1:
+            return raw
+        target_size = max(1, int(round(samples.size * target_rate_hz / source_rate_hz)))
+        source_x = np.linspace(0.0, 1.0, num=samples.size, endpoint=True)
+        target_x = np.linspace(0.0, 1.0, num=target_size, endpoint=True)
+        resampled = np.interp(target_x, source_x, samples)
+        return np.clip(resampled, -32_768, 32_767).astype("<i2").tobytes()
+    resampler = StreamingAudioResampler(source_rate_hz, target_rate_hz)
+    normalized = samples.astype(np.float32) * np.float32(1.0 / 32768.0)
+    resampled = resampler.process(normalized, final=True)
+    return np.clip(np.rint(resampled * 32768.0), -32768, 32767).astype("<i2").tobytes()
+
+
+def decode_g711_ulaw(raw: bytes) -> bytes:
+    if ulaw2lin is not None:
+        return ulaw2lin(raw, 2)
+    data = np.frombuffer(raw, dtype=np.uint8)
+    value = np.bitwise_not(data).astype(np.int16)
+    sign = value & 0x80
+    exponent = (value >> 4) & 0x07
+    mantissa = value & 0x0F
+    sample = ((mantissa << 3) + 0x84) << exponent
+    return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype("<i2").tobytes()
+
+
+def decode_g711_alaw(raw: bytes) -> bytes:
+    if alaw2lin is not None:
+        return alaw2lin(raw, 2)
+    data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16)
+    sign = data & 0x80
+    exponent = (data >> 4) & 0x07
+    mantissa = data & 0x0F
+    sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1))
+    return np.where(sign != 0, sample, -sample).astype("<i2").tobytes()
+
+
+def encode_g711_ulaw(raw: bytes) -> bytes:
+    if lin2ulaw is not None:
+        return lin2ulaw(raw, 2)
+    pcm = np.clip(np.frombuffer(raw, dtype="<i2").astype(np.int32), -32635, 32635)
+    sign = np.where(pcm < 0, 0x80, 0)
+    magnitude = np.abs(pcm) + 0x84
+    exponent = np.zeros_like(magnitude)
+    for exp in range(7):
+        exponent = np.where(magnitude > (0xFF << exp), exp + 1, exponent)
+    mantissa = (magnitude >> (exponent + 3)) & 0x0F
+    return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes()
+
+
+def encode_g711_alaw(raw: bytes) -> bytes:
+    if lin2alaw is not None:
+        return lin2alaw(raw, 2)
+    pcm = np.frombuffer(raw, dtype="<i2").astype(np.int32)
+    sign = np.where(pcm >= 0, 0x80, 0x00)
+    magnitude = np.abs(pcm)
+    exponent = np.zeros_like(magnitude)
+    for exp in range(1, 8):
+        exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent)
+    mantissa = np.where(
+        exponent == 0,
+        (magnitude >> 4) & 0x0F,
+        (magnitude >> (exponent + 3)) & 0x0F,
+    )
+    return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes()
+
+
+def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]:
+    try:
+        with wave.open(io.BytesIO(raw), "rb") as wav_file:
+            channels = wav_file.getnchannels()
+            sample_width = wav_file.getsampwidth()
+            sample_rate_hz = wav_file.getframerate()
+            frames = wav_file.readframes(wav_file.getnframes())
+        if sample_width != 2:
+            return None, sample_rate_hz
+        if channels <= 1:
+            return frames, sample_rate_hz
+        pcm = np.frombuffer(frames, dtype="<i2").reshape(-1, channels)
+        mono = np.mean(pcm.astype(np.float32), axis=1)
+        return np.clip(mono, -32768, 32767).astype("<i2").tobytes(), sample_rate_hz
+    except (EOFError, ValueError, wave.Error):
+        return None, None
+
+
+def encode_float32_mono_wav_base64(
+    samples: np.ndarray,
+    *,
+    sample_rate_hz: int,
+) -> str:
+    """Package normalized mono float32 samples as a PCM16 WAV data payload."""
+    rate = validate_input_sample_rate_hz(sample_rate_hz)
+    normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1)
+    pcm16 = np.clip(np.rint(normalized * 32768.0), -32768, 32767).astype("<i2")
+    with io.BytesIO() as buffer:
+        with wave.open(buffer, "wb") as wav_file:
+            wav_file.setnchannels(1)
+            wav_file.setsampwidth(2)
+            wav_file.setframerate(rate)
+            wav_file.writeframes(pcm16.tobytes())
+        return base64.b64encode(buffer.getvalue()).decode("ascii")
+
+
+def convert_input_audio_with_rate(
+    audio: object,
+    fmt: object,
+    *,
+    sample_rate_hz: int | float | None = None,
+    target_sample_rate_hz: int = 16_000,
+) -> tuple[object, object, int | float | None]:
+    if not isinstance(audio, str) or not isinstance(fmt, str):
+        return audio, fmt, sample_rate_hz
+    normalized = fmt.lower()
+    if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}:
+        return audio, fmt, sample_rate_hz
+    if isinstance(sample_rate_hz, int | float):
+        sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz)
+    try:
+        raw = base64.b64decode(audio.strip(), validate=False)
+    except (binascii.Error, ValueError):
+        return audio, fmt, sample_rate_hz
+    if normalized == "g711_ulaw":
+        raw = decode_g711_ulaw(raw)
+        sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
+    elif normalized == "g711_alaw":
+        raw = decode_g711_alaw(raw)
+        sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
+    elif len(raw) % 2:
+        return audio, fmt, sample_rate_hz
+    if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz:
+        raw = resample_pcm16_mono(
+            raw,
+            source_rate_hz=int(sample_rate_hz),
+            target_rate_hz=target_sample_rate_hz,
+        )
+        sample_rate_hz = target_sample_rate_hz
+    pcm = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
+    encoded = base64.b64encode(np.ascontiguousarray(pcm, dtype="<f4").tobytes()).decode("ascii")
+    return encoded, "pcm_f32le", sample_rate_hz
+
+
+def convert_output_audio(
+    audio: str,
+    *,
+    source_fmt: str,
+    target_fmt: str,
+    source_sample_rate_hz: int | None = None,
+    target_sample_rate_hz: int | None = None,
+) -> tuple[str, str, int | None]:
+    target = target_fmt.lower()
+    if target not in {"g711_ulaw", "g711_alaw"}:
+        return audio, source_fmt, source_sample_rate_hz
+    try:
+        raw = base64.b64decode(audio, validate=False)
+    except (binascii.Error, ValueError):
+        return audio, source_fmt, source_sample_rate_hz
+    source = source_fmt.lower()
+    if source == "wav":
+        pcm_raw, wav_rate = wav_payload_to_pcm16(raw)
+        if pcm_raw is None:
+            return audio, source_fmt, source_sample_rate_hz
+        raw = pcm_raw
+        source_sample_rate_hz = source_sample_rate_hz or wav_rate
+    elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2:
+        return audio, source_fmt, source_sample_rate_hz
+    target_rate = target_sample_rate_hz or 8_000
+    if source_sample_rate_hz is not None:
+        raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate)
+    encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw)
+    return base64.b64encode(encoded).decode("ascii"), target, target_rate

vllm_omni/protocol/realtime/audio_input.py

@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Decoding one ``input_audio_buffer.append``.
+
+An append is the hottest client event on a Realtime connection and the one with
+the most wire surface: base64 audio in any accepted format, an optional rate,
+optional camera frames, and a set of optional hints (``duration_ms``,
+``is_speech``, a VAD probability, a transcript) that clients attach and servers
+are free to use or ignore.
+
+:func:`decode_audio_append` turns that into :class:`RealtimeAudioAppend`, a
+value object: bytes at 16 kHz ``pcm_f32le``, the surviving hints, and a speech
+verdict. It is a pure function of the event plus the session's
+:class:`~vllm_omni.protocol.realtime.session.RealtimeInputDefaults`, so any
+consumer decodes an append the same way. Building a consumer's own command
+object out of it is one constructor call --- see
+``vllm_omni.engine.duplex.realtime_commands.build_append_audio``, which wraps it
+for the duplex ``AppendAudio``.
+
+The speech verdict here is the *client-declared* one (explicit flags, a VAD
+probability the client sent, or an RMS floor as a last resort). It is not
+server-side VAD; that is a runtime concern and lives with whoever owns the
+session.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import cast
+
+import numpy as np
+
+from vllm_omni.protocol.realtime.audio import convert_input_audio_with_rate
+from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
+from vllm_omni.protocol.realtime.formats import (
+    is_supported_realtime_input_format,
+    parse_realtime_audio_format,
+)
+from vllm_omni.protocol.realtime.items import validate_realtime_video_frames
+from vllm_omni.protocol.realtime.session import RealtimeInputDefaults
+
+__all__ = [
+    "REALTIME_INPUT_HINT_KEYS",
+    "RealtimeAudioAppend",
+    "copy_realtime_input_hints",
+    "decode_audio_append",
+    "input_explicitly_non_speech",
+    "input_looks_like_speech",
+]
+
+#: Wire hint keys copied verbatim from a client append onto the internal payload.
+REALTIME_INPUT_HINT_KEYS = (
+    "duration_ms",
+    "audio_duration_ms",
+    "audio_start_ms",
+    "audio_end_ms",
+    "is_speech",
+    "speech",
+    "speech_probability",
+    "vad",
+    "overlap_action",
+    "overlap",
+    "force_barge_in",
+    "force_listen",
+    "text",
+    "transcript",
+)
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeAudioAppend:
+    """One decoded ``input_audio_buffer.append``.
+
+    ``audio`` is raw bytes in ``format`` at ``sample_rate_hz`` --- after
+    :func:`decode_audio_append` that is 16 kHz ``pcm_f32le`` for every input
+    format the codec converts.
+    """
+
+    audio: bytes
+    format: str
+    sample_rate_hz: int | None = None
+    is_speech: bool | None = None
+    video_frames: tuple[str, ...] = ()
+    duration_ms: int | None = None
+    audio_end_ms: int | None = None
+    #: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
+    hints: dict[str, object] = field(default_factory=dict)
+    #: Client correlation id (OpenAI ``event_id``), echoed on error events.
+    event_id: str | None = None
+
+
+# ---- speech hints ----
+
+
+def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
+    for key in ("is_speech", "speech"):
+        value = event.get(key)
+        if isinstance(value, bool):
+            return not value
+    vad = event.get("vad")
+    if isinstance(vad, dict):
+        value = vad.get("is_speech")
+        if isinstance(value, bool):
+            return not value
+        probability = vad.get("speech_probability", vad.get("probability"))
+        if isinstance(probability, int | float):
+            return float(probability) < 0.5
+    probability = event.get("speech_probability")
+    return isinstance(probability, int | float) and float(probability) < 0.5
+
+
+def input_looks_like_speech(
+    event: Mapping[str, object],
+    *,
+    audio: object,
+    fmt: object,
+    overlap_silence_rms: float,
+) -> bool:
+    if input_explicitly_non_speech(event):
+        return False
+    for key in ("is_speech", "speech"):
+        value = event.get(key)
+        if isinstance(value, bool):
+            return value
+    vad = event.get("vad")
+    if isinstance(vad, dict):
+        probability = vad.get("speech_probability", vad.get("probability"))
+        if isinstance(probability, int | float):
+            return float(probability) >= 0.5
+    probability = event.get("speech_probability")
+    if isinstance(probability, int | float):
+        return float(probability) >= 0.5
+    if fmt != "pcm_f32le":
+        return True
+    if isinstance(audio, bytes | bytearray):
+        raw = bytes(audio)
+    elif isinstance(audio, str):
+        try:
+            raw = base64.b64decode(audio, validate=True)
+        except (binascii.Error, ValueError):
+            return True
+    else:
+        return True
+    if len(raw) < 4 or len(raw) % 4 != 0:
+        return True
+    samples = np.frombuffer(raw, dtype=np.float32)
+    if samples.size == 0:
+        return False
+    rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
+    threshold = event.get("overlap_silence_rms")
+    if not isinstance(threshold, int | float):
+        vad = event.get("vad")
+        if isinstance(vad, dict):
+            threshold = vad.get("silence_rms")
+    silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
+    return rms >= max(0.0, silence_rms)
+
+
+def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
+    for key in REALTIME_INPUT_HINT_KEYS:
+        if key in source:
+            target[key] = source[key]
+
+
+# ---- append decoding ----
+
+
+def decode_audio_append(
+    event: Mapping[str, object],
+    *,
+    defaults: RealtimeInputDefaults,
+    hints_source: Mapping[str, object] | None = None,
+) -> RealtimeAudioAppend:
+    """Validate, convert (to 16 kHz ``pcm_f32le``) and pack one audio append.
+
+    ``hints_source`` is the enclosing payload when the audio arrives inside
+    something larger than a bare append --- a ``conversation.item.create``
+    audio part, say --- so its hints apply unless the part overrides them.
+
+    Raises :class:`RealtimeProtocolError` for an unsupported format, undecodable
+    audio or invalid camera frames.
+    """
+    event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
+    audio = event.get("audio") or event.get("delta")
+    fmt, format_rate = parse_realtime_audio_format(
+        event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
+    )
+    sample_rate_hz = (
+        event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
+    )
+    if not is_supported_realtime_input_format(fmt):
+        raise RealtimeProtocolError(
+            f"Unsupported input_audio_format: {fmt}",
+            code="unsupported_audio_format",
+            event_id=event_id,
+        )
+    try:
+        audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
+            audio,
+            fmt,
+            sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
+        )
+    except ValueError as exc:
+        raise RealtimeProtocolError(str(exc), code="bad_event", event_id=event_id) from exc
+    hints: dict[str, object] = {}
+    if hints_source is not None:
+        copy_realtime_input_hints(hints_source, hints)
+    copy_realtime_input_hints(event, hints)
+    looks_like_speech = input_looks_like_speech(
+        {**(hints_source or {}), **event},
+        audio=audio,
+        fmt=fmt,
+        overlap_silence_rms=defaults.overlap_silence_rms,
+    )
+    video_frames: tuple[str, ...] = ()
+    raw_frames = event.get("video_frames")
+    if raw_frames is not None:
+        frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
+        if frames_error is not None:
+            raise RealtimeProtocolError(frames_error, code="invalid_video_frames", event_id=event_id)
+        if isinstance(raw_frames, list):
+            video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
+    duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
+    audio_end_ms = hints.get("audio_end_ms")
+    try:
+        audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
+    except (binascii.Error, ValueError) as exc:
+        raise RealtimeProtocolError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
+    return RealtimeAudioAppend(
+        event_id=event_id,
+        audio=audio_bytes,
+        format=str(fmt),
+        sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
+        is_speech=looks_like_speech,
+        video_frames=video_frames,
+        duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
+        audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
+        hints=hints,
+    )

vllm_omni/protocol/realtime/capabilities.py

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""What a Realtime consumer can serve, and the session check that reads it.
+
+``session.update`` is the one client event whose acceptance depends on who is
+behind the socket. The audio formats a server can decode, and whether it can do
+server-side turn detection at all, are properties of the *consumer* --- the
+duplex engine, or some other runtime --- not of the protocol. The codec still
+has to do the checking, because it owns the error envelope and the order the
+checks run in.
+
+:class:`RealtimeProtocolCapabilities` is how a consumer states those answers.
+It is the post-#7413 form of the ``supports(turn_detection, formats)`` half of
+RFC #6592's proposed ``RealtimeModelAdapter``. The other three halves of that
+proposal --- building a prompt, starting a response, cancelling one --- are not
+here and must not come here: for a duplex model they are already owned by
+``DuplexModelPlugin`` together with ``DuplexSessionRunner``, and a second
+interface over the same responsibility is exactly the fork this layer exists to
+prevent.
+
+The duplex binding of this object is
+``vllm_omni.engine.duplex.realtime_commands.DUPLEX_REALTIME_CAPABILITIES``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+
+from vllm_omni.protocol.realtime.formats import (
+    REALTIME_INPUT_AUDIO_FORMATS,
+    REALTIME_OUTPUT_AUDIO_FORMATS,
+    validate_realtime_session_audio_formats,
+)
+
+__all__ = [
+    "RealtimeProtocolCapabilities",
+    "RealtimeSessionRejection",
+    "validate_session_payload",
+]
+
+#: A consumer's answer to "can you do this turn detection?": an error message, or None.
+TurnDetectionValidator = Callable[[Mapping[str, object]], str | None]
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeProtocolCapabilities:
+    """One consumer's answer to what a ``session`` object may ask for.
+
+    The defaults are permissive, not restrictive: every format the codec can
+    decode, and **no turn-detection validation at all**. A consumer that leaves
+    ``validate_turn_detection`` at ``None`` therefore accepts whatever
+    ``turn_detection`` the session object carries --- including values it cannot
+    actually serve, such as ``semantic_vad``. A consumer that supports only some
+    turn-detection modes (or none) must supply its own validator; see
+    ``vllm_omni.engine.duplex.realtime_commands.DUPLEX_REALTIME_CAPABILITIES``.
+    """
+
+    input_audio_formats: frozenset[str] = field(default_factory=lambda: frozenset(REALTIME_INPUT_AUDIO_FORMATS))
+    output_audio_formats: frozenset[str] = field(default_factory=lambda: frozenset(REALTIME_OUTPUT_AUDIO_FORMATS))
+    #: Consumer-specific ``turn_detection`` validation. ``None`` means the
+    #: consumer takes whatever the session object says without a check of its own.
+    validate_turn_detection: TurnDetectionValidator | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeSessionRejection:
+    """Why a session object was refused: the internal error code and the message.
+
+    ``param`` names the offending session field where there is one, for the
+    ``error.param`` slot of the OpenAI error envelope.
+    """
+
+    code: str
+    message: str
+    param: str | None = None
+
+
+def validate_session_payload(
+    session_payload: Mapping[str, object],
+    *,
+    capabilities: RealtimeProtocolCapabilities,
+) -> RealtimeSessionRejection | None:
+    """Check a ``session`` object against one consumer; ``None`` when it is acceptable.
+
+    Audio formats are checked before turn detection, so a session object that
+    is wrong in both ways reports the format problem --- keep that order, it is
+    what clients already see.
+    """
+    format_error = validate_realtime_session_audio_formats(
+        session_payload,
+        input_audio_formats=capabilities.input_audio_formats,
+        output_audio_formats=capabilities.output_audio_formats,
+    )
+    if format_error is not None:
+        return RealtimeSessionRejection(code="unsupported_audio_format", message=format_error)
+    validate_turn_detection = capabilities.validate_turn_detection
+    if validate_turn_detection is not None:
+        turn_detection_error = validate_turn_detection(session_payload)
+        if turn_detection_error is not None:
+            return RealtimeSessionRejection(
+                code="unsupported_turn_detection", message=turn_detection_error, param="turn_detection"
+            )
+    return None

vllm_omni/protocol/realtime/commands.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""OpenAI Realtime client events as typed commands.
+
+A Realtime server's first job is turning a client JSON event into something
+typed. These are the 10 client events OpenAI's Realtime API defines, as frozen
+dataclasses carrying the fields that survive decoding.
+
+vLLM-Omni's own client events (barge-in, playback acknowledgement, text append,
+heartbeat, explicit close, generic turn signals) are a separate vocabulary in
+``vllm_omni.protocol.duplex.commands``.
+
+Decoding a payload *into* these stays with the consumer, because which command
+a given event becomes can depend on what that consumer supports --- see
+``vllm_omni.engine.duplex.realtime_commands.translate_realtime_command``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import ClassVar
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class RealtimeCommand:
+    """One decoded OpenAI Realtime client event.
+
+    ``wire_type`` is the event type the client sent; the typed fields are what
+    survived decoding, so a consumer reads them instead of re-parsing JSON.
+
+    There is deliberately no rendering method here. A server receives commands,
+    it does not emit them, and how a runtime represents one internally is its
+    own business --- the duplex engine renders its mailbox dictionary in
+    ``vllm_omni.engine.duplex.commands``.
+    """
+
+    #: OpenAI Realtime client event this command decodes from.
+    wire_type: ClassVar[str] = ""
+    #: Client correlation id (OpenAI ``event_id``), echoed on error events.
+    event_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class UpdateSession(RealtimeCommand):
+    """``session.update``: a Realtime ``session`` object patch."""
+
+    wire_type: ClassVar[str] = "session.update"
+    patch: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AppendAudio(RealtimeCommand):
+    wire_type: ClassVar[str] = "input_audio_buffer.append"
+    #: Raw audio bytes in ``format`` at ``sample_rate_hz`` (base64 only on the wire).
+    audio: bytes
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Commit(RealtimeCommand):
+    wire_type: ClassVar[str] = "input_audio_buffer.commit"
+    final: bool = True
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ClearInput(RealtimeCommand):
+    wire_type: ClassVar[str] = "input_audio_buffer.clear"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ClearOutputAudio(RealtimeCommand):
+    wire_type: ClassVar[str] = "output_audio_buffer.clear"
+    #: Explicit response to clear; ``None`` targets the active/last response.
+    response_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CreateResponse(RealtimeCommand):
+    wire_type: ClassVar[str] = "response.create"
+    #: Raw Realtime ``response`` object (instructions, voice, modalities, ...).
+    options: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CancelResponse(RealtimeCommand):
+    wire_type: ClassVar[str] = "response.cancel"
+    response_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class CreateItem(RealtimeCommand):
+    """``conversation.item.create`` (history injection or function-call output)."""
+
+    wire_type: ClassVar[str] = "conversation.item.create"
+    item: Mapping[str, object]
+    previous_item_id: str | None = None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class DeleteItem(RealtimeCommand):
+    wire_type: ClassVar[str] = "conversation.item.delete"
+    item_id: str
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TruncateItem(RealtimeCommand):
+    wire_type: ClassVar[str] = "conversation.item.truncate"
+    item_id: str
+    audio_end_ms: int
+    content_index: int = 0
+
+
+__all__ = [
+    "AppendAudio",
+    "CancelResponse",
+    "ClearInput",
+    "ClearOutputAudio",
+    "Commit",
+    "CreateItem",
+    "CreateResponse",
+    "DeleteItem",
+    "RealtimeCommand",
+    "TruncateItem",
+    "UpdateSession",
+]

vllm_omni/protocol/realtime/errors.py

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""The OpenAI Realtime error envelope.
+
+A Realtime server reports a rejected client event as one ``error`` event whose
+``error`` object carries ``type`` / ``code`` / ``message`` (plus the client's
+``event_id`` and an optional ``param``). ``type`` is not free-form: it is one
+of OpenAI's four buckets, derived from our internal code by
+:data:`REALTIME_ERROR_TYPES_BY_CODE`.
+
+OpenAI standardises the three ``type`` classes but not the *codes*, so the code
+vocabulary is a consumer's own. vLLM-Omni's lives in
+``vllm_omni.protocol.duplex.errors`` (Tier 3 in
+``docs/serving/realtime_duplex_api.md``); what stays here is the exception type
+every consumer raises, so the envelope is rendered identically whoever built it.
+"""
+
+from __future__ import annotations
+
+__all__ = ["RealtimeProtocolError"]
+
+
+class RealtimeProtocolError(ValueError):
+    """A client payload could not be decoded into a valid Realtime intent.
+
+    ``code`` is the internal error code that :data:`REALTIME_ERROR_TYPES_BY_CODE`
+    maps to an OpenAI ``error.type``; ``event_id`` is the *client* event id the
+    error answers. ``vllm_omni.engine.duplex.commands.DuplexCommandError`` is
+    the duplex specialization, so a consumer catching either sees the same
+    three attributes.
+    """
+
+    def __init__(self, message: str, *, code: str = "bad_event", event_id: str | None = None) -> None:
+        super().__init__(message)
+        self.code = code
+        self.event_id = event_id

vllm_omni/protocol/realtime/events.py

@@ -0,0 +1,467 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""OpenAI Realtime server events: typed classes and their wire rendering.
+
+Every event is a frozen dataclass with explicit fields, and the Realtime wire
+JSON is *derived* from those fields by :meth:`RealtimeEvent.to_realtime`. That
+rendering is pure --- it never consults session state --- which is why it can
+live here, away from whoever owns the session. The stateful part (response and
+item ids, content-part bookkeeping) is consumed when the events are
+*constructed*, by the consumer.
+
+This module holds the 30 events OpenAI's Realtime API defines. vLLM-Omni's own
+additions to the wire (listen/speak turn-taking, playback acknowledgement,
+session resume) are a separate vocabulary in
+``vllm_omni.protocol.duplex.events``, so a consumer that only speaks GA does
+not inherit them.
+
+``session_id`` / ``epoch`` are bound by the consumer when an event leaves it;
+producers build events with the defaults.
+"""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Mapping
+from dataclasses import dataclass, field, fields
+from typing import ClassVar
+from uuid import uuid4
+
+
+def new_event_id() -> str:
+    return f"event_{uuid4().hex}"
+
+
+_IDENTITY_FIELDS = frozenset({"session_id", "epoch", "event_id"})
+
+
+def wire_value(value: object) -> object:
+    if isinstance(value, Mapping):
+        return {str(k): wire_value(v) for k, v in value.items()}
+    if isinstance(value, tuple | list):
+        return [wire_value(v) for v in value]
+    return value
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class RealtimeEvent:
+    """Base of every public session event."""
+
+    #: Realtime ``type`` this class renders to.
+    wire_type: ClassVar[str] = ""
+    #: Fields that are omitted from the wire object when ``None``.
+    optional_wire_fields: ClassVar[frozenset[str]] = frozenset()
+
+    session_id: str = ""
+    epoch: int | None = None
+    #: Server event id (OpenAI ``event_id``); generated at construction.
+    event_id: str = field(default_factory=new_event_id)
+
+    # ---- identity ----
+
+    @property
+    def type(self) -> str:
+        return self.wire_type
+
+    @property
+    def is_terminal(self) -> bool:
+        return False
+
+    # ---- generic accessors (None when the class has no such field) ----
+
+    @property
+    def response_id(self) -> str | None:
+        return None
+
+    @property
+    def item_id(self) -> str | None:
+        return None
+
+    @property
+    def text(self) -> str | None:
+        return None
+
+    @property
+    def audio(self) -> bytes | None:
+        return None
+
+    # ---- wire rendering (pure) ----
+
+    def _wire_fields(self) -> dict[str, object]:
+        data: dict[str, object] = {}
+        for f in fields(self):
+            if f.name in _IDENTITY_FIELDS:
+                continue
+            value = getattr(self, f.name)
+            if value is None and f.name in self.optional_wire_fields:
+                continue
+            data[f.name] = wire_value(value)
+        return data
+
+    def to_realtime(self) -> dict[str, object]:
+        """The Realtime wire JSON object for this event (derived from the fields)."""
+        return {"type": self.wire_type, "event_id": self.event_id, **self._wire_fields()}
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging aid
+        return f"{type(self).__name__}(type={self.wire_type!r}, session_id={self.session_id!r}, epoch={self.epoch!r})"
+
+
+# ---- shared bases ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseScopedEvent(RealtimeEvent):
+    """Events addressed to one response (and usually one output item / content part)."""
+
+    response_id: str | None = None  # type: ignore[assignment]
+    item_id: str | None = None  # type: ignore[assignment]
+    output_index: int = 0
+    content_index: int = 0
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class _ItemEvent(RealtimeEvent):
+    previous_item_id: str | None = None
+    item: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def item_id(self) -> str | None:
+        value = self.item.get("id")
+        return value if isinstance(value, str) else None
+
+
+# ---- GA server events ----
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionCreated(RealtimeEvent):
+    wire_type = "session.created"
+
+    session: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SessionUpdated(RealtimeEvent):
+    wire_type = "session.updated"
+
+    session: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseCreated(RealtimeEvent):
+    wire_type = "response.created"
+
+    response: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class OutputItemAdded(RealtimeEvent):
+    wire_type = "response.output_item.added"
+
+    response_id: str | None = None  # type: ignore[assignment]
+    output_index: int = 0
+    item: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def item_id(self) -> str | None:
+        value = self.item.get("id")
+        return value if isinstance(value, str) else None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class OutputItemDone(OutputItemAdded):
+    wire_type = "response.output_item.done"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ContentPartAdded(ResponseScopedEvent):
+    wire_type = "response.content_part.added"
+
+    part: Mapping[str, object] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ContentPartDone(ContentPartAdded):
+    wire_type = "response.content_part.done"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AudioDelta(ResponseScopedEvent):
+    wire_type = "response.output_audio.delta"
+
+    #: Base64 audio in ``format``.
+    delta: str = ""
+
+    @property
+    def audio(self) -> bytes | None:
+        try:
+            return base64.b64decode(self.delta)
+        except (ValueError, TypeError):
+            return None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class AudioDone(ResponseScopedEvent):
+    wire_type = "response.output_audio.done"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TranscriptDelta(ResponseScopedEvent):
+    wire_type = "response.output_audio_transcript.delta"
+
+    delta: str = ""
+
+    @property
+    def text(self) -> str | None:
+        return self.delta
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TranscriptDone(ResponseScopedEvent):
+    wire_type = "response.output_audio_transcript.done"
+
+    transcript: str = ""
+
+    @property
+    def text(self) -> str | None:
+        return self.transcript
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TextDelta(ResponseScopedEvent):
+    wire_type = "response.output_text.delta"
+
+    delta: str = ""
+
+    @property
+    def text(self) -> str | None:
+        return self.delta
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TextDone(ResponseScopedEvent):
+    wire_type = "response.output_text.done"
+
+    text: str = ""  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class FunctionCallArgumentsDelta(RealtimeEvent):
+    wire_type = "response.function_call_arguments.delta"
+
+    response_id: str | None = None  # type: ignore[assignment]
+    item_id: str | None = None  # type: ignore[assignment]
+    output_index: int = 0
+    call_id: str = ""
+    delta: str = ""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class FunctionCallArgumentsDone(RealtimeEvent):
+    wire_type = "response.function_call_arguments.done"
+
+    response_id: str | None = None  # type: ignore[assignment]
+    item_id: str | None = None  # type: ignore[assignment]
+    output_index: int = 0
+    call_id: str = ""
+    arguments: str = ""
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ResponseDone(RealtimeEvent):
+    wire_type = "response.done"
+
+    response: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def status(self) -> str | None:
+        value = self.response.get("status")
+        return value if isinstance(value, str) else None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class RateLimitsUpdated(RealtimeEvent):
+    wire_type = "rate_limits.updated"
+
+    rate_limits: tuple[Mapping[str, object], ...] = ()
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class InputCommitted(RealtimeEvent):
+    wire_type = "input_audio_buffer.committed"
+
+    previous_item_id: str | None = None
+    item_id: str | None = None  # type: ignore[assignment]
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"previous_item_id": self.previous_item_id, "item_id": self.item_id}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class InputCleared(RealtimeEvent):
+    wire_type = "input_audio_buffer.cleared"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SpeechStarted(RealtimeEvent):
+    wire_type = "input_audio_buffer.speech_started"
+
+    audio_start_ms: int = 0
+    item_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class SpeechStopped(RealtimeEvent):
+    wire_type = "input_audio_buffer.speech_stopped"
+
+    audio_end_ms: int = 0
+    item_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemAdded(_ItemEvent):
+    wire_type = "conversation.item.added"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemCreated(_ItemEvent):
+    wire_type = "conversation.item.created"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemDone(_ItemEvent):
+    wire_type = "conversation.item.done"
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemRetrieved(RealtimeEvent):
+    wire_type = "conversation.item.retrieved"
+
+    item: Mapping[str, object] = field(default_factory=dict)
+
+    @property
+    def item_id(self) -> str | None:
+        value = self.item.get("id")
+        return value if isinstance(value, str) else None
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemDeleted(RealtimeEvent):
+    wire_type = "conversation.item.deleted"
+
+    item_id: str | None = None  # type: ignore[assignment]
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"item_id": self.item_id}
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ItemTruncated(RealtimeEvent):
+    wire_type = "conversation.item.truncated"
+
+    item_id: str | None = None  # type: ignore[assignment]
+    content_index: int = 0
+    audio_end_ms: int = 0
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {
+            "item_id": self.item_id,
+            "content_index": self.content_index,
+            "audio_end_ms": self.audio_end_ms,
+        }
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class InputTranscriptionCompleted(RealtimeEvent):
+    wire_type = "conversation.item.input_audio_transcription.completed"
+
+    item_id: str | None = None  # type: ignore[assignment]
+    content_index: int = 0
+    transcript: str = ""
+
+    @property
+    def text(self) -> str | None:
+        return self.transcript
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class OutputAudioCleared(RealtimeEvent):
+    wire_type = "output_audio_buffer.cleared"
+
+    response_id: str | None = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class ErrorEvent(RealtimeEvent):
+    wire_type = "error"
+
+    code: str = "internal_error"
+    message: str = ""
+    #: Client ``event_id`` this error answers (OpenAI ``error.event_id``).
+    related_event_id: str | None = None
+    param: str | None = None
+
+    @property
+    def error_type(self) -> str:
+        """OpenAI's error class for this error.
+
+        OpenAI standardises only the three classes, not the codes, so the base
+        reports the default. A consumer with its own code vocabulary overrides
+        this --- see ``vllm_omni.protocol.duplex.events.ErrorEvent``.
+        """
+        return "invalid_request_error"
+
+    @property
+    def error(self) -> dict[str, object]:
+        error: dict[str, object] = {"type": self.error_type, "code": self.code, "message": self.message}
+        if self.related_event_id:
+            error["event_id"] = self.related_event_id
+        if self.param:
+            error["param"] = self.param
+        return error
+
+    @property
+    def text(self) -> str | None:
+        return self.message
+
+    def _wire_fields(self) -> dict[str, object]:
+        return {"error": self.error}
+
+
+__all__ = [
+    "AudioDelta",
+    "AudioDone",
+    "ContentPartAdded",
+    "ContentPartDone",
+    "ErrorEvent",
+    "FunctionCallArgumentsDelta",
+    "FunctionCallArgumentsDone",
+    "InputCleared",
+    "InputCommitted",
+    "InputTranscriptionCompleted",
+    "ItemAdded",
+    "ItemCreated",
+    "ItemDeleted",
+    "ItemDone",
+    "ItemRetrieved",
+    "ItemTruncated",
+    "OutputAudioCleared",
+    "OutputItemAdded",
+    "OutputItemDone",
+    "RateLimitsUpdated",
+    "RealtimeEvent",
+    "ResponseCreated",
+    "ResponseDone",
+    "ResponseScopedEvent",
+    "SessionCreated",
+    "SessionUpdated",
+    "SpeechStarted",
+    "SpeechStopped",
+    "TextDelta",
+    "TextDone",
+    "TranscriptDelta",
+    "TranscriptDone",
+    "new_event_id",
+    "wire_value",
+]

vllm_omni/protocol/realtime/formats.py

@@ -0,0 +1,177 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Realtime audio-format negotiation: what a client may declare and how it is read.
+
+A Realtime client declares audio formats in three places --- the session object
+(``session.audio.input.format`` and the older flat ``input_audio_format``), a
+``response.create`` override, and a ``conversation.item.create`` audio part ---
+and may spell one format several ways (``audio/pcm``, ``pcm16``, ``s16le``).
+This module is the single place that normalizes those spellings and says which
+are supported. Folding a session's declaration into the per-connection append
+defaults is the job of ``session.py``, which builds on this module.
+
+Everything here is a pure function of wire payloads: no session state, no model,
+no engine.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Collection, Mapping
+
+__all__ = [
+    "REALTIME_INPUT_AUDIO_FORMATS",
+    "REALTIME_OUTPUT_AUDIO_FORMATS",
+    "is_supported_realtime_input_format",
+    "parse_realtime_audio_format",
+    "realtime_audio_format_object",
+    "realtime_output_format",
+    "validate_conversation_item_audio_formats",
+    "validate_realtime_response_audio_formats",
+    "validate_realtime_session_audio_formats",
+]
+
+REALTIME_INPUT_AUDIO_FORMATS = {
+    "pcm16",
+    "pcm_s16le",
+    "s16le",
+    "pcm_f32le",
+    "g711_ulaw",
+    "g711_alaw",
+}
+REALTIME_OUTPUT_AUDIO_FORMATS = {
+    "pcm16",
+    "pcm_s16le",
+    "s16le",
+    "wav",
+    "pcm",
+    "g711_ulaw",
+    "g711_alaw",
+}
+
+
+def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]:
+    def normalize_format(fmt: str) -> str:
+        normalized = fmt.lower()
+        if normalized in {"audio/pcm", "pcm"}:
+            return "pcm16"
+        if normalized in {"audio/wav", "wav"}:
+            return "wav"
+        if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}:
+            return "pcm16"
+        if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}:
+            return "pcm_f32le"
+        if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}:
+            return "g711_ulaw"
+        if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}:
+            return "g711_alaw"
+        return fmt
+
+    if isinstance(raw_format, str):
+        return normalize_format(raw_format), None
+    if not isinstance(raw_format, dict):
+        return raw_format, None
+    rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate")
+    sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None
+    fmt = raw_format.get("type") or raw_format.get("format")
+    if not isinstance(fmt, str):
+        return raw_format, sample_rate_hz
+    return normalize_format(fmt), sample_rate_hz
+
+
+def realtime_output_format(duplex_format: object) -> str:
+    if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}:
+        return duplex_format.lower()
+    if isinstance(duplex_format, str) and duplex_format.lower() == "pcm":
+        return "pcm16"
+    return str(duplex_format or "wav")
+
+
+def is_supported_realtime_input_format(fmt: object) -> bool:
+    return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS
+
+
+def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]:
+    if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}:
+        payload: dict[str, object] = {"type": "audio/pcm"}
+    elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le":
+        payload = {"type": "audio/pcm_f32le"}
+    elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw":
+        payload = {"type": "audio/g711_ulaw"}
+    elif isinstance(fmt, str) and fmt.lower() == "g711_alaw":
+        payload = {"type": "audio/g711_alaw"}
+    else:
+        payload = {"type": "audio/wav"}
+    if sample_rate_hz is not None:
+        payload["rate"] = int(sample_rate_hz)
+    return payload
+
+
+def validate_realtime_session_audio_formats(
+    session_payload: Mapping[str, object],
+    *,
+    input_audio_formats: Collection[str] | None = None,
+    output_audio_formats: Collection[str] | None = None,
+) -> str | None:
+    """Reject a session object that declares an audio format we cannot serve.
+
+    The format sets default to everything the codec can decode; a consumer that
+    serves a narrower set passes its own (see
+    ``vllm_omni.protocol.realtime.capabilities``).
+    """
+    supported_input = REALTIME_INPUT_AUDIO_FORMATS if input_audio_formats is None else input_audio_formats
+    supported_output = REALTIME_OUTPUT_AUDIO_FORMATS if output_audio_formats is None else output_audio_formats
+    audio_config = session_payload.get("audio")
+    input_format: object = session_payload.get("input_audio_format")
+    if input_format is None and isinstance(audio_config, dict):
+        audio_input = audio_config.get("input")
+        if isinstance(audio_input, dict):
+            input_format = audio_input.get("format")
+    parsed_input, _ = parse_realtime_audio_format(input_format)
+    if input_format is not None and not (isinstance(parsed_input, str) and parsed_input.lower() in supported_input):
+        return f"Unsupported input_audio_format: {input_format}"
+
+    output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
+    if output_format is None and isinstance(audio_config, dict):
+        audio_output = audio_config.get("output")
+        if isinstance(audio_output, dict):
+            output_format = audio_output.get("format")
+    parsed_output, _ = parse_realtime_audio_format(output_format)
+    if output_format is not None and not (isinstance(parsed_output, str) and parsed_output.lower() in supported_output):
+        return f"Unsupported output_audio_format: {output_format}"
+    return None
+
+
+def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None:
+    output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format")
+    audio_config = response_payload.get("audio")
+    if output_format is None and isinstance(audio_config, dict):
+        audio_output = audio_config.get("output")
+        if isinstance(audio_output, dict):
+            output_format = audio_output.get("format")
+    parsed_output, _ = parse_realtime_audio_format(output_format)
+    if output_format is not None and not (
+        isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
+    ):
+        return f"Unsupported output_audio_format: {output_format}"
+    return None
+
+
+def validate_conversation_item_audio_formats(item: object) -> str | None:
+    if not isinstance(item, dict):
+        return None
+    content = item.get("content")
+    if not isinstance(content, list):
+        return None
+    for part in content:
+        if not isinstance(part, dict):
+            continue
+        if part.get("type") not in {"input_audio", "audio"}:
+            continue
+        raw_format = part.get("format")
+        parsed_format, _ = parse_realtime_audio_format(raw_format)
+        if raw_format is not None and not (
+            isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS
+        ):
+            return f"Unsupported input_audio format in conversation.item.create: {raw_format}"
+    return None

vllm_omni/protocol/realtime/items.py

@@ -0,0 +1,197 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Realtime conversation items: shape, truncation and the transcript inside one.
+
+A Realtime conversation is a list of *items* --- a user message, an assistant
+message, a function call --- each with a ``content`` list of typed parts. The
+client creates them (``conversation.item.create``), truncates an assistant
+item's audio after a barge-in (``conversation.item.truncate``), and the server
+echoes them back inside ``conversation.item.*`` and ``response.done`` events.
+
+The rules in this module are the ones both sides have to agree on: which
+defaults a bare item gets, how far a transcript is cut when the client says it
+only heard N milliseconds of audio, and what makes a truncate request invalid.
+Item *storage* --- which items a session currently holds, in what order --- is
+the consumer's state, not this module's.
+
+``validate_realtime_video_frames`` sits here too: camera frames ride along an
+``input_audio_buffer.append`` in the vLLM-Omni extension of the protocol, and
+validating them is the same kind of wire-shape check.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+from collections.abc import Mapping
+from uuid import uuid4
+
+__all__ = [
+    "input_transcript_from_item",
+    "normalize_conversation_item",
+    "text_chars_for_audio_ms_from_marks",
+    "truncate_realtime_item_content",
+    "validate_realtime_item_truncate",
+    "validate_realtime_video_frames",
+]
+
+
+def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
+    """Validate omni-duplex camera frames on input_audio_buffer.append.
+
+    Wire contract matches the official MiniCPM-o duplex loop: one base
+    base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
+    stacked composite tiling the sub-frames captured inside it (at most 2
+    images either way). A caller-supplied ``max_slice_nums`` is rejected rather
+    than silently ignored: slicing is Stage 0's decision here, and Stage 0
+    already applies the official HD suggestion for a stacked unit
+    (``max_slice_nums=[2, 1]``). The wire simply does not let the client choose
+    it.
+    """
+    if max_slice_nums not in (None, 1):
+        return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself"
+    if not isinstance(video_frames, list):
+        return "video_frames must be a list of base64-encoded images"
+    frames = [frame for frame in video_frames if frame is not None]
+    if len(frames) > 2:
+        return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
+    for frame in frames:
+        if not isinstance(frame, str) or not frame:
+            return "video_frames entries must be non-empty base64 strings"
+        if len(frame) > 4_000_000:
+            return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
+        try:
+            header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
+        except (binascii.Error, ValueError):
+            return "video_frames entries must be valid base64"
+        if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
+            return "video_frames entries must be JPEG or PNG images"
+    return None
+
+
+def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]:
+    normalized = dict(item)
+    normalized.setdefault("id", f"item_{uuid4().hex}")
+    normalized.setdefault("object", "realtime.item")
+    normalized.setdefault("type", "message")
+    normalized.setdefault("status", "completed")
+    if "role" not in normalized and normalized.get("type") == "message":
+        normalized["role"] = "user"
+    if not isinstance(normalized.get("content"), list):
+        normalized["content"] = []
+    return normalized
+
+
+def text_chars_for_audio_ms_from_marks(
+    audio_end_ms: int,
+    text_len: int,
+    marks: list[object],
+    *,
+    final_ms: object | None = None,
+) -> int:
+    if text_len <= 0:
+        return 0
+    clean_marks: list[tuple[int, int]] = []
+    for mark in marks:
+        if not isinstance(mark, dict):
+            continue
+        raw_text_chars = mark.get("text_chars")
+        raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
+        if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
+            continue
+        clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
+    if not clean_marks:
+        return 0 if audio_end_ms <= 0 else text_len
+    clean_marks.sort(key=lambda item: item[0])
+    audio_end_ms = max(0, int(audio_end_ms))
+    if audio_end_ms <= 0:
+        return 0
+    previous_ms = 0
+    previous_chars = 0
+    for mark_ms, mark_chars in clean_marks:
+        mark_ms = max(previous_ms, mark_ms)
+        mark_chars = max(previous_chars, min(text_len, mark_chars))
+        if audio_end_ms <= mark_ms:
+            if mark_ms <= previous_ms:
+                return mark_chars
+            ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
+            return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
+        previous_ms = mark_ms
+        previous_chars = mark_chars
+    if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
+        if audio_end_ms >= int(final_ms):
+            return text_len
+        ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
+        return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
+    return text_len if audio_end_ms >= previous_ms else previous_chars
+
+
+def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None:
+    content = item.get("content")
+    if not isinstance(content, list) or not content:
+        return
+    index = max(0, int(content_index))
+    if index >= len(content):
+        return
+    part = content[index]
+    if not isinstance(part, dict):
+        return
+    transcript = part.get("transcript")
+    if not isinstance(transcript, str) or not transcript:
+        return
+    marks = part.get("audio_text_marks")
+    if isinstance(marks, list):
+        keep_chars = text_chars_for_audio_ms_from_marks(
+            audio_end_ms,
+            len(transcript),
+            marks,
+            final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
+        )
+        part["transcript"] = transcript[:keep_chars].rstrip()
+        return
+    duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
+    if isinstance(duration_ms, int | float) and duration_ms > 0:
+        keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
+        part["transcript"] = transcript[:keep_chars].rstrip()
+    elif audio_end_ms <= 0:
+        part["transcript"] = ""
+
+
+def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None:
+    if item.get("type") != "message" or item.get("role") != "assistant":
+        return "conversation.item.truncate only supports assistant message items"
+    if audio_end_ms < 0:
+        return "conversation.item.truncate requires non-negative audio_end_ms"
+    content = item.get("content")
+    if not isinstance(content, list) or not content:
+        return None
+    index = max(0, int(content_index))
+    if index >= len(content):
+        return f"conversation.item.truncate content_index out of range: {content_index}"
+    part = content[index]
+    if not isinstance(part, dict):
+        return "conversation.item.truncate target content part is invalid"
+    if part.get("type") not in {"audio", "output_audio"}:
+        return "conversation.item.truncate target content part must be audio"
+    duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
+    if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
+        return "conversation.item.truncate audio_end_ms exceeds item audio duration"
+    return None
+
+
+def input_transcript_from_item(item: Mapping[str, object]) -> str:
+    content = item.get("content")
+    if not isinstance(content, list):
+        return ""
+    parts: list[str] = []
+    for part in content:
+        if not isinstance(part, dict):
+            continue
+        for key in ("transcript", "text"):
+            value = part.get(key)
+            if isinstance(value, str) and value:
+                if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
+                    parts.append(value)
+                    break
+    return "".join(parts).strip()

vllm_omni/protocol/realtime/session.py

@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Reading the Realtime ``session`` object.
+
+``session.update`` carries one JSON object that a client may spell in either of
+two shapes: the GA nest (``session.audio.input.format``,
+``session.audio.output.sample_rate_hz``) or the older flat beta keys
+(``input_audio_format``, ``sample_rate_hz``). Everything downstream --- the
+append decoder, the projection that renders ``session.created`` back out, a
+model's runtime configuration --- has to agree on what a given session object
+means, so the readers live here once.
+
+:class:`RealtimeInputDefaults` is the resolved answer for the input path: the
+format, sample rate and silence threshold an ``input_audio_buffer.append``
+inherits when it omits them. A connection keeps one of these and re-derives it
+on every ``session.update``.
+
+``overlap_policy`` / ``playback_commit_policy`` are vLLM-Omni extension fields
+on the session object rather than GA OpenAI fields. They are read here because
+this module owns what the wire may say; what a server *does* with them is the
+consumer's business.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from typing import Any
+
+from vllm_omni.protocol.realtime.formats import (
+    REALTIME_INPUT_AUDIO_FORMATS,
+    REALTIME_OUTPUT_AUDIO_FORMATS,
+    parse_realtime_audio_format,
+    realtime_output_format,
+)
+
+__all__ = [
+    "RealtimeInputDefaults",
+    "apply_realtime_session_defaults",
+    "input_audio_transcription_config",
+    "json_safe_realtime_payload",
+    "realtime_max_output_tokens",
+    "realtime_overlap_fields",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeInputDefaults:
+    """Session-level wire defaults an append may omit (derived from the session object)."""
+
+    input_audio_format: str = "pcm16"
+    input_sample_rate_hz: int = 16000
+    output_audio_format: str = "pcm16"
+    output_sample_rate_hz: int | None = None
+    overlap_silence_rms: float = 0.003
+
+    def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults:
+        """Return defaults updated from a Realtime ``session`` object (session.update)."""
+        values = apply_realtime_session_defaults(self, session_payload)
+        return values
+
+
+def apply_realtime_session_defaults(
+    defaults: RealtimeInputDefaults,
+    session_payload: Mapping[str, object],
+) -> RealtimeInputDefaults:
+    """Derive the wire defaults a Realtime ``session`` object declares."""
+    input_format: object = session_payload.get("input_audio_format")
+    audio_config = session_payload.get("audio")
+    if input_format is None and isinstance(audio_config, dict):
+        audio_input = audio_config.get("input")
+        if isinstance(audio_input, dict):
+            input_format = audio_input.get("format")
+    input_format, input_rate = parse_realtime_audio_format(input_format)
+    # Any: heterogeneous field values splatted into ``dataclasses.replace`` below.
+    updates: dict[str, Any] = {}
+    if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS:
+        updates["input_audio_format"] = input_format
+    output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
+    output_rate_raw: object | None = None
+    if output_format is None and isinstance(audio_config, dict):
+        audio_output = audio_config.get("output")
+        if isinstance(audio_output, dict):
+            output_format = audio_output.get("format")
+            output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate")
+    output_format, output_rate = parse_realtime_audio_format(output_format)
+    if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0:
+        output_rate = int(output_rate_raw)
+    if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
+        updates["output_audio_format"] = realtime_output_format(output_format)
+    sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate")
+    if sample_rate is None and isinstance(audio_config, dict):
+        audio_input = audio_config.get("input")
+        if isinstance(audio_input, dict):
+            sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate")
+    if sample_rate is None:
+        sample_rate = input_rate
+    if isinstance(sample_rate, int | float) and sample_rate > 0:
+        updates["input_sample_rate_hz"] = int(sample_rate)
+    if isinstance(output_rate, int | float) and output_rate > 0:
+        updates["output_sample_rate_hz"] = int(output_rate)
+    overlap_fields = realtime_overlap_fields(session_payload)
+    overlap_silence_rms = overlap_fields.get("overlap_silence_rms")
+    if isinstance(overlap_silence_rms, int | float):
+        updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms))
+    return replace(defaults, **updates) if updates else defaults
+
+
+def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]:
+    fields: dict[str, object] = {}
+    if isinstance(session_payload.get("overlap_policy"), str):
+        fields["overlap_policy"] = session_payload["overlap_policy"]
+    for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"):
+        value = session_payload.get(key)
+        if isinstance(value, int | float):
+            fields[key] = value
+    if isinstance(session_payload.get("playback_commit_policy"), str):
+        fields["playback_commit_policy"] = session_payload["playback_commit_policy"]
+    return fields
+
+
+def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]:
+    clean: dict[str, object] = {}
+    for key, value in payload.items():
+        if key == "extra_body":
+            continue
+        if isinstance(value, str | int | float | bool) or value is None:
+            clean[key] = value
+        elif isinstance(value, dict):
+            clean[key] = json_safe_realtime_payload(value)
+        elif isinstance(value, list):
+            clean[key] = [
+                (json_safe_realtime_payload(item) if isinstance(item, dict) else item)
+                for item in value
+                if isinstance(item, str | int | float | bool | dict) or item is None
+            ]
+    return clean
+
+
+def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
+    transcription = session_payload.get("input_audio_transcription")
+    if isinstance(transcription, dict):
+        return transcription
+    audio_config = session_payload.get("audio")
+    if not isinstance(audio_config, dict):
+        return None
+    audio_input = audio_config.get("input")
+    if not isinstance(audio_input, dict):
+        return None
+    transcription = audio_input.get("transcription")
+    return transcription if isinstance(transcription, dict) else None
+
+
+def realtime_max_output_tokens(value: object) -> int | None:
+    """Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
+    if value is None:
+        return None
+    if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}:
+        return None
+    if isinstance(value, int) and value > 0:
+        return int(value)
+    return None