From e3be42e052e88d052d21cf7a24ee84c70f020fd6 Mon Sep 17 00:00:00 2001 From: psv666 <150513104+psv666@users.noreply.github.com> Date: Thu, 17 Sep 2026 17:33:40 +0800 Subject: [PATCH] [Core] Give the OpenAI Realtime wire codec a reusable home (RFC #6592 P0a) (#7640) Signed-off-by: psv666 <2693925048@qq.com> --- .../common/ci_source_file_dependencies.yml | 2 + .github/CODEOWNERS | 5 + docs/design/fullduplex.md | 77 +- docs/design/module/entrypoints.md | 43 + tests/protocol/__init__.py | 2 + tests/protocol/duplex/__init__.py | 2 + .../duplex/test_duplex_protocol_facade.py | 144 ++++ tests/protocol/realtime/__init__.py | 2 + .../realtime/test_protocol_import_boundary.py | 106 +++ .../realtime/test_realtime_capabilities.py | 91 +++ .../test_realtime_codec_single_source.py | 256 ++++++ tools/pre_commit/check_forbidden_imports.py | 7 +- vllm_omni/clients/inline_duplex.py | 4 +- vllm_omni/engine/duplex/audio.py | 250 +----- vllm_omni/engine/duplex/commands.py | 238 +++--- vllm_omni/engine/duplex/config.py | 36 +- vllm_omni/engine/duplex/events.py | 737 ++---------------- vllm_omni/engine/duplex/realtime_commands.py | 663 ++-------------- vllm_omni/engine/duplex/realtime_events.py | 12 +- vllm_omni/engine/duplex/session/runner.py | 2 +- .../entrypoints/duplex/realtime_input.py | 10 +- vllm_omni/entrypoints/duplex/serving.py | 2 +- vllm_omni/protocol/__init__.py | 17 + vllm_omni/protocol/duplex/__init__.py | 159 ++++ vllm_omni/protocol/duplex/commands.py | 164 ++++ vllm_omni/protocol/duplex/errors.py | 75 ++ vllm_omni/protocol/duplex/events.py | 419 ++++++++++ vllm_omni/protocol/realtime/__init__.py | 163 ++++ vllm_omni/protocol/realtime/audio.py | 218 ++++++ vllm_omni/protocol/realtime/audio_input.py | 243 ++++++ vllm_omni/protocol/realtime/capabilities.py | 105 +++ vllm_omni/protocol/realtime/commands.py | 126 +++ vllm_omni/protocol/realtime/errors.py | 37 + vllm_omni/protocol/realtime/events.py | 467 +++++++++++ vllm_omni/protocol/realtime/formats.py | 177 +++++ vllm_omni/protocol/realtime/items.py | 197 +++++ vllm_omni/protocol/realtime/session.py | 163 ++++ 37 files changed, 3792 insertions(+), 1629 deletions(-) create mode 100644 tests/protocol/__init__.py create mode 100644 tests/protocol/duplex/__init__.py create mode 100644 tests/protocol/duplex/test_duplex_protocol_facade.py create mode 100644 tests/protocol/realtime/__init__.py create mode 100644 tests/protocol/realtime/test_protocol_import_boundary.py create mode 100644 tests/protocol/realtime/test_realtime_capabilities.py create mode 100644 tests/protocol/realtime/test_realtime_codec_single_source.py create mode 100644 vllm_omni/protocol/__init__.py create mode 100644 vllm_omni/protocol/duplex/__init__.py create mode 100644 vllm_omni/protocol/duplex/commands.py create mode 100644 vllm_omni/protocol/duplex/errors.py create mode 100644 vllm_omni/protocol/duplex/events.py create mode 100644 vllm_omni/protocol/realtime/__init__.py create mode 100644 vllm_omni/protocol/realtime/audio.py create mode 100644 vllm_omni/protocol/realtime/audio_input.py create mode 100644 vllm_omni/protocol/realtime/capabilities.py create mode 100644 vllm_omni/protocol/realtime/commands.py create mode 100644 vllm_omni/protocol/realtime/errors.py create mode 100644 vllm_omni/protocol/realtime/events.py create mode 100644 vllm_omni/protocol/realtime/formats.py create mode 100644 vllm_omni/protocol/realtime/items.py create mode 100644 vllm_omni/protocol/realtime/session.py diff --git a/.buildkite/common/ci_source_file_dependencies.yml b/.buildkite/common/ci_source_file_dependencies.yml index e29001d9a..c101db48d 100644 --- a/.buildkite/common/ci_source_file_dependencies.yml +++ b/.buildkite/common/ci_source_file_dependencies.yml @@ -44,6 +44,7 @@ x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex - vllm_omni/entrypoints/duplex/ - vllm_omni/entrypoints/duplex_omni.py - vllm_omni/entrypoints/async_omni_base.py + - vllm_omni/protocol/ - vllm_omni/clients/ - vllm_omni/outputs/duplex.py - vllm_omni/model_executor/duplex_sampling.py @@ -259,6 +260,7 @@ source_file_dependencies: - tests/entrypoints/test_duplex_omni.py - tests/entrypoints/openai_api/test_duplex_api_server.py - tests/model_executor/models/minicpmo_4_5/duplex/ + - tests/protocol/ omni_minicpmo_4_5_duplex_perf: - *minicpmo_4_5 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c92c7b22f..e77f70914 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -67,6 +67,11 @@ # File-level overrides within entrypoints/ /vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao /vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter +# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex +# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the +# engine depends on it — so it is owned by the serving-boundary document rather +# than by any one consumer. — docs/design/module/entrypoints.md +/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao # Error contracts — docs/design/module/error_contracts.md /vllm_omni/errors.py @alex-jw-brooks @NickCao diff --git a/docs/design/fullduplex.md b/docs/design/fullduplex.md index 41a050a47..324cd5bdc 100644 --- a/docs/design/fullduplex.md +++ b/docs/design/fullduplex.md @@ -9,6 +9,12 @@ the owner document for `vllm_omni/engine/duplex/**`, `vllm_omni/entrypoints/duplex_omni.py`, `vllm_omni/entrypoints/duplex/**`, `vllm_omni/clients/**` and the per-model `duplex/` packages. +`vllm_omni/protocol/realtime/**` (the OpenAI Realtime API) and +`vllm_omni/protocol/duplex/**` (vLLM-Omni's extension of it) are the shared +wire codec duplex is built on. Duplex is its first consumer but not its owner: the codec +is deliberately runtime-agnostic so a second Realtime surface can reuse it +(RFC #6592 P0a). + The public wire contract lives in [`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md); this page describes how it is produced. @@ -124,6 +130,40 @@ plugin and the session runtime config to `DuplexOrchestrator` directly. attachment/resume/replay bookkeeping; it holds no session state. 6. **No `typing.Protocol`** in the duplex surfaces: the plugin, data plane, session state, PCM buffer, stage port and client transport seams are ABCs. +7. **The wire codec is not duplex, and a wire type is not a mailbox message.** + The event and command *vocabulary* plus its wire rendering, the audio format + negotiation, the conversation-item rules and the error envelope are model- + and runtime-agnostic, and split along the three tiers + [`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md) + already defines: **Tier 1** (identical to OpenAI) is + `vllm_omni/protocol/realtime/**`; **Tier 2** (OpenAI names carrying our + extensions) and **Tier 3** (ours alone) are `vllm_omni/protocol/duplex/**`. + A Tier 2 class *subclasses* its Tier 1 twin and declares only what it adds, + so the OpenAI surface stays honestly pure --- `session.created` without + `resume_token`, `input_audio_buffer.append` without `video_frames`. Our + error **codes** are Tier 3 (`protocol/duplex/errors.py`); only the envelope + shape and OpenAI's three `error.type` classes are Tier 1. + What stays engine-side is everything that is *internal representation* + rather than contract: `DuplexCommand.payload()` renders the session runner's + mailbox dictionary, whose channel genuinely differs from the client event + (`session.update` and the three `conversation.item.*` commands all travel on + `turn.signal`); `engine/duplex/realtime_commands.py` decides which command a + decoded event becomes; `engine/duplex/realtime_events.py` holds the + session's projection state. The codec may not import the engine, the + entrypoints, `model_executor` or the clients + (`tests/protocol/realtime/test_protocol_import_boundary.py`), and there is + exactly one implementation of each codec behaviour + (`tests/protocol/realtime/test_realtime_codec_single_source.py`). + The dependency chain is `protocol/realtime` <- `protocol/duplex` <- engine / + entrypoints / clients, and a duplex consumer uses **only** the middle link: + `protocol/duplex/**` re-exports the Tier 1 names it does not extend, so a + helper that later needs a duplex-specific version (`convert_input_audio_with_rate` + resamples to MiniCPM-o's 16 kHz rather than the client's rate) is overridden + in one file instead of at every call site + (`tests/protocol/duplex/test_duplex_protocol_facade.py`). + `DuplexEvent` is therefore a plain alias of `RealtimeEvent` --- an event has + no engine-internal half --- while `DuplexCommand` is a real class, because a + command does. ## Package layout @@ -142,6 +182,31 @@ vllm_omni/ │ │ ├── chat_completions.py DuplexChatCompletionsAdapter (/v1/chat/completions on a session per request) │ │ └── websocket.py websocket send/close/receive helpers │ └── openai/api_server.py builds DuplexOmni for duplex models; session-backed app state +├── protocol/ SHARED WIRE CODEC (no engine / no model / no transport) +│ ├── realtime/ the OpenAI Realtime API itself +│ │ ├── events.py RealtimeEvent + OpenAI's 30 server events (22 ship as Tier 1; +│ │ │ 8 have a Tier 2 twin in protocol/duplex) +│ │ ├── commands.py RealtimeCommand + OpenAI's 10 client commands (8 Tier 1, +│ │ │ 2 with a Tier 2 twin) --- wire_type + fields, no payload() +│ │ ├── formats.py audio format spellings, what is supported, per-session validation +│ │ ├── session.py session-object readers; RealtimeInputDefaults (append defaults) +│ │ ├── items.py conversation item shape, truncation, transcripts, camera frames +│ │ ├── audio_input.py decode_audio_append -> RealtimeAudioAppend; client speech hints +│ │ ├── audio.py PCM / G.711 / WAV decode, resample, re-encode +│ │ ├── errors.py RealtimeProtocolError (the envelope's exception type) +│ │ └── capabilities.py RealtimeProtocolCapabilities: what one consumer can serve +│ └── duplex/ Tier 2 + Tier 3, and the one door a duplex consumer uses +│ ├── __init__.py re-exports the Tier 1 helper functions (audio, formats, +│ │ items, session, capabilities) so the engine never +│ │ imports protocol/realtime directly +│ ├── events.py the whole 42-event vocabulary: 22 Tier-1 re-exports, +│ │ 8 Tier-2 subclasses (session.created + resume_token, ...), +│ │ 12 Tier-3 events (listen/speak, playback, resume, ...); +│ │ DuplexEvent = RealtimeEvent +│ ├── commands.py the whole 17-command vocabulary: 8 Tier-1 re-exports, +│ │ 2 Tier-2 subclasses (append + hints, commit + final), +│ │ 7 Tier-3 commands (barge_in, playback.ack, ...) +│ └── errors.py our error-code vocabulary (Tier 3) + RealtimeProtocolError ├── engine/ │ ├── omni_engine_base.py OmniEngineBase (stage processes, orchestrator thread, queues, RPC) │ ├── async_omni_engine.py AsyncOmniEngine (turn-based request building) @@ -149,16 +214,20 @@ vllm_omni/ │ ├── orchestrator.py OrchestratorBase + Orchestrator │ ├── duplex_orchestrator.py DuplexOrchestrator (+ DuplexOrchestratorRequestState; implements DuplexStagePort) │ └── duplex/ -│ ├── commands.py DuplexCommand dataclasses, command_from_realtime -│ ├── realtime_commands.py Realtime client event -> DuplexCommand translation -│ ├── events.py DuplexEvent dataclasses (+ to_realtime), REALTIME_ERROR_TYPES_BY_CODE +│ ├── commands.py mailbox half: DuplexCommand.payload() + the mailbox `type`, +│ │ paired with each protocol command; command_from_realtime +│ ├── realtime_commands.py duplex binding of the codec: decoded event -> DuplexCommand, +│ │ DUPLEX_REALTIME_CAPABILITIES, duplex_response_format +│ ├── events.py re-export shim (DuplexEvent = RealtimeEvent) + the runner's +│ │ epoch-filter sets DOMAIN_TERMINAL_EVENTS / MODEL_OUTPUT_EVENTS │ ├── realtime_events.py RealtimeProjectionState: internal event -> typed events │ ├── messages.py queue envelopes (Open/Close/Resume/Touch/Command/Result/Event), DuplexSessionError │ ├── config.py DuplexSessionConfig, DuplexCapabilities, ResponseCreateOptions │ ├── contracts.py DuplexFence (session_id, epoch, turn_id), stage request records, DuplexStagePort │ ├── plugin.py DuplexModelPlugin, DuplexModelSessionState, DuplexDataPlane, PcmAppendBuffer ABCs │ ├── turn_detection.py server-side VAD turn detector used by the session -│ ├── audio.py / vad.py / intermediate.py +│ ├── audio.py compatibility re-export, via protocol/duplex +│ ├── vad.py / intermediate.py │ └── session/ one engine-resident session and everything that runs it │ ├── engine_session.py DuplexEngineSession: ledgers, lease, fence, stage resources, append sequencing │ ├── runner.py DuplexSessionRunner (per-session mailbox on the orchestrator loop) diff --git a/docs/design/module/entrypoints.md b/docs/design/module/entrypoints.md index f37ddc6ba..0c0ba8439 100644 --- a/docs/design/module/entrypoints.md +++ b/docs/design/module/entrypoints.md @@ -30,6 +30,7 @@ primary_code_paths: - vllm_omni/entrypoints/utils.py - vllm_omni/entrypoints/duplex/** - vllm_omni/clients/** + - vllm_omni/protocol/** primary_path_exceptions: - path: vllm_omni/entrypoints/openai/errors.py owner: error_contracts.md @@ -53,6 +54,7 @@ depends_on: - error_contracts.md - engine_orchestration.md validation_paths: + - tests/protocol/** - tests/entrypoints/test_omni_entrypoints.py - tests/entrypoints/test_async_omni.py - tests/entrypoints/test_async_omni_pause_sleep_routing.py @@ -106,6 +108,31 @@ placement, payload implementation, or semantic error classification. `entrypoints/openai/errors.py` is an explicit primary-path exception owned by `error_contracts.md`. +It also owns `vllm_omni/protocol/**`, the shared wire codec promoted by RFC +[#6592](https://github.com/vllm-project/vllm-omni/issues/6592) P0a. It follows the three tiers +`docs/serving/realtime_duplex_api.md` already defines. `protocol/realtime/**` +is Tier 1, identical to OpenAI: 30 server events, 10 client commands, their +typed fields and pure wire rendering, audio-format negotiation, +conversation-item rules and the error-envelope shape. `protocol/duplex/**` is +Tier 2 (OpenAI names carrying vLLM-Omni extensions, each a subclass of its +Tier 1 twin declaring only what it adds) and Tier 3 (ours alone, including our +error-code vocabulary), kept separate so a GA-only consumer is not handed a +vocabulary its clients never send. That package sits +outside `entrypoints/` on purpose --- the engine depends on it, so it belongs +to neither layer --- but deciding what a Realtime client may put on the wire is +this document's subject, and the same reasoning already places +`vllm_omni/errors.py`, `inputs/` and `outputs/` under contract documents rather +than under a consumer. + +It is deliberately **not** owned by `../fullduplex.md`, even though duplex is +the codec's only consumer today. RFC #6592 asks for a named owner precisely so +a second Realtime consumer does not need duplex review to change the codec +(#6592 open question 8). Ownership here is review scope, not a dependency: the +codec may not import the engine, the entrypoints or the model code, which +`tests/protocol/realtime/test_protocol_import_boundary.py` enforces. When a +second consumer lands (#6592 P0b) the package is expected to graduate to its +own module document with owners drawn from both consumers. + ## Candidate invariants These identifiers are proposals while the document is `draft`. @@ -115,6 +142,22 @@ These identifiers are proposals while the document is `draft`. **Rule:** Entrypoints MUST NOT implement cross-stage routing or stage lifecycle policy. +### ENTRY-INV-002: The shared Realtime codec does not depend on its consumers + +**Rule:** `vllm_omni/protocol/**` MUST NOT import `vllm_omni.engine`, +`vllm_omni.entrypoints`, `vllm_omni.model_executor`, `vllm_omni.worker` or +`vllm_omni.clients`; `protocol/duplex/**` MAY depend on `protocol/realtime/**` +but never the reverse, and a Tier 1 class MUST NOT carry a vLLM-Omni extension +field; and each protocol/codec behavior MUST have exactly one +implementation that every consumer uses. A wire type MUST NOT carry a +consumer's internal representation: the duplex mailbox rendering +(`DuplexCommand.payload()`, whose channel differs from the client event for +`session.update` and the `conversation.item.*` commands) stays engine-side. +A duplex consumer (`engine/duplex/**`, `entrypoints/duplex/**`, the duplex +clients) MUST import `vllm_omni.protocol.duplex` and MUST NOT import +`vllm_omni.protocol.realtime` directly, so the tier boundary has exactly one +extension point. + ### ENTRY-INV-100: Public requests are normalized once **Rule:** Public protocol values MUST be validated and converted to internal diff --git a/tests/protocol/__init__.py b/tests/protocol/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/tests/protocol/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/tests/protocol/duplex/__init__.py b/tests/protocol/duplex/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/tests/protocol/duplex/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/tests/protocol/duplex/test_duplex_protocol_facade.py b/tests/protocol/duplex/test_duplex_protocol_facade.py new file mode 100644 index 000000000..61dd15fe1 --- /dev/null +++ b/tests/protocol/duplex/test_duplex_protocol_facade.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""``protocol.duplex`` is the only door a duplex consumer uses. + +The layering is a chain:: + + protocol.realtime <- protocol.duplex <- engine / entrypoints / clients + +The middle link only earns its keep if the last arrow is the *only* one. If the +engine may also reach past it straight into ``protocol.realtime``, then giving a +Tier 1 helper a duplex-specific version later means touching every call site +instead of one file --- which is exactly the change +``convert_input_audio_with_rate`` is expected to need (it resamples to +MiniCPM-o's 16 kHz, not the client's rate). + +So this asserts two things: nobody skips the middle link, and the middle link is +complete enough that nobody has to. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +REPO_ROOT = Path(__file__).resolve().parents[3] + +#: Every module that speaks the duplex protocol. These import +#: ``vllm_omni.protocol.duplex``; reaching into ``vllm_omni.protocol.realtime`` +#: from here skips the extension point. +CONSUMER_PATHS = ( + "vllm_omni/engine/duplex", + "vllm_omni/engine/duplex_omni_engine.py", + "vllm_omni/entrypoints/duplex", + "vllm_omni/entrypoints/duplex_omni.py", + "vllm_omni/clients/duplex.py", + "vllm_omni/clients/inline_duplex.py", +) + + +def _consumer_modules() -> list[Path]: + found: list[Path] = [] + for entry in CONSUMER_PATHS: + path = REPO_ROOT / entry + found.extend(sorted(path.rglob("*.py")) if path.is_dir() else [path]) + assert found, "consumer list is stale" + return found + + +@pytest.mark.parametrize("module", _consumer_modules(), ids=lambda p: p.name) +def test_a_duplex_consumer_does_not_skip_the_duplex_protocol_package(module: Path) -> None: + # Source-level, so a lazy import inside a function body is caught too. + tree = ast.parse(module.read_text()) + offenders = [] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and (node.module or "").startswith("vllm_omni.protocol.realtime"): + offenders.append(f"line {node.lineno}: from {node.module} import ...") + elif isinstance(node, ast.Import): + offenders.extend( + f"line {node.lineno}: import {alias.name}" + for alias in node.names + if alias.name.startswith("vllm_omni.protocol.realtime") + ) + assert not offenders, ( + f"{module.relative_to(REPO_ROOT)} imports the Tier 1 package directly; " + f"import from vllm_omni.protocol.duplex instead:\n " + "\n ".join(offenders) + ) + + +def test_the_duplex_package_carries_the_whole_command_vocabulary() -> None: + from vllm_omni.engine.duplex import commands as engine_commands + from vllm_omni.protocol.duplex import commands as duplex_wire + + # Every command the engine handles is nameable through the one door. + engine_names = { + name + for name in engine_commands.__all__ + if isinstance(getattr(engine_commands, name), type) + and issubclass(getattr(engine_commands, name), engine_commands.DuplexCommand) + and getattr(engine_commands, name) is not engine_commands.DuplexCommand + } + assert engine_names <= set(duplex_wire.__all__) + + +def test_the_duplex_package_carries_the_whole_event_vocabulary() -> None: + from vllm_omni.engine.duplex import events as engine_events + from vllm_omni.protocol.duplex import events as duplex_wire + + engine_names = { + name + for name in engine_events.__all__ + if isinstance(getattr(engine_events, name), type) + and issubclass(getattr(engine_events, name), engine_events.DuplexEvent) + } + assert engine_names <= set(duplex_wire.__all__) + + +@pytest.mark.parametrize( + "module_pair", + [("events", "events"), ("commands", "commands")], + ids=["events", "commands"], +) +def test_a_reexported_tier1_name_is_the_tier1_object(module_pair: tuple[str, str]) -> None: + """Re-export, not re-declaration: the Tier 1 class is one object.""" + import importlib + + duplex_mod = importlib.import_module(f"vllm_omni.protocol.duplex.{module_pair[0]}") + realtime_mod = importlib.import_module(f"vllm_omni.protocol.realtime.{module_pair[1]}") + + shared = [n for n in duplex_mod.__all__ if hasattr(realtime_mod, n)] + assert shared, "nothing re-exported --- the facade is not doing its job" + for name in shared: + duplex_obj = getattr(duplex_mod, name) + realtime_obj = getattr(realtime_mod, name) + if duplex_obj is realtime_obj: + continue + # The only permitted divergence is a Tier 2 class extending its twin. + assert isinstance(duplex_obj, type) and issubclass(duplex_obj, realtime_obj), ( + f"{name} in protocol.duplex is neither the Tier 1 object nor a subclass of it" + ) + + +def test_duplex_event_is_an_alias_but_duplex_command_is_not() -> None: + """The asymmetry is deliberate, and a later reader should not 'fix' it. + + An event has no engine-internal half --- what the session emits is what the + socket sends --- so ``DuplexEvent`` is simply ``RealtimeEvent``. A command + does: ``payload()`` renders the session runner's mailbox dictionary, and for + four of the seventeen the mailbox channel is not even the client event type. + """ + from vllm_omni.engine.duplex.commands import DuplexCommand + from vllm_omni.protocol.duplex.events import DuplexEvent + from vllm_omni.protocol.realtime.commands import RealtimeCommand + from vllm_omni.protocol.realtime.events import RealtimeEvent + + assert DuplexEvent is RealtimeEvent + + assert DuplexCommand is not RealtimeCommand + assert issubclass(DuplexCommand, RealtimeCommand) + assert hasattr(DuplexCommand, "payload") and not hasattr(RealtimeCommand, "payload") diff --git a/tests/protocol/realtime/__init__.py b/tests/protocol/realtime/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/tests/protocol/realtime/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/tests/protocol/realtime/test_protocol_import_boundary.py b/tests/protocol/realtime/test_protocol_import_boundary.py new file mode 100644 index 000000000..a00d4da62 --- /dev/null +++ b/tests/protocol/realtime/test_protocol_import_boundary.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""The shared Realtime codec must not depend on any runtime that uses it. + +``vllm_omni.protocol.realtime`` exists so a second Realtime surface can reuse +the wire codec without adopting the duplex session control plane. That only +holds while the arrows point one way: the engine, the entrypoints and the model +code import the protocol, never the reverse. Nothing in the package's own code +enforces that, so it is asserted here --- once on the source (which catches a +lazy import inside a function body) and once on a real import (which catches a +dependency arriving through a third module). +""" + +from __future__ import annotations + +import ast +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +REPO_ROOT = Path(__file__).resolve().parents[3] +PROTOCOL_DIR = REPO_ROOT / "vllm_omni" / "protocol" + +#: Packages that consume the protocol. A dependency on any of them would make +#: the codec unusable outside the runtime it accidentally reached into. +#: ``vllm_omni.utils`` is deliberately absent: it is leaf helpers (the audio +#: resampler) that consume nothing, so depending on it costs a reuser nothing. +FORBIDDEN_PREFIXES = ( + "vllm_omni.engine", + "vllm_omni.entrypoints", + "vllm_omni.model_executor", + "vllm_omni.worker", + "vllm_omni.clients", +) + + +def _protocol_sources() -> list[Path]: + sources = sorted(PROTOCOL_DIR.rglob("*.py")) + assert sources, f"no python sources under {PROTOCOL_DIR}" + return sources + + +def _imported_modules(source: Path) -> set[str]: + """Every module name imported anywhere in the file, lazy imports included.""" + names: set[str] = set() + for node in ast.walk(ast.parse(source.read_text())): + if isinstance(node, ast.Import): + names.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + names.add(node.module) + return names + + +@pytest.mark.parametrize("source", _protocol_sources(), ids=lambda path: path.name) +def test_protocol_sources_do_not_import_a_consumer(source: Path) -> None: + offenders = sorted( + name + for name in _imported_modules(source) + if any(name == prefix or name.startswith(prefix + ".") for prefix in FORBIDDEN_PREFIXES) + ) + assert not offenders, f"{source.relative_to(REPO_ROOT)} imports {', '.join(offenders)}" + + +def test_importing_the_protocol_package_loads_no_consumer() -> None: + # A child interpreter, because this test session has already imported the + # duplex engine: only a fresh process can tell what the package pulls in. + # ``vllm_omni`` itself is imported first, because its ``__init__`` loads a + # baseline graph (patches, the model registry) that has nothing to do with + # this package; only what the protocol import *adds* is measured. + env = os.environ.copy() + env.pop("CUDA_VISIBLE_DEVICES", None) + env.pop("HIP_VISIBLE_DEVICES", None) + script = f""" +import sys + +import vllm_omni # noqa: F401 + +baseline = set(sys.modules) + +import vllm_omni.protocol.realtime # noqa: F401 + +forbidden_prefixes = {FORBIDDEN_PREFIXES!r} +added = sorted( + name + for name in set(sys.modules) - baseline + if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes) +) +if added: + raise SystemExit("the realtime protocol package loaded consumers: " + ", ".join(added)) +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/protocol/realtime/test_realtime_capabilities.py b/tests/protocol/realtime/test_realtime_capabilities.py new file mode 100644 index 000000000..2a05fc0a9 --- /dev/null +++ b/tests/protocol/realtime/test_realtime_capabilities.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""``RealtimeProtocolCapabilities``: one consumer's answers, the codec's checking. + +This is the seam that lets a non-duplex Realtime surface validate a +``session.update`` without a ``DuplexModelPlugin``. What it has to get right is +narrow: the order the checks run in (formats before turn detection, because +that is the order clients already see), the error codes, and that a consumer +which declares a narrower format set is actually narrowed. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +import pytest + +from vllm_omni.protocol.realtime.capabilities import ( + RealtimeProtocolCapabilities, + RealtimeSessionRejection, + validate_session_payload, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_DEFAULT = RealtimeProtocolCapabilities() + + +def test_a_consumer_without_a_turn_detection_check_accepts_a_plain_session(): + assert ( + validate_session_payload({"model": "m", "audio": {"input": {"format": "pcm16"}}}, capabilities=_DEFAULT) is None + ) + + +def test_a_consumer_without_a_turn_detection_check_ignores_turn_detection(): + payload = {"turn_detection": {"type": "semantic_vad"}} + + assert validate_session_payload(payload, capabilities=_DEFAULT) is None + + +def test_an_unsupported_input_format_is_rejected_with_the_realtime_code(): + rejection = validate_session_payload({"input_audio_format": "opus"}, capabilities=_DEFAULT) + + assert rejection == RealtimeSessionRejection( + code="unsupported_audio_format", message="Unsupported input_audio_format: opus" + ) + + +def test_a_consumer_may_declare_a_narrower_format_set(): + pcm_only = RealtimeProtocolCapabilities(input_audio_formats=frozenset({"pcm16"})) + + assert validate_session_payload({"input_audio_format": "pcm16"}, capabilities=pcm_only) is None + rejection = validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=pcm_only) + assert rejection is not None + assert rejection.code == "unsupported_audio_format" + # The default capabilities still accept it: the narrowing is per consumer. + assert validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=_DEFAULT) is None + + +def test_the_turn_detection_check_runs_and_reports_its_own_code(): + def reject_everything(session_payload: Mapping[str, object]) -> str | None: + del session_payload + return "server_vad is not available" + + capabilities = RealtimeProtocolCapabilities(validate_turn_detection=reject_everything) + + rejection = validate_session_payload({"turn_detection": {"type": "server_vad"}}, capabilities=capabilities) + + assert rejection == RealtimeSessionRejection( + code="unsupported_turn_detection", message="server_vad is not available", param="turn_detection" + ) + + +def test_a_bad_format_is_reported_before_a_bad_turn_detection(): + calls: list[Mapping[str, object]] = [] + + def record(session_payload: Mapping[str, object]) -> str | None: + calls.append(session_payload) + return "turn detection is wrong too" + + capabilities = RealtimeProtocolCapabilities(validate_turn_detection=record) + + rejection = validate_session_payload( + {"input_audio_format": "opus", "turn_detection": {"type": "semantic_vad"}}, + capabilities=capabilities, + ) + + assert rejection is not None + assert rejection.code == "unsupported_audio_format" + assert calls == [] diff --git a/tests/protocol/realtime/test_realtime_codec_single_source.py b/tests/protocol/realtime/test_realtime_codec_single_source.py new file mode 100644 index 000000000..2f719750b --- /dev/null +++ b/tests/protocol/realtime/test_realtime_codec_single_source.py @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""There is one Realtime codec, and the duplex engine uses it. + +RFC #6592 P0a rejects the "copy the codec and fix it up" option because the two +copies drift. The extraction is only worth anything while the duplex names are +*the same objects* as the protocol ones, not lookalikes --- so assert identity, +which a re-implementation cannot satisfy. +""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from vllm_omni.engine.duplex import audio as duplex_audio +from vllm_omni.engine.duplex import events as duplex_events +from vllm_omni.engine.duplex import realtime_commands as duplex_codec +from vllm_omni.engine.duplex.commands import DuplexCommandError +from vllm_omni.protocol import realtime as protocol +from vllm_omni.protocol.realtime.errors import RealtimeProtocolError + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +#: Names ``engine.duplex.realtime_commands`` re-exports for its existing importers. +_REEXPORTED_FROM_PROTOCOL = ( + "REALTIME_INPUT_AUDIO_FORMATS", + "REALTIME_INPUT_HINT_KEYS", + "REALTIME_OUTPUT_AUDIO_FORMATS", + "RealtimeInputDefaults", + "apply_realtime_session_defaults", + "copy_realtime_input_hints", + "input_audio_transcription_config", + "input_explicitly_non_speech", + "input_looks_like_speech", + "input_transcript_from_item", + "is_supported_realtime_input_format", + "json_safe_realtime_payload", + "normalize_conversation_item", + "parse_realtime_audio_format", + "realtime_audio_format_object", + "realtime_max_output_tokens", + "realtime_output_format", + "realtime_overlap_fields", + "text_chars_for_audio_ms_from_marks", + "truncate_realtime_item_content", + "validate_conversation_item_audio_formats", + "validate_realtime_item_truncate", + "validate_realtime_response_audio_formats", + "validate_realtime_session_audio_formats", + "validate_realtime_video_frames", +) + + +@pytest.mark.parametrize("name", _REEXPORTED_FROM_PROTOCOL) +def test_the_duplex_codec_name_is_the_protocol_object(name: str) -> None: + assert getattr(duplex_codec, name) is getattr(protocol, name) + + +@pytest.mark.parametrize( + "name", + ("convert_input_audio_with_rate", "convert_output_audio", "resample_pcm16_mono", "wav_payload_to_pcm16"), +) +def test_the_duplex_audio_shim_is_the_protocol_object(name: str) -> None: + assert getattr(duplex_audio, name) is getattr(protocol, name) + + +def test_a_duplex_command_error_is_a_realtime_protocol_error() -> None: + # The duplex binding converts the codec's error into its own type; both + # carry the same code/event_id, which is what the error envelope renders. + error = DuplexCommandError("bad", code="bad_audio", event_id="event_1") + + assert isinstance(error, RealtimeProtocolError) + assert (error.code, error.event_id) == ("bad_audio", "event_1") + + +def test_the_duplex_append_command_is_built_from_the_shared_decoder() -> None: + defaults = protocol.RealtimeInputDefaults() + event = {"event_id": "event_1", "audio": "", "format": "pcm16", "duration_ms": 40} + + decoded = protocol.decode_audio_append(event, defaults=defaults) + command = duplex_codec.build_append_audio(event, defaults=defaults) + + assert (command.audio, command.format, command.sample_rate_hz) == ( + decoded.audio, + decoded.format, + decoded.sample_rate_hz, + ) + assert (command.is_speech, command.video_frames, command.duration_ms) == ( + decoded.is_speech, + decoded.video_frames, + decoded.duration_ms, + ) + assert (command.audio_end_ms, dict(command.hints), command.event_id) == ( + decoded.audio_end_ms, + decoded.hints, + decoded.event_id, + ) + + +def test_the_shared_decoder_error_reaches_the_client_as_a_duplex_command_error() -> None: + defaults = protocol.RealtimeInputDefaults() + event = {"event_id": "event_1", "audio": "AAAA", "format": "opus"} + + with pytest.raises(RealtimeProtocolError) as shared: + protocol.decode_audio_append(event, defaults=defaults) + with pytest.raises(DuplexCommandError) as duplex: + duplex_codec.build_append_audio(event, defaults=defaults) + + assert (duplex.value.code, str(duplex.value)) == (shared.value.code, str(shared.value)) + assert duplex.value.event_id == shared.value.event_id == "event_1" + + +def test_the_duplex_capabilities_reject_unimplemented_turn_detection() -> None: + # The duplex answer comes from engine.duplex.turn_detection, reached through + # the capability object rather than imported by the codec. + rejection = protocol.validate_session_payload( + {"turn_detection": {"type": "semantic_vad"}}, + capabilities=duplex_codec.DUPLEX_REALTIME_CAPABILITIES, + ) + + assert rejection is not None + assert rejection.code == "unsupported_turn_detection" + + +# ---- the class split (RFC #6592 P0a, second half) ---- +# +# The event and command classes moved out of the engine too. The same identity +# rule applies: the duplex names must BE the protocol objects, and the mailbox +# half must have stayed behind. + +# `docs/serving/realtime_duplex_api.md` sorts every message into three tiers. +# Tier 1 is pure OpenAI and lives in protocol/realtime; Tier 2 (OpenAI names +# carrying our extensions) and Tier 3 (ours alone) live in protocol/duplex. +_TIER1_EVENTS = ("SessionUpdated", "ItemAdded", "ItemCreated", "TextDelta", "OutputItemAdded") +_TIER2_EVENTS = ( + "SessionCreated", + "ResponseCreated", + "ResponseDone", + "AudioDelta", + "InputCommitted", + "ItemDeleted", + "ItemTruncated", + "ErrorEvent", +) +_TIER3_EVENTS = ("Listen", "Speak", "OverlapDecision", "PlaybackAcknowledged", "SessionResumed") +_TIER1_COMMANDS = ("UpdateSession", "CreateItem", "CancelResponse", "ClearInput") +_TIER2_COMMANDS = ("AppendAudio", "Commit") +_TIER3_COMMANDS = ("BargeIn", "AckPlayback", "Heartbeat", "SignalTurn", "CloseSession") + + +@pytest.mark.parametrize("name", _TIER1_EVENTS) +def test_tier1_events_are_the_pure_openai_objects(name: str) -> None: + from vllm_omni.protocol.realtime import events as realtime_events + + assert duplex_events.__dict__[name] is getattr(realtime_events, name) + + +@pytest.mark.parametrize("name", _TIER2_EVENTS + _TIER3_EVENTS) +def test_tier2_and_tier3_events_come_from_protocol_duplex(name: str) -> None: + from vllm_omni.protocol.duplex import events as duplex_wire_events + + assert duplex_events.__dict__[name] is getattr(duplex_wire_events, name) + + +@pytest.mark.parametrize("name", _TIER2_EVENTS) +def test_tier2_extends_its_tier1_twin_rather_than_replacing_it(name: str) -> None: + """A Tier 2 class must be its Tier 1 class plus fields, never a fork of it.""" + from vllm_omni.protocol.duplex import events as duplex_wire_events + from vllm_omni.protocol.realtime import events as realtime_events + + tier1 = getattr(realtime_events, name) + tier2 = getattr(duplex_wire_events, name) + + assert issubclass(tier2, tier1) + assert tier2.wire_type == tier1.wire_type + tier1_fields = {f.name for f in dataclasses.fields(tier1)} + tier2_fields = {f.name for f in dataclasses.fields(tier2)} + # Purely additive: Tier 2 adds, never drops or renames. + assert tier1_fields < tier2_fields + + +@pytest.mark.parametrize("name", _TIER1_EVENTS + _TIER1_COMMANDS) +def test_tier1_carries_no_vllm_omni_extension_fields(name: str) -> None: + """The point of the split: protocol/realtime must be honestly pure OpenAI.""" + from vllm_omni.protocol.realtime import commands as realtime_commands + from vllm_omni.protocol.realtime import events as realtime_events + + cls = getattr(realtime_events, name, None) or getattr(realtime_commands, name) + fields = {f.name for f in dataclasses.fields(cls)} + # Extension fields catalogued as Tier 2 in the serving doc. + assert not ( + fields + & { + "attachment_generation", + "resume_token", + "details", + "extra", + "is_speech", + "video_frames", + "hints", + "realtime_item_id", + } + ) + + +def test_the_error_code_vocabulary_is_tier3() -> None: + """OpenAI standardises the error classes, not our codes (doc: Tier 3).""" + from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE + from vllm_omni.protocol.realtime import errors as realtime_errors + + assert duplex_events.REALTIME_ERROR_TYPES_BY_CODE is REALTIME_ERROR_TYPES_BY_CODE + assert not hasattr(realtime_errors, "REALTIME_ERROR_TYPES_BY_CODE") + + +def test_the_tier1_error_event_reports_only_openai_classes() -> None: + from vllm_omni.protocol.duplex import events as duplex_wire_events + from vllm_omni.protocol.realtime import events as realtime_events + + # Tier 1 knows the envelope shape but not our code vocabulary. + assert realtime_events.ErrorEvent(code="resource_exhausted").error_type == "invalid_request_error" + # Tier 2 resolves it through our table. + assert duplex_wire_events.ErrorEvent(code="resource_exhausted").error_type == "rate_limit_error" + + +@pytest.mark.parametrize("name", _TIER1_COMMANDS + _TIER2_COMMANDS + _TIER3_COMMANDS) +def test_engine_commands_subclass_their_protocol_wire_type(name: str) -> None: + from vllm_omni.engine.duplex import commands as engine_commands + from vllm_omni.protocol.duplex import commands as duplex_wire_commands + from vllm_omni.protocol.realtime import commands as realtime_commands + + engine_cls = getattr(engine_commands, name) + wire_cls = getattr(duplex_wire_commands, name, None) or getattr(realtime_commands, name) + + assert issubclass(engine_cls, wire_cls) + assert engine_cls.wire_type == wire_cls.wire_type + + +def test_the_mailbox_channel_stayed_engine_side() -> None: + """``payload()``'s ``type`` is the runner's channel, not the client event. + + These four are the proof that the two halves are genuinely different, and + so the reason the classes were split rather than relocated wholesale. + """ + from vllm_omni.engine.duplex import commands as engine_commands + from vllm_omni.protocol.realtime import commands as realtime_commands + + for name in ("UpdateSession", "CreateItem", "DeleteItem", "TruncateItem"): + engine_cls = getattr(engine_commands, name) + wire_cls = getattr(realtime_commands, name) + assert engine_cls.type == "turn.signal" + assert wire_cls.wire_type != "turn.signal" + # The wire class carries no mailbox vocabulary at all. + assert "type" not in vars(wire_cls) diff --git a/tools/pre_commit/check_forbidden_imports.py b/tools/pre_commit/check_forbidden_imports.py index 959007c30..5f19123bc 100644 --- a/tools/pre_commit/check_forbidden_imports.py +++ b/tools/pre_commit/check_forbidden_imports.py @@ -79,11 +79,12 @@ CHECK_IMPORTS = { "vllm_omni/entrypoints/openai/video_stream_base.py", "vllm_omni/entrypoints/openai/video_stream_session.py", "vllm_omni/experimental/fullduplex/client.py", - "vllm_omni/engine/duplex/audio.py", "vllm_omni/engine/duplex/commands.py", "vllm_omni/engine/duplex/contracts.py", - "vllm_omni/engine/duplex/events.py", - "vllm_omni/engine/duplex/realtime_commands.py", + "vllm_omni/protocol/realtime/audio.py", + "vllm_omni/protocol/realtime/audio_input.py", + "vllm_omni/protocol/realtime/events.py", + "vllm_omni/protocol/realtime/items.py", "vllm_omni/experimental/fullduplex/joyvl/memory/memory.py", "vllm_omni/model_executor/models/minicpmo_4_5/duplex/input.py", "vllm_omni/model_executor/models/minicpmo_4_5/duplex/plugin.py", diff --git a/vllm_omni/clients/inline_duplex.py b/vllm_omni/clients/inline_duplex.py index 79c09e014..5af8efc5f 100644 --- a/vllm_omni/clients/inline_duplex.py +++ b/vllm_omni/clients/inline_duplex.py @@ -32,8 +32,8 @@ from typing import TYPE_CHECKING from vllm_omni.clients.duplex import DuplexClientBase, DuplexConnectionError, SessionConfig if TYPE_CHECKING: - from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle + from vllm_omni.protocol.duplex import RealtimeInputDefaults __all__ = ["InlineDuplexClient"] @@ -71,7 +71,7 @@ class InlineDuplexClient(DuplexClientBase): def _input_defaults(self) -> RealtimeInputDefaults: """Audio format / sample-rate / VAD defaults declared by this session's object.""" - from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults + from vllm_omni.protocol.duplex import RealtimeInputDefaults session_payload = self.session_info or self.config.to_session_payload(model=self.model) return RealtimeInputDefaults().with_session_payload(session_payload) diff --git a/vllm_omni/engine/duplex/audio.py b/vllm_omni/engine/duplex/audio.py index 5f163826a..2d46e8edb 100644 --- a/vllm_omni/engine/duplex/audio.py +++ b/vllm_omni/engine/duplex/audio.py @@ -1,218 +1,44 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Compatibility re-export: the Realtime audio codec moved out of the duplex engine. + +Decoding PCM / G.711 / WAV, resampling and re-encoding are wire-format work with +no duplex session in them, so the implementation now lives in +``vllm_omni.protocol.realtime.audio`` where a non-duplex Realtime surface can +use it too. Duplex code reaches it through ``vllm_omni.protocol.duplex``, the +one door the engine uses --- which is also where a duplex-specific conversion +would be introduced. Import from there in new code. +""" + from __future__ import annotations -import base64 -import binascii -import io -import math -import wave +from vllm_omni.protocol.duplex import ( + MAX_INPUT_SAMPLE_RATE_HZ, + MIN_INPUT_SAMPLE_RATE_HZ, + convert_input_audio_with_rate, + convert_output_audio, + decode_g711_alaw, + decode_g711_ulaw, + encode_float32_mono_wav_base64, + encode_g711_alaw, + encode_g711_ulaw, + resample_pcm16_mono, + validate_input_sample_rate_hz, + wav_payload_to_pcm16, +) -import numpy as np - -from vllm_omni.utils.audio_resample import StreamingAudioResampler - -try: - from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin -except ImportError: # pragma: no cover - audioop is removed in newer Python. - alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None - - -MIN_INPUT_SAMPLE_RATE_HZ = 8_000 -MAX_INPUT_SAMPLE_RATE_HZ = 192_000 - - -def validate_input_sample_rate_hz(sample_rate_hz: object) -> int: - if ( - isinstance(sample_rate_hz, bool) - or not isinstance(sample_rate_hz, int | float) - or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz)) - ): - raise ValueError("sample_rate_hz must be a finite integer") - rate = int(sample_rate_hz) - if rate != sample_rate_hz: - raise ValueError("sample_rate_hz must be an integer") - if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ: - raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}") - return rate - - -def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes: - if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz: - return raw - samples = np.frombuffer(raw, dtype=" StreamingAudioResampler.max_polyphase_factor: - # The FIR size scales with the reduced ratio. Keep unusual client - # rates bounded while common audio rates use the high-quality path. - if samples.size <= 1: - return raw - target_size = max(1, int(round(samples.size * target_rate_hz / source_rate_hz))) - source_x = np.linspace(0.0, 1.0, num=samples.size, endpoint=True) - target_x = np.linspace(0.0, 1.0, num=target_size, endpoint=True) - resampled = np.interp(target_x, source_x, samples) - return np.clip(resampled, -32_768, 32_767).astype(" bytes: - if ulaw2lin is not None: - return ulaw2lin(raw, 2) - data = np.frombuffer(raw, dtype=np.uint8) - value = np.bitwise_not(data).astype(np.int16) - sign = value & 0x80 - exponent = (value >> 4) & 0x07 - mantissa = value & 0x0F - sample = ((mantissa << 3) + 0x84) << exponent - return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype(" bytes: - if alaw2lin is not None: - return alaw2lin(raw, 2) - data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16) - sign = data & 0x80 - exponent = (data >> 4) & 0x07 - mantissa = data & 0x0F - sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1)) - return np.where(sign != 0, sample, -sample).astype(" bytes: - if lin2ulaw is not None: - return lin2ulaw(raw, 2) - pcm = np.clip(np.frombuffer(raw, dtype=" (0xFF << exp), exp + 1, exponent) - mantissa = (magnitude >> (exponent + 3)) & 0x0F - return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes() - - -def encode_g711_alaw(raw: bytes) -> bytes: - if lin2alaw is not None: - return lin2alaw(raw, 2) - pcm = np.frombuffer(raw, dtype="= 0, 0x80, 0x00) - magnitude = np.abs(pcm) - exponent = np.zeros_like(magnitude) - for exp in range(1, 8): - exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent) - mantissa = np.where( - exponent == 0, - (magnitude >> 4) & 0x0F, - (magnitude >> (exponent + 3)) & 0x0F, - ) - return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes() - - -def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]: - try: - with wave.open(io.BytesIO(raw), "rb") as wav_file: - channels = wav_file.getnchannels() - sample_width = wav_file.getsampwidth() - sample_rate_hz = wav_file.getframerate() - frames = wav_file.readframes(wav_file.getnframes()) - if sample_width != 2: - return None, sample_rate_hz - if channels <= 1: - return frames, sample_rate_hz - pcm = np.frombuffer(frames, dtype=" str: - """Package normalized mono float32 samples as a PCM16 WAV data payload.""" - rate = validate_input_sample_rate_hz(sample_rate_hz) - normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1) - pcm16 = np.clip(np.rint(normalized * 32768.0), -32768, 32767).astype(" tuple[object, object, int | float | None]: - if not isinstance(audio, str) or not isinstance(fmt, str): - return audio, fmt, sample_rate_hz - normalized = fmt.lower() - if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}: - return audio, fmt, sample_rate_hz - if isinstance(sample_rate_hz, int | float): - sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz) - try: - raw = base64.b64decode(audio.strip(), validate=False) - except (binascii.Error, ValueError): - return audio, fmt, sample_rate_hz - if normalized == "g711_ulaw": - raw = decode_g711_ulaw(raw) - sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000 - elif normalized == "g711_alaw": - raw = decode_g711_alaw(raw) - sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000 - elif len(raw) % 2: - return audio, fmt, sample_rate_hz - if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz: - raw = resample_pcm16_mono( - raw, - source_rate_hz=int(sample_rate_hz), - target_rate_hz=target_sample_rate_hz, - ) - sample_rate_hz = target_sample_rate_hz - pcm = np.frombuffer(raw, dtype=" tuple[str, str, int | None]: - target = target_fmt.lower() - if target not in {"g711_ulaw", "g711_alaw"}: - return audio, source_fmt, source_sample_rate_hz - try: - raw = base64.b64decode(audio, validate=False) - except (binascii.Error, ValueError): - return audio, source_fmt, source_sample_rate_hz - source = source_fmt.lower() - if source == "wav": - pcm_raw, wav_rate = wav_payload_to_pcm16(raw) - if pcm_raw is None: - return audio, source_fmt, source_sample_rate_hz - raw = pcm_raw - source_sample_rate_hz = source_sample_rate_hz or wav_rate - elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2: - return audio, source_fmt, source_sample_rate_hz - target_rate = target_sample_rate_hz or 8_000 - if source_sample_rate_hz is not None: - raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate) - encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw) - return base64.b64encode(encoded).decode("ascii"), target, target_rate +__all__ = [ + "MAX_INPUT_SAMPLE_RATE_HZ", + "MIN_INPUT_SAMPLE_RATE_HZ", + "convert_input_audio_with_rate", + "convert_output_audio", + "decode_g711_alaw", + "decode_g711_ulaw", + "encode_float32_mono_wav_base64", + "encode_g711_alaw", + "encode_g711_ulaw", + "resample_pcm16_mono", + "validate_input_sample_rate_hz", + "wav_payload_to_pcm16", +] diff --git a/vllm_omni/engine/duplex/commands.py b/vllm_omni/engine/duplex/commands.py index 90d48d050..19674948c 100644 --- a/vllm_omni/engine/duplex/commands.py +++ b/vllm_omni/engine/duplex/commands.py @@ -1,41 +1,62 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project -"""Typed commands accepted by a duplex session. +"""The command vocabulary one duplex session accepts, plus its mailbox rendering. ``DuplexSessionHandle.submit()`` takes one of these; the websocket handler and -``InlineDuplexClient`` build them with :func:`command_from_realtime` from the -OpenAI Realtime client event vocabulary. Every command can also render the -session-internal mailbox payload (``payload()``), which is the dictionary -vocabulary the session runner bodies were written against. +``InlineDuplexClient`` build them with :func:`command_from_realtime`. + +The *wire* half of each command --- which client event it decodes from and what +fields survive decoding --- now lives in ``vllm_omni.protocol.duplex.commands``, +which carries the whole vocabulary: the eight Tier 1 classes re-exported from +``vllm_omni.protocol.realtime.commands``, the two Tier 2 ones and the seven +Tier 3 ones. The engine imports that module and never the Tier 1 one directly, +so a command that later grows a duplex extension changes one file +(RFC #6592 P0a). + +The *engine* half stays here, because it is not wire contract at all: +``payload()`` renders the session-internal mailbox dictionary the runner bodies +were written against, and its ``type`` is the mailbox channel rather than the +client event. Those two genuinely differ --- ``session.update`` and +``conversation.item.create`` / ``.delete`` / ``.truncate`` all travel on the +``turn.signal`` channel --- which is why the halves are separated instead of +the classes being relocated wholesale. """ from __future__ import annotations import base64 from collections.abc import Mapping -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, fields from typing import TYPE_CHECKING, ClassVar +from vllm_omni.protocol.duplex import RealtimeProtocolError +from vllm_omni.protocol.duplex import commands as _duplex_wire +from vllm_omni.protocol.duplex.commands import RealtimeCommand + if TYPE_CHECKING: - from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults + from vllm_omni.protocol.duplex import RealtimeInputDefaults -class DuplexCommandError(ValueError): - """A client payload could not be turned into a command.""" +class DuplexCommandError(RealtimeProtocolError): + """A client payload could not be turned into a duplex command. - def __init__(self, message: str, *, code: str = "bad_event", event_id: str | None = None) -> None: - super().__init__(message) - self.code = code - self.event_id = event_id + The duplex name for a Realtime protocol error: same ``code`` / + ``event_id`` contract, so the error envelope is rendered identically + whichever consumer raised it. + """ @dataclass(frozen=True, slots=True, kw_only=True) -class DuplexCommand: +class DuplexCommand(RealtimeCommand): + """A Realtime command as the duplex engine handles it. + + Adds the mailbox channel (``type``) and its rendering on top of the wire + command; every concrete class below pairs this with its protocol twin. + """ + #: Mailbox event type this command renders to (see ``payload()``). type: ClassVar[str] = "" - #: Client correlation id (OpenAI ``event_id``), echoed on error events. - event_id: str | None = None def payload(self) -> dict[str, object]: """Render the session-internal mailbox dictionary.""" @@ -56,19 +77,23 @@ class DuplexCommand: return data +# ---- GA commands ---- + + @dataclass(frozen=True, slots=True, kw_only=True) -class AppendAudio(DuplexCommand): +class UpdateSession(DuplexCommand, _duplex_wire.UpdateSession): + type: ClassVar[str] = "turn.signal" + + def payload(self) -> dict[str, object]: + data = DuplexCommand.payload(self) + data["event"] = "session.update" + data["payload"] = dict(data.pop("patch", {}) or {}) + return data + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AppendAudio(DuplexCommand, _duplex_wire.AppendAudio): type: ClassVar[str] = "input_audio_buffer.append" - #: Raw audio bytes in ``format`` at ``sample_rate_hz`` (base64 only on the wire). - audio: bytes - format: str = "pcm16" - sample_rate_hz: int | None = None - is_speech: bool | None = None - video_frames: tuple[str, ...] = () - duration_ms: int | None = None - audio_end_ms: int | None = None - #: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...). - hints: Mapping[str, object] = field(default_factory=dict) def payload(self) -> dict[str, object]: data = DuplexCommand.payload(self) @@ -90,21 +115,8 @@ class AppendAudio(DuplexCommand): @dataclass(frozen=True, slots=True, kw_only=True) -class AppendText(DuplexCommand): - type: ClassVar[str] = "input.text.append" - text: str - - -@dataclass(frozen=True, slots=True, kw_only=True) -class Commit(DuplexCommand): +class Commit(DuplexCommand, _duplex_wire.Commit): type: ClassVar[str] = "input_audio_buffer.commit" - final: bool = True - #: ``None`` means "no explicit request": the runner decides on commit (auto-response - #: sessions answer on their own); ``True`` / ``False`` force it. - create_response: bool | None = None - is_speech: bool | None = None - #: Realtime conversation item created for this commit (wire correlation only). - realtime_item_id: str | None = None def payload(self) -> dict[str, object]: data = DuplexCommand.payload(self) @@ -114,10 +126,18 @@ class Commit(DuplexCommand): @dataclass(frozen=True, slots=True, kw_only=True) -class CreateResponse(DuplexCommand): +class ClearInput(DuplexCommand, _duplex_wire.ClearInput): + type: ClassVar[str] = "input_audio_buffer.clear" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ClearOutputAudio(DuplexCommand, _duplex_wire.ClearOutputAudio): + type: ClassVar[str] = "output_audio_buffer.clear" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CreateResponse(DuplexCommand, _duplex_wire.CreateResponse): type: ClassVar[str] = "response.create" - #: Raw Realtime ``response`` object (instructions, voice, modalities, ...). - options: Mapping[str, object] = field(default_factory=dict) def payload(self) -> dict[str, object]: data = DuplexCommand.payload(self) @@ -128,84 +148,13 @@ class CreateResponse(DuplexCommand): @dataclass(frozen=True, slots=True, kw_only=True) -class ClearInput(DuplexCommand): - type: ClassVar[str] = "input_audio_buffer.clear" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class CancelInput(DuplexCommand): - type: ClassVar[str] = "input.cancel" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class CancelResponse(DuplexCommand): +class CancelResponse(DuplexCommand, _duplex_wire.CancelResponse): type: ClassVar[str] = "response.cancel" - response_id: str | None = None @dataclass(frozen=True, slots=True, kw_only=True) -class BargeIn(DuplexCommand): - type: ClassVar[str] = "barge_in" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ClearOutputAudio(DuplexCommand): - type: ClassVar[str] = "output_audio_buffer.clear" - #: Explicit response to clear; ``None`` targets the active/last response. - response_id: str | None = None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SignalTurn(DuplexCommand): - """Generic ``turn.signal`` (local turn transitions such as ``user_started``).""" - +class CreateItem(DuplexCommand, _duplex_wire.CreateItem): type: ClassVar[str] = "turn.signal" - event: str - signal_payload: Mapping[str, object] = field(default_factory=dict) - - def payload(self) -> dict[str, object]: - data = DuplexCommand.payload(self) - signal_payload = data.pop("signal_payload", None) - if isinstance(signal_payload, Mapping) and signal_payload: - data["payload"] = dict(signal_payload) - return data - - -@dataclass(frozen=True, slots=True, kw_only=True) -class UpdateSession(DuplexCommand): - """``session.update``: a Realtime ``session`` object patch.""" - - type: ClassVar[str] = "turn.signal" - patch: Mapping[str, object] = field(default_factory=dict) - - def payload(self) -> dict[str, object]: - data = DuplexCommand.payload(self) - data["event"] = "session.update" - data["payload"] = dict(data.pop("patch", {}) or {}) - return data - - -@dataclass(frozen=True, slots=True, kw_only=True) -class AckPlayback(DuplexCommand): - type: ClassVar[str] = "playback.ack" - played_ms: int - committed_ms: int | None = None - response_id: str | None = None - item_id: str | None = None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class Heartbeat(DuplexCommand): - type: ClassVar[str] = "session.heartbeat" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class CreateItem(DuplexCommand): - """``conversation.item.create`` (history injection or function-call output).""" - - type: ClassVar[str] = "turn.signal" - item: Mapping[str, object] - previous_item_id: str | None = None def payload(self) -> dict[str, object]: data = DuplexCommand.payload(self) @@ -219,9 +168,8 @@ class CreateItem(DuplexCommand): @dataclass(frozen=True, slots=True, kw_only=True) -class DeleteItem(DuplexCommand): +class DeleteItem(DuplexCommand, _duplex_wire.DeleteItem): type: ClassVar[str] = "turn.signal" - item_id: str def payload(self) -> dict[str, object]: data = DuplexCommand.payload(self) @@ -231,11 +179,8 @@ class DeleteItem(DuplexCommand): @dataclass(frozen=True, slots=True, kw_only=True) -class TruncateItem(DuplexCommand): +class TruncateItem(DuplexCommand, _duplex_wire.TruncateItem): type: ClassVar[str] = "turn.signal" - item_id: str - audio_end_ms: int - content_index: int = 0 def payload(self) -> dict[str, object]: data = DuplexCommand.payload(self) @@ -248,15 +193,51 @@ class TruncateItem(DuplexCommand): return data +# ---- duplex extension commands ---- + + @dataclass(frozen=True, slots=True, kw_only=True) -class CloseSession(DuplexCommand): - """Graceful close requested through the command stream (``session.close``).""" +class AppendText(DuplexCommand, _duplex_wire.AppendText): + type: ClassVar[str] = "input.text.append" + +@dataclass(frozen=True, slots=True, kw_only=True) +class CancelInput(DuplexCommand, _duplex_wire.CancelInput): + type: ClassVar[str] = "input.cancel" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BargeIn(DuplexCommand, _duplex_wire.BargeIn): + type: ClassVar[str] = "barge_in" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SignalTurn(DuplexCommand, _duplex_wire.SignalTurn): + type: ClassVar[str] = "turn.signal" + + def payload(self) -> dict[str, object]: + data = DuplexCommand.payload(self) + signal_payload = data.pop("signal_payload", None) + if isinstance(signal_payload, Mapping) and signal_payload: + data["payload"] = dict(signal_payload) + return data + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AckPlayback(DuplexCommand, _duplex_wire.AckPlayback): + type: ClassVar[str] = "playback.ack" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Heartbeat(DuplexCommand, _duplex_wire.Heartbeat): + type: ClassVar[str] = "session.heartbeat" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CloseSession(DuplexCommand, _duplex_wire.CloseSession): type: ClassVar[str] = "session.close" - reason: str = "client_close" -#: Realtime client event types that map onto commands (wire vocabulary). REALTIME_COMMAND_TYPES: frozenset[str] = frozenset( { "input_audio_buffer.append", @@ -300,7 +281,6 @@ def command_from_realtime( __all__ = [ - "REALTIME_COMMAND_TYPES", "AckPlayback", "AppendAudio", "AppendText", @@ -317,6 +297,8 @@ __all__ = [ "DuplexCommand", "DuplexCommandError", "Heartbeat", + "REALTIME_COMMAND_TYPES", + "RealtimeCommand", "SignalTurn", "TruncateItem", "UpdateSession", diff --git a/vllm_omni/engine/duplex/config.py b/vllm_omni/engine/duplex/config.py index aab0fb1c5..a898a2cf5 100644 --- a/vllm_omni/engine/duplex/config.py +++ b/vllm_omni/engine/duplex/config.py @@ -339,26 +339,26 @@ class DuplexSessionConfig: ``_session_create_from_realtime``. """ from vllm_omni.engine.duplex.realtime_commands import ( - RealtimeInputDefaults, + DUPLEX_REALTIME_CAPABILITIES, duplex_response_format, + ) + from vllm_omni.engine.duplex.turn_detection import normalize_turn_detection_session_payload + from vllm_omni.protocol.duplex import ( + RealtimeInputDefaults, input_audio_transcription_config, json_safe_realtime_payload, realtime_max_output_tokens, realtime_overlap_fields, - validate_realtime_session_audio_formats, - ) - from vllm_omni.engine.duplex.turn_detection import ( - normalize_turn_detection_session_payload, - validate_realtime_turn_detection, + validate_session_payload, ) payload: dict[str, object] = dict(session_payload) - format_error = validate_realtime_session_audio_formats(payload) - if format_error is not None: - raise DuplexConfigError(format_error, code="unsupported_audio_format") - turn_detection_error = validate_realtime_turn_detection(payload) - if turn_detection_error is not None: - raise DuplexConfigError(turn_detection_error, code="unsupported_turn_detection", param="turn_detection") + # One session check for every consumer (ENTRY-INV-002): the same + # capability object ``translate_realtime_command`` uses, so a session + # object is accepted or refused identically whichever door it came in. + rejection = validate_session_payload(payload, capabilities=DUPLEX_REALTIME_CAPABILITIES) + if rejection is not None: + raise DuplexConfigError(rejection.message, code=rejection.code, param=rejection.param) normalize_turn_detection_session_payload(payload) defaults = RealtimeInputDefaults().with_session_payload(payload) payload.update(realtime_overlap_fields(payload)) @@ -446,9 +446,9 @@ class DuplexSessionConfig: when the patch changes something a live session cannot change. ``audio_started`` is ``playback.generated_ms > 0 or playback.sent_ms > 0``. """ - from vllm_omni.engine.duplex.realtime_commands import ( + from vllm_omni.engine.duplex.realtime_commands import duplex_response_format + from vllm_omni.protocol.duplex import ( REALTIME_OUTPUT_AUDIO_FORMATS, - duplex_response_format, input_audio_transcription_config, json_safe_realtime_payload, parse_realtime_audio_format, @@ -629,9 +629,9 @@ class ResponseCreateOptions: for options a model-native duplex session cannot apply per response. Private runtime keys in ``extra_body`` are dropped. """ - from vllm_omni.engine.duplex.realtime_commands import ( + from vllm_omni.engine.duplex.realtime_commands import duplex_response_format + from vllm_omni.protocol.duplex import ( REALTIME_OUTPUT_AUDIO_FORMATS, - duplex_response_format, parse_realtime_audio_format, realtime_max_output_tokens, ) @@ -782,14 +782,14 @@ def realtime_item_to_history_message(item: object) -> dict[str, object] | None: def realtime_max_output_tokens(value: object) -> int | None: """Normalize Realtime max output tokens (``"inf"`` -> ``None``).""" - from vllm_omni.engine.duplex.realtime_commands import realtime_max_output_tokens as _impl + from vllm_omni.protocol.duplex import realtime_max_output_tokens as _impl return _impl(value) def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None: """Return the ``input_audio_transcription`` object of a Realtime session payload.""" - from vllm_omni.engine.duplex.realtime_commands import input_audio_transcription_config as _impl + from vllm_omni.protocol.duplex import input_audio_transcription_config as _impl return _impl(session_payload) diff --git a/vllm_omni/engine/duplex/events.py b/vllm_omni/engine/duplex/events.py index 9c131a893..197dec613 100644 --- a/vllm_omni/engine/duplex/events.py +++ b/vllm_omni/engine/duplex/events.py @@ -1,677 +1,79 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project -"""Typed events emitted by a duplex session. +"""The event vocabulary one duplex session speaks, assembled from two layers. -Every public event is a frozen dataclass with explicit fields. The Realtime -wire JSON is *derived* from those fields by :meth:`DuplexEvent.to_realtime`, -which is pure: it never consults session state. The stateful part (response / -item ids, content-part bookkeeping) lives in -``vllm_omni.engine.duplex.realtime_events`` on the session runner and is -consumed when the events are *constructed*. +The classes themselves no longer live here. ``vllm_omni.protocol.duplex.events`` +carries the whole vocabulary --- 22 Tier 1 classes re-exported from +``vllm_omni.protocol.realtime.events``, 8 Tier 2 ones and 12 Tier 3 ones --- so +a non-duplex Realtime consumer can take the Tier 1 half and its wire rendering +without the duplex session control plane (RFC #6592 P0a). The engine imports +the duplex module and never the Tier 1 one directly. -``session_id`` / ``epoch`` are bound by -``DuplexSessionManager.emit`` when an event leaves the runner; producers -(projector, serving handshake helpers) build events with the defaults. +What is genuinely engine-side stays: :data:`DOMAIN_TERMINAL_EVENTS` and +:data:`MODEL_OUTPUT_EVENTS` are the session runner's epoch-filter policy, not +wire contract --- they name *internal* event types, and which of them may never +be dropped as stale is a runtime decision. + +``DuplexEvent`` is re-exported as the name the engine and the clients already +use. It is an alias of +:class:`~vllm_omni.protocol.realtime.events.RealtimeEvent`, not a subclass, +because an event has no engine-internal half --- what the session emits is +exactly what the socket sends. A *command* does have one (``payload()`` renders +the runner's mailbox), which is why ``DuplexCommand`` is a real class. """ from __future__ import annotations -import base64 -from collections.abc import Mapping -from dataclasses import dataclass, field, fields -from typing import ClassVar, cast -from uuid import uuid4 +from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE +from vllm_omni.protocol.duplex.events import ( + AudioDelta, + AudioDone, + ContentPartAdded, + ContentPartDone, + DuplexEvent, + DuplexRawEvent, + ErrorEvent, + FunctionCallArgumentsDelta, + FunctionCallArgumentsDone, + InputCleared, + InputCommitted, + InputTranscriptionCompleted, + ItemAdded, + ItemCreated, + ItemDeleted, + ItemDone, + ItemRetrieved, + ItemTruncated, + Listen, + OutputAudioCleared, + OutputItemAdded, + OutputItemDone, + OverlapDecision, + PlaybackAcknowledged, + RateLimitsUpdated, + ResponseCreated, + ResponseDone, + SessionClosed, + SessionCreated, + SessionExpired, + SessionHeartbeatAck, + SessionReplaced, + SessionResumed, + SessionResyncRequired, + SessionUpdated, + Speak, + SpeechStarted, + SpeechStopped, + TextDelta, + TextDone, + TranscriptDelta, + TranscriptDone, + TurnEvent, + error_event, + new_event_id, +) - -def new_event_id() -> str: - return f"event_{uuid4().hex}" - - -#: OpenAI Realtime ``error.type`` for each internal error code. -REALTIME_ERROR_TYPES_BY_CODE: dict[str, str] = { - "bad_event": "invalid_request_error", - "bad_audio": "invalid_request_error", - "config_timeout": "invalid_request_error", - "invalid_json": "invalid_request_error", - "event_too_large": "invalid_request_error", - "unknown_event": "invalid_request_error", - "internal_error": "server_error", - "runtime_append_failed": "server_error", - "runtime_append_task_failed": "server_error", - "runtime_signal_failed": "server_error", - "runtime_abort_failed": "server_error", - "runtime_data_plane_stream_failed": "server_error", - "runtime_data_plane_text_without_audio": "server_error", - "resource_exhausted": "rate_limit_error", - "session_exists": "invalid_request_error", - "session_closed": "invalid_request_error", - "unknown_session": "invalid_request_error", - "invalid_duplex_runtime_config": "invalid_request_error", - "instructions_update_unsupported": "invalid_request_error", - "persona_update_unsupported": "invalid_request_error", - "voice_update_unsupported": "invalid_request_error", - "unsupported_nemotron_duplex_mode": "invalid_request_error", - "unsupported_native_response_options": "invalid_request_error", - "runtime_touch_failed": "server_error", - "engine_error": "server_error", - "input_backpressure": "rate_limit_error", - "response_already_active": "invalid_request_error", - "response_not_active": "invalid_request_error", - "response_create_without_input": "invalid_request_error", - "text_only_turn_unsupported": "invalid_request_error", - "commit_aborted": "server_error", - "input_audio_buffer_empty": "invalid_request_error", - "missing_item_id": "invalid_request_error", - "item_not_found": "invalid_request_error", - "playback_item_mismatch": "invalid_request_error", - "playback_item_not_found": "invalid_request_error", - "playback_ack_too_late": "invalid_request_error", - "unsupported_audio_format": "invalid_request_error", - "unsupported_turn_detection": "invalid_request_error", - "unsupported_ref_audio_path": "invalid_request_error", - "ref_audio_required": "invalid_request_error", - "model_update_unsupported": "invalid_request_error", - "voice_update_after_audio_unsupported": "invalid_request_error", - "ref_audio_update_unsupported": "invalid_request_error", - "native_text_append_unsupported": "invalid_request_error", - "invalid_video_frames": "invalid_request_error", - "invalid_function_call_output": "invalid_request_error", - "server_vad_unavailable": "server_error", -} - -_IDENTITY_FIELDS = frozenset({"session_id", "epoch", "event_id"}) - - -def _wire_value(value: object) -> object: - if isinstance(value, Mapping): - return {str(k): _wire_value(v) for k, v in value.items()} - if isinstance(value, tuple | list): - return [_wire_value(v) for v in value] - return value - - -@dataclass(frozen=True, slots=True, kw_only=True) -class DuplexEvent: - """Base of every public session event.""" - - #: Realtime ``type`` this class renders to. - wire_type: ClassVar[str] = "" - #: Fields that are omitted from the wire object when ``None``. - optional_wire_fields: ClassVar[frozenset[str]] = frozenset() - - session_id: str = "" - epoch: int | None = None - #: Server event id (OpenAI ``event_id``); generated at construction. - event_id: str = field(default_factory=new_event_id) - - # ---- identity ---- - - @property - def type(self) -> str: - return self.wire_type - - @property - def is_terminal(self) -> bool: - return False - - # ---- generic accessors (None when the class has no such field) ---- - - @property - def response_id(self) -> str | None: - return None - - @property - def item_id(self) -> str | None: - return None - - @property - def text(self) -> str | None: - return None - - @property - def audio(self) -> bytes | None: - return None - - # ---- wire rendering (pure) ---- - - def _wire_fields(self) -> dict[str, object]: - data: dict[str, object] = {} - for f in fields(self): - if f.name in _IDENTITY_FIELDS: - continue - value = getattr(self, f.name) - if value is None and f.name in self.optional_wire_fields: - continue - data[f.name] = _wire_value(value) - return data - - def to_realtime(self) -> dict[str, object]: - """The Realtime wire JSON object for this event (derived from the fields).""" - return {"type": self.wire_type, "event_id": self.event_id, **self._wire_fields()} - - def __repr__(self) -> str: # pragma: no cover - debugging aid - return f"{type(self).__name__}(type={self.wire_type!r}, session_id={self.session_id!r}, epoch={self.epoch!r})" - - -# ---- session lifecycle ---- - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SessionCreated(DuplexEvent): - wire_type = "session.created" - optional_wire_fields = frozenset({"attachment_generation", "resume_token"}) - - session: Mapping[str, object] = field(default_factory=dict) - #: Transport resume credentials (set by the websocket handler when resume is supported). - attachment_generation: int | None = None - resume_token: str | None = None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SessionUpdated(DuplexEvent): - wire_type = "session.updated" - - session: Mapping[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SessionResumed(SessionCreated): - wire_type = "session.resumed" - - def _wire_fields(self) -> dict[str, object]: - return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SessionReplaced(DuplexEvent): - """A newer connection took the session over (sent to the replaced socket).""" - - wire_type = "session.replaced" - - attachment_generation: int = 0 - - def _wire_fields(self) -> dict[str, object]: - return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SessionResyncRequired(DuplexEvent): - """The replay journal cannot bridge the gap; the client must start over.""" - - wire_type = "session.resync_required" - - reason: str = "journal_gap" - - def _wire_fields(self) -> dict[str, object]: - return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SessionClosed(DuplexEvent): - wire_type = "session.closed" - - reason: str = "closed" - #: The session-internal close event (kept for clients that inspect it). - details: Mapping[str, object] = field(default_factory=dict) - - @property - def is_terminal(self) -> bool: - return True - - def _wire_fields(self) -> dict[str, object]: - return {"session_id": self.session_id, "reason": self.reason, "event": _wire_value(self.details)} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SessionExpired(SessionClosed): - wire_type = "session.expired" - - def _wire_fields(self) -> dict[str, object]: - return {"session_id": self.session_id, "reason": self.reason} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SessionHeartbeatAck(DuplexEvent): - wire_type = "session.heartbeat_ack" - - def _wire_fields(self) -> dict[str, object]: - return {"session_id": self.session_id} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class TurnEvent(DuplexEvent): - """Turn-state transition (``turn.event``).""" - - wire_type = "turn.event" - - event: str = "" - turn_state: str = "" - - def _wire_fields(self) -> dict[str, object]: - return {"session_id": self.session_id, "event": self.event, "turn_state": self.turn_state, "epoch": self.epoch} - - -# ---- response lifecycle ---- - - -@dataclass(frozen=True, slots=True, kw_only=True) -class _ResponseEvent(DuplexEvent): - """Events addressed to one response (and usually one output item / content part).""" - - response_id: str | None = None # type: ignore[assignment] - item_id: str | None = None # type: ignore[assignment] - output_index: int = 0 - content_index: int = 0 - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ResponseCreated(DuplexEvent): - wire_type = "response.created" - - response_id: str | None = None # type: ignore[assignment] - response: Mapping[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class Listen(DuplexEvent): - """The model decided to keep listening (no spoken response for this turn).""" - - wire_type = "response.listen" - optional_wire_fields = frozenset({"response_id"}) - - response_id: str | None = None # type: ignore[assignment] - details: Mapping[str, object] = field(default_factory=dict) - - def _wire_fields(self) -> dict[str, object]: - response: dict[str, object] = { - "object": "realtime.response", - "status": "listening", - "metadata": _wire_value(self.details), - } - data: dict[str, object] = {"session_id": self.session_id, "epoch": self.epoch, "response": response} - if self.response_id: - response["id"] = self.response_id - data["response_id"] = self.response_id - return data - - -@dataclass(frozen=True, slots=True, kw_only=True) -class Speak(_ResponseEvent): - wire_type = "response.speak" - - metadata: Mapping[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class OverlapDecision(DuplexEvent): - wire_type = "overlap.decision" - - policy: str | None = None - action: str | None = None - reason: str | None = None - details: Mapping[str, object] = field(default_factory=dict) - - def _wire_fields(self) -> dict[str, object]: - return { - "session_id": self.session_id, - "epoch": self.epoch, - "policy": self.policy, - "action": self.action, - "reason": self.reason, - "metadata": _wire_value(self.details), - } - - -@dataclass(frozen=True, slots=True, kw_only=True) -class OutputItemAdded(DuplexEvent): - wire_type = "response.output_item.added" - - response_id: str | None = None # type: ignore[assignment] - output_index: int = 0 - item: Mapping[str, object] = field(default_factory=dict) - - @property - def item_id(self) -> str | None: - value = self.item.get("id") - return value if isinstance(value, str) else None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class OutputItemDone(OutputItemAdded): - wire_type = "response.output_item.done" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ContentPartAdded(_ResponseEvent): - wire_type = "response.content_part.added" - - part: Mapping[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ContentPartDone(ContentPartAdded): - wire_type = "response.content_part.done" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class AudioDelta(_ResponseEvent): - wire_type = "response.output_audio.delta" - optional_wire_fields = frozenset({"sample_rate_hz", "metadata"}) - - #: Base64 audio in ``format``. - delta: str = "" - format: str = "pcm16" - sample_rate_hz: int | None = None - metadata: Mapping[str, object] | None = None - - @property - def audio(self) -> bytes | None: - try: - return base64.b64decode(self.delta) - except (ValueError, TypeError): - return None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class AudioDone(_ResponseEvent): - wire_type = "response.output_audio.done" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class TranscriptDelta(_ResponseEvent): - wire_type = "response.output_audio_transcript.delta" - - delta: str = "" - - @property - def text(self) -> str | None: - return self.delta - - -@dataclass(frozen=True, slots=True, kw_only=True) -class TranscriptDone(_ResponseEvent): - wire_type = "response.output_audio_transcript.done" - - transcript: str = "" - - @property - def text(self) -> str | None: - return self.transcript - - -@dataclass(frozen=True, slots=True, kw_only=True) -class TextDelta(_ResponseEvent): - wire_type = "response.output_text.delta" - - delta: str = "" - - @property - def text(self) -> str | None: - return self.delta - - -@dataclass(frozen=True, slots=True, kw_only=True) -class TextDone(_ResponseEvent): - wire_type = "response.output_text.done" - - text: str = "" # type: ignore[assignment] - - -@dataclass(frozen=True, slots=True, kw_only=True) -class FunctionCallArgumentsDelta(DuplexEvent): - wire_type = "response.function_call_arguments.delta" - - response_id: str | None = None # type: ignore[assignment] - item_id: str | None = None # type: ignore[assignment] - output_index: int = 0 - call_id: str = "" - delta: str = "" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class FunctionCallArgumentsDone(DuplexEvent): - wire_type = "response.function_call_arguments.done" - - response_id: str | None = None # type: ignore[assignment] - item_id: str | None = None # type: ignore[assignment] - output_index: int = 0 - call_id: str = "" - arguments: str = "" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ResponseDone(DuplexEvent): - wire_type = "response.done" - - response_id: str | None = None # type: ignore[assignment] - response: Mapping[str, object] = field(default_factory=dict) - - @property - def status(self) -> str | None: - value = self.response.get("status") - return value if isinstance(value, str) else None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class RateLimitsUpdated(DuplexEvent): - wire_type = "rate_limits.updated" - - rate_limits: tuple[Mapping[str, object], ...] = () - - -# ---- input buffer / conversation ---- - - -@dataclass(frozen=True, slots=True, kw_only=True) -class InputCommitted(DuplexEvent): - wire_type = "input_audio_buffer.committed" - - previous_item_id: str | None = None - item_id: str | None = None # type: ignore[assignment] - #: The session-internal ``input.committed`` event. - details: Mapping[str, object] = field(default_factory=dict) - - def _wire_fields(self) -> dict[str, object]: - return { - "previous_item_id": self.previous_item_id, - "item_id": self.item_id, - "event": _wire_value(self.details), - } - - -@dataclass(frozen=True, slots=True, kw_only=True) -class InputCleared(DuplexEvent): - wire_type = "input_audio_buffer.cleared" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SpeechStarted(DuplexEvent): - wire_type = "input_audio_buffer.speech_started" - - audio_start_ms: int = 0 - item_id: str | None = None # type: ignore[assignment] - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SpeechStopped(DuplexEvent): - wire_type = "input_audio_buffer.speech_stopped" - - audio_end_ms: int = 0 - item_id: str | None = None # type: ignore[assignment] - - -@dataclass(frozen=True, slots=True, kw_only=True) -class _ItemEvent(DuplexEvent): - previous_item_id: str | None = None - item: Mapping[str, object] = field(default_factory=dict) - - @property - def item_id(self) -> str | None: - value = self.item.get("id") - return value if isinstance(value, str) else None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ItemAdded(_ItemEvent): - wire_type = "conversation.item.added" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ItemCreated(_ItemEvent): - wire_type = "conversation.item.created" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ItemDone(_ItemEvent): - wire_type = "conversation.item.done" - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ItemRetrieved(DuplexEvent): - wire_type = "conversation.item.retrieved" - - item: Mapping[str, object] = field(default_factory=dict) - - @property - def item_id(self) -> str | None: - value = self.item.get("id") - return value if isinstance(value, str) else None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ItemDeleted(DuplexEvent): - wire_type = "conversation.item.deleted" - - item_id: str | None = None # type: ignore[assignment] - details: Mapping[str, object] = field(default_factory=dict) - - def _wire_fields(self) -> dict[str, object]: - return {"item_id": self.item_id, "event": _wire_value(self.details)} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ItemTruncated(DuplexEvent): - wire_type = "conversation.item.truncated" - - item_id: str | None = None # type: ignore[assignment] - content_index: int = 0 - audio_end_ms: int = 0 - details: Mapping[str, object] = field(default_factory=dict) - - def _wire_fields(self) -> dict[str, object]: - return { - "item_id": self.item_id, - "content_index": self.content_index, - "audio_end_ms": self.audio_end_ms, - "event": _wire_value(self.details), - } - - -@dataclass(frozen=True, slots=True, kw_only=True) -class InputTranscriptionCompleted(DuplexEvent): - wire_type = "conversation.item.input_audio_transcription.completed" - - item_id: str | None = None # type: ignore[assignment] - content_index: int = 0 - transcript: str = "" - - @property - def text(self) -> str | None: - return self.transcript - - -# ---- playback / control ---- - - -@dataclass(frozen=True, slots=True, kw_only=True) -class OutputAudioCleared(DuplexEvent): - wire_type = "output_audio_buffer.cleared" - - response_id: str | None = None # type: ignore[assignment] - - -@dataclass(frozen=True, slots=True, kw_only=True) -class PlaybackAcknowledged(DuplexEvent): - wire_type = "playback.acknowledged" - - #: The session-internal ``playback.acknowledged`` event (cursor, committed_ms ...). - details: Mapping[str, object] = field(default_factory=dict) - - def _wire_fields(self) -> dict[str, object]: - return {"event": _wire_value(self.details)} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ErrorEvent(DuplexEvent): - wire_type = "error" - - code: str = "internal_error" - message: str = "" - #: Client ``event_id`` this error answers (OpenAI ``error.event_id``). - related_event_id: str | None = None - param: str | None = None - #: Extra keys merged into the wire ``error`` object (``retryable`` ...). - extra: Mapping[str, object] = field(default_factory=dict) - - @property - def error_type(self) -> str: - return REALTIME_ERROR_TYPES_BY_CODE.get(self.code, "invalid_request_error") - - @property - def error(self) -> dict[str, object]: - error: dict[str, object] = {"type": self.error_type, "code": self.code, "message": self.message} - if self.related_event_id: - error["event_id"] = self.related_event_id - if self.param: - error["param"] = self.param - error.update(cast("Mapping[str, object]", _wire_value(self.extra))) - return error - - @property - def text(self) -> str | None: - return self.message - - def _wire_fields(self) -> dict[str, object]: - return {"error": self.error} - - -@dataclass(frozen=True, slots=True, kw_only=True) -class DuplexRawEvent(DuplexEvent): - """A session-internal event with no dedicated Realtime type (``duplex.``).""" - - wire_type = "duplex.raw" - - internal_type: str = "" - details: Mapping[str, object] = field(default_factory=dict) - - @property - def type(self) -> str: - return f"duplex.{self.internal_type}" - - def to_realtime(self) -> dict[str, object]: - return {"type": self.type, "event_id": self.event_id, "event": _wire_value(self.details)} - - -def error_event( - code: str, - message: str, - *, - event_id: object | None = None, - param: object | None = None, - extra: Mapping[str, object] | None = None, -) -> ErrorEvent: - """Build the error event for an internal code; ``event_id`` is the *client* event id.""" - return ErrorEvent( - code=code, - message=message, - related_event_id=event_id if isinstance(event_id, str) and event_id else None, - param=param if isinstance(param, str) and param else None, - extra=dict(extra or {}), - ) - - -#: Internal events that terminate a response/session and must never be dropped as stale. DOMAIN_TERMINAL_EVENTS = frozenset( { "response.done", @@ -682,7 +84,6 @@ DOMAIN_TERMINAL_EVENTS = frozenset( } ) -#: Internal streaming model-output events subject to epoch/close stale filtering. MODEL_OUTPUT_EVENTS = frozenset( { "response.created", @@ -705,13 +106,11 @@ MODEL_OUTPUT_EVENTS = frozenset( __all__ = [ - "DOMAIN_TERMINAL_EVENTS", - "MODEL_OUTPUT_EVENTS", - "REALTIME_ERROR_TYPES_BY_CODE", "AudioDelta", "AudioDone", "ContentPartAdded", "ContentPartDone", + "DOMAIN_TERMINAL_EVENTS", "DuplexEvent", "DuplexRawEvent", "ErrorEvent", @@ -727,11 +126,13 @@ __all__ = [ "ItemRetrieved", "ItemTruncated", "Listen", + "MODEL_OUTPUT_EVENTS", "OutputAudioCleared", "OutputItemAdded", "OutputItemDone", "OverlapDecision", "PlaybackAcknowledged", + "REALTIME_ERROR_TYPES_BY_CODE", "RateLimitsUpdated", "ResponseCreated", "ResponseDone", diff --git a/vllm_omni/engine/duplex/realtime_commands.py b/vllm_omni/engine/duplex/realtime_commands.py index 370364017..31b1c0949 100644 --- a/vllm_omni/engine/duplex/realtime_commands.py +++ b/vllm_omni/engine/duplex/realtime_commands.py @@ -3,26 +3,31 @@ """OpenAI Realtime client events -> :class:`DuplexCommand` (stateless). -This is the mapping half of the former ``entrypoints/duplex/realtime_input.py`` -translator. Everything that needed per-session state in the old translator -(input-buffer emptiness for commits, response-id fallbacks for cancels, -conversation-item lookups, VAD) is resolved by the session runner through the -helpers on :class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`; +This module is the *duplex binding* of the shared Realtime codec. Parsing, +validating and decoding a client event is not duplex-specific and lives in +``vllm_omni.protocol.realtime``, reached through ``vllm_omni.protocol.duplex``; +what lives here is the part that is: which +:class:`~vllm_omni.engine.duplex.commands.DuplexCommand` a decoded event +becomes, what the duplex engine can serve +(:data:`DUPLEX_REALTIME_CAPABILITIES`), and the mapping between a Realtime +output format and the duplex ``response_format`` vocabulary. + +Everything that needs per-session state (input-buffer emptiness for commits, +response-id fallbacks for cancels, conversation-item lookups, VAD) is resolved +by the session runner through the helpers on +:class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`; the commands produced here carry the raw client intent only. + +The shared names re-exported at the bottom are a compatibility surface for +existing importers. The canonical definitions are in +``vllm_omni.protocol.realtime`` and there is exactly one of each. """ from __future__ import annotations -import base64 -import binascii from collections.abc import Mapping -from dataclasses import dataclass, replace -from typing import Any, cast -from uuid import uuid4 +from typing import cast -import numpy as np - -from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate from vllm_omni.engine.duplex.commands import ( AckPlayback, AppendAudio, @@ -44,93 +49,58 @@ from vllm_omni.engine.duplex.commands import ( TruncateItem, UpdateSession, ) - -REALTIME_INPUT_AUDIO_FORMATS = { - "pcm16", - "pcm_s16le", - "s16le", - "pcm_f32le", - "g711_ulaw", - "g711_alaw", -} -REALTIME_OUTPUT_AUDIO_FORMATS = { - "pcm16", - "pcm_s16le", - "s16le", - "wav", - "pcm", - "g711_ulaw", - "g711_alaw", -} - -#: Wire hint keys copied verbatim from a client append onto the internal payload. -REALTIME_INPUT_HINT_KEYS = ( - "duration_ms", - "audio_duration_ms", - "audio_start_ms", - "audio_end_ms", - "is_speech", - "speech", - "speech_probability", - "vad", - "overlap_action", - "overlap", - "force_barge_in", - "force_listen", - "text", - "transcript", +from vllm_omni.protocol.duplex import ( + REALTIME_INPUT_AUDIO_FORMATS, + REALTIME_INPUT_HINT_KEYS, + REALTIME_OUTPUT_AUDIO_FORMATS, + RealtimeInputDefaults, + RealtimeProtocolCapabilities, + RealtimeProtocolError, + apply_realtime_session_defaults, + copy_realtime_input_hints, + decode_audio_append, + input_audio_transcription_config, + input_explicitly_non_speech, + input_looks_like_speech, + input_transcript_from_item, + is_supported_realtime_input_format, + json_safe_realtime_payload, + normalize_conversation_item, + parse_realtime_audio_format, + realtime_audio_format_object, + realtime_max_output_tokens, + realtime_output_format, + realtime_overlap_fields, + text_chars_for_audio_ms_from_marks, + truncate_realtime_item_content, + validate_conversation_item_audio_formats, + validate_realtime_item_truncate, + validate_realtime_response_audio_formats, + validate_realtime_session_audio_formats, + validate_realtime_video_frames, + validate_session_payload, ) -@dataclass(frozen=True, slots=True) -class RealtimeInputDefaults: - """Session-level wire defaults an append may omit (derived from the session object).""" +def _validate_duplex_turn_detection(session_payload: Mapping[str, object]) -> str | None: + """The duplex engine's answer for ``turn_detection`` on a session object. - input_audio_format: str = "pcm16" - input_sample_rate_hz: int = 16000 - output_audio_format: str = "pcm16" - output_sample_rate_hz: int | None = None - overlap_silence_rms: float = 0.003 + Imported lazily: the validator lives with the Silero VAD backend, and a + connection that never sends a session object should not pay for loading it. + """ + from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection - def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults: - """Return defaults updated from a Realtime ``session`` object (session.update).""" - values = apply_realtime_session_defaults(self, session_payload) - return values + return validate_realtime_turn_detection(session_payload) -# ---- audio format helpers ---- - - -def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]: - def normalize_format(fmt: str) -> str: - normalized = fmt.lower() - if normalized in {"audio/pcm", "pcm"}: - return "pcm16" - if normalized in {"audio/wav", "wav"}: - return "wav" - if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}: - return "pcm16" - if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}: - return "pcm_f32le" - if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}: - return "g711_ulaw" - if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}: - return "g711_alaw" - return fmt - - if isinstance(raw_format, str): - return normalize_format(raw_format), None - if not isinstance(raw_format, dict): - return raw_format, None - rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate") - sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None - fmt = raw_format.get("type") or raw_format.get("format") - if not isinstance(fmt, str): - return raw_format, sample_rate_hz - return normalize_format(fmt), sample_rate_hz +#: What the duplex engine can serve, for the shared session-object validator. +DUPLEX_REALTIME_CAPABILITIES = RealtimeProtocolCapabilities( + validate_turn_detection=_validate_duplex_turn_detection, +) def duplex_response_format(realtime_format: str) -> str: + """A Realtime output format -> the duplex ``response_format`` vocabulary.""" normalized = realtime_format.lower() if normalized in {"pcm16", "pcm_s16le", "s16le"}: return "pcm" @@ -141,433 +111,6 @@ def duplex_response_format(realtime_format: str) -> str: return "wav" -def realtime_output_format(duplex_format: object) -> str: - if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}: - return duplex_format.lower() - if isinstance(duplex_format, str) and duplex_format.lower() == "pcm": - return "pcm16" - return str(duplex_format or "wav") - - -def is_supported_realtime_input_format(fmt: object) -> bool: - return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS - - -def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]: - if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}: - payload: dict[str, object] = {"type": "audio/pcm"} - elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le": - payload = {"type": "audio/pcm_f32le"} - elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw": - payload = {"type": "audio/g711_ulaw"} - elif isinstance(fmt, str) and fmt.lower() == "g711_alaw": - payload = {"type": "audio/g711_alaw"} - else: - payload = {"type": "audio/wav"} - if sample_rate_hz is not None: - payload["rate"] = int(sample_rate_hz) - return payload - - -def validate_realtime_session_audio_formats(session_payload: Mapping[str, object]) -> str | None: - audio_config = session_payload.get("audio") - input_format: object = session_payload.get("input_audio_format") - if input_format is None and isinstance(audio_config, dict): - audio_input = audio_config.get("input") - if isinstance(audio_input, dict): - input_format = audio_input.get("format") - parsed_input, _ = parse_realtime_audio_format(input_format) - if input_format is not None and not ( - isinstance(parsed_input, str) and parsed_input.lower() in REALTIME_INPUT_AUDIO_FORMATS - ): - return f"Unsupported input_audio_format: {input_format}" - - output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format") - if output_format is None and isinstance(audio_config, dict): - audio_output = audio_config.get("output") - if isinstance(audio_output, dict): - output_format = audio_output.get("format") - parsed_output, _ = parse_realtime_audio_format(output_format) - if output_format is not None and not ( - isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS - ): - return f"Unsupported output_audio_format: {output_format}" - return None - - -def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None: - output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format") - audio_config = response_payload.get("audio") - if output_format is None and isinstance(audio_config, dict): - audio_output = audio_config.get("output") - if isinstance(audio_output, dict): - output_format = audio_output.get("format") - parsed_output, _ = parse_realtime_audio_format(output_format) - if output_format is not None and not ( - isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS - ): - return f"Unsupported output_audio_format: {output_format}" - return None - - -def validate_conversation_item_audio_formats(item: object) -> str | None: - if not isinstance(item, dict): - return None - content = item.get("content") - if not isinstance(content, list): - return None - for part in content: - if not isinstance(part, dict): - continue - if part.get("type") not in {"input_audio", "audio"}: - continue - raw_format = part.get("format") - parsed_format, _ = parse_realtime_audio_format(raw_format) - if raw_format is not None and not ( - isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS - ): - return f"Unsupported input_audio format in conversation.item.create: {raw_format}" - return None - - -def apply_realtime_session_defaults( - defaults: RealtimeInputDefaults, - session_payload: Mapping[str, object], -) -> RealtimeInputDefaults: - """Derive the wire defaults a Realtime ``session`` object declares.""" - input_format: object = session_payload.get("input_audio_format") - audio_config = session_payload.get("audio") - if input_format is None and isinstance(audio_config, dict): - audio_input = audio_config.get("input") - if isinstance(audio_input, dict): - input_format = audio_input.get("format") - input_format, input_rate = parse_realtime_audio_format(input_format) - # Any: heterogeneous field values splatted into ``dataclasses.replace`` below. - updates: dict[str, Any] = {} - if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS: - updates["input_audio_format"] = input_format - output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format") - output_rate_raw: object | None = None - if output_format is None and isinstance(audio_config, dict): - audio_output = audio_config.get("output") - if isinstance(audio_output, dict): - output_format = audio_output.get("format") - output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate") - output_format, output_rate = parse_realtime_audio_format(output_format) - if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0: - output_rate = int(output_rate_raw) - if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: - updates["output_audio_format"] = realtime_output_format(output_format) - sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate") - if sample_rate is None and isinstance(audio_config, dict): - audio_input = audio_config.get("input") - if isinstance(audio_input, dict): - sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate") - if sample_rate is None: - sample_rate = input_rate - if isinstance(sample_rate, int | float) and sample_rate > 0: - updates["input_sample_rate_hz"] = int(sample_rate) - if isinstance(output_rate, int | float) and output_rate > 0: - updates["output_sample_rate_hz"] = int(output_rate) - overlap_fields = realtime_overlap_fields(session_payload) - overlap_silence_rms = overlap_fields.get("overlap_silence_rms") - if isinstance(overlap_silence_rms, int | float): - updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms)) - return replace(defaults, **updates) if updates else defaults - - -def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]: - fields: dict[str, object] = {} - if isinstance(session_payload.get("overlap_policy"), str): - fields["overlap_policy"] = session_payload["overlap_policy"] - for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"): - value = session_payload.get(key) - if isinstance(value, int | float): - fields[key] = value - if isinstance(session_payload.get("playback_commit_policy"), str): - fields["playback_commit_policy"] = session_payload["playback_commit_policy"] - return fields - - -def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]: - clean: dict[str, object] = {} - for key, value in payload.items(): - if key == "extra_body": - continue - if isinstance(value, str | int | float | bool) or value is None: - clean[key] = value - elif isinstance(value, dict): - clean[key] = json_safe_realtime_payload(value) - elif isinstance(value, list): - clean[key] = [ - (json_safe_realtime_payload(item) if isinstance(item, dict) else item) - for item in value - if isinstance(item, str | int | float | bool | dict) or item is None - ] - return clean - - -def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None: - transcription = session_payload.get("input_audio_transcription") - if isinstance(transcription, dict): - return transcription - audio_config = session_payload.get("audio") - if not isinstance(audio_config, dict): - return None - audio_input = audio_config.get("input") - if not isinstance(audio_input, dict): - return None - transcription = audio_input.get("transcription") - return transcription if isinstance(transcription, dict) else None - - -def realtime_max_output_tokens(value: object) -> int | None: - """Normalize Realtime max output tokens (``"inf"`` -> ``None``).""" - if value is None: - return None - if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}: - return None - if isinstance(value, int) and value > 0: - return int(value) - return None - - -# ---- speech hints ---- - - -def input_explicitly_non_speech(event: Mapping[str, object]) -> bool: - for key in ("is_speech", "speech"): - value = event.get(key) - if isinstance(value, bool): - return not value - vad = event.get("vad") - if isinstance(vad, dict): - value = vad.get("is_speech") - if isinstance(value, bool): - return not value - probability = vad.get("speech_probability", vad.get("probability")) - if isinstance(probability, int | float): - return float(probability) < 0.5 - probability = event.get("speech_probability") - return isinstance(probability, int | float) and float(probability) < 0.5 - - -def input_looks_like_speech( - event: Mapping[str, object], - *, - audio: object, - fmt: object, - overlap_silence_rms: float, -) -> bool: - if input_explicitly_non_speech(event): - return False - for key in ("is_speech", "speech"): - value = event.get(key) - if isinstance(value, bool): - return value - vad = event.get("vad") - if isinstance(vad, dict): - probability = vad.get("speech_probability", vad.get("probability")) - if isinstance(probability, int | float): - return float(probability) >= 0.5 - probability = event.get("speech_probability") - if isinstance(probability, int | float): - return float(probability) >= 0.5 - if fmt != "pcm_f32le": - return True - if isinstance(audio, bytes | bytearray): - raw = bytes(audio) - elif isinstance(audio, str): - try: - raw = base64.b64decode(audio, validate=True) - except (binascii.Error, ValueError): - return True - else: - return True - if len(raw) < 4 or len(raw) % 4 != 0: - return True - samples = np.frombuffer(raw, dtype=np.float32) - if samples.size == 0: - return False - rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32))))) - threshold = event.get("overlap_silence_rms") - if not isinstance(threshold, int | float): - vad = event.get("vad") - if isinstance(vad, dict): - threshold = vad.get("silence_rms") - silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms - return rms >= max(0.0, silence_rms) - - -def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None: - for key in REALTIME_INPUT_HINT_KEYS: - if key in source: - target[key] = source[key] - - -def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None: - """Validate omni-duplex camera frames on input_audio_buffer.append. - - Wire contract matches the official MiniCPM-o duplex loop: one base - base64 JPEG per ~1 s audio chunk, optionally followed by that unit's - stacked composite tiling the sub-frames captured inside it (at most 2 - images either way). A caller-supplied ``max_slice_nums`` is rejected rather - than silently ignored: slicing is Stage 0's decision here, and Stage 0 - already applies the official HD suggestion for a stacked unit - (``max_slice_nums=[2, 1]``). The wire simply does not let the client choose - it. - """ - if max_slice_nums not in (None, 1): - return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself" - if not isinstance(video_frames, list): - return "video_frames must be a list of base64-encoded images" - frames = [frame for frame in video_frames if frame is not None] - if len(frames) > 2: - return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk" - for frame in frames: - if not isinstance(frame, str) or not frame: - return "video_frames entries must be non-empty base64 strings" - if len(frame) > 4_000_000: - return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality" - try: - header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4)) - except (binascii.Error, ValueError): - return "video_frames entries must be valid base64" - if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")): - return "video_frames entries must be JPEG or PNG images" - return None - - -# ---- conversation items ---- - - -def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]: - normalized = dict(item) - normalized.setdefault("id", f"item_{uuid4().hex}") - normalized.setdefault("object", "realtime.item") - normalized.setdefault("type", "message") - normalized.setdefault("status", "completed") - if "role" not in normalized and normalized.get("type") == "message": - normalized["role"] = "user" - if not isinstance(normalized.get("content"), list): - normalized["content"] = [] - return normalized - - -def text_chars_for_audio_ms_from_marks( - audio_end_ms: int, - text_len: int, - marks: list[object], - *, - final_ms: object | None = None, -) -> int: - if text_len <= 0: - return 0 - clean_marks: list[tuple[int, int]] = [] - for mark in marks: - if not isinstance(mark, dict): - continue - raw_text_chars = mark.get("text_chars") - raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms")) - if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float): - continue - clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars))))) - if not clean_marks: - return 0 if audio_end_ms <= 0 else text_len - clean_marks.sort(key=lambda item: item[0]) - audio_end_ms = max(0, int(audio_end_ms)) - if audio_end_ms <= 0: - return 0 - previous_ms = 0 - previous_chars = 0 - for mark_ms, mark_chars in clean_marks: - mark_ms = max(previous_ms, mark_ms) - mark_chars = max(previous_chars, min(text_len, mark_chars)) - if audio_end_ms <= mark_ms: - if mark_ms <= previous_ms: - return mark_chars - ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms) - return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio))) - previous_ms = mark_ms - previous_chars = mark_chars - if isinstance(final_ms, int | float) and int(final_ms) > previous_ms: - if audio_end_ms >= int(final_ms): - return text_len - ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms) - return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio))) - return text_len if audio_end_ms >= previous_ms else previous_chars - - -def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None: - content = item.get("content") - if not isinstance(content, list) or not content: - return - index = max(0, int(content_index)) - if index >= len(content): - return - part = content[index] - if not isinstance(part, dict): - return - transcript = part.get("transcript") - if not isinstance(transcript, str) or not transcript: - return - marks = part.get("audio_text_marks") - if isinstance(marks, list): - keep_chars = text_chars_for_audio_ms_from_marks( - audio_end_ms, - len(transcript), - marks, - final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"), - ) - part["transcript"] = transcript[:keep_chars].rstrip() - return - duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms") - if isinstance(duration_ms, int | float) and duration_ms > 0: - keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms)))) - part["transcript"] = transcript[:keep_chars].rstrip() - elif audio_end_ms <= 0: - part["transcript"] = "" - - -def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None: - if item.get("type") != "message" or item.get("role") != "assistant": - return "conversation.item.truncate only supports assistant message items" - if audio_end_ms < 0: - return "conversation.item.truncate requires non-negative audio_end_ms" - content = item.get("content") - if not isinstance(content, list) or not content: - return None - index = max(0, int(content_index)) - if index >= len(content): - return f"conversation.item.truncate content_index out of range: {content_index}" - part = content[index] - if not isinstance(part, dict): - return "conversation.item.truncate target content part is invalid" - if part.get("type") not in {"audio", "output_audio"}: - return "conversation.item.truncate target content part must be audio" - duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms") - if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms): - return "conversation.item.truncate audio_end_ms exceeds item audio duration" - return None - - -def input_transcript_from_item(item: Mapping[str, object]) -> str: - content = item.get("content") - if not isinstance(content, list): - return "" - parts: list[str] = [] - for part in content: - if not isinstance(part, dict): - continue - for key in ("transcript", "text"): - value = part.get(key) - if isinstance(value, str) and value: - if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}: - parts.append(value) - break - return "".join(parts).strip() - - # ---- append / item audio conversion ---- @@ -577,67 +120,21 @@ def build_append_audio( defaults: RealtimeInputDefaults, hints_source: Mapping[str, object] | None = None, ) -> AppendAudio: - """Validate, convert (to 16 kHz ``pcm_f32le`` base64) and pack one audio append. - - Speech classification without VAD (``is_speech`` on the command) uses the - explicit wire hints and the RMS fallback exactly like the old translator. - """ - event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None - audio = event.get("audio") or event.get("delta") - fmt, format_rate = parse_realtime_audio_format( - event.get("format") or event.get("input_audio_format") or defaults.input_audio_format - ) - sample_rate_hz = ( - event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz - ) - if not is_supported_realtime_input_format(fmt): - raise DuplexCommandError( - f"Unsupported input_audio_format: {fmt}", - code="unsupported_audio_format", - event_id=event_id, - ) + """Decode one audio append (shared codec) and pack it as the duplex command.""" try: - audio, fmt, sample_rate_hz = convert_input_audio_with_rate( - audio, - fmt, - sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None, - ) - except ValueError as exc: - raise DuplexCommandError(str(exc), code="bad_event", event_id=event_id) from exc - hints: dict[str, object] = {} - if hints_source is not None: - copy_realtime_input_hints(hints_source, hints) - copy_realtime_input_hints(event, hints) - looks_like_speech = input_looks_like_speech( - {**(hints_source or {}), **event}, - audio=audio, - fmt=fmt, - overlap_silence_rms=defaults.overlap_silence_rms, - ) - video_frames: tuple[str, ...] = () - raw_frames = event.get("video_frames") - if raw_frames is not None: - frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums")) - if frames_error is not None: - raise DuplexCommandError(frames_error, code="invalid_video_frames", event_id=event_id) - if isinstance(raw_frames, list): - video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame) - duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms")) - audio_end_ms = hints.get("audio_end_ms") - try: - audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b"" - except (binascii.Error, ValueError) as exc: - raise DuplexCommandError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc + decoded = decode_audio_append(event, defaults=defaults, hints_source=hints_source) + except RealtimeProtocolError as exc: + raise DuplexCommandError(str(exc), code=exc.code, event_id=exc.event_id) from exc return AppendAudio( - event_id=event_id, - audio=audio_bytes, - format=str(fmt), - sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None, - is_speech=looks_like_speech, - video_frames=video_frames, - duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None, - audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None, - hints=hints, + event_id=decoded.event_id, + audio=decoded.audio, + format=decoded.format, + sample_rate_hz=decoded.sample_rate_hz, + is_speech=decoded.is_speech, + video_frames=decoded.video_frames, + duration_ms=decoded.duration_ms, + audio_end_ms=decoded.audio_end_ms, + hints=decoded.hints, ) @@ -664,14 +161,9 @@ def translate_realtime_command( if event_type == "session.update": session = payload.get("session") session_payload: Mapping[str, object] = session if isinstance(session, dict) else payload - format_error = validate_realtime_session_audio_formats(session_payload) - if format_error is not None: - raise DuplexCommandError(format_error, code="unsupported_audio_format", event_id=event_id) - from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection - - turn_detection_error = validate_realtime_turn_detection(session_payload) - if turn_detection_error is not None: - raise DuplexCommandError(turn_detection_error, code="unsupported_turn_detection", event_id=event_id) + rejection = validate_session_payload(session_payload, capabilities=DUPLEX_REALTIME_CAPABILITIES) + if rejection is not None: + raise DuplexCommandError(rejection.message, code=rejection.code, event_id=event_id) return UpdateSession(event_id=event_id, patch=dict(session_payload)) if event_type == "conversation.item.create": @@ -855,7 +347,10 @@ def translate_realtime_command( raise DuplexCommandError(f"Unknown duplex event type: {event_type}", code="unknown_event", event_id=event_id) +#: Names this module used to define, re-exported so existing importers keep +#: working. The canonical definitions live in ``vllm_omni.protocol.realtime``. __all__ = [ + "DUPLEX_REALTIME_CAPABILITIES", "REALTIME_INPUT_AUDIO_FORMATS", "REALTIME_INPUT_HINT_KEYS", "REALTIME_OUTPUT_AUDIO_FORMATS", diff --git a/vllm_omni/engine/duplex/realtime_events.py b/vllm_omni/engine/duplex/realtime_events.py index 91359f297..f67abbbc9 100644 --- a/vllm_omni/engine/duplex/realtime_events.py +++ b/vllm_omni/engine/duplex/realtime_events.py @@ -15,6 +15,12 @@ carries the input-side bookkeeping the old input translator kept (input buffer flags, conversation items, response-id fallbacks); the ``resolve_*`` / ``note_*`` helpers give the runner the same behaviour for the corresponding commands. + +What is *not* here is the model- and runtime-agnostic half of the codec --- +audio format negotiation, conversation-item shape and truncation, transcript +extraction, audio conversion. That lives in ``vllm_omni.protocol.realtime`` so +a non-duplex Realtime surface can use it without the duplex session; this +module is the duplex consumer of it (RFC #6592 P0a). """ from __future__ import annotations @@ -24,7 +30,6 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, cast from uuid import uuid4 -from vllm_omni.engine.duplex.audio import convert_output_audio from vllm_omni.engine.duplex.commands import ( AppendAudio, CancelResponse, @@ -74,10 +79,11 @@ from vllm_omni.engine.duplex.events import ( TranscriptDone, error_event, ) -from vllm_omni.engine.duplex.realtime_commands import ( +from vllm_omni.engine.duplex.realtime_commands import build_append_audio +from vllm_omni.protocol.duplex import ( RealtimeInputDefaults, apply_realtime_session_defaults, - build_append_audio, + convert_output_audio, copy_realtime_input_hints, input_looks_like_speech, input_transcript_from_item, diff --git a/vllm_omni/engine/duplex/session/runner.py b/vllm_omni/engine/duplex/session/runner.py index 7915500b7..0b8af49ab 100644 --- a/vllm_omni/engine/duplex/session/runner.py +++ b/vllm_omni/engine/duplex/session/runner.py @@ -28,7 +28,6 @@ from typing import TYPE_CHECKING, TypeVar from vllm.logger import init_logger -from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate from vllm_omni.engine.duplex.commands import ( AckPlayback, AppendAudio, @@ -103,6 +102,7 @@ from vllm_omni.engine.duplex.turn_detection import ( TurnDetectionResult, ) from vllm_omni.metrics.stats import StageRequestStats +from vllm_omni.protocol.duplex import convert_input_audio_with_rate if TYPE_CHECKING: from vllm.config import ModelConfig diff --git a/vllm_omni/entrypoints/duplex/realtime_input.py b/vllm_omni/entrypoints/duplex/realtime_input.py index be700ef62..79a081696 100644 --- a/vllm_omni/entrypoints/duplex/realtime_input.py +++ b/vllm_omni/entrypoints/duplex/realtime_input.py @@ -5,7 +5,9 @@ Everything that needed session state now lives engine-side (``vllm_omni.engine.duplex.realtime_commands`` for the command mapping, -``vllm_omni.engine.duplex.realtime_events`` for the projection state). What is +``vllm_omni.engine.duplex.realtime_events`` for the projection state), and the +model-agnostic parsing both of those build on lives in +``vllm_omni.protocol.realtime``. What is left here is the per-connection handshake policy (query-param defaults, autostart / resume-only rules, ``session.resume`` parsing), the wire defaults used to translate appends, and error rendering through the typed @@ -20,10 +22,8 @@ from typing import TYPE_CHECKING from vllm_omni.engine.duplex.commands import DuplexCommand, DuplexCommandError from vllm_omni.engine.duplex.events import error_event -from vllm_omni.engine.duplex.realtime_commands import ( - RealtimeInputDefaults, - translate_realtime_command, -) +from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command +from vllm_omni.protocol.duplex import RealtimeInputDefaults if TYPE_CHECKING: from starlette.websockets import WebSocket diff --git a/vllm_omni/entrypoints/duplex/serving.py b/vllm_omni/entrypoints/duplex/serving.py index 364f715c0..ff1abbe25 100644 --- a/vllm_omni/entrypoints/duplex/serving.py +++ b/vllm_omni/entrypoints/duplex/serving.py @@ -36,7 +36,6 @@ from vllm_omni.engine.duplex.events import ( SessionResyncRequired, ) from vllm_omni.engine.duplex.messages import DuplexSessionError -from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults from vllm_omni.entrypoints.duplex.realtime_input import RealtimeEnvelope, parse_resume_request from vllm_omni.entrypoints.duplex.session_attachment import ( DuplexJournalGapError, @@ -52,6 +51,7 @@ from vllm_omni.entrypoints.duplex.websocket import ( receive_text_with_timeout, ) from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle +from vllm_omni.protocol.duplex import RealtimeInputDefaults logger = init_logger(__name__) diff --git a/vllm_omni/protocol/__init__.py b/vllm_omni/protocol/__init__.py new file mode 100644 index 000000000..e241a7924 --- /dev/null +++ b/vllm_omni/protocol/__init__.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Wire protocols vLLM-Omni speaks, independent of who serves or runs them. + +A module under ``vllm_omni.protocol`` describes *what is on the wire*: how a +client payload is parsed and validated, and what shape a server payload has. +It never decides how a request is executed, which model answers it, or where +the session state lives. That keeps one protocol implementation usable by more +than one runtime (see ``vllm_omni.protocol.realtime``). + +Dependency rule, asserted by ``tests/protocol/realtime/test_protocol_import_boundary.py``: +nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``, +``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``. +The arrows point inwards --- those packages import the protocol, never the +other way round. +""" diff --git a/vllm_omni/protocol/duplex/__init__.py b/vllm_omni/protocol/duplex/__init__.py new file mode 100644 index 000000000..d89a01322 --- /dev/null +++ b/vllm_omni/protocol/duplex/__init__.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""vLLM-Omni's full-duplex extension of the OpenAI Realtime wire protocol. + +OpenAI's Realtime API assumes one turn at a time: the client commits input, the +server answers. A full-duplex session does not --- the model listens while it +speaks, it decides when to take a turn, the client reports playback progress, +and a dropped socket re-attaches to a session that kept running. Those need +events and commands the GA vocabulary does not have. + +This package is that delta. It builds on ``vllm_omni.protocol.realtime`` (same +base classes, same pure wire rendering, no duplication) and, like it, owns no +session and imports no runtime: the duplex engine depends on this package, not +the other way round. + +The one door +------------ +It is also the *only* door a duplex consumer uses. ``protocol.duplex.commands`` +and ``protocol.duplex.events`` carry the complete vocabulary --- Tier 1 classes +re-exported unchanged alongside the Tier 2 and Tier 3 ones --- and this module +re-exports the Tier 1 helper functions. So the layering is a chain, not a mesh:: + + protocol.realtime (OpenAI's surface) + ^ + | imports + | + protocol.duplex (this package: re-exports Tier 1, adds Tier 2 and 3) + ^ + | imports + | + duplex engine / entrypoints / clients + +``tests/protocol/duplex/test_duplex_protocol_facade.py`` asserts that the last +arrow is the only one: no duplex consumer imports ``protocol.realtime`` +directly. The payoff is that a helper which later needs a duplex-specific +version --- ``convert_input_audio_with_rate`` is the standing example, it +resamples to MiniCPM-o's 16 kHz rather than the client's rate --- is overridden +in one file instead of at every call site. + +What stays outside +------------------ +The *mailbox* half of a command does not live here. A duplex command carries +two different things: the client event it decodes from (wire, this package) and +the dictionary the session runner consumes (engine). They genuinely differ --- +``session.update``, ``conversation.item.create`` / ``.delete`` / ``.truncate`` +all travel on the runner's ``turn.signal`` channel --- so ``payload()`` and its +``type`` stay in ``vllm_omni.engine.duplex.commands``. Events have no such +half, which is why :data:`~vllm_omni.protocol.duplex.events.DuplexEvent` can be +a plain alias of ``RealtimeEvent`` while ``DuplexCommand`` cannot. +""" + +from vllm_omni.protocol.duplex.errors import ( + REALTIME_ERROR_TYPES_BY_CODE, + RealtimeProtocolError, + realtime_error_type, +) +from vllm_omni.protocol.realtime.audio import ( + MAX_INPUT_SAMPLE_RATE_HZ, + MIN_INPUT_SAMPLE_RATE_HZ, + convert_input_audio_with_rate, + convert_output_audio, + decode_g711_alaw, + decode_g711_ulaw, + encode_float32_mono_wav_base64, + encode_g711_alaw, + encode_g711_ulaw, + resample_pcm16_mono, + validate_input_sample_rate_hz, + wav_payload_to_pcm16, +) +from vllm_omni.protocol.realtime.audio_input import ( + REALTIME_INPUT_HINT_KEYS, + RealtimeAudioAppend, + copy_realtime_input_hints, + decode_audio_append, + input_explicitly_non_speech, + input_looks_like_speech, +) +from vllm_omni.protocol.realtime.capabilities import ( + RealtimeProtocolCapabilities, + RealtimeSessionRejection, + validate_session_payload, +) +from vllm_omni.protocol.realtime.formats import ( + REALTIME_INPUT_AUDIO_FORMATS, + REALTIME_OUTPUT_AUDIO_FORMATS, + is_supported_realtime_input_format, + parse_realtime_audio_format, + realtime_audio_format_object, + realtime_output_format, + validate_conversation_item_audio_formats, + validate_realtime_response_audio_formats, + validate_realtime_session_audio_formats, +) +from vllm_omni.protocol.realtime.items import ( + input_transcript_from_item, + normalize_conversation_item, + text_chars_for_audio_ms_from_marks, + truncate_realtime_item_content, + validate_realtime_item_truncate, + validate_realtime_video_frames, +) +from vllm_omni.protocol.realtime.session import ( + RealtimeInputDefaults, + apply_realtime_session_defaults, + input_audio_transcription_config, + json_safe_realtime_payload, + realtime_max_output_tokens, + realtime_overlap_fields, +) + +__all__ = [ + "MAX_INPUT_SAMPLE_RATE_HZ", + "MIN_INPUT_SAMPLE_RATE_HZ", + "REALTIME_ERROR_TYPES_BY_CODE", + "REALTIME_INPUT_AUDIO_FORMATS", + "REALTIME_INPUT_HINT_KEYS", + "REALTIME_OUTPUT_AUDIO_FORMATS", + "RealtimeAudioAppend", + "RealtimeInputDefaults", + "RealtimeProtocolCapabilities", + "RealtimeProtocolError", + "RealtimeSessionRejection", + "apply_realtime_session_defaults", + "convert_input_audio_with_rate", + "convert_output_audio", + "copy_realtime_input_hints", + "decode_audio_append", + "decode_g711_alaw", + "decode_g711_ulaw", + "encode_float32_mono_wav_base64", + "encode_g711_alaw", + "encode_g711_ulaw", + "input_audio_transcription_config", + "input_explicitly_non_speech", + "input_looks_like_speech", + "input_transcript_from_item", + "is_supported_realtime_input_format", + "json_safe_realtime_payload", + "normalize_conversation_item", + "parse_realtime_audio_format", + "realtime_audio_format_object", + "realtime_error_type", + "realtime_max_output_tokens", + "realtime_output_format", + "realtime_overlap_fields", + "resample_pcm16_mono", + "text_chars_for_audio_ms_from_marks", + "truncate_realtime_item_content", + "validate_conversation_item_audio_formats", + "validate_input_sample_rate_hz", + "validate_realtime_item_truncate", + "validate_realtime_response_audio_formats", + "validate_realtime_session_audio_formats", + "validate_realtime_video_frames", + "validate_session_payload", + "wav_payload_to_pcm16", +] diff --git a/vllm_omni/protocol/duplex/commands.py b/vllm_omni/protocol/duplex/commands.py new file mode 100644 index 000000000..1dbb9bf25 --- /dev/null +++ b/vllm_omni/protocol/duplex/commands.py @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""The client-event vocabulary a full-duplex session accepts. + +Seven of the seventeen are events OpenAI's Realtime API has no equivalent for, +because they only make sense when the model and the user can talk at the same +time: the user cuts in (``barge_in``), the client reports how much audio it +played (``playback.ack``), a turn transition is signalled locally +(``turn.signal``), input is abandoned without committing (``input.cancel``), +text is pushed into a live turn (``input.text.append``), and the session is +kept alive or closed explicitly (``session.heartbeat`` / ``session.close``). +Two more are OpenAI commands carrying duplex-only fields (Tier 2). + +The remaining eight are OpenAI's, unchanged, and re-exported from +``vllm_omni.protocol.realtime.commands`` rather than redeclared --- they are +the same objects. They are re-exported so this module is the *whole* +vocabulary and a duplex consumer never has to import the Tier 1 package +directly; keeping them declared there means a GA-only consumer is still not +handed events its clients never send. + +None of these carries the engine's mailbox rendering: ``payload()`` and its +channel live in ``vllm_omni.engine.duplex.commands``, because for four of the +seventeen the runner's channel is not the client event type at all. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import ClassVar + +from vllm_omni.protocol.realtime import commands as realtime_commands +from vllm_omni.protocol.realtime.commands import ( + CancelResponse, + ClearInput, + ClearOutputAudio, + CreateItem, + CreateResponse, + DeleteItem, + RealtimeCommand, + TruncateItem, + UpdateSession, +) + +# ---- Tier 2: OpenAI names carrying vLLM-Omni extensions ---- + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AppendAudio(realtime_commands.AppendAudio): + """Tier 2: OpenAI's ``input_audio_buffer.append`` plus the duplex hints. + + OpenAI's append is just base64 audio. The duplex lane also lets a client + declare this chunk's own format and rate, whether it believes the chunk is + speech, camera frames captured alongside it, and its timing --- all + additive, all ignored by a stock client. + """ + + #: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...). + format: str = "pcm16" + sample_rate_hz: int | None = None + is_speech: bool | None = None + video_frames: tuple[str, ...] = () + duration_ms: int | None = None + audio_end_ms: int | None = None + hints: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Commit(realtime_commands.Commit): + """Tier 2: OpenAI's ``input_audio_buffer.commit`` plus duplex turn control. + + OpenAI's commit carries nothing and always produces an item. Here a commit + may be non-final, may decline to start a response, and may declare itself + silence --- the semantic divergence documented in + ``docs/serving/realtime_duplex_api.md`` ("Commit != response"). + """ + + #: Realtime conversation item created for this commit (wire correlation only). + #: ``None`` means "no explicit request": the runner decides on commit (auto-response + #: sessions answer on their own); ``True`` / ``False`` force it. + create_response: bool | None = None + is_speech: bool | None = None + realtime_item_id: str | None = None + + +# ---- Tier 3: vLLM-Omni only ---- + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AppendText(RealtimeCommand): + wire_type: ClassVar[str] = "input.text.append" + text: str + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CancelInput(RealtimeCommand): + wire_type: ClassVar[str] = "input.cancel" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BargeIn(RealtimeCommand): + wire_type: ClassVar[str] = "barge_in" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SignalTurn(RealtimeCommand): + """Generic ``turn.signal`` (local turn transitions such as ``user_started``).""" + + wire_type: ClassVar[str] = "turn.signal" + event: str + signal_payload: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AckPlayback(RealtimeCommand): + wire_type: ClassVar[str] = "playback.ack" + played_ms: int + committed_ms: int | None = None + response_id: str | None = None + item_id: str | None = None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Heartbeat(RealtimeCommand): + wire_type: ClassVar[str] = "session.heartbeat" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CloseSession(RealtimeCommand): + """Graceful close requested through the command stream (``session.close``).""" + + wire_type: ClassVar[str] = "session.close" + reason: str = "client_close" + + +#: The complete command vocabulary a duplex client may send: the eight Tier 1 +#: classes re-exported unchanged, the two Tier 2 classes defined above, and the +#: seven Tier 3 ones. A duplex consumer imports from here and never reaches +#: past this module into ``vllm_omni.protocol.realtime``, so a command that +#: later grows a duplex extension changes only this file. +__all__ = [ + # Tier 1 --- re-exported from ``vllm_omni.protocol.realtime.commands``. + "CancelResponse", + "ClearInput", + "ClearOutputAudio", + "CreateItem", + "CreateResponse", + "DeleteItem", + "RealtimeCommand", + "TruncateItem", + "UpdateSession", + # Tier 2 --- an OpenAI command plus duplex-only fields. + "AppendAudio", + "Commit", + # Tier 3 --- vLLM-Omni only. + "AckPlayback", + "AppendText", + "BargeIn", + "CancelInput", + "CloseSession", + "Heartbeat", + "SignalTurn", +] diff --git a/vllm_omni/protocol/duplex/errors.py b/vllm_omni/protocol/duplex/errors.py new file mode 100644 index 000000000..418c98712 --- /dev/null +++ b/vllm_omni/protocol/duplex/errors.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""vLLM-Omni's Realtime error-code vocabulary. + +OpenAI's Realtime API standardises the ``error.type`` classes +(``invalid_request_error`` / ``server_error`` / ``rate_limit_error``) but not +the ``code`` values inside them. Ours are listed here --- Tier 3 in +``docs/serving/realtime_duplex_api.md``, "the full error-code vocabulary" --- +so ``protocol/realtime`` stays the OpenAI surface and nothing more. +""" + +from __future__ import annotations + +from vllm_omni.protocol.realtime.errors import RealtimeProtocolError + +__all__ = ["REALTIME_ERROR_TYPES_BY_CODE", "RealtimeProtocolError", "realtime_error_type"] + + +#: OpenAI Realtime ``error.type`` for each internal error code. +REALTIME_ERROR_TYPES_BY_CODE: dict[str, str] = { + "bad_event": "invalid_request_error", + "bad_audio": "invalid_request_error", + "config_timeout": "invalid_request_error", + "invalid_json": "invalid_request_error", + "event_too_large": "invalid_request_error", + "unknown_event": "invalid_request_error", + "internal_error": "server_error", + "runtime_append_failed": "server_error", + "runtime_append_task_failed": "server_error", + "runtime_signal_failed": "server_error", + "runtime_abort_failed": "server_error", + "runtime_data_plane_stream_failed": "server_error", + "runtime_data_plane_text_without_audio": "server_error", + "resource_exhausted": "rate_limit_error", + "session_exists": "invalid_request_error", + "session_closed": "invalid_request_error", + "unknown_session": "invalid_request_error", + "invalid_duplex_runtime_config": "invalid_request_error", + "instructions_update_unsupported": "invalid_request_error", + "persona_update_unsupported": "invalid_request_error", + "voice_update_unsupported": "invalid_request_error", + "unsupported_nemotron_duplex_mode": "invalid_request_error", + "unsupported_native_response_options": "invalid_request_error", + "runtime_touch_failed": "server_error", + "engine_error": "server_error", + "input_backpressure": "rate_limit_error", + "response_already_active": "invalid_request_error", + "response_not_active": "invalid_request_error", + "response_create_without_input": "invalid_request_error", + "text_only_turn_unsupported": "invalid_request_error", + "commit_aborted": "server_error", + "input_audio_buffer_empty": "invalid_request_error", + "missing_item_id": "invalid_request_error", + "item_not_found": "invalid_request_error", + "playback_item_mismatch": "invalid_request_error", + "playback_item_not_found": "invalid_request_error", + "playback_ack_too_late": "invalid_request_error", + "unsupported_audio_format": "invalid_request_error", + "unsupported_turn_detection": "invalid_request_error", + "unsupported_ref_audio_path": "invalid_request_error", + "ref_audio_required": "invalid_request_error", + "model_update_unsupported": "invalid_request_error", + "voice_update_after_audio_unsupported": "invalid_request_error", + "ref_audio_update_unsupported": "invalid_request_error", + "native_text_append_unsupported": "invalid_request_error", + "invalid_video_frames": "invalid_request_error", + "invalid_function_call_output": "invalid_request_error", + "server_vad_unavailable": "server_error", +} + + +def realtime_error_type(code: str) -> str: + """The OpenAI ``error.type`` bucket for an internal error code.""" + return REALTIME_ERROR_TYPES_BY_CODE.get(code, "invalid_request_error") diff --git a/vllm_omni/protocol/duplex/events.py b/vllm_omni/protocol/duplex/events.py new file mode 100644 index 000000000..f2abe748e --- /dev/null +++ b/vllm_omni/protocol/duplex/events.py @@ -0,0 +1,419 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""vLLM-Omni's half of the Realtime server-event vocabulary. + +``docs/serving/realtime_duplex_api.md`` sorts every message into three tiers. +Tier 1 --- identical to OpenAI --- is ``vllm_omni.protocol.realtime.events``. +The other two are here: + +**Tier 2, OpenAI names carrying vLLM-Omni extensions.** Same wire type, same +OpenAI fields, plus additive keys a stock client ignores. Each one subclasses +its Tier 1 twin and declares only what it adds, so the OpenAI surface stays +honest: a consumer that wants pure GA takes ``protocol/realtime`` and gets +exactly OpenAI's fields. + +**Tier 3, vLLM-Omni only.** Events OpenAI has no equivalent for, because they +only make sense when the model and the user can talk at the same time: the +model decides when to listen and when to speak, the client reports how much +audio it played, overlapping speech needs a recorded decision, and a dropped +socket can re-attach to a live session. + +Same-named classes are deliberate --- ``duplex.events.SessionCreated`` *is* +``realtime.events.SessionCreated`` with our extensions. Import whichever +surface you mean. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import cast + +from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE +from vllm_omni.protocol.realtime import events as realtime_events +from vllm_omni.protocol.realtime.events import ( + AudioDone, + ContentPartAdded, + ContentPartDone, + FunctionCallArgumentsDelta, + FunctionCallArgumentsDone, + InputCleared, + InputTranscriptionCompleted, + ItemAdded, + ItemCreated, + ItemDone, + ItemRetrieved, + OutputAudioCleared, + OutputItemAdded, + OutputItemDone, + RateLimitsUpdated, + RealtimeEvent, + ResponseScopedEvent, + SessionUpdated, + SpeechStarted, + SpeechStopped, + TextDelta, + TextDone, + TranscriptDelta, + TranscriptDone, + new_event_id, + wire_value, +) + +#: The duplex event base class *is* the Realtime one: unlike a command, an +#: event has no engine-internal half to carry --- what the session emits is +#: exactly what the socket sends. Named here rather than in the engine so the +#: duplex vocabulary is described in one place, and kept as an alias rather +#: than an empty subclass so ``isinstance`` and the class identity tests in +#: ``tests/protocol/`` stay meaningful. +DuplexEvent = RealtimeEvent + +# ---- Tier 2: OpenAI names carrying vLLM-Omni extensions ---- + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionCreated(realtime_events.SessionCreated): + """Tier 2: OpenAI's ``SessionCreated`` plus the resume credentials a reconnecting client needs.""" + + #: Transport resume credentials (set by the websocket handler when resume is supported). + optional_wire_fields = frozenset({"attachment_generation", "resume_token"}) + attachment_generation: int | None = None + resume_token: str | None = None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ResponseCreated(realtime_events.ResponseCreated): + """Tier 2: OpenAI's ``ResponseCreated`` plus a top-level ``response_id``. + + OpenAI carries the id nested as ``response.id``; the duplex lane also + reports it at the top level, where every other response-scoped event + already has one. + """ + + response_id: str | None = None # type: ignore[assignment] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ResponseDone(realtime_events.ResponseDone): + """Tier 2: OpenAI's ``ResponseDone`` plus a top-level ``response_id``. + + OpenAI carries the id nested as ``response.id``; the duplex lane also + reports it at the top level, where every other response-scoped event + already has one. + """ + + response_id: str | None = None # type: ignore[assignment] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AudioDelta(realtime_events.AudioDelta): + """Tier 2: OpenAI's ``AudioDelta`` plus the chunk's own format/rate and the duplex playback metadata.""" + + optional_wire_fields = frozenset({"sample_rate_hz", "metadata"}) + format: str = "pcm16" + sample_rate_hz: int | None = None + metadata: Mapping[str, object] | None = None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class InputCommitted(realtime_events.InputCommitted): + """Tier 2: OpenAI's ``InputCommitted`` plus the native ``input.committed`` event echoed under ``event``.""" + + #: The session-internal ``input.committed`` event. + details: Mapping[str, object] = field(default_factory=dict) + + def _wire_fields(self) -> dict[str, object]: + # Explicit parent call, not ``super()``: ``@dataclass(slots=True)`` + # rebuilds the class, so the zero-argument form's ``__class__`` cell + # points at the pre-slots class and is no longer in the MRO. + base = realtime_events.__dict__[type(self).__name__]._wire_fields(self) + return {**base, "event": wire_value(self.details)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ItemDeleted(realtime_events.ItemDeleted): + """Tier 2: OpenAI's ``ItemDeleted`` plus the originating event echoed under ``event``.""" + + details: Mapping[str, object] = field(default_factory=dict) + + def _wire_fields(self) -> dict[str, object]: + # Explicit parent call, not ``super()``: ``@dataclass(slots=True)`` + # rebuilds the class, so the zero-argument form's ``__class__`` cell + # points at the pre-slots class and is no longer in the MRO. + base = realtime_events.__dict__[type(self).__name__]._wire_fields(self) + return {**base, "event": wire_value(self.details)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ItemTruncated(realtime_events.ItemTruncated): + """Tier 2: OpenAI's ``ItemTruncated`` plus the originating event echoed under ``event``.""" + + details: Mapping[str, object] = field(default_factory=dict) + + def _wire_fields(self) -> dict[str, object]: + # Explicit parent call, not ``super()``: ``@dataclass(slots=True)`` + # rebuilds the class, so the zero-argument form's ``__class__`` cell + # points at the pre-slots class and is no longer in the MRO. + base = realtime_events.__dict__[type(self).__name__]._wire_fields(self) + return {**base, "event": wire_value(self.details)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ErrorEvent(realtime_events.ErrorEvent): + """Tier 2: OpenAI's ``ErrorEvent`` plus vLLM-Omni's error codes and any extra keys merged into the envelope.""" + + #: Extra keys merged into the wire ``error`` object (``retryable`` ...). + extra: Mapping[str, object] = field(default_factory=dict) + + @property + def error_type(self) -> str: + return REALTIME_ERROR_TYPES_BY_CODE.get(self.code, "invalid_request_error") + + @property + def error(self) -> dict[str, object]: + # Explicit parent call, not ``super()``: ``@dataclass(slots=True)`` + # rebuilds the class, so the zero-argument form's ``__class__`` cell + # points at the pre-slots class and is no longer in the MRO. Reached + # through ``__dict__`` because attribute access on the class gives the + # getter function, not the ``property`` that owns it. + error = dict(realtime_events.ErrorEvent.__dict__["error"].fget(self)) + error.update(cast("Mapping[str, object]", wire_value(self.extra))) + return error + + +# ---- Tier 3: vLLM-Omni only ---- + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionResumed(SessionCreated): + wire_type = "session.resumed" + + def _wire_fields(self) -> dict[str, object]: + return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionReplaced(RealtimeEvent): + """A newer connection took the session over (sent to the replaced socket).""" + + wire_type = "session.replaced" + + attachment_generation: int = 0 + + def _wire_fields(self) -> dict[str, object]: + return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionResyncRequired(RealtimeEvent): + """The replay journal cannot bridge the gap; the client must start over.""" + + wire_type = "session.resync_required" + + reason: str = "journal_gap" + + def _wire_fields(self) -> dict[str, object]: + return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionClosed(RealtimeEvent): + wire_type = "session.closed" + + reason: str = "closed" + #: The session-internal close event (kept for clients that inspect it). + details: Mapping[str, object] = field(default_factory=dict) + + @property + def is_terminal(self) -> bool: + return True + + def _wire_fields(self) -> dict[str, object]: + return {"session_id": self.session_id, "reason": self.reason, "event": wire_value(self.details)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionExpired(SessionClosed): + wire_type = "session.expired" + + def _wire_fields(self) -> dict[str, object]: + return {"session_id": self.session_id, "reason": self.reason} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionHeartbeatAck(RealtimeEvent): + wire_type = "session.heartbeat_ack" + + def _wire_fields(self) -> dict[str, object]: + return {"session_id": self.session_id} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TurnEvent(RealtimeEvent): + """Turn-state transition (``turn.event``).""" + + wire_type = "turn.event" + + event: str = "" + turn_state: str = "" + + def _wire_fields(self) -> dict[str, object]: + return {"session_id": self.session_id, "event": self.event, "turn_state": self.turn_state, "epoch": self.epoch} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Listen(RealtimeEvent): + """The model decided to keep listening (no spoken response for this turn).""" + + wire_type = "response.listen" + optional_wire_fields = frozenset({"response_id"}) + + response_id: str | None = None # type: ignore[assignment] + details: Mapping[str, object] = field(default_factory=dict) + + def _wire_fields(self) -> dict[str, object]: + response: dict[str, object] = { + "object": "realtime.response", + "status": "listening", + "metadata": wire_value(self.details), + } + data: dict[str, object] = {"session_id": self.session_id, "epoch": self.epoch, "response": response} + if self.response_id: + response["id"] = self.response_id + data["response_id"] = self.response_id + return data + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Speak(ResponseScopedEvent): + wire_type = "response.speak" + + metadata: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class OverlapDecision(RealtimeEvent): + wire_type = "overlap.decision" + + policy: str | None = None + action: str | None = None + reason: str | None = None + details: Mapping[str, object] = field(default_factory=dict) + + def _wire_fields(self) -> dict[str, object]: + return { + "session_id": self.session_id, + "epoch": self.epoch, + "policy": self.policy, + "action": self.action, + "reason": self.reason, + "metadata": wire_value(self.details), + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class PlaybackAcknowledged(RealtimeEvent): + wire_type = "playback.acknowledged" + + #: The session-internal ``playback.acknowledged`` event (cursor, committed_ms ...). + details: Mapping[str, object] = field(default_factory=dict) + + def _wire_fields(self) -> dict[str, object]: + return {"event": wire_value(self.details)} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class DuplexRawEvent(RealtimeEvent): + """A session-internal event with no dedicated Realtime type (``duplex.``).""" + + wire_type = "duplex.raw" + + internal_type: str = "" + details: Mapping[str, object] = field(default_factory=dict) + + @property + def type(self) -> str: + return f"duplex.{self.internal_type}" + + def to_realtime(self) -> dict[str, object]: + return {"type": self.type, "event_id": self.event_id, "event": wire_value(self.details)} + + +def error_event( + code: str, + message: str, + *, + event_id: object | None = None, + param: object | None = None, + extra: Mapping[str, object] | None = None, +) -> ErrorEvent: + """Build the error event for an internal code; ``event_id`` is the *client* event id.""" + return ErrorEvent( + code=code, + message=message, + related_event_id=event_id if isinstance(event_id, str) and event_id else None, + param=param if isinstance(param, str) and param else None, + extra=dict(extra or {}), + ) + + +#: The complete event vocabulary a duplex client may receive: 22 Tier 1 +#: classes re-exported unchanged, the 8 Tier 2 ones defined above, and 12 +#: Tier 3 ones. A duplex consumer imports from here and never reaches past +#: this module into ``vllm_omni.protocol.realtime``. +__all__ = [ + # Tier 1 --- re-exported from ``vllm_omni.protocol.realtime.events``. + "AudioDone", + "ContentPartAdded", + "ContentPartDone", + "FunctionCallArgumentsDelta", + "FunctionCallArgumentsDone", + "InputCleared", + "InputTranscriptionCompleted", + "ItemAdded", + "ItemCreated", + "ItemDone", + "ItemRetrieved", + "OutputAudioCleared", + "OutputItemAdded", + "OutputItemDone", + "RateLimitsUpdated", + "SessionUpdated", + "SpeechStarted", + "SpeechStopped", + "TextDelta", + "TextDone", + "TranscriptDelta", + "TranscriptDone", + # Tier 2 --- an OpenAI event plus duplex-only fields. + "AudioDelta", + "ErrorEvent", + "InputCommitted", + "ItemDeleted", + "ItemTruncated", + "ResponseCreated", + "ResponseDone", + "SessionCreated", + # Tier 3 --- vLLM-Omni only. + "DuplexRawEvent", + "Listen", + "OverlapDecision", + "PlaybackAcknowledged", + "SessionClosed", + "SessionExpired", + "SessionHeartbeatAck", + "SessionReplaced", + "SessionResumed", + "SessionResyncRequired", + "Speak", + "TurnEvent", + # Bases and helpers shared with the Tier 1 surface. + "DuplexEvent", + "REALTIME_ERROR_TYPES_BY_CODE", + "RealtimeEvent", + "ResponseScopedEvent", + "error_event", + "new_event_id", + "wire_value", +] diff --git a/vllm_omni/protocol/realtime/__init__.py b/vllm_omni/protocol/realtime/__init__.py new file mode 100644 index 000000000..af7194cb5 --- /dev/null +++ b/vllm_omni/protocol/realtime/__init__.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""The OpenAI Realtime wire codec, shared by every vLLM-Omni Realtime surface. + +What this is +------------ +The model-agnostic half of ``WS /v1/realtime``: how a client event is parsed, +which audio formats and session fields are accepted, what a conversation item +means, how audio is decoded and resampled, and the shape of the error envelope +(the *codes* inside it are a consumer's own --- ours are in +``vllm_omni.protocol.duplex.errors``). It is a library of pure functions and value objects. + +What it is not +-------------- +It owns no session. It does not decide when a response starts, what a model is +prompted with, whether a turn is over, or where committed audio goes. Those are +runtime decisions, and after PR #7413 a duplex session's runtime decisions all +belong to ``vllm_omni.engine.duplex`` --- the session runner, the typed +``DuplexCommand`` / ``DuplexEvent`` boundary and the ``DuplexModelPlugin`` seam. +This package sits *under* that: the duplex engine binds the codec to its own +command and event vocabulary in +``vllm_omni.engine.duplex.realtime_commands`` and +``vllm_omni.engine.duplex.realtime_events``. + +Why it is separate (RFC #6592, P0a) +----------------------------------- +So that a second Realtime surface --- a Qwen3-Omni GA profile, say --- can reuse +the parsing, validation, format negotiation and error envelope without adopting +the duplex session control plane (lease manager, commit policy, model channel), +and without a second copy of the codec drifting away from this one. A consumer +states what it can serve through +:class:`~vllm_omni.protocol.realtime.capabilities.RealtimeProtocolCapabilities` +and keeps its own session state. + +Known duplex-shaped defaults +---------------------------- +This package is model-agnostic in its *dependencies*, not yet in every +*default*. Two constants still carry the assumptions of its first consumer and +a second one will have to parameterize them: + +* :func:`~vllm_omni.protocol.realtime.audio.convert_input_audio_with_rate` + resamples to 16 kHz, which is MiniCPM-o's feature-extractor rate. A GA client + sending 24 kHz is converted down rather than served at its own rate. +* :func:`~vllm_omni.protocol.realtime.items.validate_realtime_video_frames` + encodes the MiniCPM-o duplex camera contract (at most two frames per append, + ``max_slice_nums`` not selectable on the wire). + +Both are behaviour this extraction deliberately did **not** change (RFC #6592 +P0a is behaviour-preserving); making them per-consumer is P0b/P1 work. + +Dependency rule +--------------- +Nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``, +``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``; +``tests/protocol/realtime/test_protocol_import_boundary.py`` asserts it. +""" + +from vllm_omni.protocol.realtime.audio import ( + MAX_INPUT_SAMPLE_RATE_HZ, + MIN_INPUT_SAMPLE_RATE_HZ, + convert_input_audio_with_rate, + convert_output_audio, + encode_float32_mono_wav_base64, + resample_pcm16_mono, + validate_input_sample_rate_hz, + wav_payload_to_pcm16, +) +from vllm_omni.protocol.realtime.audio_input import ( + REALTIME_INPUT_HINT_KEYS, + RealtimeAudioAppend, + copy_realtime_input_hints, + decode_audio_append, + input_explicitly_non_speech, + input_looks_like_speech, +) +from vllm_omni.protocol.realtime.capabilities import ( + RealtimeProtocolCapabilities, + RealtimeSessionRejection, + validate_session_payload, +) +from vllm_omni.protocol.realtime.commands import RealtimeCommand +from vllm_omni.protocol.realtime.errors import RealtimeProtocolError +from vllm_omni.protocol.realtime.events import ( + RealtimeEvent, + ResponseScopedEvent, + new_event_id, + wire_value, +) +from vllm_omni.protocol.realtime.formats import ( + REALTIME_INPUT_AUDIO_FORMATS, + REALTIME_OUTPUT_AUDIO_FORMATS, + is_supported_realtime_input_format, + parse_realtime_audio_format, + realtime_audio_format_object, + realtime_output_format, + validate_conversation_item_audio_formats, + validate_realtime_response_audio_formats, + validate_realtime_session_audio_formats, +) +from vllm_omni.protocol.realtime.items import ( + input_transcript_from_item, + normalize_conversation_item, + text_chars_for_audio_ms_from_marks, + truncate_realtime_item_content, + validate_realtime_item_truncate, + validate_realtime_video_frames, +) +from vllm_omni.protocol.realtime.session import ( + RealtimeInputDefaults, + apply_realtime_session_defaults, + input_audio_transcription_config, + json_safe_realtime_payload, + realtime_max_output_tokens, + realtime_overlap_fields, +) + +__all__ = [ + "MAX_INPUT_SAMPLE_RATE_HZ", + "MIN_INPUT_SAMPLE_RATE_HZ", + "REALTIME_INPUT_AUDIO_FORMATS", + "REALTIME_INPUT_HINT_KEYS", + "REALTIME_OUTPUT_AUDIO_FORMATS", + "RealtimeAudioAppend", + "RealtimeCommand", + "RealtimeEvent", + "RealtimeInputDefaults", + "RealtimeProtocolCapabilities", + "RealtimeProtocolError", + "RealtimeSessionRejection", + "ResponseScopedEvent", + "apply_realtime_session_defaults", + "convert_input_audio_with_rate", + "convert_output_audio", + "copy_realtime_input_hints", + "decode_audio_append", + "encode_float32_mono_wav_base64", + "input_audio_transcription_config", + "input_explicitly_non_speech", + "input_looks_like_speech", + "input_transcript_from_item", + "is_supported_realtime_input_format", + "json_safe_realtime_payload", + "new_event_id", + "normalize_conversation_item", + "parse_realtime_audio_format", + "realtime_audio_format_object", + "realtime_max_output_tokens", + "realtime_output_format", + "realtime_overlap_fields", + "resample_pcm16_mono", + "text_chars_for_audio_ms_from_marks", + "truncate_realtime_item_content", + "validate_conversation_item_audio_formats", + "validate_input_sample_rate_hz", + "validate_realtime_item_truncate", + "validate_realtime_response_audio_formats", + "validate_realtime_session_audio_formats", + "validate_realtime_video_frames", + "validate_session_payload", + "wav_payload_to_pcm16", + "wire_value", +] diff --git a/vllm_omni/protocol/realtime/audio.py b/vllm_omni/protocol/realtime/audio.py new file mode 100644 index 000000000..5f163826a --- /dev/null +++ b/vllm_omni/protocol/realtime/audio.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from __future__ import annotations + +import base64 +import binascii +import io +import math +import wave + +import numpy as np + +from vllm_omni.utils.audio_resample import StreamingAudioResampler + +try: + from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin +except ImportError: # pragma: no cover - audioop is removed in newer Python. + alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None + + +MIN_INPUT_SAMPLE_RATE_HZ = 8_000 +MAX_INPUT_SAMPLE_RATE_HZ = 192_000 + + +def validate_input_sample_rate_hz(sample_rate_hz: object) -> int: + if ( + isinstance(sample_rate_hz, bool) + or not isinstance(sample_rate_hz, int | float) + or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz)) + ): + raise ValueError("sample_rate_hz must be a finite integer") + rate = int(sample_rate_hz) + if rate != sample_rate_hz: + raise ValueError("sample_rate_hz must be an integer") + if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ: + raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}") + return rate + + +def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes: + if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz: + return raw + samples = np.frombuffer(raw, dtype=" StreamingAudioResampler.max_polyphase_factor: + # The FIR size scales with the reduced ratio. Keep unusual client + # rates bounded while common audio rates use the high-quality path. + if samples.size <= 1: + return raw + target_size = max(1, int(round(samples.size * target_rate_hz / source_rate_hz))) + source_x = np.linspace(0.0, 1.0, num=samples.size, endpoint=True) + target_x = np.linspace(0.0, 1.0, num=target_size, endpoint=True) + resampled = np.interp(target_x, source_x, samples) + return np.clip(resampled, -32_768, 32_767).astype(" bytes: + if ulaw2lin is not None: + return ulaw2lin(raw, 2) + data = np.frombuffer(raw, dtype=np.uint8) + value = np.bitwise_not(data).astype(np.int16) + sign = value & 0x80 + exponent = (value >> 4) & 0x07 + mantissa = value & 0x0F + sample = ((mantissa << 3) + 0x84) << exponent + return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype(" bytes: + if alaw2lin is not None: + return alaw2lin(raw, 2) + data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16) + sign = data & 0x80 + exponent = (data >> 4) & 0x07 + mantissa = data & 0x0F + sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1)) + return np.where(sign != 0, sample, -sample).astype(" bytes: + if lin2ulaw is not None: + return lin2ulaw(raw, 2) + pcm = np.clip(np.frombuffer(raw, dtype=" (0xFF << exp), exp + 1, exponent) + mantissa = (magnitude >> (exponent + 3)) & 0x0F + return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes() + + +def encode_g711_alaw(raw: bytes) -> bytes: + if lin2alaw is not None: + return lin2alaw(raw, 2) + pcm = np.frombuffer(raw, dtype="= 0, 0x80, 0x00) + magnitude = np.abs(pcm) + exponent = np.zeros_like(magnitude) + for exp in range(1, 8): + exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent) + mantissa = np.where( + exponent == 0, + (magnitude >> 4) & 0x0F, + (magnitude >> (exponent + 3)) & 0x0F, + ) + return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes() + + +def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]: + try: + with wave.open(io.BytesIO(raw), "rb") as wav_file: + channels = wav_file.getnchannels() + sample_width = wav_file.getsampwidth() + sample_rate_hz = wav_file.getframerate() + frames = wav_file.readframes(wav_file.getnframes()) + if sample_width != 2: + return None, sample_rate_hz + if channels <= 1: + return frames, sample_rate_hz + pcm = np.frombuffer(frames, dtype=" str: + """Package normalized mono float32 samples as a PCM16 WAV data payload.""" + rate = validate_input_sample_rate_hz(sample_rate_hz) + normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1) + pcm16 = np.clip(np.rint(normalized * 32768.0), -32768, 32767).astype(" tuple[object, object, int | float | None]: + if not isinstance(audio, str) or not isinstance(fmt, str): + return audio, fmt, sample_rate_hz + normalized = fmt.lower() + if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}: + return audio, fmt, sample_rate_hz + if isinstance(sample_rate_hz, int | float): + sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz) + try: + raw = base64.b64decode(audio.strip(), validate=False) + except (binascii.Error, ValueError): + return audio, fmt, sample_rate_hz + if normalized == "g711_ulaw": + raw = decode_g711_ulaw(raw) + sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000 + elif normalized == "g711_alaw": + raw = decode_g711_alaw(raw) + sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000 + elif len(raw) % 2: + return audio, fmt, sample_rate_hz + if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz: + raw = resample_pcm16_mono( + raw, + source_rate_hz=int(sample_rate_hz), + target_rate_hz=target_sample_rate_hz, + ) + sample_rate_hz = target_sample_rate_hz + pcm = np.frombuffer(raw, dtype=" tuple[str, str, int | None]: + target = target_fmt.lower() + if target not in {"g711_ulaw", "g711_alaw"}: + return audio, source_fmt, source_sample_rate_hz + try: + raw = base64.b64decode(audio, validate=False) + except (binascii.Error, ValueError): + return audio, source_fmt, source_sample_rate_hz + source = source_fmt.lower() + if source == "wav": + pcm_raw, wav_rate = wav_payload_to_pcm16(raw) + if pcm_raw is None: + return audio, source_fmt, source_sample_rate_hz + raw = pcm_raw + source_sample_rate_hz = source_sample_rate_hz or wav_rate + elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2: + return audio, source_fmt, source_sample_rate_hz + target_rate = target_sample_rate_hz or 8_000 + if source_sample_rate_hz is not None: + raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate) + encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw) + return base64.b64encode(encoded).decode("ascii"), target, target_rate diff --git a/vllm_omni/protocol/realtime/audio_input.py b/vllm_omni/protocol/realtime/audio_input.py new file mode 100644 index 000000000..634d32c34 --- /dev/null +++ b/vllm_omni/protocol/realtime/audio_input.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Decoding one ``input_audio_buffer.append``. + +An append is the hottest client event on a Realtime connection and the one with +the most wire surface: base64 audio in any accepted format, an optional rate, +optional camera frames, and a set of optional hints (``duration_ms``, +``is_speech``, a VAD probability, a transcript) that clients attach and servers +are free to use or ignore. + +:func:`decode_audio_append` turns that into :class:`RealtimeAudioAppend`, a +value object: bytes at 16 kHz ``pcm_f32le``, the surviving hints, and a speech +verdict. It is a pure function of the event plus the session's +:class:`~vllm_omni.protocol.realtime.session.RealtimeInputDefaults`, so any +consumer decodes an append the same way. Building a consumer's own command +object out of it is one constructor call --- see +``vllm_omni.engine.duplex.realtime_commands.build_append_audio``, which wraps it +for the duplex ``AppendAudio``. + +The speech verdict here is the *client-declared* one (explicit flags, a VAD +probability the client sent, or an RMS floor as a last resort). It is not +server-side VAD; that is a runtime concern and lives with whoever owns the +session. +""" + +from __future__ import annotations + +import base64 +import binascii +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import cast + +import numpy as np + +from vllm_omni.protocol.realtime.audio import convert_input_audio_with_rate +from vllm_omni.protocol.realtime.errors import RealtimeProtocolError +from vllm_omni.protocol.realtime.formats import ( + is_supported_realtime_input_format, + parse_realtime_audio_format, +) +from vllm_omni.protocol.realtime.items import validate_realtime_video_frames +from vllm_omni.protocol.realtime.session import RealtimeInputDefaults + +__all__ = [ + "REALTIME_INPUT_HINT_KEYS", + "RealtimeAudioAppend", + "copy_realtime_input_hints", + "decode_audio_append", + "input_explicitly_non_speech", + "input_looks_like_speech", +] + +#: Wire hint keys copied verbatim from a client append onto the internal payload. +REALTIME_INPUT_HINT_KEYS = ( + "duration_ms", + "audio_duration_ms", + "audio_start_ms", + "audio_end_ms", + "is_speech", + "speech", + "speech_probability", + "vad", + "overlap_action", + "overlap", + "force_barge_in", + "force_listen", + "text", + "transcript", +) + + +@dataclass(frozen=True, slots=True) +class RealtimeAudioAppend: + """One decoded ``input_audio_buffer.append``. + + ``audio`` is raw bytes in ``format`` at ``sample_rate_hz`` --- after + :func:`decode_audio_append` that is 16 kHz ``pcm_f32le`` for every input + format the codec converts. + """ + + audio: bytes + format: str + sample_rate_hz: int | None = None + is_speech: bool | None = None + video_frames: tuple[str, ...] = () + duration_ms: int | None = None + audio_end_ms: int | None = None + #: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...). + hints: dict[str, object] = field(default_factory=dict) + #: Client correlation id (OpenAI ``event_id``), echoed on error events. + event_id: str | None = None + + +# ---- speech hints ---- + + +def input_explicitly_non_speech(event: Mapping[str, object]) -> bool: + for key in ("is_speech", "speech"): + value = event.get(key) + if isinstance(value, bool): + return not value + vad = event.get("vad") + if isinstance(vad, dict): + value = vad.get("is_speech") + if isinstance(value, bool): + return not value + probability = vad.get("speech_probability", vad.get("probability")) + if isinstance(probability, int | float): + return float(probability) < 0.5 + probability = event.get("speech_probability") + return isinstance(probability, int | float) and float(probability) < 0.5 + + +def input_looks_like_speech( + event: Mapping[str, object], + *, + audio: object, + fmt: object, + overlap_silence_rms: float, +) -> bool: + if input_explicitly_non_speech(event): + return False + for key in ("is_speech", "speech"): + value = event.get(key) + if isinstance(value, bool): + return value + vad = event.get("vad") + if isinstance(vad, dict): + probability = vad.get("speech_probability", vad.get("probability")) + if isinstance(probability, int | float): + return float(probability) >= 0.5 + probability = event.get("speech_probability") + if isinstance(probability, int | float): + return float(probability) >= 0.5 + if fmt != "pcm_f32le": + return True + if isinstance(audio, bytes | bytearray): + raw = bytes(audio) + elif isinstance(audio, str): + try: + raw = base64.b64decode(audio, validate=True) + except (binascii.Error, ValueError): + return True + else: + return True + if len(raw) < 4 or len(raw) % 4 != 0: + return True + samples = np.frombuffer(raw, dtype=np.float32) + if samples.size == 0: + return False + rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32))))) + threshold = event.get("overlap_silence_rms") + if not isinstance(threshold, int | float): + vad = event.get("vad") + if isinstance(vad, dict): + threshold = vad.get("silence_rms") + silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms + return rms >= max(0.0, silence_rms) + + +def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None: + for key in REALTIME_INPUT_HINT_KEYS: + if key in source: + target[key] = source[key] + + +# ---- append decoding ---- + + +def decode_audio_append( + event: Mapping[str, object], + *, + defaults: RealtimeInputDefaults, + hints_source: Mapping[str, object] | None = None, +) -> RealtimeAudioAppend: + """Validate, convert (to 16 kHz ``pcm_f32le``) and pack one audio append. + + ``hints_source`` is the enclosing payload when the audio arrives inside + something larger than a bare append --- a ``conversation.item.create`` + audio part, say --- so its hints apply unless the part overrides them. + + Raises :class:`RealtimeProtocolError` for an unsupported format, undecodable + audio or invalid camera frames. + """ + event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None + audio = event.get("audio") or event.get("delta") + fmt, format_rate = parse_realtime_audio_format( + event.get("format") or event.get("input_audio_format") or defaults.input_audio_format + ) + sample_rate_hz = ( + event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz + ) + if not is_supported_realtime_input_format(fmt): + raise RealtimeProtocolError( + f"Unsupported input_audio_format: {fmt}", + code="unsupported_audio_format", + event_id=event_id, + ) + try: + audio, fmt, sample_rate_hz = convert_input_audio_with_rate( + audio, + fmt, + sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None, + ) + except ValueError as exc: + raise RealtimeProtocolError(str(exc), code="bad_event", event_id=event_id) from exc + hints: dict[str, object] = {} + if hints_source is not None: + copy_realtime_input_hints(hints_source, hints) + copy_realtime_input_hints(event, hints) + looks_like_speech = input_looks_like_speech( + {**(hints_source or {}), **event}, + audio=audio, + fmt=fmt, + overlap_silence_rms=defaults.overlap_silence_rms, + ) + video_frames: tuple[str, ...] = () + raw_frames = event.get("video_frames") + if raw_frames is not None: + frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums")) + if frames_error is not None: + raise RealtimeProtocolError(frames_error, code="invalid_video_frames", event_id=event_id) + if isinstance(raw_frames, list): + video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame) + duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms")) + audio_end_ms = hints.get("audio_end_ms") + try: + audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b"" + except (binascii.Error, ValueError) as exc: + raise RealtimeProtocolError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc + return RealtimeAudioAppend( + event_id=event_id, + audio=audio_bytes, + format=str(fmt), + sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None, + is_speech=looks_like_speech, + video_frames=video_frames, + duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None, + audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None, + hints=hints, + ) diff --git a/vllm_omni/protocol/realtime/capabilities.py b/vllm_omni/protocol/realtime/capabilities.py new file mode 100644 index 000000000..658a57e56 --- /dev/null +++ b/vllm_omni/protocol/realtime/capabilities.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""What a Realtime consumer can serve, and the session check that reads it. + +``session.update`` is the one client event whose acceptance depends on who is +behind the socket. The audio formats a server can decode, and whether it can do +server-side turn detection at all, are properties of the *consumer* --- the +duplex engine, or some other runtime --- not of the protocol. The codec still +has to do the checking, because it owns the error envelope and the order the +checks run in. + +:class:`RealtimeProtocolCapabilities` is how a consumer states those answers. +It is the post-#7413 form of the ``supports(turn_detection, formats)`` half of +RFC #6592's proposed ``RealtimeModelAdapter``. The other three halves of that +proposal --- building a prompt, starting a response, cancelling one --- are not +here and must not come here: for a duplex model they are already owned by +``DuplexModelPlugin`` together with ``DuplexSessionRunner``, and a second +interface over the same responsibility is exactly the fork this layer exists to +prevent. + +The duplex binding of this object is +``vllm_omni.engine.duplex.realtime_commands.DUPLEX_REALTIME_CAPABILITIES``. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field + +from vllm_omni.protocol.realtime.formats import ( + REALTIME_INPUT_AUDIO_FORMATS, + REALTIME_OUTPUT_AUDIO_FORMATS, + validate_realtime_session_audio_formats, +) + +__all__ = [ + "RealtimeProtocolCapabilities", + "RealtimeSessionRejection", + "validate_session_payload", +] + +#: A consumer's answer to "can you do this turn detection?": an error message, or None. +TurnDetectionValidator = Callable[[Mapping[str, object]], str | None] + + +@dataclass(frozen=True, slots=True) +class RealtimeProtocolCapabilities: + """One consumer's answer to what a ``session`` object may ask for. + + The defaults are permissive, not restrictive: every format the codec can + decode, and **no turn-detection validation at all**. A consumer that leaves + ``validate_turn_detection`` at ``None`` therefore accepts whatever + ``turn_detection`` the session object carries --- including values it cannot + actually serve, such as ``semantic_vad``. A consumer that supports only some + turn-detection modes (or none) must supply its own validator; see + ``vllm_omni.engine.duplex.realtime_commands.DUPLEX_REALTIME_CAPABILITIES``. + """ + + input_audio_formats: frozenset[str] = field(default_factory=lambda: frozenset(REALTIME_INPUT_AUDIO_FORMATS)) + output_audio_formats: frozenset[str] = field(default_factory=lambda: frozenset(REALTIME_OUTPUT_AUDIO_FORMATS)) + #: Consumer-specific ``turn_detection`` validation. ``None`` means the + #: consumer takes whatever the session object says without a check of its own. + validate_turn_detection: TurnDetectionValidator | None = None + + +@dataclass(frozen=True, slots=True) +class RealtimeSessionRejection: + """Why a session object was refused: the internal error code and the message. + + ``param`` names the offending session field where there is one, for the + ``error.param`` slot of the OpenAI error envelope. + """ + + code: str + message: str + param: str | None = None + + +def validate_session_payload( + session_payload: Mapping[str, object], + *, + capabilities: RealtimeProtocolCapabilities, +) -> RealtimeSessionRejection | None: + """Check a ``session`` object against one consumer; ``None`` when it is acceptable. + + Audio formats are checked before turn detection, so a session object that + is wrong in both ways reports the format problem --- keep that order, it is + what clients already see. + """ + format_error = validate_realtime_session_audio_formats( + session_payload, + input_audio_formats=capabilities.input_audio_formats, + output_audio_formats=capabilities.output_audio_formats, + ) + if format_error is not None: + return RealtimeSessionRejection(code="unsupported_audio_format", message=format_error) + validate_turn_detection = capabilities.validate_turn_detection + if validate_turn_detection is not None: + turn_detection_error = validate_turn_detection(session_payload) + if turn_detection_error is not None: + return RealtimeSessionRejection( + code="unsupported_turn_detection", message=turn_detection_error, param="turn_detection" + ) + return None diff --git a/vllm_omni/protocol/realtime/commands.py b/vllm_omni/protocol/realtime/commands.py new file mode 100644 index 000000000..590f3ee93 --- /dev/null +++ b/vllm_omni/protocol/realtime/commands.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""OpenAI Realtime client events as typed commands. + +A Realtime server's first job is turning a client JSON event into something +typed. These are the 10 client events OpenAI's Realtime API defines, as frozen +dataclasses carrying the fields that survive decoding. + +vLLM-Omni's own client events (barge-in, playback acknowledgement, text append, +heartbeat, explicit close, generic turn signals) are a separate vocabulary in +``vllm_omni.protocol.duplex.commands``. + +Decoding a payload *into* these stays with the consumer, because which command +a given event becomes can depend on what that consumer supports --- see +``vllm_omni.engine.duplex.realtime_commands.translate_realtime_command``. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import ClassVar + + +@dataclass(frozen=True, slots=True, kw_only=True) +class RealtimeCommand: + """One decoded OpenAI Realtime client event. + + ``wire_type`` is the event type the client sent; the typed fields are what + survived decoding, so a consumer reads them instead of re-parsing JSON. + + There is deliberately no rendering method here. A server receives commands, + it does not emit them, and how a runtime represents one internally is its + own business --- the duplex engine renders its mailbox dictionary in + ``vllm_omni.engine.duplex.commands``. + """ + + #: OpenAI Realtime client event this command decodes from. + wire_type: ClassVar[str] = "" + #: Client correlation id (OpenAI ``event_id``), echoed on error events. + event_id: str | None = None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class UpdateSession(RealtimeCommand): + """``session.update``: a Realtime ``session`` object patch.""" + + wire_type: ClassVar[str] = "session.update" + patch: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AppendAudio(RealtimeCommand): + wire_type: ClassVar[str] = "input_audio_buffer.append" + #: Raw audio bytes in ``format`` at ``sample_rate_hz`` (base64 only on the wire). + audio: bytes + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Commit(RealtimeCommand): + wire_type: ClassVar[str] = "input_audio_buffer.commit" + final: bool = True + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ClearInput(RealtimeCommand): + wire_type: ClassVar[str] = "input_audio_buffer.clear" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ClearOutputAudio(RealtimeCommand): + wire_type: ClassVar[str] = "output_audio_buffer.clear" + #: Explicit response to clear; ``None`` targets the active/last response. + response_id: str | None = None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CreateResponse(RealtimeCommand): + wire_type: ClassVar[str] = "response.create" + #: Raw Realtime ``response`` object (instructions, voice, modalities, ...). + options: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CancelResponse(RealtimeCommand): + wire_type: ClassVar[str] = "response.cancel" + response_id: str | None = None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CreateItem(RealtimeCommand): + """``conversation.item.create`` (history injection or function-call output).""" + + wire_type: ClassVar[str] = "conversation.item.create" + item: Mapping[str, object] + previous_item_id: str | None = None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class DeleteItem(RealtimeCommand): + wire_type: ClassVar[str] = "conversation.item.delete" + item_id: str + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TruncateItem(RealtimeCommand): + wire_type: ClassVar[str] = "conversation.item.truncate" + item_id: str + audio_end_ms: int + content_index: int = 0 + + +__all__ = [ + "AppendAudio", + "CancelResponse", + "ClearInput", + "ClearOutputAudio", + "Commit", + "CreateItem", + "CreateResponse", + "DeleteItem", + "RealtimeCommand", + "TruncateItem", + "UpdateSession", +] diff --git a/vllm_omni/protocol/realtime/errors.py b/vllm_omni/protocol/realtime/errors.py new file mode 100644 index 000000000..f9ba67bf4 --- /dev/null +++ b/vllm_omni/protocol/realtime/errors.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""The OpenAI Realtime error envelope. + +A Realtime server reports a rejected client event as one ``error`` event whose +``error`` object carries ``type`` / ``code`` / ``message`` (plus the client's +``event_id`` and an optional ``param``). ``type`` is not free-form: it is one +of OpenAI's four buckets, derived from our internal code by +:data:`REALTIME_ERROR_TYPES_BY_CODE`. + +OpenAI standardises the three ``type`` classes but not the *codes*, so the code +vocabulary is a consumer's own. vLLM-Omni's lives in +``vllm_omni.protocol.duplex.errors`` (Tier 3 in +``docs/serving/realtime_duplex_api.md``); what stays here is the exception type +every consumer raises, so the envelope is rendered identically whoever built it. +""" + +from __future__ import annotations + +__all__ = ["RealtimeProtocolError"] + + +class RealtimeProtocolError(ValueError): + """A client payload could not be decoded into a valid Realtime intent. + + ``code`` is the internal error code that :data:`REALTIME_ERROR_TYPES_BY_CODE` + maps to an OpenAI ``error.type``; ``event_id`` is the *client* event id the + error answers. ``vllm_omni.engine.duplex.commands.DuplexCommandError`` is + the duplex specialization, so a consumer catching either sees the same + three attributes. + """ + + def __init__(self, message: str, *, code: str = "bad_event", event_id: str | None = None) -> None: + super().__init__(message) + self.code = code + self.event_id = event_id diff --git a/vllm_omni/protocol/realtime/events.py b/vllm_omni/protocol/realtime/events.py new file mode 100644 index 000000000..47bab6372 --- /dev/null +++ b/vllm_omni/protocol/realtime/events.py @@ -0,0 +1,467 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""OpenAI Realtime server events: typed classes and their wire rendering. + +Every event is a frozen dataclass with explicit fields, and the Realtime wire +JSON is *derived* from those fields by :meth:`RealtimeEvent.to_realtime`. That +rendering is pure --- it never consults session state --- which is why it can +live here, away from whoever owns the session. The stateful part (response and +item ids, content-part bookkeeping) is consumed when the events are +*constructed*, by the consumer. + +This module holds the 30 events OpenAI's Realtime API defines. vLLM-Omni's own +additions to the wire (listen/speak turn-taking, playback acknowledgement, +session resume) are a separate vocabulary in +``vllm_omni.protocol.duplex.events``, so a consumer that only speaks GA does +not inherit them. + +``session_id`` / ``epoch`` are bound by the consumer when an event leaves it; +producers build events with the defaults. +""" + +from __future__ import annotations + +import base64 +from collections.abc import Mapping +from dataclasses import dataclass, field, fields +from typing import ClassVar +from uuid import uuid4 + + +def new_event_id() -> str: + return f"event_{uuid4().hex}" + + +_IDENTITY_FIELDS = frozenset({"session_id", "epoch", "event_id"}) + + +def wire_value(value: object) -> object: + if isinstance(value, Mapping): + return {str(k): wire_value(v) for k, v in value.items()} + if isinstance(value, tuple | list): + return [wire_value(v) for v in value] + return value + + +@dataclass(frozen=True, slots=True, kw_only=True) +class RealtimeEvent: + """Base of every public session event.""" + + #: Realtime ``type`` this class renders to. + wire_type: ClassVar[str] = "" + #: Fields that are omitted from the wire object when ``None``. + optional_wire_fields: ClassVar[frozenset[str]] = frozenset() + + session_id: str = "" + epoch: int | None = None + #: Server event id (OpenAI ``event_id``); generated at construction. + event_id: str = field(default_factory=new_event_id) + + # ---- identity ---- + + @property + def type(self) -> str: + return self.wire_type + + @property + def is_terminal(self) -> bool: + return False + + # ---- generic accessors (None when the class has no such field) ---- + + @property + def response_id(self) -> str | None: + return None + + @property + def item_id(self) -> str | None: + return None + + @property + def text(self) -> str | None: + return None + + @property + def audio(self) -> bytes | None: + return None + + # ---- wire rendering (pure) ---- + + def _wire_fields(self) -> dict[str, object]: + data: dict[str, object] = {} + for f in fields(self): + if f.name in _IDENTITY_FIELDS: + continue + value = getattr(self, f.name) + if value is None and f.name in self.optional_wire_fields: + continue + data[f.name] = wire_value(value) + return data + + def to_realtime(self) -> dict[str, object]: + """The Realtime wire JSON object for this event (derived from the fields).""" + return {"type": self.wire_type, "event_id": self.event_id, **self._wire_fields()} + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"{type(self).__name__}(type={self.wire_type!r}, session_id={self.session_id!r}, epoch={self.epoch!r})" + + +# ---- shared bases ---- + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ResponseScopedEvent(RealtimeEvent): + """Events addressed to one response (and usually one output item / content part).""" + + response_id: str | None = None # type: ignore[assignment] + item_id: str | None = None # type: ignore[assignment] + output_index: int = 0 + content_index: int = 0 + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _ItemEvent(RealtimeEvent): + previous_item_id: str | None = None + item: Mapping[str, object] = field(default_factory=dict) + + @property + def item_id(self) -> str | None: + value = self.item.get("id") + return value if isinstance(value, str) else None + + +# ---- GA server events ---- + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionCreated(RealtimeEvent): + wire_type = "session.created" + + session: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionUpdated(RealtimeEvent): + wire_type = "session.updated" + + session: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ResponseCreated(RealtimeEvent): + wire_type = "response.created" + + response: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class OutputItemAdded(RealtimeEvent): + wire_type = "response.output_item.added" + + response_id: str | None = None # type: ignore[assignment] + output_index: int = 0 + item: Mapping[str, object] = field(default_factory=dict) + + @property + def item_id(self) -> str | None: + value = self.item.get("id") + return value if isinstance(value, str) else None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class OutputItemDone(OutputItemAdded): + wire_type = "response.output_item.done" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ContentPartAdded(ResponseScopedEvent): + wire_type = "response.content_part.added" + + part: Mapping[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ContentPartDone(ContentPartAdded): + wire_type = "response.content_part.done" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AudioDelta(ResponseScopedEvent): + wire_type = "response.output_audio.delta" + + #: Base64 audio in ``format``. + delta: str = "" + + @property + def audio(self) -> bytes | None: + try: + return base64.b64decode(self.delta) + except (ValueError, TypeError): + return None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class AudioDone(ResponseScopedEvent): + wire_type = "response.output_audio.done" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TranscriptDelta(ResponseScopedEvent): + wire_type = "response.output_audio_transcript.delta" + + delta: str = "" + + @property + def text(self) -> str | None: + return self.delta + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TranscriptDone(ResponseScopedEvent): + wire_type = "response.output_audio_transcript.done" + + transcript: str = "" + + @property + def text(self) -> str | None: + return self.transcript + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TextDelta(ResponseScopedEvent): + wire_type = "response.output_text.delta" + + delta: str = "" + + @property + def text(self) -> str | None: + return self.delta + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TextDone(ResponseScopedEvent): + wire_type = "response.output_text.done" + + text: str = "" # type: ignore[assignment] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class FunctionCallArgumentsDelta(RealtimeEvent): + wire_type = "response.function_call_arguments.delta" + + response_id: str | None = None # type: ignore[assignment] + item_id: str | None = None # type: ignore[assignment] + output_index: int = 0 + call_id: str = "" + delta: str = "" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class FunctionCallArgumentsDone(RealtimeEvent): + wire_type = "response.function_call_arguments.done" + + response_id: str | None = None # type: ignore[assignment] + item_id: str | None = None # type: ignore[assignment] + output_index: int = 0 + call_id: str = "" + arguments: str = "" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ResponseDone(RealtimeEvent): + wire_type = "response.done" + + response: Mapping[str, object] = field(default_factory=dict) + + @property + def status(self) -> str | None: + value = self.response.get("status") + return value if isinstance(value, str) else None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class RateLimitsUpdated(RealtimeEvent): + wire_type = "rate_limits.updated" + + rate_limits: tuple[Mapping[str, object], ...] = () + + +@dataclass(frozen=True, slots=True, kw_only=True) +class InputCommitted(RealtimeEvent): + wire_type = "input_audio_buffer.committed" + + previous_item_id: str | None = None + item_id: str | None = None # type: ignore[assignment] + + def _wire_fields(self) -> dict[str, object]: + return {"previous_item_id": self.previous_item_id, "item_id": self.item_id} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class InputCleared(RealtimeEvent): + wire_type = "input_audio_buffer.cleared" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SpeechStarted(RealtimeEvent): + wire_type = "input_audio_buffer.speech_started" + + audio_start_ms: int = 0 + item_id: str | None = None # type: ignore[assignment] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SpeechStopped(RealtimeEvent): + wire_type = "input_audio_buffer.speech_stopped" + + audio_end_ms: int = 0 + item_id: str | None = None # type: ignore[assignment] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ItemAdded(_ItemEvent): + wire_type = "conversation.item.added" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ItemCreated(_ItemEvent): + wire_type = "conversation.item.created" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ItemDone(_ItemEvent): + wire_type = "conversation.item.done" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ItemRetrieved(RealtimeEvent): + wire_type = "conversation.item.retrieved" + + item: Mapping[str, object] = field(default_factory=dict) + + @property + def item_id(self) -> str | None: + value = self.item.get("id") + return value if isinstance(value, str) else None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ItemDeleted(RealtimeEvent): + wire_type = "conversation.item.deleted" + + item_id: str | None = None # type: ignore[assignment] + + def _wire_fields(self) -> dict[str, object]: + return {"item_id": self.item_id} + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ItemTruncated(RealtimeEvent): + wire_type = "conversation.item.truncated" + + item_id: str | None = None # type: ignore[assignment] + content_index: int = 0 + audio_end_ms: int = 0 + + def _wire_fields(self) -> dict[str, object]: + return { + "item_id": self.item_id, + "content_index": self.content_index, + "audio_end_ms": self.audio_end_ms, + } + + +@dataclass(frozen=True, slots=True, kw_only=True) +class InputTranscriptionCompleted(RealtimeEvent): + wire_type = "conversation.item.input_audio_transcription.completed" + + item_id: str | None = None # type: ignore[assignment] + content_index: int = 0 + transcript: str = "" + + @property + def text(self) -> str | None: + return self.transcript + + +@dataclass(frozen=True, slots=True, kw_only=True) +class OutputAudioCleared(RealtimeEvent): + wire_type = "output_audio_buffer.cleared" + + response_id: str | None = None # type: ignore[assignment] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ErrorEvent(RealtimeEvent): + wire_type = "error" + + code: str = "internal_error" + message: str = "" + #: Client ``event_id`` this error answers (OpenAI ``error.event_id``). + related_event_id: str | None = None + param: str | None = None + + @property + def error_type(self) -> str: + """OpenAI's error class for this error. + + OpenAI standardises only the three classes, not the codes, so the base + reports the default. A consumer with its own code vocabulary overrides + this --- see ``vllm_omni.protocol.duplex.events.ErrorEvent``. + """ + return "invalid_request_error" + + @property + def error(self) -> dict[str, object]: + error: dict[str, object] = {"type": self.error_type, "code": self.code, "message": self.message} + if self.related_event_id: + error["event_id"] = self.related_event_id + if self.param: + error["param"] = self.param + return error + + @property + def text(self) -> str | None: + return self.message + + def _wire_fields(self) -> dict[str, object]: + return {"error": self.error} + + +__all__ = [ + "AudioDelta", + "AudioDone", + "ContentPartAdded", + "ContentPartDone", + "ErrorEvent", + "FunctionCallArgumentsDelta", + "FunctionCallArgumentsDone", + "InputCleared", + "InputCommitted", + "InputTranscriptionCompleted", + "ItemAdded", + "ItemCreated", + "ItemDeleted", + "ItemDone", + "ItemRetrieved", + "ItemTruncated", + "OutputAudioCleared", + "OutputItemAdded", + "OutputItemDone", + "RateLimitsUpdated", + "RealtimeEvent", + "ResponseCreated", + "ResponseDone", + "ResponseScopedEvent", + "SessionCreated", + "SessionUpdated", + "SpeechStarted", + "SpeechStopped", + "TextDelta", + "TextDone", + "TranscriptDelta", + "TranscriptDone", + "new_event_id", + "wire_value", +] diff --git a/vllm_omni/protocol/realtime/formats.py b/vllm_omni/protocol/realtime/formats.py new file mode 100644 index 000000000..161aab11b --- /dev/null +++ b/vllm_omni/protocol/realtime/formats.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Realtime audio-format negotiation: what a client may declare and how it is read. + +A Realtime client declares audio formats in three places --- the session object +(``session.audio.input.format`` and the older flat ``input_audio_format``), a +``response.create`` override, and a ``conversation.item.create`` audio part --- +and may spell one format several ways (``audio/pcm``, ``pcm16``, ``s16le``). +This module is the single place that normalizes those spellings and says which +are supported. Folding a session's declaration into the per-connection append +defaults is the job of ``session.py``, which builds on this module. + +Everything here is a pure function of wire payloads: no session state, no model, +no engine. +""" + +from __future__ import annotations + +from collections.abc import Collection, Mapping + +__all__ = [ + "REALTIME_INPUT_AUDIO_FORMATS", + "REALTIME_OUTPUT_AUDIO_FORMATS", + "is_supported_realtime_input_format", + "parse_realtime_audio_format", + "realtime_audio_format_object", + "realtime_output_format", + "validate_conversation_item_audio_formats", + "validate_realtime_response_audio_formats", + "validate_realtime_session_audio_formats", +] + +REALTIME_INPUT_AUDIO_FORMATS = { + "pcm16", + "pcm_s16le", + "s16le", + "pcm_f32le", + "g711_ulaw", + "g711_alaw", +} +REALTIME_OUTPUT_AUDIO_FORMATS = { + "pcm16", + "pcm_s16le", + "s16le", + "wav", + "pcm", + "g711_ulaw", + "g711_alaw", +} + + +def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]: + def normalize_format(fmt: str) -> str: + normalized = fmt.lower() + if normalized in {"audio/pcm", "pcm"}: + return "pcm16" + if normalized in {"audio/wav", "wav"}: + return "wav" + if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}: + return "pcm16" + if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}: + return "pcm_f32le" + if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}: + return "g711_ulaw" + if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}: + return "g711_alaw" + return fmt + + if isinstance(raw_format, str): + return normalize_format(raw_format), None + if not isinstance(raw_format, dict): + return raw_format, None + rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate") + sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None + fmt = raw_format.get("type") or raw_format.get("format") + if not isinstance(fmt, str): + return raw_format, sample_rate_hz + return normalize_format(fmt), sample_rate_hz + + +def realtime_output_format(duplex_format: object) -> str: + if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}: + return duplex_format.lower() + if isinstance(duplex_format, str) and duplex_format.lower() == "pcm": + return "pcm16" + return str(duplex_format or "wav") + + +def is_supported_realtime_input_format(fmt: object) -> bool: + return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS + + +def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]: + if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}: + payload: dict[str, object] = {"type": "audio/pcm"} + elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le": + payload = {"type": "audio/pcm_f32le"} + elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw": + payload = {"type": "audio/g711_ulaw"} + elif isinstance(fmt, str) and fmt.lower() == "g711_alaw": + payload = {"type": "audio/g711_alaw"} + else: + payload = {"type": "audio/wav"} + if sample_rate_hz is not None: + payload["rate"] = int(sample_rate_hz) + return payload + + +def validate_realtime_session_audio_formats( + session_payload: Mapping[str, object], + *, + input_audio_formats: Collection[str] | None = None, + output_audio_formats: Collection[str] | None = None, +) -> str | None: + """Reject a session object that declares an audio format we cannot serve. + + The format sets default to everything the codec can decode; a consumer that + serves a narrower set passes its own (see + ``vllm_omni.protocol.realtime.capabilities``). + """ + supported_input = REALTIME_INPUT_AUDIO_FORMATS if input_audio_formats is None else input_audio_formats + supported_output = REALTIME_OUTPUT_AUDIO_FORMATS if output_audio_formats is None else output_audio_formats + audio_config = session_payload.get("audio") + input_format: object = session_payload.get("input_audio_format") + if input_format is None and isinstance(audio_config, dict): + audio_input = audio_config.get("input") + if isinstance(audio_input, dict): + input_format = audio_input.get("format") + parsed_input, _ = parse_realtime_audio_format(input_format) + if input_format is not None and not (isinstance(parsed_input, str) and parsed_input.lower() in supported_input): + return f"Unsupported input_audio_format: {input_format}" + + output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format") + if output_format is None and isinstance(audio_config, dict): + audio_output = audio_config.get("output") + if isinstance(audio_output, dict): + output_format = audio_output.get("format") + parsed_output, _ = parse_realtime_audio_format(output_format) + if output_format is not None and not (isinstance(parsed_output, str) and parsed_output.lower() in supported_output): + return f"Unsupported output_audio_format: {output_format}" + return None + + +def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None: + output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format") + audio_config = response_payload.get("audio") + if output_format is None and isinstance(audio_config, dict): + audio_output = audio_config.get("output") + if isinstance(audio_output, dict): + output_format = audio_output.get("format") + parsed_output, _ = parse_realtime_audio_format(output_format) + if output_format is not None and not ( + isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS + ): + return f"Unsupported output_audio_format: {output_format}" + return None + + +def validate_conversation_item_audio_formats(item: object) -> str | None: + if not isinstance(item, dict): + return None + content = item.get("content") + if not isinstance(content, list): + return None + for part in content: + if not isinstance(part, dict): + continue + if part.get("type") not in {"input_audio", "audio"}: + continue + raw_format = part.get("format") + parsed_format, _ = parse_realtime_audio_format(raw_format) + if raw_format is not None and not ( + isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS + ): + return f"Unsupported input_audio format in conversation.item.create: {raw_format}" + return None diff --git a/vllm_omni/protocol/realtime/items.py b/vllm_omni/protocol/realtime/items.py new file mode 100644 index 000000000..cbcb9c652 --- /dev/null +++ b/vllm_omni/protocol/realtime/items.py @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Realtime conversation items: shape, truncation and the transcript inside one. + +A Realtime conversation is a list of *items* --- a user message, an assistant +message, a function call --- each with a ``content`` list of typed parts. The +client creates them (``conversation.item.create``), truncates an assistant +item's audio after a barge-in (``conversation.item.truncate``), and the server +echoes them back inside ``conversation.item.*`` and ``response.done`` events. + +The rules in this module are the ones both sides have to agree on: which +defaults a bare item gets, how far a transcript is cut when the client says it +only heard N milliseconds of audio, and what makes a truncate request invalid. +Item *storage* --- which items a session currently holds, in what order --- is +the consumer's state, not this module's. + +``validate_realtime_video_frames`` sits here too: camera frames ride along an +``input_audio_buffer.append`` in the vLLM-Omni extension of the protocol, and +validating them is the same kind of wire-shape check. +""" + +from __future__ import annotations + +import base64 +import binascii +from collections.abc import Mapping +from uuid import uuid4 + +__all__ = [ + "input_transcript_from_item", + "normalize_conversation_item", + "text_chars_for_audio_ms_from_marks", + "truncate_realtime_item_content", + "validate_realtime_item_truncate", + "validate_realtime_video_frames", +] + + +def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None: + """Validate omni-duplex camera frames on input_audio_buffer.append. + + Wire contract matches the official MiniCPM-o duplex loop: one base + base64 JPEG per ~1 s audio chunk, optionally followed by that unit's + stacked composite tiling the sub-frames captured inside it (at most 2 + images either way). A caller-supplied ``max_slice_nums`` is rejected rather + than silently ignored: slicing is Stage 0's decision here, and Stage 0 + already applies the official HD suggestion for a stacked unit + (``max_slice_nums=[2, 1]``). The wire simply does not let the client choose + it. + """ + if max_slice_nums not in (None, 1): + return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself" + if not isinstance(video_frames, list): + return "video_frames must be a list of base64-encoded images" + frames = [frame for frame in video_frames if frame is not None] + if len(frames) > 2: + return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk" + for frame in frames: + if not isinstance(frame, str) or not frame: + return "video_frames entries must be non-empty base64 strings" + if len(frame) > 4_000_000: + return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality" + try: + header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4)) + except (binascii.Error, ValueError): + return "video_frames entries must be valid base64" + if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")): + return "video_frames entries must be JPEG or PNG images" + return None + + +def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]: + normalized = dict(item) + normalized.setdefault("id", f"item_{uuid4().hex}") + normalized.setdefault("object", "realtime.item") + normalized.setdefault("type", "message") + normalized.setdefault("status", "completed") + if "role" not in normalized and normalized.get("type") == "message": + normalized["role"] = "user" + if not isinstance(normalized.get("content"), list): + normalized["content"] = [] + return normalized + + +def text_chars_for_audio_ms_from_marks( + audio_end_ms: int, + text_len: int, + marks: list[object], + *, + final_ms: object | None = None, +) -> int: + if text_len <= 0: + return 0 + clean_marks: list[tuple[int, int]] = [] + for mark in marks: + if not isinstance(mark, dict): + continue + raw_text_chars = mark.get("text_chars") + raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms")) + if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float): + continue + clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars))))) + if not clean_marks: + return 0 if audio_end_ms <= 0 else text_len + clean_marks.sort(key=lambda item: item[0]) + audio_end_ms = max(0, int(audio_end_ms)) + if audio_end_ms <= 0: + return 0 + previous_ms = 0 + previous_chars = 0 + for mark_ms, mark_chars in clean_marks: + mark_ms = max(previous_ms, mark_ms) + mark_chars = max(previous_chars, min(text_len, mark_chars)) + if audio_end_ms <= mark_ms: + if mark_ms <= previous_ms: + return mark_chars + ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms) + return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio))) + previous_ms = mark_ms + previous_chars = mark_chars + if isinstance(final_ms, int | float) and int(final_ms) > previous_ms: + if audio_end_ms >= int(final_ms): + return text_len + ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms) + return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio))) + return text_len if audio_end_ms >= previous_ms else previous_chars + + +def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None: + content = item.get("content") + if not isinstance(content, list) or not content: + return + index = max(0, int(content_index)) + if index >= len(content): + return + part = content[index] + if not isinstance(part, dict): + return + transcript = part.get("transcript") + if not isinstance(transcript, str) or not transcript: + return + marks = part.get("audio_text_marks") + if isinstance(marks, list): + keep_chars = text_chars_for_audio_ms_from_marks( + audio_end_ms, + len(transcript), + marks, + final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"), + ) + part["transcript"] = transcript[:keep_chars].rstrip() + return + duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms") + if isinstance(duration_ms, int | float) and duration_ms > 0: + keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms)))) + part["transcript"] = transcript[:keep_chars].rstrip() + elif audio_end_ms <= 0: + part["transcript"] = "" + + +def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None: + if item.get("type") != "message" or item.get("role") != "assistant": + return "conversation.item.truncate only supports assistant message items" + if audio_end_ms < 0: + return "conversation.item.truncate requires non-negative audio_end_ms" + content = item.get("content") + if not isinstance(content, list) or not content: + return None + index = max(0, int(content_index)) + if index >= len(content): + return f"conversation.item.truncate content_index out of range: {content_index}" + part = content[index] + if not isinstance(part, dict): + return "conversation.item.truncate target content part is invalid" + if part.get("type") not in {"audio", "output_audio"}: + return "conversation.item.truncate target content part must be audio" + duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms") + if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms): + return "conversation.item.truncate audio_end_ms exceeds item audio duration" + return None + + +def input_transcript_from_item(item: Mapping[str, object]) -> str: + content = item.get("content") + if not isinstance(content, list): + return "" + parts: list[str] = [] + for part in content: + if not isinstance(part, dict): + continue + for key in ("transcript", "text"): + value = part.get(key) + if isinstance(value, str) and value: + if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}: + parts.append(value) + break + return "".join(parts).strip() diff --git a/vllm_omni/protocol/realtime/session.py b/vllm_omni/protocol/realtime/session.py new file mode 100644 index 000000000..1c73ea926 --- /dev/null +++ b/vllm_omni/protocol/realtime/session.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Reading the Realtime ``session`` object. + +``session.update`` carries one JSON object that a client may spell in either of +two shapes: the GA nest (``session.audio.input.format``, +``session.audio.output.sample_rate_hz``) or the older flat beta keys +(``input_audio_format``, ``sample_rate_hz``). Everything downstream --- the +append decoder, the projection that renders ``session.created`` back out, a +model's runtime configuration --- has to agree on what a given session object +means, so the readers live here once. + +:class:`RealtimeInputDefaults` is the resolved answer for the input path: the +format, sample rate and silence threshold an ``input_audio_buffer.append`` +inherits when it omits them. A connection keeps one of these and re-derives it +on every ``session.update``. + +``overlap_policy`` / ``playback_commit_policy`` are vLLM-Omni extension fields +on the session object rather than GA OpenAI fields. They are read here because +this module owns what the wire may say; what a server *does* with them is the +consumer's business. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, replace +from typing import Any + +from vllm_omni.protocol.realtime.formats import ( + REALTIME_INPUT_AUDIO_FORMATS, + REALTIME_OUTPUT_AUDIO_FORMATS, + parse_realtime_audio_format, + realtime_output_format, +) + +__all__ = [ + "RealtimeInputDefaults", + "apply_realtime_session_defaults", + "input_audio_transcription_config", + "json_safe_realtime_payload", + "realtime_max_output_tokens", + "realtime_overlap_fields", +] + + +@dataclass(frozen=True, slots=True) +class RealtimeInputDefaults: + """Session-level wire defaults an append may omit (derived from the session object).""" + + input_audio_format: str = "pcm16" + input_sample_rate_hz: int = 16000 + output_audio_format: str = "pcm16" + output_sample_rate_hz: int | None = None + overlap_silence_rms: float = 0.003 + + def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults: + """Return defaults updated from a Realtime ``session`` object (session.update).""" + values = apply_realtime_session_defaults(self, session_payload) + return values + + +def apply_realtime_session_defaults( + defaults: RealtimeInputDefaults, + session_payload: Mapping[str, object], +) -> RealtimeInputDefaults: + """Derive the wire defaults a Realtime ``session`` object declares.""" + input_format: object = session_payload.get("input_audio_format") + audio_config = session_payload.get("audio") + if input_format is None and isinstance(audio_config, dict): + audio_input = audio_config.get("input") + if isinstance(audio_input, dict): + input_format = audio_input.get("format") + input_format, input_rate = parse_realtime_audio_format(input_format) + # Any: heterogeneous field values splatted into ``dataclasses.replace`` below. + updates: dict[str, Any] = {} + if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS: + updates["input_audio_format"] = input_format + output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format") + output_rate_raw: object | None = None + if output_format is None and isinstance(audio_config, dict): + audio_output = audio_config.get("output") + if isinstance(audio_output, dict): + output_format = audio_output.get("format") + output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate") + output_format, output_rate = parse_realtime_audio_format(output_format) + if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0: + output_rate = int(output_rate_raw) + if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: + updates["output_audio_format"] = realtime_output_format(output_format) + sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate") + if sample_rate is None and isinstance(audio_config, dict): + audio_input = audio_config.get("input") + if isinstance(audio_input, dict): + sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate") + if sample_rate is None: + sample_rate = input_rate + if isinstance(sample_rate, int | float) and sample_rate > 0: + updates["input_sample_rate_hz"] = int(sample_rate) + if isinstance(output_rate, int | float) and output_rate > 0: + updates["output_sample_rate_hz"] = int(output_rate) + overlap_fields = realtime_overlap_fields(session_payload) + overlap_silence_rms = overlap_fields.get("overlap_silence_rms") + if isinstance(overlap_silence_rms, int | float): + updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms)) + return replace(defaults, **updates) if updates else defaults + + +def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]: + fields: dict[str, object] = {} + if isinstance(session_payload.get("overlap_policy"), str): + fields["overlap_policy"] = session_payload["overlap_policy"] + for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"): + value = session_payload.get(key) + if isinstance(value, int | float): + fields[key] = value + if isinstance(session_payload.get("playback_commit_policy"), str): + fields["playback_commit_policy"] = session_payload["playback_commit_policy"] + return fields + + +def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]: + clean: dict[str, object] = {} + for key, value in payload.items(): + if key == "extra_body": + continue + if isinstance(value, str | int | float | bool) or value is None: + clean[key] = value + elif isinstance(value, dict): + clean[key] = json_safe_realtime_payload(value) + elif isinstance(value, list): + clean[key] = [ + (json_safe_realtime_payload(item) if isinstance(item, dict) else item) + for item in value + if isinstance(item, str | int | float | bool | dict) or item is None + ] + return clean + + +def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None: + transcription = session_payload.get("input_audio_transcription") + if isinstance(transcription, dict): + return transcription + audio_config = session_payload.get("audio") + if not isinstance(audio_config, dict): + return None + audio_input = audio_config.get("input") + if not isinstance(audio_input, dict): + return None + transcription = audio_input.get("transcription") + return transcription if isinstance(transcription, dict) else None + + +def realtime_max_output_tokens(value: object) -> int | None: + """Normalize Realtime max output tokens (``"inf"`` -> ``None``).""" + if value is None: + return None + if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}: + return None + if isinstance(value, int) and value > 0: + return int(value) + return None