Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
psv666 authored and GitHub committed
1 parent
825ab20b07
commit
e3be42e052
37 files changed
+3792
-1629
No files matched your search
@@ -44,6 +44,7 @@ x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
|
||||
- vllm_omni/entrypoints/duplex/
|
||||
- vllm_omni/entrypoints/duplex_omni.py
|
||||
- vllm_omni/entrypoints/async_omni_base.py
|
||||
- vllm_omni/protocol/
|
||||
- vllm_omni/clients/
|
||||
- vllm_omni/outputs/duplex.py
|
||||
- vllm_omni/model_executor/duplex_sampling.py
|
||||
@@ -259,6 +260,7 @@ source_file_dependencies:
|
||||
- tests/entrypoints/test_duplex_omni.py
|
||||
- tests/entrypoints/openai_api/test_duplex_api_server.py
|
||||
- tests/model_executor/models/minicpmo_4_5/duplex/
|
||||
- tests/protocol/
|
||||
|
||||
omni_minicpmo_4_5_duplex_perf:
|
||||
- *minicpmo_4_5
|
||||
|
||||
@@ -67,6 +67,11 @@
|
||||
# File-level overrides within entrypoints/
|
||||
/vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
|
||||
/vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
|
||||
# Shared wire codec: protocol/realtime (OpenAI Realtime GA) and protocol/duplex
|
||||
# (vLLM-Omni's extension of it) — RFC #6592 P0a. Layer-neutral by design — the
|
||||
# engine depends on it — so it is owned by the serving-boundary document rather
|
||||
# than by any one consumer. — docs/design/module/entrypoints.md
|
||||
/vllm_omni/protocol/ @alex-jw-brooks @linyueqian @NickCao
|
||||
|
||||
# Error contracts — docs/design/module/error_contracts.md
|
||||
/vllm_omni/errors.py @alex-jw-brooks @NickCao
|
||||
|
||||
@@ -9,6 +9,12 @@ the owner document for `vllm_omni/engine/duplex/**`,
|
||||
`vllm_omni/entrypoints/duplex_omni.py`, `vllm_omni/entrypoints/duplex/**`,
|
||||
`vllm_omni/clients/**` and the per-model `duplex/` packages.
|
||||
|
||||
`vllm_omni/protocol/realtime/**` (the OpenAI Realtime API) and
|
||||
`vllm_omni/protocol/duplex/**` (vLLM-Omni's extension of it) are the shared
|
||||
wire codec duplex is built on. Duplex is its first consumer but not its owner: the codec
|
||||
is deliberately runtime-agnostic so a second Realtime surface can reuse it
|
||||
(RFC #6592 P0a).
|
||||
|
||||
The public wire contract lives in
|
||||
[`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md);
|
||||
this page describes how it is produced.
|
||||
@@ -124,6 +130,40 @@ plugin and the session runtime config to `DuplexOrchestrator` directly.
|
||||
attachment/resume/replay bookkeeping; it holds no session state.
|
||||
6. **No `typing.Protocol`** in the duplex surfaces: the plugin, data plane,
|
||||
session state, PCM buffer, stage port and client transport seams are ABCs.
|
||||
7. **The wire codec is not duplex, and a wire type is not a mailbox message.**
|
||||
The event and command *vocabulary* plus its wire rendering, the audio format
|
||||
negotiation, the conversation-item rules and the error envelope are model-
|
||||
and runtime-agnostic, and split along the three tiers
|
||||
[`docs/serving/realtime_duplex_api.md`](../serving/realtime_duplex_api.md)
|
||||
already defines: **Tier 1** (identical to OpenAI) is
|
||||
`vllm_omni/protocol/realtime/**`; **Tier 2** (OpenAI names carrying our
|
||||
extensions) and **Tier 3** (ours alone) are `vllm_omni/protocol/duplex/**`.
|
||||
A Tier 2 class *subclasses* its Tier 1 twin and declares only what it adds,
|
||||
so the OpenAI surface stays honestly pure --- `session.created` without
|
||||
`resume_token`, `input_audio_buffer.append` without `video_frames`. Our
|
||||
error **codes** are Tier 3 (`protocol/duplex/errors.py`); only the envelope
|
||||
shape and OpenAI's three `error.type` classes are Tier 1.
|
||||
What stays engine-side is everything that is *internal representation*
|
||||
rather than contract: `DuplexCommand.payload()` renders the session runner's
|
||||
mailbox dictionary, whose channel genuinely differs from the client event
|
||||
(`session.update` and the three `conversation.item.*` commands all travel on
|
||||
`turn.signal`); `engine/duplex/realtime_commands.py` decides which command a
|
||||
decoded event becomes; `engine/duplex/realtime_events.py` holds the
|
||||
session's projection state. The codec may not import the engine, the
|
||||
entrypoints, `model_executor` or the clients
|
||||
(`tests/protocol/realtime/test_protocol_import_boundary.py`), and there is
|
||||
exactly one implementation of each codec behaviour
|
||||
(`tests/protocol/realtime/test_realtime_codec_single_source.py`).
|
||||
The dependency chain is `protocol/realtime` <- `protocol/duplex` <- engine /
|
||||
entrypoints / clients, and a duplex consumer uses **only** the middle link:
|
||||
`protocol/duplex/**` re-exports the Tier 1 names it does not extend, so a
|
||||
helper that later needs a duplex-specific version (`convert_input_audio_with_rate`
|
||||
resamples to MiniCPM-o's 16 kHz rather than the client's rate) is overridden
|
||||
in one file instead of at every call site
|
||||
(`tests/protocol/duplex/test_duplex_protocol_facade.py`).
|
||||
`DuplexEvent` is therefore a plain alias of `RealtimeEvent` --- an event has
|
||||
no engine-internal half --- while `DuplexCommand` is a real class, because a
|
||||
command does.
|
||||
|
||||
## Package layout
|
||||
|
||||
@@ -142,6 +182,31 @@ vllm_omni/
|
||||
│ │ ├── chat_completions.py DuplexChatCompletionsAdapter (/v1/chat/completions on a session per request)
|
||||
│ │ └── websocket.py websocket send/close/receive helpers
|
||||
│ └── openai/api_server.py builds DuplexOmni for duplex models; session-backed app state
|
||||
├── protocol/ SHARED WIRE CODEC (no engine / no model / no transport)
|
||||
│ ├── realtime/ the OpenAI Realtime API itself
|
||||
│ │ ├── events.py RealtimeEvent + OpenAI's 30 server events (22 ship as Tier 1;
|
||||
│ │ │ 8 have a Tier 2 twin in protocol/duplex)
|
||||
│ │ ├── commands.py RealtimeCommand + OpenAI's 10 client commands (8 Tier 1,
|
||||
│ │ │ 2 with a Tier 2 twin) --- wire_type + fields, no payload()
|
||||
│ │ ├── formats.py audio format spellings, what is supported, per-session validation
|
||||
│ │ ├── session.py session-object readers; RealtimeInputDefaults (append defaults)
|
||||
│ │ ├── items.py conversation item shape, truncation, transcripts, camera frames
|
||||
│ │ ├── audio_input.py decode_audio_append -> RealtimeAudioAppend; client speech hints
|
||||
│ │ ├── audio.py PCM / G.711 / WAV decode, resample, re-encode
|
||||
│ │ ├── errors.py RealtimeProtocolError (the envelope's exception type)
|
||||
│ │ └── capabilities.py RealtimeProtocolCapabilities: what one consumer can serve
|
||||
│ └── duplex/ Tier 2 + Tier 3, and the one door a duplex consumer uses
|
||||
│ ├── __init__.py re-exports the Tier 1 helper functions (audio, formats,
|
||||
│ │ items, session, capabilities) so the engine never
|
||||
│ │ imports protocol/realtime directly
|
||||
│ ├── events.py the whole 42-event vocabulary: 22 Tier-1 re-exports,
|
||||
│ │ 8 Tier-2 subclasses (session.created + resume_token, ...),
|
||||
│ │ 12 Tier-3 events (listen/speak, playback, resume, ...);
|
||||
│ │ DuplexEvent = RealtimeEvent
|
||||
│ ├── commands.py the whole 17-command vocabulary: 8 Tier-1 re-exports,
|
||||
│ │ 2 Tier-2 subclasses (append + hints, commit + final),
|
||||
│ │ 7 Tier-3 commands (barge_in, playback.ack, ...)
|
||||
│ └── errors.py our error-code vocabulary (Tier 3) + RealtimeProtocolError
|
||||
├── engine/
|
||||
│ ├── omni_engine_base.py OmniEngineBase (stage processes, orchestrator thread, queues, RPC)
|
||||
│ ├── async_omni_engine.py AsyncOmniEngine (turn-based request building)
|
||||
@@ -149,16 +214,20 @@ vllm_omni/
|
||||
│ ├── orchestrator.py OrchestratorBase + Orchestrator
|
||||
│ ├── duplex_orchestrator.py DuplexOrchestrator (+ DuplexOrchestratorRequestState; implements DuplexStagePort)
|
||||
│ └── duplex/
|
||||
│ ├── commands.py DuplexCommand dataclasses, command_from_realtime
|
||||
│ ├── realtime_commands.py Realtime client event -> DuplexCommand translation
|
||||
│ ├── events.py DuplexEvent dataclasses (+ to_realtime), REALTIME_ERROR_TYPES_BY_CODE
|
||||
│ ├── commands.py mailbox half: DuplexCommand.payload() + the mailbox `type`,
|
||||
│ │ paired with each protocol command; command_from_realtime
|
||||
│ ├── realtime_commands.py duplex binding of the codec: decoded event -> DuplexCommand,
|
||||
│ │ DUPLEX_REALTIME_CAPABILITIES, duplex_response_format
|
||||
│ ├── events.py re-export shim (DuplexEvent = RealtimeEvent) + the runner's
|
||||
│ │ epoch-filter sets DOMAIN_TERMINAL_EVENTS / MODEL_OUTPUT_EVENTS
|
||||
│ ├── realtime_events.py RealtimeProjectionState: internal event -> typed events
|
||||
│ ├── messages.py queue envelopes (Open/Close/Resume/Touch/Command/Result/Event), DuplexSessionError
|
||||
│ ├── config.py DuplexSessionConfig, DuplexCapabilities, ResponseCreateOptions
|
||||
│ ├── contracts.py DuplexFence (session_id, epoch, turn_id), stage request records, DuplexStagePort
|
||||
│ ├── plugin.py DuplexModelPlugin, DuplexModelSessionState, DuplexDataPlane, PcmAppendBuffer ABCs
|
||||
│ ├── turn_detection.py server-side VAD turn detector used by the session
|
||||
│ ├── audio.py / vad.py / intermediate.py
|
||||
│ ├── audio.py compatibility re-export, via protocol/duplex
|
||||
│ ├── vad.py / intermediate.py
|
||||
│ └── session/ one engine-resident session and everything that runs it
|
||||
│ ├── engine_session.py DuplexEngineSession: ledgers, lease, fence, stage resources, append sequencing
|
||||
│ ├── runner.py DuplexSessionRunner (per-session mailbox on the orchestrator loop)
|
||||
|
||||
@@ -30,6 +30,7 @@ primary_code_paths:
|
||||
- vllm_omni/entrypoints/utils.py
|
||||
- vllm_omni/entrypoints/duplex/**
|
||||
- vllm_omni/clients/**
|
||||
- vllm_omni/protocol/**
|
||||
primary_path_exceptions:
|
||||
- path: vllm_omni/entrypoints/openai/errors.py
|
||||
owner: error_contracts.md
|
||||
@@ -53,6 +54,7 @@ depends_on:
|
||||
- error_contracts.md
|
||||
- engine_orchestration.md
|
||||
validation_paths:
|
||||
- tests/protocol/**
|
||||
- tests/entrypoints/test_omni_entrypoints.py
|
||||
- tests/entrypoints/test_async_omni.py
|
||||
- tests/entrypoints/test_async_omni_pause_sleep_routing.py
|
||||
@@ -106,6 +108,31 @@ placement, payload implementation, or semantic error classification.
|
||||
`entrypoints/openai/errors.py` is an explicit primary-path exception owned by
|
||||
`error_contracts.md`.
|
||||
|
||||
It also owns `vllm_omni/protocol/**`, the shared wire codec promoted by RFC
|
||||
[#6592](https://github.com/vllm-project/vllm-omni/issues/6592) P0a. It follows the three tiers
|
||||
`docs/serving/realtime_duplex_api.md` already defines. `protocol/realtime/**`
|
||||
is Tier 1, identical to OpenAI: 30 server events, 10 client commands, their
|
||||
typed fields and pure wire rendering, audio-format negotiation,
|
||||
conversation-item rules and the error-envelope shape. `protocol/duplex/**` is
|
||||
Tier 2 (OpenAI names carrying vLLM-Omni extensions, each a subclass of its
|
||||
Tier 1 twin declaring only what it adds) and Tier 3 (ours alone, including our
|
||||
error-code vocabulary), kept separate so a GA-only consumer is not handed a
|
||||
vocabulary its clients never send. That package sits
|
||||
outside `entrypoints/` on purpose --- the engine depends on it, so it belongs
|
||||
to neither layer --- but deciding what a Realtime client may put on the wire is
|
||||
this document's subject, and the same reasoning already places
|
||||
`vllm_omni/errors.py`, `inputs/` and `outputs/` under contract documents rather
|
||||
than under a consumer.
|
||||
|
||||
It is deliberately **not** owned by `../fullduplex.md`, even though duplex is
|
||||
the codec's only consumer today. RFC #6592 asks for a named owner precisely so
|
||||
a second Realtime consumer does not need duplex review to change the codec
|
||||
(#6592 open question 8). Ownership here is review scope, not a dependency: the
|
||||
codec may not import the engine, the entrypoints or the model code, which
|
||||
`tests/protocol/realtime/test_protocol_import_boundary.py` enforces. When a
|
||||
second consumer lands (#6592 P0b) the package is expected to graduate to its
|
||||
own module document with owners drawn from both consumers.
|
||||
|
||||
## Candidate invariants
|
||||
|
||||
These identifiers are proposals while the document is `draft`.
|
||||
@@ -115,6 +142,22 @@ These identifiers are proposals while the document is `draft`.
|
||||
**Rule:** Entrypoints MUST NOT implement cross-stage routing or stage lifecycle
|
||||
policy.
|
||||
|
||||
### ENTRY-INV-002: The shared Realtime codec does not depend on its consumers
|
||||
|
||||
**Rule:** `vllm_omni/protocol/**` MUST NOT import `vllm_omni.engine`,
|
||||
`vllm_omni.entrypoints`, `vllm_omni.model_executor`, `vllm_omni.worker` or
|
||||
`vllm_omni.clients`; `protocol/duplex/**` MAY depend on `protocol/realtime/**`
|
||||
but never the reverse, and a Tier 1 class MUST NOT carry a vLLM-Omni extension
|
||||
field; and each protocol/codec behavior MUST have exactly one
|
||||
implementation that every consumer uses. A wire type MUST NOT carry a
|
||||
consumer's internal representation: the duplex mailbox rendering
|
||||
(`DuplexCommand.payload()`, whose channel differs from the client event for
|
||||
`session.update` and the `conversation.item.*` commands) stays engine-side.
|
||||
A duplex consumer (`engine/duplex/**`, `entrypoints/duplex/**`, the duplex
|
||||
clients) MUST import `vllm_omni.protocol.duplex` and MUST NOT import
|
||||
`vllm_omni.protocol.realtime` directly, so the tier boundary has exactly one
|
||||
extension point.
|
||||
|
||||
### ENTRY-INV-100: Public requests are normalized once
|
||||
|
||||
**Rule:** Public protocol values MUST be validated and converted to internal
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
@@ -0,0 +1,144 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""``protocol.duplex`` is the only door a duplex consumer uses.
|
||||
|
||||
The layering is a chain::
|
||||
|
||||
protocol.realtime <- protocol.duplex <- engine / entrypoints / clients
|
||||
|
||||
The middle link only earns its keep if the last arrow is the *only* one. If the
|
||||
engine may also reach past it straight into ``protocol.realtime``, then giving a
|
||||
Tier 1 helper a duplex-specific version later means touching every call site
|
||||
instead of one file --- which is exactly the change
|
||||
``convert_input_audio_with_rate`` is expected to need (it resamples to
|
||||
MiniCPM-o's 16 kHz, not the client's rate).
|
||||
|
||||
So this asserts two things: nobody skips the middle link, and the middle link is
|
||||
complete enough that nobody has to.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
#: Every module that speaks the duplex protocol. These import
|
||||
#: ``vllm_omni.protocol.duplex``; reaching into ``vllm_omni.protocol.realtime``
|
||||
#: from here skips the extension point.
|
||||
CONSUMER_PATHS = (
|
||||
"vllm_omni/engine/duplex",
|
||||
"vllm_omni/engine/duplex_omni_engine.py",
|
||||
"vllm_omni/entrypoints/duplex",
|
||||
"vllm_omni/entrypoints/duplex_omni.py",
|
||||
"vllm_omni/clients/duplex.py",
|
||||
"vllm_omni/clients/inline_duplex.py",
|
||||
)
|
||||
|
||||
|
||||
def _consumer_modules() -> list[Path]:
|
||||
found: list[Path] = []
|
||||
for entry in CONSUMER_PATHS:
|
||||
path = REPO_ROOT / entry
|
||||
found.extend(sorted(path.rglob("*.py")) if path.is_dir() else [path])
|
||||
assert found, "consumer list is stale"
|
||||
return found
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", _consumer_modules(), ids=lambda p: p.name)
|
||||
def test_a_duplex_consumer_does_not_skip_the_duplex_protocol_package(module: Path) -> None:
|
||||
# Source-level, so a lazy import inside a function body is caught too.
|
||||
tree = ast.parse(module.read_text())
|
||||
offenders = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and (node.module or "").startswith("vllm_omni.protocol.realtime"):
|
||||
offenders.append(f"line {node.lineno}: from {node.module} import ...")
|
||||
elif isinstance(node, ast.Import):
|
||||
offenders.extend(
|
||||
f"line {node.lineno}: import {alias.name}"
|
||||
for alias in node.names
|
||||
if alias.name.startswith("vllm_omni.protocol.realtime")
|
||||
)
|
||||
assert not offenders, (
|
||||
f"{module.relative_to(REPO_ROOT)} imports the Tier 1 package directly; "
|
||||
f"import from vllm_omni.protocol.duplex instead:\n " + "\n ".join(offenders)
|
||||
)
|
||||
|
||||
|
||||
def test_the_duplex_package_carries_the_whole_command_vocabulary() -> None:
|
||||
from vllm_omni.engine.duplex import commands as engine_commands
|
||||
from vllm_omni.protocol.duplex import commands as duplex_wire
|
||||
|
||||
# Every command the engine handles is nameable through the one door.
|
||||
engine_names = {
|
||||
name
|
||||
for name in engine_commands.__all__
|
||||
if isinstance(getattr(engine_commands, name), type)
|
||||
and issubclass(getattr(engine_commands, name), engine_commands.DuplexCommand)
|
||||
and getattr(engine_commands, name) is not engine_commands.DuplexCommand
|
||||
}
|
||||
assert engine_names <= set(duplex_wire.__all__)
|
||||
|
||||
|
||||
def test_the_duplex_package_carries_the_whole_event_vocabulary() -> None:
|
||||
from vllm_omni.engine.duplex import events as engine_events
|
||||
from vllm_omni.protocol.duplex import events as duplex_wire
|
||||
|
||||
engine_names = {
|
||||
name
|
||||
for name in engine_events.__all__
|
||||
if isinstance(getattr(engine_events, name), type)
|
||||
and issubclass(getattr(engine_events, name), engine_events.DuplexEvent)
|
||||
}
|
||||
assert engine_names <= set(duplex_wire.__all__)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module_pair",
|
||||
[("events", "events"), ("commands", "commands")],
|
||||
ids=["events", "commands"],
|
||||
)
|
||||
def test_a_reexported_tier1_name_is_the_tier1_object(module_pair: tuple[str, str]) -> None:
|
||||
"""Re-export, not re-declaration: the Tier 1 class is one object."""
|
||||
import importlib
|
||||
|
||||
duplex_mod = importlib.import_module(f"vllm_omni.protocol.duplex.{module_pair[0]}")
|
||||
realtime_mod = importlib.import_module(f"vllm_omni.protocol.realtime.{module_pair[1]}")
|
||||
|
||||
shared = [n for n in duplex_mod.__all__ if hasattr(realtime_mod, n)]
|
||||
assert shared, "nothing re-exported --- the facade is not doing its job"
|
||||
for name in shared:
|
||||
duplex_obj = getattr(duplex_mod, name)
|
||||
realtime_obj = getattr(realtime_mod, name)
|
||||
if duplex_obj is realtime_obj:
|
||||
continue
|
||||
# The only permitted divergence is a Tier 2 class extending its twin.
|
||||
assert isinstance(duplex_obj, type) and issubclass(duplex_obj, realtime_obj), (
|
||||
f"{name} in protocol.duplex is neither the Tier 1 object nor a subclass of it"
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_event_is_an_alias_but_duplex_command_is_not() -> None:
|
||||
"""The asymmetry is deliberate, and a later reader should not 'fix' it.
|
||||
|
||||
An event has no engine-internal half --- what the session emits is what the
|
||||
socket sends --- so ``DuplexEvent`` is simply ``RealtimeEvent``. A command
|
||||
does: ``payload()`` renders the session runner's mailbox dictionary, and for
|
||||
four of the seventeen the mailbox channel is not even the client event type.
|
||||
"""
|
||||
from vllm_omni.engine.duplex.commands import DuplexCommand
|
||||
from vllm_omni.protocol.duplex.events import DuplexEvent
|
||||
from vllm_omni.protocol.realtime.commands import RealtimeCommand
|
||||
from vllm_omni.protocol.realtime.events import RealtimeEvent
|
||||
|
||||
assert DuplexEvent is RealtimeEvent
|
||||
|
||||
assert DuplexCommand is not RealtimeCommand
|
||||
assert issubclass(DuplexCommand, RealtimeCommand)
|
||||
assert hasattr(DuplexCommand, "payload") and not hasattr(RealtimeCommand, "payload")
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The shared Realtime codec must not depend on any runtime that uses it.
|
||||
|
||||
``vllm_omni.protocol.realtime`` exists so a second Realtime surface can reuse
|
||||
the wire codec without adopting the duplex session control plane. That only
|
||||
holds while the arrows point one way: the engine, the entrypoints and the model
|
||||
code import the protocol, never the reverse. Nothing in the package's own code
|
||||
enforces that, so it is asserted here --- once on the source (which catches a
|
||||
lazy import inside a function body) and once on a real import (which catches a
|
||||
dependency arriving through a third module).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
PROTOCOL_DIR = REPO_ROOT / "vllm_omni" / "protocol"
|
||||
|
||||
#: Packages that consume the protocol. A dependency on any of them would make
|
||||
#: the codec unusable outside the runtime it accidentally reached into.
|
||||
#: ``vllm_omni.utils`` is deliberately absent: it is leaf helpers (the audio
|
||||
#: resampler) that consume nothing, so depending on it costs a reuser nothing.
|
||||
FORBIDDEN_PREFIXES = (
|
||||
"vllm_omni.engine",
|
||||
"vllm_omni.entrypoints",
|
||||
"vllm_omni.model_executor",
|
||||
"vllm_omni.worker",
|
||||
"vllm_omni.clients",
|
||||
)
|
||||
|
||||
|
||||
def _protocol_sources() -> list[Path]:
|
||||
sources = sorted(PROTOCOL_DIR.rglob("*.py"))
|
||||
assert sources, f"no python sources under {PROTOCOL_DIR}"
|
||||
return sources
|
||||
|
||||
|
||||
def _imported_modules(source: Path) -> set[str]:
|
||||
"""Every module name imported anywhere in the file, lazy imports included."""
|
||||
names: set[str] = set()
|
||||
for node in ast.walk(ast.parse(source.read_text())):
|
||||
if isinstance(node, ast.Import):
|
||||
names.update(alias.name for alias in node.names)
|
||||
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
|
||||
names.add(node.module)
|
||||
return names
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", _protocol_sources(), ids=lambda path: path.name)
|
||||
def test_protocol_sources_do_not_import_a_consumer(source: Path) -> None:
|
||||
offenders = sorted(
|
||||
name
|
||||
for name in _imported_modules(source)
|
||||
if any(name == prefix or name.startswith(prefix + ".") for prefix in FORBIDDEN_PREFIXES)
|
||||
)
|
||||
assert not offenders, f"{source.relative_to(REPO_ROOT)} imports {', '.join(offenders)}"
|
||||
|
||||
|
||||
def test_importing_the_protocol_package_loads_no_consumer() -> None:
|
||||
# A child interpreter, because this test session has already imported the
|
||||
# duplex engine: only a fresh process can tell what the package pulls in.
|
||||
# ``vllm_omni`` itself is imported first, because its ``__init__`` loads a
|
||||
# baseline graph (patches, the model registry) that has nothing to do with
|
||||
# this package; only what the protocol import *adds* is measured.
|
||||
env = os.environ.copy()
|
||||
env.pop("CUDA_VISIBLE_DEVICES", None)
|
||||
env.pop("HIP_VISIBLE_DEVICES", None)
|
||||
script = f"""
|
||||
import sys
|
||||
|
||||
import vllm_omni # noqa: F401
|
||||
|
||||
baseline = set(sys.modules)
|
||||
|
||||
import vllm_omni.protocol.realtime # noqa: F401
|
||||
|
||||
forbidden_prefixes = {FORBIDDEN_PREFIXES!r}
|
||||
added = sorted(
|
||||
name
|
||||
for name in set(sys.modules) - baseline
|
||||
if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes)
|
||||
)
|
||||
if added:
|
||||
raise SystemExit("the realtime protocol package loaded consumers: " + ", ".join(added))
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=180,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""``RealtimeProtocolCapabilities``: one consumer's answers, the codec's checking.
|
||||
|
||||
This is the seam that lets a non-duplex Realtime surface validate a
|
||||
``session.update`` without a ``DuplexModelPlugin``. What it has to get right is
|
||||
narrow: the order the checks run in (formats before turn detection, because
|
||||
that is the order clients already see), the error codes, and that a consumer
|
||||
which declares a narrower format set is actually narrowed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.protocol.realtime.capabilities import (
|
||||
RealtimeProtocolCapabilities,
|
||||
RealtimeSessionRejection,
|
||||
validate_session_payload,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_DEFAULT = RealtimeProtocolCapabilities()
|
||||
|
||||
|
||||
def test_a_consumer_without_a_turn_detection_check_accepts_a_plain_session():
|
||||
assert (
|
||||
validate_session_payload({"model": "m", "audio": {"input": {"format": "pcm16"}}}, capabilities=_DEFAULT) is None
|
||||
)
|
||||
|
||||
|
||||
def test_a_consumer_without_a_turn_detection_check_ignores_turn_detection():
|
||||
payload = {"turn_detection": {"type": "semantic_vad"}}
|
||||
|
||||
assert validate_session_payload(payload, capabilities=_DEFAULT) is None
|
||||
|
||||
|
||||
def test_an_unsupported_input_format_is_rejected_with_the_realtime_code():
|
||||
rejection = validate_session_payload({"input_audio_format": "opus"}, capabilities=_DEFAULT)
|
||||
|
||||
assert rejection == RealtimeSessionRejection(
|
||||
code="unsupported_audio_format", message="Unsupported input_audio_format: opus"
|
||||
)
|
||||
|
||||
|
||||
def test_a_consumer_may_declare_a_narrower_format_set():
|
||||
pcm_only = RealtimeProtocolCapabilities(input_audio_formats=frozenset({"pcm16"}))
|
||||
|
||||
assert validate_session_payload({"input_audio_format": "pcm16"}, capabilities=pcm_only) is None
|
||||
rejection = validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=pcm_only)
|
||||
assert rejection is not None
|
||||
assert rejection.code == "unsupported_audio_format"
|
||||
# The default capabilities still accept it: the narrowing is per consumer.
|
||||
assert validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=_DEFAULT) is None
|
||||
|
||||
|
||||
def test_the_turn_detection_check_runs_and_reports_its_own_code():
|
||||
def reject_everything(session_payload: Mapping[str, object]) -> str | None:
|
||||
del session_payload
|
||||
return "server_vad is not available"
|
||||
|
||||
capabilities = RealtimeProtocolCapabilities(validate_turn_detection=reject_everything)
|
||||
|
||||
rejection = validate_session_payload({"turn_detection": {"type": "server_vad"}}, capabilities=capabilities)
|
||||
|
||||
assert rejection == RealtimeSessionRejection(
|
||||
code="unsupported_turn_detection", message="server_vad is not available", param="turn_detection"
|
||||
)
|
||||
|
||||
|
||||
def test_a_bad_format_is_reported_before_a_bad_turn_detection():
|
||||
calls: list[Mapping[str, object]] = []
|
||||
|
||||
def record(session_payload: Mapping[str, object]) -> str | None:
|
||||
calls.append(session_payload)
|
||||
return "turn detection is wrong too"
|
||||
|
||||
capabilities = RealtimeProtocolCapabilities(validate_turn_detection=record)
|
||||
|
||||
rejection = validate_session_payload(
|
||||
{"input_audio_format": "opus", "turn_detection": {"type": "semantic_vad"}},
|
||||
capabilities=capabilities,
|
||||
)
|
||||
|
||||
assert rejection is not None
|
||||
assert rejection.code == "unsupported_audio_format"
|
||||
assert calls == []
|
||||
@@ -0,0 +1,256 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""There is one Realtime codec, and the duplex engine uses it.
|
||||
|
||||
RFC #6592 P0a rejects the "copy the codec and fix it up" option because the two
|
||||
copies drift. The extraction is only worth anything while the duplex names are
|
||||
*the same objects* as the protocol ones, not lookalikes --- so assert identity,
|
||||
which a re-implementation cannot satisfy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex import audio as duplex_audio
|
||||
from vllm_omni.engine.duplex import events as duplex_events
|
||||
from vllm_omni.engine.duplex import realtime_commands as duplex_codec
|
||||
from vllm_omni.engine.duplex.commands import DuplexCommandError
|
||||
from vllm_omni.protocol import realtime as protocol
|
||||
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
#: Names ``engine.duplex.realtime_commands`` re-exports for its existing importers.
|
||||
_REEXPORTED_FROM_PROTOCOL = (
|
||||
"REALTIME_INPUT_AUDIO_FORMATS",
|
||||
"REALTIME_INPUT_HINT_KEYS",
|
||||
"REALTIME_OUTPUT_AUDIO_FORMATS",
|
||||
"RealtimeInputDefaults",
|
||||
"apply_realtime_session_defaults",
|
||||
"copy_realtime_input_hints",
|
||||
"input_audio_transcription_config",
|
||||
"input_explicitly_non_speech",
|
||||
"input_looks_like_speech",
|
||||
"input_transcript_from_item",
|
||||
"is_supported_realtime_input_format",
|
||||
"json_safe_realtime_payload",
|
||||
"normalize_conversation_item",
|
||||
"parse_realtime_audio_format",
|
||||
"realtime_audio_format_object",
|
||||
"realtime_max_output_tokens",
|
||||
"realtime_output_format",
|
||||
"realtime_overlap_fields",
|
||||
"text_chars_for_audio_ms_from_marks",
|
||||
"truncate_realtime_item_content",
|
||||
"validate_conversation_item_audio_formats",
|
||||
"validate_realtime_item_truncate",
|
||||
"validate_realtime_response_audio_formats",
|
||||
"validate_realtime_session_audio_formats",
|
||||
"validate_realtime_video_frames",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _REEXPORTED_FROM_PROTOCOL)
|
||||
def test_the_duplex_codec_name_is_the_protocol_object(name: str) -> None:
|
||||
assert getattr(duplex_codec, name) is getattr(protocol, name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
("convert_input_audio_with_rate", "convert_output_audio", "resample_pcm16_mono", "wav_payload_to_pcm16"),
|
||||
)
|
||||
def test_the_duplex_audio_shim_is_the_protocol_object(name: str) -> None:
|
||||
assert getattr(duplex_audio, name) is getattr(protocol, name)
|
||||
|
||||
|
||||
def test_a_duplex_command_error_is_a_realtime_protocol_error() -> None:
|
||||
# The duplex binding converts the codec's error into its own type; both
|
||||
# carry the same code/event_id, which is what the error envelope renders.
|
||||
error = DuplexCommandError("bad", code="bad_audio", event_id="event_1")
|
||||
|
||||
assert isinstance(error, RealtimeProtocolError)
|
||||
assert (error.code, error.event_id) == ("bad_audio", "event_1")
|
||||
|
||||
|
||||
def test_the_duplex_append_command_is_built_from_the_shared_decoder() -> None:
|
||||
defaults = protocol.RealtimeInputDefaults()
|
||||
event = {"event_id": "event_1", "audio": "", "format": "pcm16", "duration_ms": 40}
|
||||
|
||||
decoded = protocol.decode_audio_append(event, defaults=defaults)
|
||||
command = duplex_codec.build_append_audio(event, defaults=defaults)
|
||||
|
||||
assert (command.audio, command.format, command.sample_rate_hz) == (
|
||||
decoded.audio,
|
||||
decoded.format,
|
||||
decoded.sample_rate_hz,
|
||||
)
|
||||
assert (command.is_speech, command.video_frames, command.duration_ms) == (
|
||||
decoded.is_speech,
|
||||
decoded.video_frames,
|
||||
decoded.duration_ms,
|
||||
)
|
||||
assert (command.audio_end_ms, dict(command.hints), command.event_id) == (
|
||||
decoded.audio_end_ms,
|
||||
decoded.hints,
|
||||
decoded.event_id,
|
||||
)
|
||||
|
||||
|
||||
def test_the_shared_decoder_error_reaches_the_client_as_a_duplex_command_error() -> None:
|
||||
defaults = protocol.RealtimeInputDefaults()
|
||||
event = {"event_id": "event_1", "audio": "AAAA", "format": "opus"}
|
||||
|
||||
with pytest.raises(RealtimeProtocolError) as shared:
|
||||
protocol.decode_audio_append(event, defaults=defaults)
|
||||
with pytest.raises(DuplexCommandError) as duplex:
|
||||
duplex_codec.build_append_audio(event, defaults=defaults)
|
||||
|
||||
assert (duplex.value.code, str(duplex.value)) == (shared.value.code, str(shared.value))
|
||||
assert duplex.value.event_id == shared.value.event_id == "event_1"
|
||||
|
||||
|
||||
def test_the_duplex_capabilities_reject_unimplemented_turn_detection() -> None:
|
||||
# The duplex answer comes from engine.duplex.turn_detection, reached through
|
||||
# the capability object rather than imported by the codec.
|
||||
rejection = protocol.validate_session_payload(
|
||||
{"turn_detection": {"type": "semantic_vad"}},
|
||||
capabilities=duplex_codec.DUPLEX_REALTIME_CAPABILITIES,
|
||||
)
|
||||
|
||||
assert rejection is not None
|
||||
assert rejection.code == "unsupported_turn_detection"
|
||||
|
||||
|
||||
# ---- the class split (RFC #6592 P0a, second half) ----
|
||||
#
|
||||
# The event and command classes moved out of the engine too. The same identity
|
||||
# rule applies: the duplex names must BE the protocol objects, and the mailbox
|
||||
# half must have stayed behind.
|
||||
|
||||
# `docs/serving/realtime_duplex_api.md` sorts every message into three tiers.
|
||||
# Tier 1 is pure OpenAI and lives in protocol/realtime; Tier 2 (OpenAI names
|
||||
# carrying our extensions) and Tier 3 (ours alone) live in protocol/duplex.
|
||||
_TIER1_EVENTS = ("SessionUpdated", "ItemAdded", "ItemCreated", "TextDelta", "OutputItemAdded")
|
||||
_TIER2_EVENTS = (
|
||||
"SessionCreated",
|
||||
"ResponseCreated",
|
||||
"ResponseDone",
|
||||
"AudioDelta",
|
||||
"InputCommitted",
|
||||
"ItemDeleted",
|
||||
"ItemTruncated",
|
||||
"ErrorEvent",
|
||||
)
|
||||
_TIER3_EVENTS = ("Listen", "Speak", "OverlapDecision", "PlaybackAcknowledged", "SessionResumed")
|
||||
_TIER1_COMMANDS = ("UpdateSession", "CreateItem", "CancelResponse", "ClearInput")
|
||||
_TIER2_COMMANDS = ("AppendAudio", "Commit")
|
||||
_TIER3_COMMANDS = ("BargeIn", "AckPlayback", "Heartbeat", "SignalTurn", "CloseSession")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _TIER1_EVENTS)
|
||||
def test_tier1_events_are_the_pure_openai_objects(name: str) -> None:
|
||||
from vllm_omni.protocol.realtime import events as realtime_events
|
||||
|
||||
assert duplex_events.__dict__[name] is getattr(realtime_events, name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _TIER2_EVENTS + _TIER3_EVENTS)
|
||||
def test_tier2_and_tier3_events_come_from_protocol_duplex(name: str) -> None:
|
||||
from vllm_omni.protocol.duplex import events as duplex_wire_events
|
||||
|
||||
assert duplex_events.__dict__[name] is getattr(duplex_wire_events, name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _TIER2_EVENTS)
|
||||
def test_tier2_extends_its_tier1_twin_rather_than_replacing_it(name: str) -> None:
|
||||
"""A Tier 2 class must be its Tier 1 class plus fields, never a fork of it."""
|
||||
from vllm_omni.protocol.duplex import events as duplex_wire_events
|
||||
from vllm_omni.protocol.realtime import events as realtime_events
|
||||
|
||||
tier1 = getattr(realtime_events, name)
|
||||
tier2 = getattr(duplex_wire_events, name)
|
||||
|
||||
assert issubclass(tier2, tier1)
|
||||
assert tier2.wire_type == tier1.wire_type
|
||||
tier1_fields = {f.name for f in dataclasses.fields(tier1)}
|
||||
tier2_fields = {f.name for f in dataclasses.fields(tier2)}
|
||||
# Purely additive: Tier 2 adds, never drops or renames.
|
||||
assert tier1_fields < tier2_fields
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _TIER1_EVENTS + _TIER1_COMMANDS)
|
||||
def test_tier1_carries_no_vllm_omni_extension_fields(name: str) -> None:
|
||||
"""The point of the split: protocol/realtime must be honestly pure OpenAI."""
|
||||
from vllm_omni.protocol.realtime import commands as realtime_commands
|
||||
from vllm_omni.protocol.realtime import events as realtime_events
|
||||
|
||||
cls = getattr(realtime_events, name, None) or getattr(realtime_commands, name)
|
||||
fields = {f.name for f in dataclasses.fields(cls)}
|
||||
# Extension fields catalogued as Tier 2 in the serving doc.
|
||||
assert not (
|
||||
fields
|
||||
& {
|
||||
"attachment_generation",
|
||||
"resume_token",
|
||||
"details",
|
||||
"extra",
|
||||
"is_speech",
|
||||
"video_frames",
|
||||
"hints",
|
||||
"realtime_item_id",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_the_error_code_vocabulary_is_tier3() -> None:
|
||||
"""OpenAI standardises the error classes, not our codes (doc: Tier 3)."""
|
||||
from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
|
||||
from vllm_omni.protocol.realtime import errors as realtime_errors
|
||||
|
||||
assert duplex_events.REALTIME_ERROR_TYPES_BY_CODE is REALTIME_ERROR_TYPES_BY_CODE
|
||||
assert not hasattr(realtime_errors, "REALTIME_ERROR_TYPES_BY_CODE")
|
||||
|
||||
|
||||
def test_the_tier1_error_event_reports_only_openai_classes() -> None:
|
||||
from vllm_omni.protocol.duplex import events as duplex_wire_events
|
||||
from vllm_omni.protocol.realtime import events as realtime_events
|
||||
|
||||
# Tier 1 knows the envelope shape but not our code vocabulary.
|
||||
assert realtime_events.ErrorEvent(code="resource_exhausted").error_type == "invalid_request_error"
|
||||
# Tier 2 resolves it through our table.
|
||||
assert duplex_wire_events.ErrorEvent(code="resource_exhausted").error_type == "rate_limit_error"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", _TIER1_COMMANDS + _TIER2_COMMANDS + _TIER3_COMMANDS)
|
||||
def test_engine_commands_subclass_their_protocol_wire_type(name: str) -> None:
|
||||
from vllm_omni.engine.duplex import commands as engine_commands
|
||||
from vllm_omni.protocol.duplex import commands as duplex_wire_commands
|
||||
from vllm_omni.protocol.realtime import commands as realtime_commands
|
||||
|
||||
engine_cls = getattr(engine_commands, name)
|
||||
wire_cls = getattr(duplex_wire_commands, name, None) or getattr(realtime_commands, name)
|
||||
|
||||
assert issubclass(engine_cls, wire_cls)
|
||||
assert engine_cls.wire_type == wire_cls.wire_type
|
||||
|
||||
|
||||
def test_the_mailbox_channel_stayed_engine_side() -> None:
|
||||
"""``payload()``'s ``type`` is the runner's channel, not the client event.
|
||||
|
||||
These four are the proof that the two halves are genuinely different, and
|
||||
so the reason the classes were split rather than relocated wholesale.
|
||||
"""
|
||||
from vllm_omni.engine.duplex import commands as engine_commands
|
||||
from vllm_omni.protocol.realtime import commands as realtime_commands
|
||||
|
||||
for name in ("UpdateSession", "CreateItem", "DeleteItem", "TruncateItem"):
|
||||
engine_cls = getattr(engine_commands, name)
|
||||
wire_cls = getattr(realtime_commands, name)
|
||||
assert engine_cls.type == "turn.signal"
|
||||
assert wire_cls.wire_type != "turn.signal"
|
||||
# The wire class carries no mailbox vocabulary at all.
|
||||
assert "type" not in vars(wire_cls)
|
||||
@@ -79,11 +79,12 @@ CHECK_IMPORTS = {
|
||||
"vllm_omni/entrypoints/openai/video_stream_base.py",
|
||||
"vllm_omni/entrypoints/openai/video_stream_session.py",
|
||||
"vllm_omni/experimental/fullduplex/client.py",
|
||||
"vllm_omni/engine/duplex/audio.py",
|
||||
"vllm_omni/engine/duplex/commands.py",
|
||||
"vllm_omni/engine/duplex/contracts.py",
|
||||
"vllm_omni/engine/duplex/events.py",
|
||||
"vllm_omni/engine/duplex/realtime_commands.py",
|
||||
"vllm_omni/protocol/realtime/audio.py",
|
||||
"vllm_omni/protocol/realtime/audio_input.py",
|
||||
"vllm_omni/protocol/realtime/events.py",
|
||||
"vllm_omni/protocol/realtime/items.py",
|
||||
"vllm_omni/experimental/fullduplex/joyvl/memory/memory.py",
|
||||
"vllm_omni/model_executor/models/minicpmo_4_5/duplex/input.py",
|
||||
"vllm_omni/model_executor/models/minicpmo_4_5/duplex/plugin.py",
|
||||
|
||||
@@ -32,8 +32,8 @@ from typing import TYPE_CHECKING
|
||||
from vllm_omni.clients.duplex import DuplexClientBase, DuplexConnectionError, SessionConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle
|
||||
from vllm_omni.protocol.duplex import RealtimeInputDefaults
|
||||
|
||||
__all__ = ["InlineDuplexClient"]
|
||||
|
||||
@@ -71,7 +71,7 @@ class InlineDuplexClient(DuplexClientBase):
|
||||
|
||||
def _input_defaults(self) -> RealtimeInputDefaults:
|
||||
"""Audio format / sample-rate / VAD defaults declared by this session's object."""
|
||||
from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
|
||||
from vllm_omni.protocol.duplex import RealtimeInputDefaults
|
||||
|
||||
session_payload = self.session_info or self.config.to_session_payload(model=self.model)
|
||||
return RealtimeInputDefaults().with_session_payload(session_payload)
|
||||
|
||||
@@ -1,218 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Compatibility re-export: the Realtime audio codec moved out of the duplex engine.
|
||||
|
||||
Decoding PCM / G.711 / WAV, resampling and re-encoding are wire-format work with
|
||||
no duplex session in them, so the implementation now lives in
|
||||
``vllm_omni.protocol.realtime.audio`` where a non-duplex Realtime surface can
|
||||
use it too. Duplex code reaches it through ``vllm_omni.protocol.duplex``, the
|
||||
one door the engine uses --- which is also where a duplex-specific conversion
|
||||
would be introduced. Import from there in new code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import io
|
||||
import math
|
||||
import wave
|
||||
from vllm_omni.protocol.duplex import (
|
||||
MAX_INPUT_SAMPLE_RATE_HZ,
|
||||
MIN_INPUT_SAMPLE_RATE_HZ,
|
||||
convert_input_audio_with_rate,
|
||||
convert_output_audio,
|
||||
decode_g711_alaw,
|
||||
decode_g711_ulaw,
|
||||
encode_float32_mono_wav_base64,
|
||||
encode_g711_alaw,
|
||||
encode_g711_ulaw,
|
||||
resample_pcm16_mono,
|
||||
validate_input_sample_rate_hz,
|
||||
wav_payload_to_pcm16,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm_omni.utils.audio_resample import StreamingAudioResampler
|
||||
|
||||
try:
|
||||
from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin
|
||||
except ImportError: # pragma: no cover - audioop is removed in newer Python.
|
||||
alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None
|
||||
|
||||
|
||||
MIN_INPUT_SAMPLE_RATE_HZ = 8_000
|
||||
MAX_INPUT_SAMPLE_RATE_HZ = 192_000
|
||||
|
||||
|
||||
def validate_input_sample_rate_hz(sample_rate_hz: object) -> int:
|
||||
if (
|
||||
isinstance(sample_rate_hz, bool)
|
||||
or not isinstance(sample_rate_hz, int | float)
|
||||
or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz))
|
||||
):
|
||||
raise ValueError("sample_rate_hz must be a finite integer")
|
||||
rate = int(sample_rate_hz)
|
||||
if rate != sample_rate_hz:
|
||||
raise ValueError("sample_rate_hz must be an integer")
|
||||
if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ:
|
||||
raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}")
|
||||
return rate
|
||||
|
||||
|
||||
def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes:
|
||||
if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz:
|
||||
return raw
|
||||
samples = np.frombuffer(raw, dtype="<i2")
|
||||
rate_gcd = math.gcd(source_rate_hz, target_rate_hz)
|
||||
reduced_factor = max(source_rate_hz // rate_gcd, target_rate_hz // rate_gcd)
|
||||
if reduced_factor > StreamingAudioResampler.max_polyphase_factor:
|
||||
# The FIR size scales with the reduced ratio. Keep unusual client
|
||||
# rates bounded while common audio rates use the high-quality path.
|
||||
if samples.size <= 1:
|
||||
return raw
|
||||
target_size = max(1, int(round(samples.size * target_rate_hz / source_rate_hz)))
|
||||
source_x = np.linspace(0.0, 1.0, num=samples.size, endpoint=True)
|
||||
target_x = np.linspace(0.0, 1.0, num=target_size, endpoint=True)
|
||||
resampled = np.interp(target_x, source_x, samples)
|
||||
return np.clip(resampled, -32_768, 32_767).astype("<i2").tobytes()
|
||||
resampler = StreamingAudioResampler(source_rate_hz, target_rate_hz)
|
||||
normalized = samples.astype(np.float32) * np.float32(1.0 / 32768.0)
|
||||
resampled = resampler.process(normalized, final=True)
|
||||
return np.clip(np.rint(resampled * 32768.0), -32768, 32767).astype("<i2").tobytes()
|
||||
|
||||
|
||||
def decode_g711_ulaw(raw: bytes) -> bytes:
|
||||
if ulaw2lin is not None:
|
||||
return ulaw2lin(raw, 2)
|
||||
data = np.frombuffer(raw, dtype=np.uint8)
|
||||
value = np.bitwise_not(data).astype(np.int16)
|
||||
sign = value & 0x80
|
||||
exponent = (value >> 4) & 0x07
|
||||
mantissa = value & 0x0F
|
||||
sample = ((mantissa << 3) + 0x84) << exponent
|
||||
return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype("<i2").tobytes()
|
||||
|
||||
|
||||
def decode_g711_alaw(raw: bytes) -> bytes:
|
||||
if alaw2lin is not None:
|
||||
return alaw2lin(raw, 2)
|
||||
data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16)
|
||||
sign = data & 0x80
|
||||
exponent = (data >> 4) & 0x07
|
||||
mantissa = data & 0x0F
|
||||
sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1))
|
||||
return np.where(sign != 0, sample, -sample).astype("<i2").tobytes()
|
||||
|
||||
|
||||
def encode_g711_ulaw(raw: bytes) -> bytes:
|
||||
if lin2ulaw is not None:
|
||||
return lin2ulaw(raw, 2)
|
||||
pcm = np.clip(np.frombuffer(raw, dtype="<i2").astype(np.int32), -32635, 32635)
|
||||
sign = np.where(pcm < 0, 0x80, 0)
|
||||
magnitude = np.abs(pcm) + 0x84
|
||||
exponent = np.zeros_like(magnitude)
|
||||
for exp in range(7):
|
||||
exponent = np.where(magnitude > (0xFF << exp), exp + 1, exponent)
|
||||
mantissa = (magnitude >> (exponent + 3)) & 0x0F
|
||||
return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes()
|
||||
|
||||
|
||||
def encode_g711_alaw(raw: bytes) -> bytes:
|
||||
if lin2alaw is not None:
|
||||
return lin2alaw(raw, 2)
|
||||
pcm = np.frombuffer(raw, dtype="<i2").astype(np.int32)
|
||||
sign = np.where(pcm >= 0, 0x80, 0x00)
|
||||
magnitude = np.abs(pcm)
|
||||
exponent = np.zeros_like(magnitude)
|
||||
for exp in range(1, 8):
|
||||
exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent)
|
||||
mantissa = np.where(
|
||||
exponent == 0,
|
||||
(magnitude >> 4) & 0x0F,
|
||||
(magnitude >> (exponent + 3)) & 0x0F,
|
||||
)
|
||||
return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes()
|
||||
|
||||
|
||||
def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]:
|
||||
try:
|
||||
with wave.open(io.BytesIO(raw), "rb") as wav_file:
|
||||
channels = wav_file.getnchannels()
|
||||
sample_width = wav_file.getsampwidth()
|
||||
sample_rate_hz = wav_file.getframerate()
|
||||
frames = wav_file.readframes(wav_file.getnframes())
|
||||
if sample_width != 2:
|
||||
return None, sample_rate_hz
|
||||
if channels <= 1:
|
||||
return frames, sample_rate_hz
|
||||
pcm = np.frombuffer(frames, dtype="<i2").reshape(-1, channels)
|
||||
mono = np.mean(pcm.astype(np.float32), axis=1)
|
||||
return np.clip(mono, -32768, 32767).astype("<i2").tobytes(), sample_rate_hz
|
||||
except (EOFError, ValueError, wave.Error):
|
||||
return None, None
|
||||
|
||||
|
||||
def encode_float32_mono_wav_base64(
|
||||
samples: np.ndarray,
|
||||
*,
|
||||
sample_rate_hz: int,
|
||||
) -> str:
|
||||
"""Package normalized mono float32 samples as a PCM16 WAV data payload."""
|
||||
rate = validate_input_sample_rate_hz(sample_rate_hz)
|
||||
normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1)
|
||||
pcm16 = np.clip(np.rint(normalized * 32768.0), -32768, 32767).astype("<i2")
|
||||
with io.BytesIO() as buffer:
|
||||
with wave.open(buffer, "wb") as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(rate)
|
||||
wav_file.writeframes(pcm16.tobytes())
|
||||
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
def convert_input_audio_with_rate(
|
||||
audio: object,
|
||||
fmt: object,
|
||||
*,
|
||||
sample_rate_hz: int | float | None = None,
|
||||
target_sample_rate_hz: int = 16_000,
|
||||
) -> tuple[object, object, int | float | None]:
|
||||
if not isinstance(audio, str) or not isinstance(fmt, str):
|
||||
return audio, fmt, sample_rate_hz
|
||||
normalized = fmt.lower()
|
||||
if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}:
|
||||
return audio, fmt, sample_rate_hz
|
||||
if isinstance(sample_rate_hz, int | float):
|
||||
sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz)
|
||||
try:
|
||||
raw = base64.b64decode(audio.strip(), validate=False)
|
||||
except (binascii.Error, ValueError):
|
||||
return audio, fmt, sample_rate_hz
|
||||
if normalized == "g711_ulaw":
|
||||
raw = decode_g711_ulaw(raw)
|
||||
sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
|
||||
elif normalized == "g711_alaw":
|
||||
raw = decode_g711_alaw(raw)
|
||||
sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
|
||||
elif len(raw) % 2:
|
||||
return audio, fmt, sample_rate_hz
|
||||
if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz:
|
||||
raw = resample_pcm16_mono(
|
||||
raw,
|
||||
source_rate_hz=int(sample_rate_hz),
|
||||
target_rate_hz=target_sample_rate_hz,
|
||||
)
|
||||
sample_rate_hz = target_sample_rate_hz
|
||||
pcm = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
|
||||
encoded = base64.b64encode(np.ascontiguousarray(pcm, dtype="<f4").tobytes()).decode("ascii")
|
||||
return encoded, "pcm_f32le", sample_rate_hz
|
||||
|
||||
|
||||
def convert_output_audio(
|
||||
audio: str,
|
||||
*,
|
||||
source_fmt: str,
|
||||
target_fmt: str,
|
||||
source_sample_rate_hz: int | None = None,
|
||||
target_sample_rate_hz: int | None = None,
|
||||
) -> tuple[str, str, int | None]:
|
||||
target = target_fmt.lower()
|
||||
if target not in {"g711_ulaw", "g711_alaw"}:
|
||||
return audio, source_fmt, source_sample_rate_hz
|
||||
try:
|
||||
raw = base64.b64decode(audio, validate=False)
|
||||
except (binascii.Error, ValueError):
|
||||
return audio, source_fmt, source_sample_rate_hz
|
||||
source = source_fmt.lower()
|
||||
if source == "wav":
|
||||
pcm_raw, wav_rate = wav_payload_to_pcm16(raw)
|
||||
if pcm_raw is None:
|
||||
return audio, source_fmt, source_sample_rate_hz
|
||||
raw = pcm_raw
|
||||
source_sample_rate_hz = source_sample_rate_hz or wav_rate
|
||||
elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2:
|
||||
return audio, source_fmt, source_sample_rate_hz
|
||||
target_rate = target_sample_rate_hz or 8_000
|
||||
if source_sample_rate_hz is not None:
|
||||
raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate)
|
||||
encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw)
|
||||
return base64.b64encode(encoded).decode("ascii"), target, target_rate
|
||||
__all__ = [
|
||||
"MAX_INPUT_SAMPLE_RATE_HZ",
|
||||
"MIN_INPUT_SAMPLE_RATE_HZ",
|
||||
"convert_input_audio_with_rate",
|
||||
"convert_output_audio",
|
||||
"decode_g711_alaw",
|
||||
"decode_g711_ulaw",
|
||||
"encode_float32_mono_wav_base64",
|
||||
"encode_g711_alaw",
|
||||
"encode_g711_ulaw",
|
||||
"resample_pcm16_mono",
|
||||
"validate_input_sample_rate_hz",
|
||||
"wav_payload_to_pcm16",
|
||||
]
|
||||
+110
-128
@@ -1,41 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Typed commands accepted by a duplex session.
|
||||
"""The command vocabulary one duplex session accepts, plus its mailbox rendering.
|
||||
|
||||
``DuplexSessionHandle.submit()`` takes one of these; the websocket handler and
|
||||
``InlineDuplexClient`` build them with :func:`command_from_realtime` from the
|
||||
OpenAI Realtime client event vocabulary. Every command can also render the
|
||||
session-internal mailbox payload (``payload()``), which is the dictionary
|
||||
vocabulary the session runner bodies were written against.
|
||||
``InlineDuplexClient`` build them with :func:`command_from_realtime`.
|
||||
|
||||
The *wire* half of each command --- which client event it decodes from and what
|
||||
fields survive decoding --- now lives in ``vllm_omni.protocol.duplex.commands``,
|
||||
which carries the whole vocabulary: the eight Tier 1 classes re-exported from
|
||||
``vllm_omni.protocol.realtime.commands``, the two Tier 2 ones and the seven
|
||||
Tier 3 ones. The engine imports that module and never the Tier 1 one directly,
|
||||
so a command that later grows a duplex extension changes one file
|
||||
(RFC #6592 P0a).
|
||||
|
||||
The *engine* half stays here, because it is not wire contract at all:
|
||||
``payload()`` renders the session-internal mailbox dictionary the runner bodies
|
||||
were written against, and its ``type`` is the mailbox channel rather than the
|
||||
client event. Those two genuinely differ --- ``session.update`` and
|
||||
``conversation.item.create`` / ``.delete`` / ``.truncate`` all travel on the
|
||||
``turn.signal`` channel --- which is why the halves are separated instead of
|
||||
the classes being relocated wholesale.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field, fields
|
||||
from dataclasses import dataclass, fields
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from vllm_omni.protocol.duplex import RealtimeProtocolError
|
||||
from vllm_omni.protocol.duplex import commands as _duplex_wire
|
||||
from vllm_omni.protocol.duplex.commands import RealtimeCommand
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
|
||||
from vllm_omni.protocol.duplex import RealtimeInputDefaults
|
||||
|
||||
|
||||
class DuplexCommandError(ValueError):
|
||||
"""A client payload could not be turned into a command."""
|
||||
class DuplexCommandError(RealtimeProtocolError):
|
||||
"""A client payload could not be turned into a duplex command.
|
||||
|
||||
def __init__(self, message: str, *, code: str = "bad_event", event_id: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.event_id = event_id
|
||||
The duplex name for a Realtime protocol error: same ``code`` /
|
||||
``event_id`` contract, so the error envelope is rendered identically
|
||||
whichever consumer raised it.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class DuplexCommand:
|
||||
class DuplexCommand(RealtimeCommand):
|
||||
"""A Realtime command as the duplex engine handles it.
|
||||
|
||||
Adds the mailbox channel (``type``) and its rendering on top of the wire
|
||||
command; every concrete class below pairs this with its protocol twin.
|
||||
"""
|
||||
|
||||
#: Mailbox event type this command renders to (see ``payload()``).
|
||||
type: ClassVar[str] = ""
|
||||
#: Client correlation id (OpenAI ``event_id``), echoed on error events.
|
||||
event_id: str | None = None
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
"""Render the session-internal mailbox dictionary."""
|
||||
@@ -56,19 +77,23 @@ class DuplexCommand:
|
||||
return data
|
||||
|
||||
|
||||
# ---- GA commands ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AppendAudio(DuplexCommand):
|
||||
class UpdateSession(DuplexCommand, _duplex_wire.UpdateSession):
|
||||
type: ClassVar[str] = "turn.signal"
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
data["event"] = "session.update"
|
||||
data["payload"] = dict(data.pop("patch", {}) or {})
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AppendAudio(DuplexCommand, _duplex_wire.AppendAudio):
|
||||
type: ClassVar[str] = "input_audio_buffer.append"
|
||||
#: Raw audio bytes in ``format`` at ``sample_rate_hz`` (base64 only on the wire).
|
||||
audio: bytes
|
||||
format: str = "pcm16"
|
||||
sample_rate_hz: int | None = None
|
||||
is_speech: bool | None = None
|
||||
video_frames: tuple[str, ...] = ()
|
||||
duration_ms: int | None = None
|
||||
audio_end_ms: int | None = None
|
||||
#: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
|
||||
hints: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
@@ -90,21 +115,8 @@ class AppendAudio(DuplexCommand):
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AppendText(DuplexCommand):
|
||||
type: ClassVar[str] = "input.text.append"
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Commit(DuplexCommand):
|
||||
class Commit(DuplexCommand, _duplex_wire.Commit):
|
||||
type: ClassVar[str] = "input_audio_buffer.commit"
|
||||
final: bool = True
|
||||
#: ``None`` means "no explicit request": the runner decides on commit (auto-response
|
||||
#: sessions answer on their own); ``True`` / ``False`` force it.
|
||||
create_response: bool | None = None
|
||||
is_speech: bool | None = None
|
||||
#: Realtime conversation item created for this commit (wire correlation only).
|
||||
realtime_item_id: str | None = None
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
@@ -114,10 +126,18 @@ class Commit(DuplexCommand):
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CreateResponse(DuplexCommand):
|
||||
class ClearInput(DuplexCommand, _duplex_wire.ClearInput):
|
||||
type: ClassVar[str] = "input_audio_buffer.clear"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ClearOutputAudio(DuplexCommand, _duplex_wire.ClearOutputAudio):
|
||||
type: ClassVar[str] = "output_audio_buffer.clear"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CreateResponse(DuplexCommand, _duplex_wire.CreateResponse):
|
||||
type: ClassVar[str] = "response.create"
|
||||
#: Raw Realtime ``response`` object (instructions, voice, modalities, ...).
|
||||
options: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
@@ -128,84 +148,13 @@ class CreateResponse(DuplexCommand):
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ClearInput(DuplexCommand):
|
||||
type: ClassVar[str] = "input_audio_buffer.clear"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CancelInput(DuplexCommand):
|
||||
type: ClassVar[str] = "input.cancel"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CancelResponse(DuplexCommand):
|
||||
class CancelResponse(DuplexCommand, _duplex_wire.CancelResponse):
|
||||
type: ClassVar[str] = "response.cancel"
|
||||
response_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class BargeIn(DuplexCommand):
|
||||
type: ClassVar[str] = "barge_in"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ClearOutputAudio(DuplexCommand):
|
||||
type: ClassVar[str] = "output_audio_buffer.clear"
|
||||
#: Explicit response to clear; ``None`` targets the active/last response.
|
||||
response_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SignalTurn(DuplexCommand):
|
||||
"""Generic ``turn.signal`` (local turn transitions such as ``user_started``)."""
|
||||
|
||||
class CreateItem(DuplexCommand, _duplex_wire.CreateItem):
|
||||
type: ClassVar[str] = "turn.signal"
|
||||
event: str
|
||||
signal_payload: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
signal_payload = data.pop("signal_payload", None)
|
||||
if isinstance(signal_payload, Mapping) and signal_payload:
|
||||
data["payload"] = dict(signal_payload)
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class UpdateSession(DuplexCommand):
|
||||
"""``session.update``: a Realtime ``session`` object patch."""
|
||||
|
||||
type: ClassVar[str] = "turn.signal"
|
||||
patch: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
data["event"] = "session.update"
|
||||
data["payload"] = dict(data.pop("patch", {}) or {})
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AckPlayback(DuplexCommand):
|
||||
type: ClassVar[str] = "playback.ack"
|
||||
played_ms: int
|
||||
committed_ms: int | None = None
|
||||
response_id: str | None = None
|
||||
item_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Heartbeat(DuplexCommand):
|
||||
type: ClassVar[str] = "session.heartbeat"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CreateItem(DuplexCommand):
|
||||
"""``conversation.item.create`` (history injection or function-call output)."""
|
||||
|
||||
type: ClassVar[str] = "turn.signal"
|
||||
item: Mapping[str, object]
|
||||
previous_item_id: str | None = None
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
@@ -219,9 +168,8 @@ class CreateItem(DuplexCommand):
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class DeleteItem(DuplexCommand):
|
||||
class DeleteItem(DuplexCommand, _duplex_wire.DeleteItem):
|
||||
type: ClassVar[str] = "turn.signal"
|
||||
item_id: str
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
@@ -231,11 +179,8 @@ class DeleteItem(DuplexCommand):
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TruncateItem(DuplexCommand):
|
||||
class TruncateItem(DuplexCommand, _duplex_wire.TruncateItem):
|
||||
type: ClassVar[str] = "turn.signal"
|
||||
item_id: str
|
||||
audio_end_ms: int
|
||||
content_index: int = 0
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
@@ -248,15 +193,51 @@ class TruncateItem(DuplexCommand):
|
||||
return data
|
||||
|
||||
|
||||
# ---- duplex extension commands ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CloseSession(DuplexCommand):
|
||||
"""Graceful close requested through the command stream (``session.close``)."""
|
||||
class AppendText(DuplexCommand, _duplex_wire.AppendText):
|
||||
type: ClassVar[str] = "input.text.append"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CancelInput(DuplexCommand, _duplex_wire.CancelInput):
|
||||
type: ClassVar[str] = "input.cancel"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class BargeIn(DuplexCommand, _duplex_wire.BargeIn):
|
||||
type: ClassVar[str] = "barge_in"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SignalTurn(DuplexCommand, _duplex_wire.SignalTurn):
|
||||
type: ClassVar[str] = "turn.signal"
|
||||
|
||||
def payload(self) -> dict[str, object]:
|
||||
data = DuplexCommand.payload(self)
|
||||
signal_payload = data.pop("signal_payload", None)
|
||||
if isinstance(signal_payload, Mapping) and signal_payload:
|
||||
data["payload"] = dict(signal_payload)
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AckPlayback(DuplexCommand, _duplex_wire.AckPlayback):
|
||||
type: ClassVar[str] = "playback.ack"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Heartbeat(DuplexCommand, _duplex_wire.Heartbeat):
|
||||
type: ClassVar[str] = "session.heartbeat"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CloseSession(DuplexCommand, _duplex_wire.CloseSession):
|
||||
type: ClassVar[str] = "session.close"
|
||||
reason: str = "client_close"
|
||||
|
||||
|
||||
#: Realtime client event types that map onto commands (wire vocabulary).
|
||||
REALTIME_COMMAND_TYPES: frozenset[str] = frozenset(
|
||||
{
|
||||
"input_audio_buffer.append",
|
||||
@@ -300,7 +281,6 @@ def command_from_realtime(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"REALTIME_COMMAND_TYPES",
|
||||
"AckPlayback",
|
||||
"AppendAudio",
|
||||
"AppendText",
|
||||
@@ -317,6 +297,8 @@ __all__ = [
|
||||
"DuplexCommand",
|
||||
"DuplexCommandError",
|
||||
"Heartbeat",
|
||||
"REALTIME_COMMAND_TYPES",
|
||||
"RealtimeCommand",
|
||||
"SignalTurn",
|
||||
"TruncateItem",
|
||||
"UpdateSession",
|
||||
|
||||
@@ -339,26 +339,26 @@ class DuplexSessionConfig:
|
||||
``_session_create_from_realtime``.
|
||||
"""
|
||||
from vllm_omni.engine.duplex.realtime_commands import (
|
||||
RealtimeInputDefaults,
|
||||
DUPLEX_REALTIME_CAPABILITIES,
|
||||
duplex_response_format,
|
||||
)
|
||||
from vllm_omni.engine.duplex.turn_detection import normalize_turn_detection_session_payload
|
||||
from vllm_omni.protocol.duplex import (
|
||||
RealtimeInputDefaults,
|
||||
input_audio_transcription_config,
|
||||
json_safe_realtime_payload,
|
||||
realtime_max_output_tokens,
|
||||
realtime_overlap_fields,
|
||||
validate_realtime_session_audio_formats,
|
||||
)
|
||||
from vllm_omni.engine.duplex.turn_detection import (
|
||||
normalize_turn_detection_session_payload,
|
||||
validate_realtime_turn_detection,
|
||||
validate_session_payload,
|
||||
)
|
||||
|
||||
payload: dict[str, object] = dict(session_payload)
|
||||
format_error = validate_realtime_session_audio_formats(payload)
|
||||
if format_error is not None:
|
||||
raise DuplexConfigError(format_error, code="unsupported_audio_format")
|
||||
turn_detection_error = validate_realtime_turn_detection(payload)
|
||||
if turn_detection_error is not None:
|
||||
raise DuplexConfigError(turn_detection_error, code="unsupported_turn_detection", param="turn_detection")
|
||||
# One session check for every consumer (ENTRY-INV-002): the same
|
||||
# capability object ``translate_realtime_command`` uses, so a session
|
||||
# object is accepted or refused identically whichever door it came in.
|
||||
rejection = validate_session_payload(payload, capabilities=DUPLEX_REALTIME_CAPABILITIES)
|
||||
if rejection is not None:
|
||||
raise DuplexConfigError(rejection.message, code=rejection.code, param=rejection.param)
|
||||
normalize_turn_detection_session_payload(payload)
|
||||
defaults = RealtimeInputDefaults().with_session_payload(payload)
|
||||
payload.update(realtime_overlap_fields(payload))
|
||||
@@ -446,9 +446,9 @@ class DuplexSessionConfig:
|
||||
when the patch changes something a live session cannot change.
|
||||
``audio_started`` is ``playback.generated_ms > 0 or playback.sent_ms > 0``.
|
||||
"""
|
||||
from vllm_omni.engine.duplex.realtime_commands import (
|
||||
from vllm_omni.engine.duplex.realtime_commands import duplex_response_format
|
||||
from vllm_omni.protocol.duplex import (
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS,
|
||||
duplex_response_format,
|
||||
input_audio_transcription_config,
|
||||
json_safe_realtime_payload,
|
||||
parse_realtime_audio_format,
|
||||
@@ -629,9 +629,9 @@ class ResponseCreateOptions:
|
||||
for options a model-native duplex session cannot apply per response.
|
||||
Private runtime keys in ``extra_body`` are dropped.
|
||||
"""
|
||||
from vllm_omni.engine.duplex.realtime_commands import (
|
||||
from vllm_omni.engine.duplex.realtime_commands import duplex_response_format
|
||||
from vllm_omni.protocol.duplex import (
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS,
|
||||
duplex_response_format,
|
||||
parse_realtime_audio_format,
|
||||
realtime_max_output_tokens,
|
||||
)
|
||||
@@ -782,14 +782,14 @@ def realtime_item_to_history_message(item: object) -> dict[str, object] | None:
|
||||
|
||||
def realtime_max_output_tokens(value: object) -> int | None:
|
||||
"""Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
|
||||
from vllm_omni.engine.duplex.realtime_commands import realtime_max_output_tokens as _impl
|
||||
from vllm_omni.protocol.duplex import realtime_max_output_tokens as _impl
|
||||
|
||||
return _impl(value)
|
||||
|
||||
|
||||
def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
|
||||
"""Return the ``input_audio_transcription`` object of a Realtime session payload."""
|
||||
from vllm_omni.engine.duplex.realtime_commands import input_audio_transcription_config as _impl
|
||||
from vllm_omni.protocol.duplex import input_audio_transcription_config as _impl
|
||||
|
||||
return _impl(session_payload)
|
||||
|
||||
|
||||
@@ -1,677 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Typed events emitted by a duplex session.
|
||||
"""The event vocabulary one duplex session speaks, assembled from two layers.
|
||||
|
||||
Every public event is a frozen dataclass with explicit fields. The Realtime
|
||||
wire JSON is *derived* from those fields by :meth:`DuplexEvent.to_realtime`,
|
||||
which is pure: it never consults session state. The stateful part (response /
|
||||
item ids, content-part bookkeeping) lives in
|
||||
``vllm_omni.engine.duplex.realtime_events`` on the session runner and is
|
||||
consumed when the events are *constructed*.
|
||||
The classes themselves no longer live here. ``vllm_omni.protocol.duplex.events``
|
||||
carries the whole vocabulary --- 22 Tier 1 classes re-exported from
|
||||
``vllm_omni.protocol.realtime.events``, 8 Tier 2 ones and 12 Tier 3 ones --- so
|
||||
a non-duplex Realtime consumer can take the Tier 1 half and its wire rendering
|
||||
without the duplex session control plane (RFC #6592 P0a). The engine imports
|
||||
the duplex module and never the Tier 1 one directly.
|
||||
|
||||
``session_id`` / ``epoch`` are bound by
|
||||
``DuplexSessionManager.emit`` when an event leaves the runner; producers
|
||||
(projector, serving handshake helpers) build events with the defaults.
|
||||
What is genuinely engine-side stays: :data:`DOMAIN_TERMINAL_EVENTS` and
|
||||
:data:`MODEL_OUTPUT_EVENTS` are the session runner's epoch-filter policy, not
|
||||
wire contract --- they name *internal* event types, and which of them may never
|
||||
be dropped as stale is a runtime decision.
|
||||
|
||||
``DuplexEvent`` is re-exported as the name the engine and the clients already
|
||||
use. It is an alias of
|
||||
:class:`~vllm_omni.protocol.realtime.events.RealtimeEvent`, not a subclass,
|
||||
because an event has no engine-internal half --- what the session emits is
|
||||
exactly what the socket sends. A *command* does have one (``payload()`` renders
|
||||
the runner's mailbox), which is why ``DuplexCommand`` is a real class.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field, fields
|
||||
from typing import ClassVar, cast
|
||||
from uuid import uuid4
|
||||
from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
|
||||
from vllm_omni.protocol.duplex.events import (
|
||||
AudioDelta,
|
||||
AudioDone,
|
||||
ContentPartAdded,
|
||||
ContentPartDone,
|
||||
DuplexEvent,
|
||||
DuplexRawEvent,
|
||||
ErrorEvent,
|
||||
FunctionCallArgumentsDelta,
|
||||
FunctionCallArgumentsDone,
|
||||
InputCleared,
|
||||
InputCommitted,
|
||||
InputTranscriptionCompleted,
|
||||
ItemAdded,
|
||||
ItemCreated,
|
||||
ItemDeleted,
|
||||
ItemDone,
|
||||
ItemRetrieved,
|
||||
ItemTruncated,
|
||||
Listen,
|
||||
OutputAudioCleared,
|
||||
OutputItemAdded,
|
||||
OutputItemDone,
|
||||
OverlapDecision,
|
||||
PlaybackAcknowledged,
|
||||
RateLimitsUpdated,
|
||||
ResponseCreated,
|
||||
ResponseDone,
|
||||
SessionClosed,
|
||||
SessionCreated,
|
||||
SessionExpired,
|
||||
SessionHeartbeatAck,
|
||||
SessionReplaced,
|
||||
SessionResumed,
|
||||
SessionResyncRequired,
|
||||
SessionUpdated,
|
||||
Speak,
|
||||
SpeechStarted,
|
||||
SpeechStopped,
|
||||
TextDelta,
|
||||
TextDone,
|
||||
TranscriptDelta,
|
||||
TranscriptDone,
|
||||
TurnEvent,
|
||||
error_event,
|
||||
new_event_id,
|
||||
)
|
||||
|
||||
|
||||
def new_event_id() -> str:
|
||||
return f"event_{uuid4().hex}"
|
||||
|
||||
|
||||
#: OpenAI Realtime ``error.type`` for each internal error code.
|
||||
REALTIME_ERROR_TYPES_BY_CODE: dict[str, str] = {
|
||||
"bad_event": "invalid_request_error",
|
||||
"bad_audio": "invalid_request_error",
|
||||
"config_timeout": "invalid_request_error",
|
||||
"invalid_json": "invalid_request_error",
|
||||
"event_too_large": "invalid_request_error",
|
||||
"unknown_event": "invalid_request_error",
|
||||
"internal_error": "server_error",
|
||||
"runtime_append_failed": "server_error",
|
||||
"runtime_append_task_failed": "server_error",
|
||||
"runtime_signal_failed": "server_error",
|
||||
"runtime_abort_failed": "server_error",
|
||||
"runtime_data_plane_stream_failed": "server_error",
|
||||
"runtime_data_plane_text_without_audio": "server_error",
|
||||
"resource_exhausted": "rate_limit_error",
|
||||
"session_exists": "invalid_request_error",
|
||||
"session_closed": "invalid_request_error",
|
||||
"unknown_session": "invalid_request_error",
|
||||
"invalid_duplex_runtime_config": "invalid_request_error",
|
||||
"instructions_update_unsupported": "invalid_request_error",
|
||||
"persona_update_unsupported": "invalid_request_error",
|
||||
"voice_update_unsupported": "invalid_request_error",
|
||||
"unsupported_nemotron_duplex_mode": "invalid_request_error",
|
||||
"unsupported_native_response_options": "invalid_request_error",
|
||||
"runtime_touch_failed": "server_error",
|
||||
"engine_error": "server_error",
|
||||
"input_backpressure": "rate_limit_error",
|
||||
"response_already_active": "invalid_request_error",
|
||||
"response_not_active": "invalid_request_error",
|
||||
"response_create_without_input": "invalid_request_error",
|
||||
"text_only_turn_unsupported": "invalid_request_error",
|
||||
"commit_aborted": "server_error",
|
||||
"input_audio_buffer_empty": "invalid_request_error",
|
||||
"missing_item_id": "invalid_request_error",
|
||||
"item_not_found": "invalid_request_error",
|
||||
"playback_item_mismatch": "invalid_request_error",
|
||||
"playback_item_not_found": "invalid_request_error",
|
||||
"playback_ack_too_late": "invalid_request_error",
|
||||
"unsupported_audio_format": "invalid_request_error",
|
||||
"unsupported_turn_detection": "invalid_request_error",
|
||||
"unsupported_ref_audio_path": "invalid_request_error",
|
||||
"ref_audio_required": "invalid_request_error",
|
||||
"model_update_unsupported": "invalid_request_error",
|
||||
"voice_update_after_audio_unsupported": "invalid_request_error",
|
||||
"ref_audio_update_unsupported": "invalid_request_error",
|
||||
"native_text_append_unsupported": "invalid_request_error",
|
||||
"invalid_video_frames": "invalid_request_error",
|
||||
"invalid_function_call_output": "invalid_request_error",
|
||||
"server_vad_unavailable": "server_error",
|
||||
}
|
||||
|
||||
_IDENTITY_FIELDS = frozenset({"session_id", "epoch", "event_id"})
|
||||
|
||||
|
||||
def _wire_value(value: object) -> object:
|
||||
if isinstance(value, Mapping):
|
||||
return {str(k): _wire_value(v) for k, v in value.items()}
|
||||
if isinstance(value, tuple | list):
|
||||
return [_wire_value(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class DuplexEvent:
|
||||
"""Base of every public session event."""
|
||||
|
||||
#: Realtime ``type`` this class renders to.
|
||||
wire_type: ClassVar[str] = ""
|
||||
#: Fields that are omitted from the wire object when ``None``.
|
||||
optional_wire_fields: ClassVar[frozenset[str]] = frozenset()
|
||||
|
||||
session_id: str = ""
|
||||
epoch: int | None = None
|
||||
#: Server event id (OpenAI ``event_id``); generated at construction.
|
||||
event_id: str = field(default_factory=new_event_id)
|
||||
|
||||
# ---- identity ----
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return self.wire_type
|
||||
|
||||
@property
|
||||
def is_terminal(self) -> bool:
|
||||
return False
|
||||
|
||||
# ---- generic accessors (None when the class has no such field) ----
|
||||
|
||||
@property
|
||||
def response_id(self) -> str | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def item_id(self) -> str | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def audio(self) -> bytes | None:
|
||||
return None
|
||||
|
||||
# ---- wire rendering (pure) ----
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
data: dict[str, object] = {}
|
||||
for f in fields(self):
|
||||
if f.name in _IDENTITY_FIELDS:
|
||||
continue
|
||||
value = getattr(self, f.name)
|
||||
if value is None and f.name in self.optional_wire_fields:
|
||||
continue
|
||||
data[f.name] = _wire_value(value)
|
||||
return data
|
||||
|
||||
def to_realtime(self) -> dict[str, object]:
|
||||
"""The Realtime wire JSON object for this event (derived from the fields)."""
|
||||
return {"type": self.wire_type, "event_id": self.event_id, **self._wire_fields()}
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - debugging aid
|
||||
return f"{type(self).__name__}(type={self.wire_type!r}, session_id={self.session_id!r}, epoch={self.epoch!r})"
|
||||
|
||||
|
||||
# ---- session lifecycle ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionCreated(DuplexEvent):
|
||||
wire_type = "session.created"
|
||||
optional_wire_fields = frozenset({"attachment_generation", "resume_token"})
|
||||
|
||||
session: Mapping[str, object] = field(default_factory=dict)
|
||||
#: Transport resume credentials (set by the websocket handler when resume is supported).
|
||||
attachment_generation: int | None = None
|
||||
resume_token: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionUpdated(DuplexEvent):
|
||||
wire_type = "session.updated"
|
||||
|
||||
session: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionResumed(SessionCreated):
|
||||
wire_type = "session.resumed"
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionReplaced(DuplexEvent):
|
||||
"""A newer connection took the session over (sent to the replaced socket)."""
|
||||
|
||||
wire_type = "session.replaced"
|
||||
|
||||
attachment_generation: int = 0
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionResyncRequired(DuplexEvent):
|
||||
"""The replay journal cannot bridge the gap; the client must start over."""
|
||||
|
||||
wire_type = "session.resync_required"
|
||||
|
||||
reason: str = "journal_gap"
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionClosed(DuplexEvent):
|
||||
wire_type = "session.closed"
|
||||
|
||||
reason: str = "closed"
|
||||
#: The session-internal close event (kept for clients that inspect it).
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def is_terminal(self) -> bool:
|
||||
return True
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, "reason": self.reason, "event": _wire_value(self.details)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionExpired(SessionClosed):
|
||||
wire_type = "session.expired"
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, "reason": self.reason}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionHeartbeatAck(DuplexEvent):
|
||||
wire_type = "session.heartbeat_ack"
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TurnEvent(DuplexEvent):
|
||||
"""Turn-state transition (``turn.event``)."""
|
||||
|
||||
wire_type = "turn.event"
|
||||
|
||||
event: str = ""
|
||||
turn_state: str = ""
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, "event": self.event, "turn_state": self.turn_state, "epoch": self.epoch}
|
||||
|
||||
|
||||
# ---- response lifecycle ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class _ResponseEvent(DuplexEvent):
|
||||
"""Events addressed to one response (and usually one output item / content part)."""
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
output_index: int = 0
|
||||
content_index: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseCreated(DuplexEvent):
|
||||
wire_type = "response.created"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
response: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Listen(DuplexEvent):
|
||||
"""The model decided to keep listening (no spoken response for this turn)."""
|
||||
|
||||
wire_type = "response.listen"
|
||||
optional_wire_fields = frozenset({"response_id"})
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
response: dict[str, object] = {
|
||||
"object": "realtime.response",
|
||||
"status": "listening",
|
||||
"metadata": _wire_value(self.details),
|
||||
}
|
||||
data: dict[str, object] = {"session_id": self.session_id, "epoch": self.epoch, "response": response}
|
||||
if self.response_id:
|
||||
response["id"] = self.response_id
|
||||
data["response_id"] = self.response_id
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Speak(_ResponseEvent):
|
||||
wire_type = "response.speak"
|
||||
|
||||
metadata: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class OverlapDecision(DuplexEvent):
|
||||
wire_type = "overlap.decision"
|
||||
|
||||
policy: str | None = None
|
||||
action: str | None = None
|
||||
reason: str | None = None
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"epoch": self.epoch,
|
||||
"policy": self.policy,
|
||||
"action": self.action,
|
||||
"reason": self.reason,
|
||||
"metadata": _wire_value(self.details),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class OutputItemAdded(DuplexEvent):
|
||||
wire_type = "response.output_item.added"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
output_index: int = 0
|
||||
item: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def item_id(self) -> str | None:
|
||||
value = self.item.get("id")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class OutputItemDone(OutputItemAdded):
|
||||
wire_type = "response.output_item.done"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ContentPartAdded(_ResponseEvent):
|
||||
wire_type = "response.content_part.added"
|
||||
|
||||
part: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ContentPartDone(ContentPartAdded):
|
||||
wire_type = "response.content_part.done"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AudioDelta(_ResponseEvent):
|
||||
wire_type = "response.output_audio.delta"
|
||||
optional_wire_fields = frozenset({"sample_rate_hz", "metadata"})
|
||||
|
||||
#: Base64 audio in ``format``.
|
||||
delta: str = ""
|
||||
format: str = "pcm16"
|
||||
sample_rate_hz: int | None = None
|
||||
metadata: Mapping[str, object] | None = None
|
||||
|
||||
@property
|
||||
def audio(self) -> bytes | None:
|
||||
try:
|
||||
return base64.b64decode(self.delta)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AudioDone(_ResponseEvent):
|
||||
wire_type = "response.output_audio.done"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TranscriptDelta(_ResponseEvent):
|
||||
wire_type = "response.output_audio_transcript.delta"
|
||||
|
||||
delta: str = ""
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.delta
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TranscriptDone(_ResponseEvent):
|
||||
wire_type = "response.output_audio_transcript.done"
|
||||
|
||||
transcript: str = ""
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.transcript
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TextDelta(_ResponseEvent):
|
||||
wire_type = "response.output_text.delta"
|
||||
|
||||
delta: str = ""
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.delta
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TextDone(_ResponseEvent):
|
||||
wire_type = "response.output_text.done"
|
||||
|
||||
text: str = "" # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class FunctionCallArgumentsDelta(DuplexEvent):
|
||||
wire_type = "response.function_call_arguments.delta"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
output_index: int = 0
|
||||
call_id: str = ""
|
||||
delta: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class FunctionCallArgumentsDone(DuplexEvent):
|
||||
wire_type = "response.function_call_arguments.done"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
output_index: int = 0
|
||||
call_id: str = ""
|
||||
arguments: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseDone(DuplexEvent):
|
||||
wire_type = "response.done"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
response: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def status(self) -> str | None:
|
||||
value = self.response.get("status")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class RateLimitsUpdated(DuplexEvent):
|
||||
wire_type = "rate_limits.updated"
|
||||
|
||||
rate_limits: tuple[Mapping[str, object], ...] = ()
|
||||
|
||||
|
||||
# ---- input buffer / conversation ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class InputCommitted(DuplexEvent):
|
||||
wire_type = "input_audio_buffer.committed"
|
||||
|
||||
previous_item_id: str | None = None
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
#: The session-internal ``input.committed`` event.
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {
|
||||
"previous_item_id": self.previous_item_id,
|
||||
"item_id": self.item_id,
|
||||
"event": _wire_value(self.details),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class InputCleared(DuplexEvent):
|
||||
wire_type = "input_audio_buffer.cleared"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SpeechStarted(DuplexEvent):
|
||||
wire_type = "input_audio_buffer.speech_started"
|
||||
|
||||
audio_start_ms: int = 0
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SpeechStopped(DuplexEvent):
|
||||
wire_type = "input_audio_buffer.speech_stopped"
|
||||
|
||||
audio_end_ms: int = 0
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class _ItemEvent(DuplexEvent):
|
||||
previous_item_id: str | None = None
|
||||
item: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def item_id(self) -> str | None:
|
||||
value = self.item.get("id")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemAdded(_ItemEvent):
|
||||
wire_type = "conversation.item.added"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemCreated(_ItemEvent):
|
||||
wire_type = "conversation.item.created"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemDone(_ItemEvent):
|
||||
wire_type = "conversation.item.done"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemRetrieved(DuplexEvent):
|
||||
wire_type = "conversation.item.retrieved"
|
||||
|
||||
item: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def item_id(self) -> str | None:
|
||||
value = self.item.get("id")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemDeleted(DuplexEvent):
|
||||
wire_type = "conversation.item.deleted"
|
||||
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"item_id": self.item_id, "event": _wire_value(self.details)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemTruncated(DuplexEvent):
|
||||
wire_type = "conversation.item.truncated"
|
||||
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
content_index: int = 0
|
||||
audio_end_ms: int = 0
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {
|
||||
"item_id": self.item_id,
|
||||
"content_index": self.content_index,
|
||||
"audio_end_ms": self.audio_end_ms,
|
||||
"event": _wire_value(self.details),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class InputTranscriptionCompleted(DuplexEvent):
|
||||
wire_type = "conversation.item.input_audio_transcription.completed"
|
||||
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
content_index: int = 0
|
||||
transcript: str = ""
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.transcript
|
||||
|
||||
|
||||
# ---- playback / control ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class OutputAudioCleared(DuplexEvent):
|
||||
wire_type = "output_audio_buffer.cleared"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class PlaybackAcknowledged(DuplexEvent):
|
||||
wire_type = "playback.acknowledged"
|
||||
|
||||
#: The session-internal ``playback.acknowledged`` event (cursor, committed_ms ...).
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"event": _wire_value(self.details)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ErrorEvent(DuplexEvent):
|
||||
wire_type = "error"
|
||||
|
||||
code: str = "internal_error"
|
||||
message: str = ""
|
||||
#: Client ``event_id`` this error answers (OpenAI ``error.event_id``).
|
||||
related_event_id: str | None = None
|
||||
param: str | None = None
|
||||
#: Extra keys merged into the wire ``error`` object (``retryable`` ...).
|
||||
extra: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def error_type(self) -> str:
|
||||
return REALTIME_ERROR_TYPES_BY_CODE.get(self.code, "invalid_request_error")
|
||||
|
||||
@property
|
||||
def error(self) -> dict[str, object]:
|
||||
error: dict[str, object] = {"type": self.error_type, "code": self.code, "message": self.message}
|
||||
if self.related_event_id:
|
||||
error["event_id"] = self.related_event_id
|
||||
if self.param:
|
||||
error["param"] = self.param
|
||||
error.update(cast("Mapping[str, object]", _wire_value(self.extra)))
|
||||
return error
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.message
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"error": self.error}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class DuplexRawEvent(DuplexEvent):
|
||||
"""A session-internal event with no dedicated Realtime type (``duplex.<internal type>``)."""
|
||||
|
||||
wire_type = "duplex.raw"
|
||||
|
||||
internal_type: str = ""
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return f"duplex.{self.internal_type}"
|
||||
|
||||
def to_realtime(self) -> dict[str, object]:
|
||||
return {"type": self.type, "event_id": self.event_id, "event": _wire_value(self.details)}
|
||||
|
||||
|
||||
def error_event(
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
event_id: object | None = None,
|
||||
param: object | None = None,
|
||||
extra: Mapping[str, object] | None = None,
|
||||
) -> ErrorEvent:
|
||||
"""Build the error event for an internal code; ``event_id`` is the *client* event id."""
|
||||
return ErrorEvent(
|
||||
code=code,
|
||||
message=message,
|
||||
related_event_id=event_id if isinstance(event_id, str) and event_id else None,
|
||||
param=param if isinstance(param, str) and param else None,
|
||||
extra=dict(extra or {}),
|
||||
)
|
||||
|
||||
|
||||
#: Internal events that terminate a response/session and must never be dropped as stale.
|
||||
DOMAIN_TERMINAL_EVENTS = frozenset(
|
||||
{
|
||||
"response.done",
|
||||
@@ -682,7 +84,6 @@ DOMAIN_TERMINAL_EVENTS = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
#: Internal streaming model-output events subject to epoch/close stale filtering.
|
||||
MODEL_OUTPUT_EVENTS = frozenset(
|
||||
{
|
||||
"response.created",
|
||||
@@ -705,13 +106,11 @@ MODEL_OUTPUT_EVENTS = frozenset(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DOMAIN_TERMINAL_EVENTS",
|
||||
"MODEL_OUTPUT_EVENTS",
|
||||
"REALTIME_ERROR_TYPES_BY_CODE",
|
||||
"AudioDelta",
|
||||
"AudioDone",
|
||||
"ContentPartAdded",
|
||||
"ContentPartDone",
|
||||
"DOMAIN_TERMINAL_EVENTS",
|
||||
"DuplexEvent",
|
||||
"DuplexRawEvent",
|
||||
"ErrorEvent",
|
||||
@@ -727,11 +126,13 @@ __all__ = [
|
||||
"ItemRetrieved",
|
||||
"ItemTruncated",
|
||||
"Listen",
|
||||
"MODEL_OUTPUT_EVENTS",
|
||||
"OutputAudioCleared",
|
||||
"OutputItemAdded",
|
||||
"OutputItemDone",
|
||||
"OverlapDecision",
|
||||
"PlaybackAcknowledged",
|
||||
"REALTIME_ERROR_TYPES_BY_CODE",
|
||||
"RateLimitsUpdated",
|
||||
"ResponseCreated",
|
||||
"ResponseDone",
|
||||
|
||||
@@ -3,26 +3,31 @@
|
||||
|
||||
"""OpenAI Realtime client events -> :class:`DuplexCommand` (stateless).
|
||||
|
||||
This is the mapping half of the former ``entrypoints/duplex/realtime_input.py``
|
||||
translator. Everything that needed per-session state in the old translator
|
||||
(input-buffer emptiness for commits, response-id fallbacks for cancels,
|
||||
conversation-item lookups, VAD) is resolved by the session runner through the
|
||||
helpers on :class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
|
||||
This module is the *duplex binding* of the shared Realtime codec. Parsing,
|
||||
validating and decoding a client event is not duplex-specific and lives in
|
||||
``vllm_omni.protocol.realtime``, reached through ``vllm_omni.protocol.duplex``;
|
||||
what lives here is the part that is: which
|
||||
:class:`~vllm_omni.engine.duplex.commands.DuplexCommand` a decoded event
|
||||
becomes, what the duplex engine can serve
|
||||
(:data:`DUPLEX_REALTIME_CAPABILITIES`), and the mapping between a Realtime
|
||||
output format and the duplex ``response_format`` vocabulary.
|
||||
|
||||
Everything that needs per-session state (input-buffer emptiness for commits,
|
||||
response-id fallbacks for cancels, conversation-item lookups, VAD) is resolved
|
||||
by the session runner through the helpers on
|
||||
:class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
|
||||
the commands produced here carry the raw client intent only.
|
||||
|
||||
The shared names re-exported at the bottom are a compatibility surface for
|
||||
existing importers. The canonical definitions are in
|
||||
``vllm_omni.protocol.realtime`` and there is exactly one of each.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any, cast
|
||||
from uuid import uuid4
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate
|
||||
from vllm_omni.engine.duplex.commands import (
|
||||
AckPlayback,
|
||||
AppendAudio,
|
||||
@@ -44,93 +49,58 @@ from vllm_omni.engine.duplex.commands import (
|
||||
TruncateItem,
|
||||
UpdateSession,
|
||||
)
|
||||
|
||||
REALTIME_INPUT_AUDIO_FORMATS = {
|
||||
"pcm16",
|
||||
"pcm_s16le",
|
||||
"s16le",
|
||||
"pcm_f32le",
|
||||
"g711_ulaw",
|
||||
"g711_alaw",
|
||||
}
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS = {
|
||||
"pcm16",
|
||||
"pcm_s16le",
|
||||
"s16le",
|
||||
"wav",
|
||||
"pcm",
|
||||
"g711_ulaw",
|
||||
"g711_alaw",
|
||||
}
|
||||
|
||||
#: Wire hint keys copied verbatim from a client append onto the internal payload.
|
||||
REALTIME_INPUT_HINT_KEYS = (
|
||||
"duration_ms",
|
||||
"audio_duration_ms",
|
||||
"audio_start_ms",
|
||||
"audio_end_ms",
|
||||
"is_speech",
|
||||
"speech",
|
||||
"speech_probability",
|
||||
"vad",
|
||||
"overlap_action",
|
||||
"overlap",
|
||||
"force_barge_in",
|
||||
"force_listen",
|
||||
"text",
|
||||
"transcript",
|
||||
from vllm_omni.protocol.duplex import (
|
||||
REALTIME_INPUT_AUDIO_FORMATS,
|
||||
REALTIME_INPUT_HINT_KEYS,
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS,
|
||||
RealtimeInputDefaults,
|
||||
RealtimeProtocolCapabilities,
|
||||
RealtimeProtocolError,
|
||||
apply_realtime_session_defaults,
|
||||
copy_realtime_input_hints,
|
||||
decode_audio_append,
|
||||
input_audio_transcription_config,
|
||||
input_explicitly_non_speech,
|
||||
input_looks_like_speech,
|
||||
input_transcript_from_item,
|
||||
is_supported_realtime_input_format,
|
||||
json_safe_realtime_payload,
|
||||
normalize_conversation_item,
|
||||
parse_realtime_audio_format,
|
||||
realtime_audio_format_object,
|
||||
realtime_max_output_tokens,
|
||||
realtime_output_format,
|
||||
realtime_overlap_fields,
|
||||
text_chars_for_audio_ms_from_marks,
|
||||
truncate_realtime_item_content,
|
||||
validate_conversation_item_audio_formats,
|
||||
validate_realtime_item_truncate,
|
||||
validate_realtime_response_audio_formats,
|
||||
validate_realtime_session_audio_formats,
|
||||
validate_realtime_video_frames,
|
||||
validate_session_payload,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RealtimeInputDefaults:
|
||||
"""Session-level wire defaults an append may omit (derived from the session object)."""
|
||||
def _validate_duplex_turn_detection(session_payload: Mapping[str, object]) -> str | None:
|
||||
"""The duplex engine's answer for ``turn_detection`` on a session object.
|
||||
|
||||
input_audio_format: str = "pcm16"
|
||||
input_sample_rate_hz: int = 16000
|
||||
output_audio_format: str = "pcm16"
|
||||
output_sample_rate_hz: int | None = None
|
||||
overlap_silence_rms: float = 0.003
|
||||
Imported lazily: the validator lives with the Silero VAD backend, and a
|
||||
connection that never sends a session object should not pay for loading it.
|
||||
"""
|
||||
from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
|
||||
|
||||
def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults:
|
||||
"""Return defaults updated from a Realtime ``session`` object (session.update)."""
|
||||
values = apply_realtime_session_defaults(self, session_payload)
|
||||
return values
|
||||
return validate_realtime_turn_detection(session_payload)
|
||||
|
||||
|
||||
# ---- audio format helpers ----
|
||||
|
||||
|
||||
def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]:
|
||||
def normalize_format(fmt: str) -> str:
|
||||
normalized = fmt.lower()
|
||||
if normalized in {"audio/pcm", "pcm"}:
|
||||
return "pcm16"
|
||||
if normalized in {"audio/wav", "wav"}:
|
||||
return "wav"
|
||||
if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}:
|
||||
return "pcm16"
|
||||
if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}:
|
||||
return "pcm_f32le"
|
||||
if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}:
|
||||
return "g711_ulaw"
|
||||
if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}:
|
||||
return "g711_alaw"
|
||||
return fmt
|
||||
|
||||
if isinstance(raw_format, str):
|
||||
return normalize_format(raw_format), None
|
||||
if not isinstance(raw_format, dict):
|
||||
return raw_format, None
|
||||
rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate")
|
||||
sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None
|
||||
fmt = raw_format.get("type") or raw_format.get("format")
|
||||
if not isinstance(fmt, str):
|
||||
return raw_format, sample_rate_hz
|
||||
return normalize_format(fmt), sample_rate_hz
|
||||
#: What the duplex engine can serve, for the shared session-object validator.
|
||||
DUPLEX_REALTIME_CAPABILITIES = RealtimeProtocolCapabilities(
|
||||
validate_turn_detection=_validate_duplex_turn_detection,
|
||||
)
|
||||
|
||||
|
||||
def duplex_response_format(realtime_format: str) -> str:
|
||||
"""A Realtime output format -> the duplex ``response_format`` vocabulary."""
|
||||
normalized = realtime_format.lower()
|
||||
if normalized in {"pcm16", "pcm_s16le", "s16le"}:
|
||||
return "pcm"
|
||||
@@ -141,433 +111,6 @@ def duplex_response_format(realtime_format: str) -> str:
|
||||
return "wav"
|
||||
|
||||
|
||||
def realtime_output_format(duplex_format: object) -> str:
|
||||
if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}:
|
||||
return duplex_format.lower()
|
||||
if isinstance(duplex_format, str) and duplex_format.lower() == "pcm":
|
||||
return "pcm16"
|
||||
return str(duplex_format or "wav")
|
||||
|
||||
|
||||
def is_supported_realtime_input_format(fmt: object) -> bool:
|
||||
return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS
|
||||
|
||||
|
||||
def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]:
|
||||
if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}:
|
||||
payload: dict[str, object] = {"type": "audio/pcm"}
|
||||
elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le":
|
||||
payload = {"type": "audio/pcm_f32le"}
|
||||
elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw":
|
||||
payload = {"type": "audio/g711_ulaw"}
|
||||
elif isinstance(fmt, str) and fmt.lower() == "g711_alaw":
|
||||
payload = {"type": "audio/g711_alaw"}
|
||||
else:
|
||||
payload = {"type": "audio/wav"}
|
||||
if sample_rate_hz is not None:
|
||||
payload["rate"] = int(sample_rate_hz)
|
||||
return payload
|
||||
|
||||
|
||||
def validate_realtime_session_audio_formats(session_payload: Mapping[str, object]) -> str | None:
|
||||
audio_config = session_payload.get("audio")
|
||||
input_format: object = session_payload.get("input_audio_format")
|
||||
if input_format is None and isinstance(audio_config, dict):
|
||||
audio_input = audio_config.get("input")
|
||||
if isinstance(audio_input, dict):
|
||||
input_format = audio_input.get("format")
|
||||
parsed_input, _ = parse_realtime_audio_format(input_format)
|
||||
if input_format is not None and not (
|
||||
isinstance(parsed_input, str) and parsed_input.lower() in REALTIME_INPUT_AUDIO_FORMATS
|
||||
):
|
||||
return f"Unsupported input_audio_format: {input_format}"
|
||||
|
||||
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
|
||||
if output_format is None and isinstance(audio_config, dict):
|
||||
audio_output = audio_config.get("output")
|
||||
if isinstance(audio_output, dict):
|
||||
output_format = audio_output.get("format")
|
||||
parsed_output, _ = parse_realtime_audio_format(output_format)
|
||||
if output_format is not None and not (
|
||||
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
|
||||
):
|
||||
return f"Unsupported output_audio_format: {output_format}"
|
||||
return None
|
||||
|
||||
|
||||
def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None:
|
||||
output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format")
|
||||
audio_config = response_payload.get("audio")
|
||||
if output_format is None and isinstance(audio_config, dict):
|
||||
audio_output = audio_config.get("output")
|
||||
if isinstance(audio_output, dict):
|
||||
output_format = audio_output.get("format")
|
||||
parsed_output, _ = parse_realtime_audio_format(output_format)
|
||||
if output_format is not None and not (
|
||||
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
|
||||
):
|
||||
return f"Unsupported output_audio_format: {output_format}"
|
||||
return None
|
||||
|
||||
|
||||
def validate_conversation_item_audio_formats(item: object) -> str | None:
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list):
|
||||
return None
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
if part.get("type") not in {"input_audio", "audio"}:
|
||||
continue
|
||||
raw_format = part.get("format")
|
||||
parsed_format, _ = parse_realtime_audio_format(raw_format)
|
||||
if raw_format is not None and not (
|
||||
isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS
|
||||
):
|
||||
return f"Unsupported input_audio format in conversation.item.create: {raw_format}"
|
||||
return None
|
||||
|
||||
|
||||
def apply_realtime_session_defaults(
|
||||
defaults: RealtimeInputDefaults,
|
||||
session_payload: Mapping[str, object],
|
||||
) -> RealtimeInputDefaults:
|
||||
"""Derive the wire defaults a Realtime ``session`` object declares."""
|
||||
input_format: object = session_payload.get("input_audio_format")
|
||||
audio_config = session_payload.get("audio")
|
||||
if input_format is None and isinstance(audio_config, dict):
|
||||
audio_input = audio_config.get("input")
|
||||
if isinstance(audio_input, dict):
|
||||
input_format = audio_input.get("format")
|
||||
input_format, input_rate = parse_realtime_audio_format(input_format)
|
||||
# Any: heterogeneous field values splatted into ``dataclasses.replace`` below.
|
||||
updates: dict[str, Any] = {}
|
||||
if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS:
|
||||
updates["input_audio_format"] = input_format
|
||||
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
|
||||
output_rate_raw: object | None = None
|
||||
if output_format is None and isinstance(audio_config, dict):
|
||||
audio_output = audio_config.get("output")
|
||||
if isinstance(audio_output, dict):
|
||||
output_format = audio_output.get("format")
|
||||
output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate")
|
||||
output_format, output_rate = parse_realtime_audio_format(output_format)
|
||||
if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0:
|
||||
output_rate = int(output_rate_raw)
|
||||
if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
|
||||
updates["output_audio_format"] = realtime_output_format(output_format)
|
||||
sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate")
|
||||
if sample_rate is None and isinstance(audio_config, dict):
|
||||
audio_input = audio_config.get("input")
|
||||
if isinstance(audio_input, dict):
|
||||
sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate")
|
||||
if sample_rate is None:
|
||||
sample_rate = input_rate
|
||||
if isinstance(sample_rate, int | float) and sample_rate > 0:
|
||||
updates["input_sample_rate_hz"] = int(sample_rate)
|
||||
if isinstance(output_rate, int | float) and output_rate > 0:
|
||||
updates["output_sample_rate_hz"] = int(output_rate)
|
||||
overlap_fields = realtime_overlap_fields(session_payload)
|
||||
overlap_silence_rms = overlap_fields.get("overlap_silence_rms")
|
||||
if isinstance(overlap_silence_rms, int | float):
|
||||
updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms))
|
||||
return replace(defaults, **updates) if updates else defaults
|
||||
|
||||
|
||||
def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]:
|
||||
fields: dict[str, object] = {}
|
||||
if isinstance(session_payload.get("overlap_policy"), str):
|
||||
fields["overlap_policy"] = session_payload["overlap_policy"]
|
||||
for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"):
|
||||
value = session_payload.get(key)
|
||||
if isinstance(value, int | float):
|
||||
fields[key] = value
|
||||
if isinstance(session_payload.get("playback_commit_policy"), str):
|
||||
fields["playback_commit_policy"] = session_payload["playback_commit_policy"]
|
||||
return fields
|
||||
|
||||
|
||||
def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]:
|
||||
clean: dict[str, object] = {}
|
||||
for key, value in payload.items():
|
||||
if key == "extra_body":
|
||||
continue
|
||||
if isinstance(value, str | int | float | bool) or value is None:
|
||||
clean[key] = value
|
||||
elif isinstance(value, dict):
|
||||
clean[key] = json_safe_realtime_payload(value)
|
||||
elif isinstance(value, list):
|
||||
clean[key] = [
|
||||
(json_safe_realtime_payload(item) if isinstance(item, dict) else item)
|
||||
for item in value
|
||||
if isinstance(item, str | int | float | bool | dict) or item is None
|
||||
]
|
||||
return clean
|
||||
|
||||
|
||||
def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
|
||||
transcription = session_payload.get("input_audio_transcription")
|
||||
if isinstance(transcription, dict):
|
||||
return transcription
|
||||
audio_config = session_payload.get("audio")
|
||||
if not isinstance(audio_config, dict):
|
||||
return None
|
||||
audio_input = audio_config.get("input")
|
||||
if not isinstance(audio_input, dict):
|
||||
return None
|
||||
transcription = audio_input.get("transcription")
|
||||
return transcription if isinstance(transcription, dict) else None
|
||||
|
||||
|
||||
def realtime_max_output_tokens(value: object) -> int | None:
|
||||
"""Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}:
|
||||
return None
|
||||
if isinstance(value, int) and value > 0:
|
||||
return int(value)
|
||||
return None
|
||||
|
||||
|
||||
# ---- speech hints ----
|
||||
|
||||
|
||||
def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
|
||||
for key in ("is_speech", "speech"):
|
||||
value = event.get(key)
|
||||
if isinstance(value, bool):
|
||||
return not value
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
value = vad.get("is_speech")
|
||||
if isinstance(value, bool):
|
||||
return not value
|
||||
probability = vad.get("speech_probability", vad.get("probability"))
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) < 0.5
|
||||
probability = event.get("speech_probability")
|
||||
return isinstance(probability, int | float) and float(probability) < 0.5
|
||||
|
||||
|
||||
def input_looks_like_speech(
|
||||
event: Mapping[str, object],
|
||||
*,
|
||||
audio: object,
|
||||
fmt: object,
|
||||
overlap_silence_rms: float,
|
||||
) -> bool:
|
||||
if input_explicitly_non_speech(event):
|
||||
return False
|
||||
for key in ("is_speech", "speech"):
|
||||
value = event.get(key)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
probability = vad.get("speech_probability", vad.get("probability"))
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) >= 0.5
|
||||
probability = event.get("speech_probability")
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) >= 0.5
|
||||
if fmt != "pcm_f32le":
|
||||
return True
|
||||
if isinstance(audio, bytes | bytearray):
|
||||
raw = bytes(audio)
|
||||
elif isinstance(audio, str):
|
||||
try:
|
||||
raw = base64.b64decode(audio, validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
if len(raw) < 4 or len(raw) % 4 != 0:
|
||||
return True
|
||||
samples = np.frombuffer(raw, dtype=np.float32)
|
||||
if samples.size == 0:
|
||||
return False
|
||||
rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
|
||||
threshold = event.get("overlap_silence_rms")
|
||||
if not isinstance(threshold, int | float):
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
threshold = vad.get("silence_rms")
|
||||
silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
|
||||
return rms >= max(0.0, silence_rms)
|
||||
|
||||
|
||||
def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
|
||||
for key in REALTIME_INPUT_HINT_KEYS:
|
||||
if key in source:
|
||||
target[key] = source[key]
|
||||
|
||||
|
||||
def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
|
||||
"""Validate omni-duplex camera frames on input_audio_buffer.append.
|
||||
|
||||
Wire contract matches the official MiniCPM-o duplex loop: one base
|
||||
base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
|
||||
stacked composite tiling the sub-frames captured inside it (at most 2
|
||||
images either way). A caller-supplied ``max_slice_nums`` is rejected rather
|
||||
than silently ignored: slicing is Stage 0's decision here, and Stage 0
|
||||
already applies the official HD suggestion for a stacked unit
|
||||
(``max_slice_nums=[2, 1]``). The wire simply does not let the client choose
|
||||
it.
|
||||
"""
|
||||
if max_slice_nums not in (None, 1):
|
||||
return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself"
|
||||
if not isinstance(video_frames, list):
|
||||
return "video_frames must be a list of base64-encoded images"
|
||||
frames = [frame for frame in video_frames if frame is not None]
|
||||
if len(frames) > 2:
|
||||
return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
|
||||
for frame in frames:
|
||||
if not isinstance(frame, str) or not frame:
|
||||
return "video_frames entries must be non-empty base64 strings"
|
||||
if len(frame) > 4_000_000:
|
||||
return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
|
||||
try:
|
||||
header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
|
||||
except (binascii.Error, ValueError):
|
||||
return "video_frames entries must be valid base64"
|
||||
if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
|
||||
return "video_frames entries must be JPEG or PNG images"
|
||||
return None
|
||||
|
||||
|
||||
# ---- conversation items ----
|
||||
|
||||
|
||||
def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]:
|
||||
normalized = dict(item)
|
||||
normalized.setdefault("id", f"item_{uuid4().hex}")
|
||||
normalized.setdefault("object", "realtime.item")
|
||||
normalized.setdefault("type", "message")
|
||||
normalized.setdefault("status", "completed")
|
||||
if "role" not in normalized and normalized.get("type") == "message":
|
||||
normalized["role"] = "user"
|
||||
if not isinstance(normalized.get("content"), list):
|
||||
normalized["content"] = []
|
||||
return normalized
|
||||
|
||||
|
||||
def text_chars_for_audio_ms_from_marks(
|
||||
audio_end_ms: int,
|
||||
text_len: int,
|
||||
marks: list[object],
|
||||
*,
|
||||
final_ms: object | None = None,
|
||||
) -> int:
|
||||
if text_len <= 0:
|
||||
return 0
|
||||
clean_marks: list[tuple[int, int]] = []
|
||||
for mark in marks:
|
||||
if not isinstance(mark, dict):
|
||||
continue
|
||||
raw_text_chars = mark.get("text_chars")
|
||||
raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
|
||||
if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
|
||||
continue
|
||||
clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
|
||||
if not clean_marks:
|
||||
return 0 if audio_end_ms <= 0 else text_len
|
||||
clean_marks.sort(key=lambda item: item[0])
|
||||
audio_end_ms = max(0, int(audio_end_ms))
|
||||
if audio_end_ms <= 0:
|
||||
return 0
|
||||
previous_ms = 0
|
||||
previous_chars = 0
|
||||
for mark_ms, mark_chars in clean_marks:
|
||||
mark_ms = max(previous_ms, mark_ms)
|
||||
mark_chars = max(previous_chars, min(text_len, mark_chars))
|
||||
if audio_end_ms <= mark_ms:
|
||||
if mark_ms <= previous_ms:
|
||||
return mark_chars
|
||||
ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
|
||||
return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
|
||||
previous_ms = mark_ms
|
||||
previous_chars = mark_chars
|
||||
if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
|
||||
if audio_end_ms >= int(final_ms):
|
||||
return text_len
|
||||
ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
|
||||
return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
|
||||
return text_len if audio_end_ms >= previous_ms else previous_chars
|
||||
|
||||
|
||||
def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None:
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list) or not content:
|
||||
return
|
||||
index = max(0, int(content_index))
|
||||
if index >= len(content):
|
||||
return
|
||||
part = content[index]
|
||||
if not isinstance(part, dict):
|
||||
return
|
||||
transcript = part.get("transcript")
|
||||
if not isinstance(transcript, str) or not transcript:
|
||||
return
|
||||
marks = part.get("audio_text_marks")
|
||||
if isinstance(marks, list):
|
||||
keep_chars = text_chars_for_audio_ms_from_marks(
|
||||
audio_end_ms,
|
||||
len(transcript),
|
||||
marks,
|
||||
final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
|
||||
)
|
||||
part["transcript"] = transcript[:keep_chars].rstrip()
|
||||
return
|
||||
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
|
||||
if isinstance(duration_ms, int | float) and duration_ms > 0:
|
||||
keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
|
||||
part["transcript"] = transcript[:keep_chars].rstrip()
|
||||
elif audio_end_ms <= 0:
|
||||
part["transcript"] = ""
|
||||
|
||||
|
||||
def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None:
|
||||
if item.get("type") != "message" or item.get("role") != "assistant":
|
||||
return "conversation.item.truncate only supports assistant message items"
|
||||
if audio_end_ms < 0:
|
||||
return "conversation.item.truncate requires non-negative audio_end_ms"
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list) or not content:
|
||||
return None
|
||||
index = max(0, int(content_index))
|
||||
if index >= len(content):
|
||||
return f"conversation.item.truncate content_index out of range: {content_index}"
|
||||
part = content[index]
|
||||
if not isinstance(part, dict):
|
||||
return "conversation.item.truncate target content part is invalid"
|
||||
if part.get("type") not in {"audio", "output_audio"}:
|
||||
return "conversation.item.truncate target content part must be audio"
|
||||
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
|
||||
if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
|
||||
return "conversation.item.truncate audio_end_ms exceeds item audio duration"
|
||||
return None
|
||||
|
||||
|
||||
def input_transcript_from_item(item: Mapping[str, object]) -> str:
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list):
|
||||
return ""
|
||||
parts: list[str] = []
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
for key in ("transcript", "text"):
|
||||
value = part.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
|
||||
parts.append(value)
|
||||
break
|
||||
return "".join(parts).strip()
|
||||
|
||||
|
||||
# ---- append / item audio conversion ----
|
||||
|
||||
|
||||
@@ -577,67 +120,21 @@ def build_append_audio(
|
||||
defaults: RealtimeInputDefaults,
|
||||
hints_source: Mapping[str, object] | None = None,
|
||||
) -> AppendAudio:
|
||||
"""Validate, convert (to 16 kHz ``pcm_f32le`` base64) and pack one audio append.
|
||||
|
||||
Speech classification without VAD (``is_speech`` on the command) uses the
|
||||
explicit wire hints and the RMS fallback exactly like the old translator.
|
||||
"""
|
||||
event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
|
||||
audio = event.get("audio") or event.get("delta")
|
||||
fmt, format_rate = parse_realtime_audio_format(
|
||||
event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
|
||||
)
|
||||
sample_rate_hz = (
|
||||
event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
|
||||
)
|
||||
if not is_supported_realtime_input_format(fmt):
|
||||
raise DuplexCommandError(
|
||||
f"Unsupported input_audio_format: {fmt}",
|
||||
code="unsupported_audio_format",
|
||||
event_id=event_id,
|
||||
)
|
||||
"""Decode one audio append (shared codec) and pack it as the duplex command."""
|
||||
try:
|
||||
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
|
||||
audio,
|
||||
fmt,
|
||||
sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise DuplexCommandError(str(exc), code="bad_event", event_id=event_id) from exc
|
||||
hints: dict[str, object] = {}
|
||||
if hints_source is not None:
|
||||
copy_realtime_input_hints(hints_source, hints)
|
||||
copy_realtime_input_hints(event, hints)
|
||||
looks_like_speech = input_looks_like_speech(
|
||||
{**(hints_source or {}), **event},
|
||||
audio=audio,
|
||||
fmt=fmt,
|
||||
overlap_silence_rms=defaults.overlap_silence_rms,
|
||||
)
|
||||
video_frames: tuple[str, ...] = ()
|
||||
raw_frames = event.get("video_frames")
|
||||
if raw_frames is not None:
|
||||
frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
|
||||
if frames_error is not None:
|
||||
raise DuplexCommandError(frames_error, code="invalid_video_frames", event_id=event_id)
|
||||
if isinstance(raw_frames, list):
|
||||
video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
|
||||
duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
|
||||
audio_end_ms = hints.get("audio_end_ms")
|
||||
try:
|
||||
audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise DuplexCommandError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
|
||||
decoded = decode_audio_append(event, defaults=defaults, hints_source=hints_source)
|
||||
except RealtimeProtocolError as exc:
|
||||
raise DuplexCommandError(str(exc), code=exc.code, event_id=exc.event_id) from exc
|
||||
return AppendAudio(
|
||||
event_id=event_id,
|
||||
audio=audio_bytes,
|
||||
format=str(fmt),
|
||||
sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
|
||||
is_speech=looks_like_speech,
|
||||
video_frames=video_frames,
|
||||
duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
|
||||
audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
|
||||
hints=hints,
|
||||
event_id=decoded.event_id,
|
||||
audio=decoded.audio,
|
||||
format=decoded.format,
|
||||
sample_rate_hz=decoded.sample_rate_hz,
|
||||
is_speech=decoded.is_speech,
|
||||
video_frames=decoded.video_frames,
|
||||
duration_ms=decoded.duration_ms,
|
||||
audio_end_ms=decoded.audio_end_ms,
|
||||
hints=decoded.hints,
|
||||
)
|
||||
|
||||
|
||||
@@ -664,14 +161,9 @@ def translate_realtime_command(
|
||||
if event_type == "session.update":
|
||||
session = payload.get("session")
|
||||
session_payload: Mapping[str, object] = session if isinstance(session, dict) else payload
|
||||
format_error = validate_realtime_session_audio_formats(session_payload)
|
||||
if format_error is not None:
|
||||
raise DuplexCommandError(format_error, code="unsupported_audio_format", event_id=event_id)
|
||||
from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
|
||||
|
||||
turn_detection_error = validate_realtime_turn_detection(session_payload)
|
||||
if turn_detection_error is not None:
|
||||
raise DuplexCommandError(turn_detection_error, code="unsupported_turn_detection", event_id=event_id)
|
||||
rejection = validate_session_payload(session_payload, capabilities=DUPLEX_REALTIME_CAPABILITIES)
|
||||
if rejection is not None:
|
||||
raise DuplexCommandError(rejection.message, code=rejection.code, event_id=event_id)
|
||||
return UpdateSession(event_id=event_id, patch=dict(session_payload))
|
||||
|
||||
if event_type == "conversation.item.create":
|
||||
@@ -855,7 +347,10 @@ def translate_realtime_command(
|
||||
raise DuplexCommandError(f"Unknown duplex event type: {event_type}", code="unknown_event", event_id=event_id)
|
||||
|
||||
|
||||
#: Names this module used to define, re-exported so existing importers keep
|
||||
#: working. The canonical definitions live in ``vllm_omni.protocol.realtime``.
|
||||
__all__ = [
|
||||
"DUPLEX_REALTIME_CAPABILITIES",
|
||||
"REALTIME_INPUT_AUDIO_FORMATS",
|
||||
"REALTIME_INPUT_HINT_KEYS",
|
||||
"REALTIME_OUTPUT_AUDIO_FORMATS",
|
||||
|
||||
@@ -15,6 +15,12 @@ carries the input-side bookkeeping the old input translator kept (input buffer
|
||||
flags, conversation items, response-id fallbacks); the ``resolve_*`` /
|
||||
``note_*`` helpers give the runner the same behaviour for the corresponding
|
||||
commands.
|
||||
|
||||
What is *not* here is the model- and runtime-agnostic half of the codec ---
|
||||
audio format negotiation, conversation-item shape and truncation, transcript
|
||||
extraction, audio conversion. That lives in ``vllm_omni.protocol.realtime`` so
|
||||
a non-duplex Realtime surface can use it without the duplex session; this
|
||||
module is the duplex consumer of it (RFC #6592 P0a).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,7 +30,6 @@ from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from vllm_omni.engine.duplex.audio import convert_output_audio
|
||||
from vllm_omni.engine.duplex.commands import (
|
||||
AppendAudio,
|
||||
CancelResponse,
|
||||
@@ -74,10 +79,11 @@ from vllm_omni.engine.duplex.events import (
|
||||
TranscriptDone,
|
||||
error_event,
|
||||
)
|
||||
from vllm_omni.engine.duplex.realtime_commands import (
|
||||
from vllm_omni.engine.duplex.realtime_commands import build_append_audio
|
||||
from vllm_omni.protocol.duplex import (
|
||||
RealtimeInputDefaults,
|
||||
apply_realtime_session_defaults,
|
||||
build_append_audio,
|
||||
convert_output_audio,
|
||||
copy_realtime_input_hints,
|
||||
input_looks_like_speech,
|
||||
input_transcript_from_item,
|
||||
|
||||
@@ -28,7 +28,6 @@ from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate
|
||||
from vllm_omni.engine.duplex.commands import (
|
||||
AckPlayback,
|
||||
AppendAudio,
|
||||
@@ -103,6 +102,7 @@ from vllm_omni.engine.duplex.turn_detection import (
|
||||
TurnDetectionResult,
|
||||
)
|
||||
from vllm_omni.metrics.stats import StageRequestStats
|
||||
from vllm_omni.protocol.duplex import convert_input_audio_with_rate
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
|
||||
Everything that needed session state now lives engine-side
|
||||
(``vllm_omni.engine.duplex.realtime_commands`` for the command mapping,
|
||||
``vllm_omni.engine.duplex.realtime_events`` for the projection state). What is
|
||||
``vllm_omni.engine.duplex.realtime_events`` for the projection state), and the
|
||||
model-agnostic parsing both of those build on lives in
|
||||
``vllm_omni.protocol.realtime``. What is
|
||||
left here is the per-connection handshake policy (query-param defaults,
|
||||
autostart / resume-only rules, ``session.resume`` parsing), the wire defaults
|
||||
used to translate appends, and error rendering through the typed
|
||||
@@ -20,10 +22,8 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from vllm_omni.engine.duplex.commands import DuplexCommand, DuplexCommandError
|
||||
from vllm_omni.engine.duplex.events import error_event
|
||||
from vllm_omni.engine.duplex.realtime_commands import (
|
||||
RealtimeInputDefaults,
|
||||
translate_realtime_command,
|
||||
)
|
||||
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
|
||||
from vllm_omni.protocol.duplex import RealtimeInputDefaults
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from starlette.websockets import WebSocket
|
||||
|
||||
@@ -36,7 +36,6 @@ from vllm_omni.engine.duplex.events import (
|
||||
SessionResyncRequired,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import DuplexSessionError
|
||||
from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
|
||||
from vllm_omni.entrypoints.duplex.realtime_input import RealtimeEnvelope, parse_resume_request
|
||||
from vllm_omni.entrypoints.duplex.session_attachment import (
|
||||
DuplexJournalGapError,
|
||||
@@ -52,6 +51,7 @@ from vllm_omni.entrypoints.duplex.websocket import (
|
||||
receive_text_with_timeout,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle
|
||||
from vllm_omni.protocol.duplex import RealtimeInputDefaults
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Wire protocols vLLM-Omni speaks, independent of who serves or runs them.
|
||||
|
||||
A module under ``vllm_omni.protocol`` describes *what is on the wire*: how a
|
||||
client payload is parsed and validated, and what shape a server payload has.
|
||||
It never decides how a request is executed, which model answers it, or where
|
||||
the session state lives. That keeps one protocol implementation usable by more
|
||||
than one runtime (see ``vllm_omni.protocol.realtime``).
|
||||
|
||||
Dependency rule, asserted by ``tests/protocol/realtime/test_protocol_import_boundary.py``:
|
||||
nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``,
|
||||
``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``.
|
||||
The arrows point inwards --- those packages import the protocol, never the
|
||||
other way round.
|
||||
"""
|
||||
@@ -0,0 +1,159 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""vLLM-Omni's full-duplex extension of the OpenAI Realtime wire protocol.
|
||||
|
||||
OpenAI's Realtime API assumes one turn at a time: the client commits input, the
|
||||
server answers. A full-duplex session does not --- the model listens while it
|
||||
speaks, it decides when to take a turn, the client reports playback progress,
|
||||
and a dropped socket re-attaches to a session that kept running. Those need
|
||||
events and commands the GA vocabulary does not have.
|
||||
|
||||
This package is that delta. It builds on ``vllm_omni.protocol.realtime`` (same
|
||||
base classes, same pure wire rendering, no duplication) and, like it, owns no
|
||||
session and imports no runtime: the duplex engine depends on this package, not
|
||||
the other way round.
|
||||
|
||||
The one door
|
||||
------------
|
||||
It is also the *only* door a duplex consumer uses. ``protocol.duplex.commands``
|
||||
and ``protocol.duplex.events`` carry the complete vocabulary --- Tier 1 classes
|
||||
re-exported unchanged alongside the Tier 2 and Tier 3 ones --- and this module
|
||||
re-exports the Tier 1 helper functions. So the layering is a chain, not a mesh::
|
||||
|
||||
protocol.realtime (OpenAI's surface)
|
||||
^
|
||||
| imports
|
||||
|
|
||||
protocol.duplex (this package: re-exports Tier 1, adds Tier 2 and 3)
|
||||
^
|
||||
| imports
|
||||
|
|
||||
duplex engine / entrypoints / clients
|
||||
|
||||
``tests/protocol/duplex/test_duplex_protocol_facade.py`` asserts that the last
|
||||
arrow is the only one: no duplex consumer imports ``protocol.realtime``
|
||||
directly. The payoff is that a helper which later needs a duplex-specific
|
||||
version --- ``convert_input_audio_with_rate`` is the standing example, it
|
||||
resamples to MiniCPM-o's 16 kHz rather than the client's rate --- is overridden
|
||||
in one file instead of at every call site.
|
||||
|
||||
What stays outside
|
||||
------------------
|
||||
The *mailbox* half of a command does not live here. A duplex command carries
|
||||
two different things: the client event it decodes from (wire, this package) and
|
||||
the dictionary the session runner consumes (engine). They genuinely differ ---
|
||||
``session.update``, ``conversation.item.create`` / ``.delete`` / ``.truncate``
|
||||
all travel on the runner's ``turn.signal`` channel --- so ``payload()`` and its
|
||||
``type`` stay in ``vllm_omni.engine.duplex.commands``. Events have no such
|
||||
half, which is why :data:`~vllm_omni.protocol.duplex.events.DuplexEvent` can be
|
||||
a plain alias of ``RealtimeEvent`` while ``DuplexCommand`` cannot.
|
||||
"""
|
||||
|
||||
from vllm_omni.protocol.duplex.errors import (
|
||||
REALTIME_ERROR_TYPES_BY_CODE,
|
||||
RealtimeProtocolError,
|
||||
realtime_error_type,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.audio import (
|
||||
MAX_INPUT_SAMPLE_RATE_HZ,
|
||||
MIN_INPUT_SAMPLE_RATE_HZ,
|
||||
convert_input_audio_with_rate,
|
||||
convert_output_audio,
|
||||
decode_g711_alaw,
|
||||
decode_g711_ulaw,
|
||||
encode_float32_mono_wav_base64,
|
||||
encode_g711_alaw,
|
||||
encode_g711_ulaw,
|
||||
resample_pcm16_mono,
|
||||
validate_input_sample_rate_hz,
|
||||
wav_payload_to_pcm16,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.audio_input import (
|
||||
REALTIME_INPUT_HINT_KEYS,
|
||||
RealtimeAudioAppend,
|
||||
copy_realtime_input_hints,
|
||||
decode_audio_append,
|
||||
input_explicitly_non_speech,
|
||||
input_looks_like_speech,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.capabilities import (
|
||||
RealtimeProtocolCapabilities,
|
||||
RealtimeSessionRejection,
|
||||
validate_session_payload,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.formats import (
|
||||
REALTIME_INPUT_AUDIO_FORMATS,
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS,
|
||||
is_supported_realtime_input_format,
|
||||
parse_realtime_audio_format,
|
||||
realtime_audio_format_object,
|
||||
realtime_output_format,
|
||||
validate_conversation_item_audio_formats,
|
||||
validate_realtime_response_audio_formats,
|
||||
validate_realtime_session_audio_formats,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.items import (
|
||||
input_transcript_from_item,
|
||||
normalize_conversation_item,
|
||||
text_chars_for_audio_ms_from_marks,
|
||||
truncate_realtime_item_content,
|
||||
validate_realtime_item_truncate,
|
||||
validate_realtime_video_frames,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.session import (
|
||||
RealtimeInputDefaults,
|
||||
apply_realtime_session_defaults,
|
||||
input_audio_transcription_config,
|
||||
json_safe_realtime_payload,
|
||||
realtime_max_output_tokens,
|
||||
realtime_overlap_fields,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MAX_INPUT_SAMPLE_RATE_HZ",
|
||||
"MIN_INPUT_SAMPLE_RATE_HZ",
|
||||
"REALTIME_ERROR_TYPES_BY_CODE",
|
||||
"REALTIME_INPUT_AUDIO_FORMATS",
|
||||
"REALTIME_INPUT_HINT_KEYS",
|
||||
"REALTIME_OUTPUT_AUDIO_FORMATS",
|
||||
"RealtimeAudioAppend",
|
||||
"RealtimeInputDefaults",
|
||||
"RealtimeProtocolCapabilities",
|
||||
"RealtimeProtocolError",
|
||||
"RealtimeSessionRejection",
|
||||
"apply_realtime_session_defaults",
|
||||
"convert_input_audio_with_rate",
|
||||
"convert_output_audio",
|
||||
"copy_realtime_input_hints",
|
||||
"decode_audio_append",
|
||||
"decode_g711_alaw",
|
||||
"decode_g711_ulaw",
|
||||
"encode_float32_mono_wav_base64",
|
||||
"encode_g711_alaw",
|
||||
"encode_g711_ulaw",
|
||||
"input_audio_transcription_config",
|
||||
"input_explicitly_non_speech",
|
||||
"input_looks_like_speech",
|
||||
"input_transcript_from_item",
|
||||
"is_supported_realtime_input_format",
|
||||
"json_safe_realtime_payload",
|
||||
"normalize_conversation_item",
|
||||
"parse_realtime_audio_format",
|
||||
"realtime_audio_format_object",
|
||||
"realtime_error_type",
|
||||
"realtime_max_output_tokens",
|
||||
"realtime_output_format",
|
||||
"realtime_overlap_fields",
|
||||
"resample_pcm16_mono",
|
||||
"text_chars_for_audio_ms_from_marks",
|
||||
"truncate_realtime_item_content",
|
||||
"validate_conversation_item_audio_formats",
|
||||
"validate_input_sample_rate_hz",
|
||||
"validate_realtime_item_truncate",
|
||||
"validate_realtime_response_audio_formats",
|
||||
"validate_realtime_session_audio_formats",
|
||||
"validate_realtime_video_frames",
|
||||
"validate_session_payload",
|
||||
"wav_payload_to_pcm16",
|
||||
]
|
||||
@@ -0,0 +1,164 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The client-event vocabulary a full-duplex session accepts.
|
||||
|
||||
Seven of the seventeen are events OpenAI's Realtime API has no equivalent for,
|
||||
because they only make sense when the model and the user can talk at the same
|
||||
time: the user cuts in (``barge_in``), the client reports how much audio it
|
||||
played (``playback.ack``), a turn transition is signalled locally
|
||||
(``turn.signal``), input is abandoned without committing (``input.cancel``),
|
||||
text is pushed into a live turn (``input.text.append``), and the session is
|
||||
kept alive or closed explicitly (``session.heartbeat`` / ``session.close``).
|
||||
Two more are OpenAI commands carrying duplex-only fields (Tier 2).
|
||||
|
||||
The remaining eight are OpenAI's, unchanged, and re-exported from
|
||||
``vllm_omni.protocol.realtime.commands`` rather than redeclared --- they are
|
||||
the same objects. They are re-exported so this module is the *whole*
|
||||
vocabulary and a duplex consumer never has to import the Tier 1 package
|
||||
directly; keeping them declared there means a GA-only consumer is still not
|
||||
handed events its clients never send.
|
||||
|
||||
None of these carries the engine's mailbox rendering: ``payload()`` and its
|
||||
channel live in ``vllm_omni.engine.duplex.commands``, because for four of the
|
||||
seventeen the runner's channel is not the client event type at all.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import ClassVar
|
||||
|
||||
from vllm_omni.protocol.realtime import commands as realtime_commands
|
||||
from vllm_omni.protocol.realtime.commands import (
|
||||
CancelResponse,
|
||||
ClearInput,
|
||||
ClearOutputAudio,
|
||||
CreateItem,
|
||||
CreateResponse,
|
||||
DeleteItem,
|
||||
RealtimeCommand,
|
||||
TruncateItem,
|
||||
UpdateSession,
|
||||
)
|
||||
|
||||
# ---- Tier 2: OpenAI names carrying vLLM-Omni extensions ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AppendAudio(realtime_commands.AppendAudio):
|
||||
"""Tier 2: OpenAI's ``input_audio_buffer.append`` plus the duplex hints.
|
||||
|
||||
OpenAI's append is just base64 audio. The duplex lane also lets a client
|
||||
declare this chunk's own format and rate, whether it believes the chunk is
|
||||
speech, camera frames captured alongside it, and its timing --- all
|
||||
additive, all ignored by a stock client.
|
||||
"""
|
||||
|
||||
#: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
|
||||
format: str = "pcm16"
|
||||
sample_rate_hz: int | None = None
|
||||
is_speech: bool | None = None
|
||||
video_frames: tuple[str, ...] = ()
|
||||
duration_ms: int | None = None
|
||||
audio_end_ms: int | None = None
|
||||
hints: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Commit(realtime_commands.Commit):
|
||||
"""Tier 2: OpenAI's ``input_audio_buffer.commit`` plus duplex turn control.
|
||||
|
||||
OpenAI's commit carries nothing and always produces an item. Here a commit
|
||||
may be non-final, may decline to start a response, and may declare itself
|
||||
silence --- the semantic divergence documented in
|
||||
``docs/serving/realtime_duplex_api.md`` ("Commit != response").
|
||||
"""
|
||||
|
||||
#: Realtime conversation item created for this commit (wire correlation only).
|
||||
#: ``None`` means "no explicit request": the runner decides on commit (auto-response
|
||||
#: sessions answer on their own); ``True`` / ``False`` force it.
|
||||
create_response: bool | None = None
|
||||
is_speech: bool | None = None
|
||||
realtime_item_id: str | None = None
|
||||
|
||||
|
||||
# ---- Tier 3: vLLM-Omni only ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AppendText(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "input.text.append"
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CancelInput(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "input.cancel"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class BargeIn(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "barge_in"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SignalTurn(RealtimeCommand):
|
||||
"""Generic ``turn.signal`` (local turn transitions such as ``user_started``)."""
|
||||
|
||||
wire_type: ClassVar[str] = "turn.signal"
|
||||
event: str
|
||||
signal_payload: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AckPlayback(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "playback.ack"
|
||||
played_ms: int
|
||||
committed_ms: int | None = None
|
||||
response_id: str | None = None
|
||||
item_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Heartbeat(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "session.heartbeat"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CloseSession(RealtimeCommand):
|
||||
"""Graceful close requested through the command stream (``session.close``)."""
|
||||
|
||||
wire_type: ClassVar[str] = "session.close"
|
||||
reason: str = "client_close"
|
||||
|
||||
|
||||
#: The complete command vocabulary a duplex client may send: the eight Tier 1
|
||||
#: classes re-exported unchanged, the two Tier 2 classes defined above, and the
|
||||
#: seven Tier 3 ones. A duplex consumer imports from here and never reaches
|
||||
#: past this module into ``vllm_omni.protocol.realtime``, so a command that
|
||||
#: later grows a duplex extension changes only this file.
|
||||
__all__ = [
|
||||
# Tier 1 --- re-exported from ``vllm_omni.protocol.realtime.commands``.
|
||||
"CancelResponse",
|
||||
"ClearInput",
|
||||
"ClearOutputAudio",
|
||||
"CreateItem",
|
||||
"CreateResponse",
|
||||
"DeleteItem",
|
||||
"RealtimeCommand",
|
||||
"TruncateItem",
|
||||
"UpdateSession",
|
||||
# Tier 2 --- an OpenAI command plus duplex-only fields.
|
||||
"AppendAudio",
|
||||
"Commit",
|
||||
# Tier 3 --- vLLM-Omni only.
|
||||
"AckPlayback",
|
||||
"AppendText",
|
||||
"BargeIn",
|
||||
"CancelInput",
|
||||
"CloseSession",
|
||||
"Heartbeat",
|
||||
"SignalTurn",
|
||||
]
|
||||
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""vLLM-Omni's Realtime error-code vocabulary.
|
||||
|
||||
OpenAI's Realtime API standardises the ``error.type`` classes
|
||||
(``invalid_request_error`` / ``server_error`` / ``rate_limit_error``) but not
|
||||
the ``code`` values inside them. Ours are listed here --- Tier 3 in
|
||||
``docs/serving/realtime_duplex_api.md``, "the full error-code vocabulary" ---
|
||||
so ``protocol/realtime`` stays the OpenAI surface and nothing more.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
|
||||
|
||||
__all__ = ["REALTIME_ERROR_TYPES_BY_CODE", "RealtimeProtocolError", "realtime_error_type"]
|
||||
|
||||
|
||||
#: OpenAI Realtime ``error.type`` for each internal error code.
|
||||
REALTIME_ERROR_TYPES_BY_CODE: dict[str, str] = {
|
||||
"bad_event": "invalid_request_error",
|
||||
"bad_audio": "invalid_request_error",
|
||||
"config_timeout": "invalid_request_error",
|
||||
"invalid_json": "invalid_request_error",
|
||||
"event_too_large": "invalid_request_error",
|
||||
"unknown_event": "invalid_request_error",
|
||||
"internal_error": "server_error",
|
||||
"runtime_append_failed": "server_error",
|
||||
"runtime_append_task_failed": "server_error",
|
||||
"runtime_signal_failed": "server_error",
|
||||
"runtime_abort_failed": "server_error",
|
||||
"runtime_data_plane_stream_failed": "server_error",
|
||||
"runtime_data_plane_text_without_audio": "server_error",
|
||||
"resource_exhausted": "rate_limit_error",
|
||||
"session_exists": "invalid_request_error",
|
||||
"session_closed": "invalid_request_error",
|
||||
"unknown_session": "invalid_request_error",
|
||||
"invalid_duplex_runtime_config": "invalid_request_error",
|
||||
"instructions_update_unsupported": "invalid_request_error",
|
||||
"persona_update_unsupported": "invalid_request_error",
|
||||
"voice_update_unsupported": "invalid_request_error",
|
||||
"unsupported_nemotron_duplex_mode": "invalid_request_error",
|
||||
"unsupported_native_response_options": "invalid_request_error",
|
||||
"runtime_touch_failed": "server_error",
|
||||
"engine_error": "server_error",
|
||||
"input_backpressure": "rate_limit_error",
|
||||
"response_already_active": "invalid_request_error",
|
||||
"response_not_active": "invalid_request_error",
|
||||
"response_create_without_input": "invalid_request_error",
|
||||
"text_only_turn_unsupported": "invalid_request_error",
|
||||
"commit_aborted": "server_error",
|
||||
"input_audio_buffer_empty": "invalid_request_error",
|
||||
"missing_item_id": "invalid_request_error",
|
||||
"item_not_found": "invalid_request_error",
|
||||
"playback_item_mismatch": "invalid_request_error",
|
||||
"playback_item_not_found": "invalid_request_error",
|
||||
"playback_ack_too_late": "invalid_request_error",
|
||||
"unsupported_audio_format": "invalid_request_error",
|
||||
"unsupported_turn_detection": "invalid_request_error",
|
||||
"unsupported_ref_audio_path": "invalid_request_error",
|
||||
"ref_audio_required": "invalid_request_error",
|
||||
"model_update_unsupported": "invalid_request_error",
|
||||
"voice_update_after_audio_unsupported": "invalid_request_error",
|
||||
"ref_audio_update_unsupported": "invalid_request_error",
|
||||
"native_text_append_unsupported": "invalid_request_error",
|
||||
"invalid_video_frames": "invalid_request_error",
|
||||
"invalid_function_call_output": "invalid_request_error",
|
||||
"server_vad_unavailable": "server_error",
|
||||
}
|
||||
|
||||
|
||||
def realtime_error_type(code: str) -> str:
|
||||
"""The OpenAI ``error.type`` bucket for an internal error code."""
|
||||
return REALTIME_ERROR_TYPES_BY_CODE.get(code, "invalid_request_error")
|
||||
@@ -0,0 +1,419 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""vLLM-Omni's half of the Realtime server-event vocabulary.
|
||||
|
||||
``docs/serving/realtime_duplex_api.md`` sorts every message into three tiers.
|
||||
Tier 1 --- identical to OpenAI --- is ``vllm_omni.protocol.realtime.events``.
|
||||
The other two are here:
|
||||
|
||||
**Tier 2, OpenAI names carrying vLLM-Omni extensions.** Same wire type, same
|
||||
OpenAI fields, plus additive keys a stock client ignores. Each one subclasses
|
||||
its Tier 1 twin and declares only what it adds, so the OpenAI surface stays
|
||||
honest: a consumer that wants pure GA takes ``protocol/realtime`` and gets
|
||||
exactly OpenAI's fields.
|
||||
|
||||
**Tier 3, vLLM-Omni only.** Events OpenAI has no equivalent for, because they
|
||||
only make sense when the model and the user can talk at the same time: the
|
||||
model decides when to listen and when to speak, the client reports how much
|
||||
audio it played, overlapping speech needs a recorded decision, and a dropped
|
||||
socket can re-attach to a live session.
|
||||
|
||||
Same-named classes are deliberate --- ``duplex.events.SessionCreated`` *is*
|
||||
``realtime.events.SessionCreated`` with our extensions. Import whichever
|
||||
surface you mean.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import cast
|
||||
|
||||
from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
|
||||
from vllm_omni.protocol.realtime import events as realtime_events
|
||||
from vllm_omni.protocol.realtime.events import (
|
||||
AudioDone,
|
||||
ContentPartAdded,
|
||||
ContentPartDone,
|
||||
FunctionCallArgumentsDelta,
|
||||
FunctionCallArgumentsDone,
|
||||
InputCleared,
|
||||
InputTranscriptionCompleted,
|
||||
ItemAdded,
|
||||
ItemCreated,
|
||||
ItemDone,
|
||||
ItemRetrieved,
|
||||
OutputAudioCleared,
|
||||
OutputItemAdded,
|
||||
OutputItemDone,
|
||||
RateLimitsUpdated,
|
||||
RealtimeEvent,
|
||||
ResponseScopedEvent,
|
||||
SessionUpdated,
|
||||
SpeechStarted,
|
||||
SpeechStopped,
|
||||
TextDelta,
|
||||
TextDone,
|
||||
TranscriptDelta,
|
||||
TranscriptDone,
|
||||
new_event_id,
|
||||
wire_value,
|
||||
)
|
||||
|
||||
#: The duplex event base class *is* the Realtime one: unlike a command, an
|
||||
#: event has no engine-internal half to carry --- what the session emits is
|
||||
#: exactly what the socket sends. Named here rather than in the engine so the
|
||||
#: duplex vocabulary is described in one place, and kept as an alias rather
|
||||
#: than an empty subclass so ``isinstance`` and the class identity tests in
|
||||
#: ``tests/protocol/`` stay meaningful.
|
||||
DuplexEvent = RealtimeEvent
|
||||
|
||||
# ---- Tier 2: OpenAI names carrying vLLM-Omni extensions ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionCreated(realtime_events.SessionCreated):
|
||||
"""Tier 2: OpenAI's ``SessionCreated`` plus the resume credentials a reconnecting client needs."""
|
||||
|
||||
#: Transport resume credentials (set by the websocket handler when resume is supported).
|
||||
optional_wire_fields = frozenset({"attachment_generation", "resume_token"})
|
||||
attachment_generation: int | None = None
|
||||
resume_token: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseCreated(realtime_events.ResponseCreated):
|
||||
"""Tier 2: OpenAI's ``ResponseCreated`` plus a top-level ``response_id``.
|
||||
|
||||
OpenAI carries the id nested as ``response.id``; the duplex lane also
|
||||
reports it at the top level, where every other response-scoped event
|
||||
already has one.
|
||||
"""
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseDone(realtime_events.ResponseDone):
|
||||
"""Tier 2: OpenAI's ``ResponseDone`` plus a top-level ``response_id``.
|
||||
|
||||
OpenAI carries the id nested as ``response.id``; the duplex lane also
|
||||
reports it at the top level, where every other response-scoped event
|
||||
already has one.
|
||||
"""
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AudioDelta(realtime_events.AudioDelta):
|
||||
"""Tier 2: OpenAI's ``AudioDelta`` plus the chunk's own format/rate and the duplex playback metadata."""
|
||||
|
||||
optional_wire_fields = frozenset({"sample_rate_hz", "metadata"})
|
||||
format: str = "pcm16"
|
||||
sample_rate_hz: int | None = None
|
||||
metadata: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class InputCommitted(realtime_events.InputCommitted):
|
||||
"""Tier 2: OpenAI's ``InputCommitted`` plus the native ``input.committed`` event echoed under ``event``."""
|
||||
|
||||
#: The session-internal ``input.committed`` event.
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
# Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
|
||||
# rebuilds the class, so the zero-argument form's ``__class__`` cell
|
||||
# points at the pre-slots class and is no longer in the MRO.
|
||||
base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
|
||||
return {**base, "event": wire_value(self.details)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemDeleted(realtime_events.ItemDeleted):
|
||||
"""Tier 2: OpenAI's ``ItemDeleted`` plus the originating event echoed under ``event``."""
|
||||
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
# Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
|
||||
# rebuilds the class, so the zero-argument form's ``__class__`` cell
|
||||
# points at the pre-slots class and is no longer in the MRO.
|
||||
base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
|
||||
return {**base, "event": wire_value(self.details)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemTruncated(realtime_events.ItemTruncated):
|
||||
"""Tier 2: OpenAI's ``ItemTruncated`` plus the originating event echoed under ``event``."""
|
||||
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
# Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
|
||||
# rebuilds the class, so the zero-argument form's ``__class__`` cell
|
||||
# points at the pre-slots class and is no longer in the MRO.
|
||||
base = realtime_events.__dict__[type(self).__name__]._wire_fields(self)
|
||||
return {**base, "event": wire_value(self.details)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ErrorEvent(realtime_events.ErrorEvent):
|
||||
"""Tier 2: OpenAI's ``ErrorEvent`` plus vLLM-Omni's error codes and any extra keys merged into the envelope."""
|
||||
|
||||
#: Extra keys merged into the wire ``error`` object (``retryable`` ...).
|
||||
extra: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def error_type(self) -> str:
|
||||
return REALTIME_ERROR_TYPES_BY_CODE.get(self.code, "invalid_request_error")
|
||||
|
||||
@property
|
||||
def error(self) -> dict[str, object]:
|
||||
# Explicit parent call, not ``super()``: ``@dataclass(slots=True)``
|
||||
# rebuilds the class, so the zero-argument form's ``__class__`` cell
|
||||
# points at the pre-slots class and is no longer in the MRO. Reached
|
||||
# through ``__dict__`` because attribute access on the class gives the
|
||||
# getter function, not the ``property`` that owns it.
|
||||
error = dict(realtime_events.ErrorEvent.__dict__["error"].fget(self))
|
||||
error.update(cast("Mapping[str, object]", wire_value(self.extra)))
|
||||
return error
|
||||
|
||||
|
||||
# ---- Tier 3: vLLM-Omni only ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionResumed(SessionCreated):
|
||||
wire_type = "session.resumed"
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionReplaced(RealtimeEvent):
|
||||
"""A newer connection took the session over (sent to the replaced socket)."""
|
||||
|
||||
wire_type = "session.replaced"
|
||||
|
||||
attachment_generation: int = 0
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionResyncRequired(RealtimeEvent):
|
||||
"""The replay journal cannot bridge the gap; the client must start over."""
|
||||
|
||||
wire_type = "session.resync_required"
|
||||
|
||||
reason: str = "journal_gap"
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, **RealtimeEvent._wire_fields(self)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionClosed(RealtimeEvent):
|
||||
wire_type = "session.closed"
|
||||
|
||||
reason: str = "closed"
|
||||
#: The session-internal close event (kept for clients that inspect it).
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def is_terminal(self) -> bool:
|
||||
return True
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, "reason": self.reason, "event": wire_value(self.details)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionExpired(SessionClosed):
|
||||
wire_type = "session.expired"
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, "reason": self.reason}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionHeartbeatAck(RealtimeEvent):
|
||||
wire_type = "session.heartbeat_ack"
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TurnEvent(RealtimeEvent):
|
||||
"""Turn-state transition (``turn.event``)."""
|
||||
|
||||
wire_type = "turn.event"
|
||||
|
||||
event: str = ""
|
||||
turn_state: str = ""
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"session_id": self.session_id, "event": self.event, "turn_state": self.turn_state, "epoch": self.epoch}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Listen(RealtimeEvent):
|
||||
"""The model decided to keep listening (no spoken response for this turn)."""
|
||||
|
||||
wire_type = "response.listen"
|
||||
optional_wire_fields = frozenset({"response_id"})
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
response: dict[str, object] = {
|
||||
"object": "realtime.response",
|
||||
"status": "listening",
|
||||
"metadata": wire_value(self.details),
|
||||
}
|
||||
data: dict[str, object] = {"session_id": self.session_id, "epoch": self.epoch, "response": response}
|
||||
if self.response_id:
|
||||
response["id"] = self.response_id
|
||||
data["response_id"] = self.response_id
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Speak(ResponseScopedEvent):
|
||||
wire_type = "response.speak"
|
||||
|
||||
metadata: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class OverlapDecision(RealtimeEvent):
|
||||
wire_type = "overlap.decision"
|
||||
|
||||
policy: str | None = None
|
||||
action: str | None = None
|
||||
reason: str | None = None
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"epoch": self.epoch,
|
||||
"policy": self.policy,
|
||||
"action": self.action,
|
||||
"reason": self.reason,
|
||||
"metadata": wire_value(self.details),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class PlaybackAcknowledged(RealtimeEvent):
|
||||
wire_type = "playback.acknowledged"
|
||||
|
||||
#: The session-internal ``playback.acknowledged`` event (cursor, committed_ms ...).
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"event": wire_value(self.details)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class DuplexRawEvent(RealtimeEvent):
|
||||
"""A session-internal event with no dedicated Realtime type (``duplex.<internal type>``)."""
|
||||
|
||||
wire_type = "duplex.raw"
|
||||
|
||||
internal_type: str = ""
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return f"duplex.{self.internal_type}"
|
||||
|
||||
def to_realtime(self) -> dict[str, object]:
|
||||
return {"type": self.type, "event_id": self.event_id, "event": wire_value(self.details)}
|
||||
|
||||
|
||||
def error_event(
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
event_id: object | None = None,
|
||||
param: object | None = None,
|
||||
extra: Mapping[str, object] | None = None,
|
||||
) -> ErrorEvent:
|
||||
"""Build the error event for an internal code; ``event_id`` is the *client* event id."""
|
||||
return ErrorEvent(
|
||||
code=code,
|
||||
message=message,
|
||||
related_event_id=event_id if isinstance(event_id, str) and event_id else None,
|
||||
param=param if isinstance(param, str) and param else None,
|
||||
extra=dict(extra or {}),
|
||||
)
|
||||
|
||||
|
||||
#: The complete event vocabulary a duplex client may receive: 22 Tier 1
|
||||
#: classes re-exported unchanged, the 8 Tier 2 ones defined above, and 12
|
||||
#: Tier 3 ones. A duplex consumer imports from here and never reaches past
|
||||
#: this module into ``vllm_omni.protocol.realtime``.
|
||||
__all__ = [
|
||||
# Tier 1 --- re-exported from ``vllm_omni.protocol.realtime.events``.
|
||||
"AudioDone",
|
||||
"ContentPartAdded",
|
||||
"ContentPartDone",
|
||||
"FunctionCallArgumentsDelta",
|
||||
"FunctionCallArgumentsDone",
|
||||
"InputCleared",
|
||||
"InputTranscriptionCompleted",
|
||||
"ItemAdded",
|
||||
"ItemCreated",
|
||||
"ItemDone",
|
||||
"ItemRetrieved",
|
||||
"OutputAudioCleared",
|
||||
"OutputItemAdded",
|
||||
"OutputItemDone",
|
||||
"RateLimitsUpdated",
|
||||
"SessionUpdated",
|
||||
"SpeechStarted",
|
||||
"SpeechStopped",
|
||||
"TextDelta",
|
||||
"TextDone",
|
||||
"TranscriptDelta",
|
||||
"TranscriptDone",
|
||||
# Tier 2 --- an OpenAI event plus duplex-only fields.
|
||||
"AudioDelta",
|
||||
"ErrorEvent",
|
||||
"InputCommitted",
|
||||
"ItemDeleted",
|
||||
"ItemTruncated",
|
||||
"ResponseCreated",
|
||||
"ResponseDone",
|
||||
"SessionCreated",
|
||||
# Tier 3 --- vLLM-Omni only.
|
||||
"DuplexRawEvent",
|
||||
"Listen",
|
||||
"OverlapDecision",
|
||||
"PlaybackAcknowledged",
|
||||
"SessionClosed",
|
||||
"SessionExpired",
|
||||
"SessionHeartbeatAck",
|
||||
"SessionReplaced",
|
||||
"SessionResumed",
|
||||
"SessionResyncRequired",
|
||||
"Speak",
|
||||
"TurnEvent",
|
||||
# Bases and helpers shared with the Tier 1 surface.
|
||||
"DuplexEvent",
|
||||
"REALTIME_ERROR_TYPES_BY_CODE",
|
||||
"RealtimeEvent",
|
||||
"ResponseScopedEvent",
|
||||
"error_event",
|
||||
"new_event_id",
|
||||
"wire_value",
|
||||
]
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The OpenAI Realtime wire codec, shared by every vLLM-Omni Realtime surface.
|
||||
|
||||
What this is
|
||||
------------
|
||||
The model-agnostic half of ``WS /v1/realtime``: how a client event is parsed,
|
||||
which audio formats and session fields are accepted, what a conversation item
|
||||
means, how audio is decoded and resampled, and the shape of the error envelope
|
||||
(the *codes* inside it are a consumer's own --- ours are in
|
||||
``vllm_omni.protocol.duplex.errors``). It is a library of pure functions and value objects.
|
||||
|
||||
What it is not
|
||||
--------------
|
||||
It owns no session. It does not decide when a response starts, what a model is
|
||||
prompted with, whether a turn is over, or where committed audio goes. Those are
|
||||
runtime decisions, and after PR #7413 a duplex session's runtime decisions all
|
||||
belong to ``vllm_omni.engine.duplex`` --- the session runner, the typed
|
||||
``DuplexCommand`` / ``DuplexEvent`` boundary and the ``DuplexModelPlugin`` seam.
|
||||
This package sits *under* that: the duplex engine binds the codec to its own
|
||||
command and event vocabulary in
|
||||
``vllm_omni.engine.duplex.realtime_commands`` and
|
||||
``vllm_omni.engine.duplex.realtime_events``.
|
||||
|
||||
Why it is separate (RFC #6592, P0a)
|
||||
-----------------------------------
|
||||
So that a second Realtime surface --- a Qwen3-Omni GA profile, say --- can reuse
|
||||
the parsing, validation, format negotiation and error envelope without adopting
|
||||
the duplex session control plane (lease manager, commit policy, model channel),
|
||||
and without a second copy of the codec drifting away from this one. A consumer
|
||||
states what it can serve through
|
||||
:class:`~vllm_omni.protocol.realtime.capabilities.RealtimeProtocolCapabilities`
|
||||
and keeps its own session state.
|
||||
|
||||
Known duplex-shaped defaults
|
||||
----------------------------
|
||||
This package is model-agnostic in its *dependencies*, not yet in every
|
||||
*default*. Two constants still carry the assumptions of its first consumer and
|
||||
a second one will have to parameterize them:
|
||||
|
||||
* :func:`~vllm_omni.protocol.realtime.audio.convert_input_audio_with_rate`
|
||||
resamples to 16 kHz, which is MiniCPM-o's feature-extractor rate. A GA client
|
||||
sending 24 kHz is converted down rather than served at its own rate.
|
||||
* :func:`~vllm_omni.protocol.realtime.items.validate_realtime_video_frames`
|
||||
encodes the MiniCPM-o duplex camera contract (at most two frames per append,
|
||||
``max_slice_nums`` not selectable on the wire).
|
||||
|
||||
Both are behaviour this extraction deliberately did **not** change (RFC #6592
|
||||
P0a is behaviour-preserving); making them per-consumer is P0b/P1 work.
|
||||
|
||||
Dependency rule
|
||||
---------------
|
||||
Nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``,
|
||||
``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``;
|
||||
``tests/protocol/realtime/test_protocol_import_boundary.py`` asserts it.
|
||||
"""
|
||||
|
||||
from vllm_omni.protocol.realtime.audio import (
|
||||
MAX_INPUT_SAMPLE_RATE_HZ,
|
||||
MIN_INPUT_SAMPLE_RATE_HZ,
|
||||
convert_input_audio_with_rate,
|
||||
convert_output_audio,
|
||||
encode_float32_mono_wav_base64,
|
||||
resample_pcm16_mono,
|
||||
validate_input_sample_rate_hz,
|
||||
wav_payload_to_pcm16,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.audio_input import (
|
||||
REALTIME_INPUT_HINT_KEYS,
|
||||
RealtimeAudioAppend,
|
||||
copy_realtime_input_hints,
|
||||
decode_audio_append,
|
||||
input_explicitly_non_speech,
|
||||
input_looks_like_speech,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.capabilities import (
|
||||
RealtimeProtocolCapabilities,
|
||||
RealtimeSessionRejection,
|
||||
validate_session_payload,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.commands import RealtimeCommand
|
||||
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
|
||||
from vllm_omni.protocol.realtime.events import (
|
||||
RealtimeEvent,
|
||||
ResponseScopedEvent,
|
||||
new_event_id,
|
||||
wire_value,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.formats import (
|
||||
REALTIME_INPUT_AUDIO_FORMATS,
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS,
|
||||
is_supported_realtime_input_format,
|
||||
parse_realtime_audio_format,
|
||||
realtime_audio_format_object,
|
||||
realtime_output_format,
|
||||
validate_conversation_item_audio_formats,
|
||||
validate_realtime_response_audio_formats,
|
||||
validate_realtime_session_audio_formats,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.items import (
|
||||
input_transcript_from_item,
|
||||
normalize_conversation_item,
|
||||
text_chars_for_audio_ms_from_marks,
|
||||
truncate_realtime_item_content,
|
||||
validate_realtime_item_truncate,
|
||||
validate_realtime_video_frames,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.session import (
|
||||
RealtimeInputDefaults,
|
||||
apply_realtime_session_defaults,
|
||||
input_audio_transcription_config,
|
||||
json_safe_realtime_payload,
|
||||
realtime_max_output_tokens,
|
||||
realtime_overlap_fields,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MAX_INPUT_SAMPLE_RATE_HZ",
|
||||
"MIN_INPUT_SAMPLE_RATE_HZ",
|
||||
"REALTIME_INPUT_AUDIO_FORMATS",
|
||||
"REALTIME_INPUT_HINT_KEYS",
|
||||
"REALTIME_OUTPUT_AUDIO_FORMATS",
|
||||
"RealtimeAudioAppend",
|
||||
"RealtimeCommand",
|
||||
"RealtimeEvent",
|
||||
"RealtimeInputDefaults",
|
||||
"RealtimeProtocolCapabilities",
|
||||
"RealtimeProtocolError",
|
||||
"RealtimeSessionRejection",
|
||||
"ResponseScopedEvent",
|
||||
"apply_realtime_session_defaults",
|
||||
"convert_input_audio_with_rate",
|
||||
"convert_output_audio",
|
||||
"copy_realtime_input_hints",
|
||||
"decode_audio_append",
|
||||
"encode_float32_mono_wav_base64",
|
||||
"input_audio_transcription_config",
|
||||
"input_explicitly_non_speech",
|
||||
"input_looks_like_speech",
|
||||
"input_transcript_from_item",
|
||||
"is_supported_realtime_input_format",
|
||||
"json_safe_realtime_payload",
|
||||
"new_event_id",
|
||||
"normalize_conversation_item",
|
||||
"parse_realtime_audio_format",
|
||||
"realtime_audio_format_object",
|
||||
"realtime_max_output_tokens",
|
||||
"realtime_output_format",
|
||||
"realtime_overlap_fields",
|
||||
"resample_pcm16_mono",
|
||||
"text_chars_for_audio_ms_from_marks",
|
||||
"truncate_realtime_item_content",
|
||||
"validate_conversation_item_audio_formats",
|
||||
"validate_input_sample_rate_hz",
|
||||
"validate_realtime_item_truncate",
|
||||
"validate_realtime_response_audio_formats",
|
||||
"validate_realtime_session_audio_formats",
|
||||
"validate_realtime_video_frames",
|
||||
"validate_session_payload",
|
||||
"wav_payload_to_pcm16",
|
||||
"wire_value",
|
||||
]
|
||||
@@ -0,0 +1,218 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import io
|
||||
import math
|
||||
import wave
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm_omni.utils.audio_resample import StreamingAudioResampler
|
||||
|
||||
try:
|
||||
from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin
|
||||
except ImportError: # pragma: no cover - audioop is removed in newer Python.
|
||||
alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None
|
||||
|
||||
|
||||
MIN_INPUT_SAMPLE_RATE_HZ = 8_000
|
||||
MAX_INPUT_SAMPLE_RATE_HZ = 192_000
|
||||
|
||||
|
||||
def validate_input_sample_rate_hz(sample_rate_hz: object) -> int:
|
||||
if (
|
||||
isinstance(sample_rate_hz, bool)
|
||||
or not isinstance(sample_rate_hz, int | float)
|
||||
or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz))
|
||||
):
|
||||
raise ValueError("sample_rate_hz must be a finite integer")
|
||||
rate = int(sample_rate_hz)
|
||||
if rate != sample_rate_hz:
|
||||
raise ValueError("sample_rate_hz must be an integer")
|
||||
if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ:
|
||||
raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}")
|
||||
return rate
|
||||
|
||||
|
||||
def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes:
|
||||
if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz:
|
||||
return raw
|
||||
samples = np.frombuffer(raw, dtype="<i2")
|
||||
rate_gcd = math.gcd(source_rate_hz, target_rate_hz)
|
||||
reduced_factor = max(source_rate_hz // rate_gcd, target_rate_hz // rate_gcd)
|
||||
if reduced_factor > StreamingAudioResampler.max_polyphase_factor:
|
||||
# The FIR size scales with the reduced ratio. Keep unusual client
|
||||
# rates bounded while common audio rates use the high-quality path.
|
||||
if samples.size <= 1:
|
||||
return raw
|
||||
target_size = max(1, int(round(samples.size * target_rate_hz / source_rate_hz)))
|
||||
source_x = np.linspace(0.0, 1.0, num=samples.size, endpoint=True)
|
||||
target_x = np.linspace(0.0, 1.0, num=target_size, endpoint=True)
|
||||
resampled = np.interp(target_x, source_x, samples)
|
||||
return np.clip(resampled, -32_768, 32_767).astype("<i2").tobytes()
|
||||
resampler = StreamingAudioResampler(source_rate_hz, target_rate_hz)
|
||||
normalized = samples.astype(np.float32) * np.float32(1.0 / 32768.0)
|
||||
resampled = resampler.process(normalized, final=True)
|
||||
return np.clip(np.rint(resampled * 32768.0), -32768, 32767).astype("<i2").tobytes()
|
||||
|
||||
|
||||
def decode_g711_ulaw(raw: bytes) -> bytes:
|
||||
if ulaw2lin is not None:
|
||||
return ulaw2lin(raw, 2)
|
||||
data = np.frombuffer(raw, dtype=np.uint8)
|
||||
value = np.bitwise_not(data).astype(np.int16)
|
||||
sign = value & 0x80
|
||||
exponent = (value >> 4) & 0x07
|
||||
mantissa = value & 0x0F
|
||||
sample = ((mantissa << 3) + 0x84) << exponent
|
||||
return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype("<i2").tobytes()
|
||||
|
||||
|
||||
def decode_g711_alaw(raw: bytes) -> bytes:
|
||||
if alaw2lin is not None:
|
||||
return alaw2lin(raw, 2)
|
||||
data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16)
|
||||
sign = data & 0x80
|
||||
exponent = (data >> 4) & 0x07
|
||||
mantissa = data & 0x0F
|
||||
sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1))
|
||||
return np.where(sign != 0, sample, -sample).astype("<i2").tobytes()
|
||||
|
||||
|
||||
def encode_g711_ulaw(raw: bytes) -> bytes:
|
||||
if lin2ulaw is not None:
|
||||
return lin2ulaw(raw, 2)
|
||||
pcm = np.clip(np.frombuffer(raw, dtype="<i2").astype(np.int32), -32635, 32635)
|
||||
sign = np.where(pcm < 0, 0x80, 0)
|
||||
magnitude = np.abs(pcm) + 0x84
|
||||
exponent = np.zeros_like(magnitude)
|
||||
for exp in range(7):
|
||||
exponent = np.where(magnitude > (0xFF << exp), exp + 1, exponent)
|
||||
mantissa = (magnitude >> (exponent + 3)) & 0x0F
|
||||
return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes()
|
||||
|
||||
|
||||
def encode_g711_alaw(raw: bytes) -> bytes:
|
||||
if lin2alaw is not None:
|
||||
return lin2alaw(raw, 2)
|
||||
pcm = np.frombuffer(raw, dtype="<i2").astype(np.int32)
|
||||
sign = np.where(pcm >= 0, 0x80, 0x00)
|
||||
magnitude = np.abs(pcm)
|
||||
exponent = np.zeros_like(magnitude)
|
||||
for exp in range(1, 8):
|
||||
exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent)
|
||||
mantissa = np.where(
|
||||
exponent == 0,
|
||||
(magnitude >> 4) & 0x0F,
|
||||
(magnitude >> (exponent + 3)) & 0x0F,
|
||||
)
|
||||
return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes()
|
||||
|
||||
|
||||
def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]:
|
||||
try:
|
||||
with wave.open(io.BytesIO(raw), "rb") as wav_file:
|
||||
channels = wav_file.getnchannels()
|
||||
sample_width = wav_file.getsampwidth()
|
||||
sample_rate_hz = wav_file.getframerate()
|
||||
frames = wav_file.readframes(wav_file.getnframes())
|
||||
if sample_width != 2:
|
||||
return None, sample_rate_hz
|
||||
if channels <= 1:
|
||||
return frames, sample_rate_hz
|
||||
pcm = np.frombuffer(frames, dtype="<i2").reshape(-1, channels)
|
||||
mono = np.mean(pcm.astype(np.float32), axis=1)
|
||||
return np.clip(mono, -32768, 32767).astype("<i2").tobytes(), sample_rate_hz
|
||||
except (EOFError, ValueError, wave.Error):
|
||||
return None, None
|
||||
|
||||
|
||||
def encode_float32_mono_wav_base64(
|
||||
samples: np.ndarray,
|
||||
*,
|
||||
sample_rate_hz: int,
|
||||
) -> str:
|
||||
"""Package normalized mono float32 samples as a PCM16 WAV data payload."""
|
||||
rate = validate_input_sample_rate_hz(sample_rate_hz)
|
||||
normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1)
|
||||
pcm16 = np.clip(np.rint(normalized * 32768.0), -32768, 32767).astype("<i2")
|
||||
with io.BytesIO() as buffer:
|
||||
with wave.open(buffer, "wb") as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(rate)
|
||||
wav_file.writeframes(pcm16.tobytes())
|
||||
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
def convert_input_audio_with_rate(
|
||||
audio: object,
|
||||
fmt: object,
|
||||
*,
|
||||
sample_rate_hz: int | float | None = None,
|
||||
target_sample_rate_hz: int = 16_000,
|
||||
) -> tuple[object, object, int | float | None]:
|
||||
if not isinstance(audio, str) or not isinstance(fmt, str):
|
||||
return audio, fmt, sample_rate_hz
|
||||
normalized = fmt.lower()
|
||||
if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}:
|
||||
return audio, fmt, sample_rate_hz
|
||||
if isinstance(sample_rate_hz, int | float):
|
||||
sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz)
|
||||
try:
|
||||
raw = base64.b64decode(audio.strip(), validate=False)
|
||||
except (binascii.Error, ValueError):
|
||||
return audio, fmt, sample_rate_hz
|
||||
if normalized == "g711_ulaw":
|
||||
raw = decode_g711_ulaw(raw)
|
||||
sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
|
||||
elif normalized == "g711_alaw":
|
||||
raw = decode_g711_alaw(raw)
|
||||
sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
|
||||
elif len(raw) % 2:
|
||||
return audio, fmt, sample_rate_hz
|
||||
if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz:
|
||||
raw = resample_pcm16_mono(
|
||||
raw,
|
||||
source_rate_hz=int(sample_rate_hz),
|
||||
target_rate_hz=target_sample_rate_hz,
|
||||
)
|
||||
sample_rate_hz = target_sample_rate_hz
|
||||
pcm = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
|
||||
encoded = base64.b64encode(np.ascontiguousarray(pcm, dtype="<f4").tobytes()).decode("ascii")
|
||||
return encoded, "pcm_f32le", sample_rate_hz
|
||||
|
||||
|
||||
def convert_output_audio(
|
||||
audio: str,
|
||||
*,
|
||||
source_fmt: str,
|
||||
target_fmt: str,
|
||||
source_sample_rate_hz: int | None = None,
|
||||
target_sample_rate_hz: int | None = None,
|
||||
) -> tuple[str, str, int | None]:
|
||||
target = target_fmt.lower()
|
||||
if target not in {"g711_ulaw", "g711_alaw"}:
|
||||
return audio, source_fmt, source_sample_rate_hz
|
||||
try:
|
||||
raw = base64.b64decode(audio, validate=False)
|
||||
except (binascii.Error, ValueError):
|
||||
return audio, source_fmt, source_sample_rate_hz
|
||||
source = source_fmt.lower()
|
||||
if source == "wav":
|
||||
pcm_raw, wav_rate = wav_payload_to_pcm16(raw)
|
||||
if pcm_raw is None:
|
||||
return audio, source_fmt, source_sample_rate_hz
|
||||
raw = pcm_raw
|
||||
source_sample_rate_hz = source_sample_rate_hz or wav_rate
|
||||
elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2:
|
||||
return audio, source_fmt, source_sample_rate_hz
|
||||
target_rate = target_sample_rate_hz or 8_000
|
||||
if source_sample_rate_hz is not None:
|
||||
raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate)
|
||||
encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw)
|
||||
return base64.b64encode(encoded).decode("ascii"), target, target_rate
|
||||
@@ -0,0 +1,243 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Decoding one ``input_audio_buffer.append``.
|
||||
|
||||
An append is the hottest client event on a Realtime connection and the one with
|
||||
the most wire surface: base64 audio in any accepted format, an optional rate,
|
||||
optional camera frames, and a set of optional hints (``duration_ms``,
|
||||
``is_speech``, a VAD probability, a transcript) that clients attach and servers
|
||||
are free to use or ignore.
|
||||
|
||||
:func:`decode_audio_append` turns that into :class:`RealtimeAudioAppend`, a
|
||||
value object: bytes at 16 kHz ``pcm_f32le``, the surviving hints, and a speech
|
||||
verdict. It is a pure function of the event plus the session's
|
||||
:class:`~vllm_omni.protocol.realtime.session.RealtimeInputDefaults`, so any
|
||||
consumer decodes an append the same way. Building a consumer's own command
|
||||
object out of it is one constructor call --- see
|
||||
``vllm_omni.engine.duplex.realtime_commands.build_append_audio``, which wraps it
|
||||
for the duplex ``AppendAudio``.
|
||||
|
||||
The speech verdict here is the *client-declared* one (explicit flags, a VAD
|
||||
probability the client sent, or an RMS floor as a last resort). It is not
|
||||
server-side VAD; that is a runtime concern and lives with whoever owns the
|
||||
session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm_omni.protocol.realtime.audio import convert_input_audio_with_rate
|
||||
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
|
||||
from vllm_omni.protocol.realtime.formats import (
|
||||
is_supported_realtime_input_format,
|
||||
parse_realtime_audio_format,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.items import validate_realtime_video_frames
|
||||
from vllm_omni.protocol.realtime.session import RealtimeInputDefaults
|
||||
|
||||
__all__ = [
|
||||
"REALTIME_INPUT_HINT_KEYS",
|
||||
"RealtimeAudioAppend",
|
||||
"copy_realtime_input_hints",
|
||||
"decode_audio_append",
|
||||
"input_explicitly_non_speech",
|
||||
"input_looks_like_speech",
|
||||
]
|
||||
|
||||
#: Wire hint keys copied verbatim from a client append onto the internal payload.
|
||||
REALTIME_INPUT_HINT_KEYS = (
|
||||
"duration_ms",
|
||||
"audio_duration_ms",
|
||||
"audio_start_ms",
|
||||
"audio_end_ms",
|
||||
"is_speech",
|
||||
"speech",
|
||||
"speech_probability",
|
||||
"vad",
|
||||
"overlap_action",
|
||||
"overlap",
|
||||
"force_barge_in",
|
||||
"force_listen",
|
||||
"text",
|
||||
"transcript",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RealtimeAudioAppend:
|
||||
"""One decoded ``input_audio_buffer.append``.
|
||||
|
||||
``audio`` is raw bytes in ``format`` at ``sample_rate_hz`` --- after
|
||||
:func:`decode_audio_append` that is 16 kHz ``pcm_f32le`` for every input
|
||||
format the codec converts.
|
||||
"""
|
||||
|
||||
audio: bytes
|
||||
format: str
|
||||
sample_rate_hz: int | None = None
|
||||
is_speech: bool | None = None
|
||||
video_frames: tuple[str, ...] = ()
|
||||
duration_ms: int | None = None
|
||||
audio_end_ms: int | None = None
|
||||
#: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
|
||||
hints: dict[str, object] = field(default_factory=dict)
|
||||
#: Client correlation id (OpenAI ``event_id``), echoed on error events.
|
||||
event_id: str | None = None
|
||||
|
||||
|
||||
# ---- speech hints ----
|
||||
|
||||
|
||||
def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
|
||||
for key in ("is_speech", "speech"):
|
||||
value = event.get(key)
|
||||
if isinstance(value, bool):
|
||||
return not value
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
value = vad.get("is_speech")
|
||||
if isinstance(value, bool):
|
||||
return not value
|
||||
probability = vad.get("speech_probability", vad.get("probability"))
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) < 0.5
|
||||
probability = event.get("speech_probability")
|
||||
return isinstance(probability, int | float) and float(probability) < 0.5
|
||||
|
||||
|
||||
def input_looks_like_speech(
|
||||
event: Mapping[str, object],
|
||||
*,
|
||||
audio: object,
|
||||
fmt: object,
|
||||
overlap_silence_rms: float,
|
||||
) -> bool:
|
||||
if input_explicitly_non_speech(event):
|
||||
return False
|
||||
for key in ("is_speech", "speech"):
|
||||
value = event.get(key)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
probability = vad.get("speech_probability", vad.get("probability"))
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) >= 0.5
|
||||
probability = event.get("speech_probability")
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) >= 0.5
|
||||
if fmt != "pcm_f32le":
|
||||
return True
|
||||
if isinstance(audio, bytes | bytearray):
|
||||
raw = bytes(audio)
|
||||
elif isinstance(audio, str):
|
||||
try:
|
||||
raw = base64.b64decode(audio, validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
if len(raw) < 4 or len(raw) % 4 != 0:
|
||||
return True
|
||||
samples = np.frombuffer(raw, dtype=np.float32)
|
||||
if samples.size == 0:
|
||||
return False
|
||||
rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
|
||||
threshold = event.get("overlap_silence_rms")
|
||||
if not isinstance(threshold, int | float):
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
threshold = vad.get("silence_rms")
|
||||
silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
|
||||
return rms >= max(0.0, silence_rms)
|
||||
|
||||
|
||||
def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
|
||||
for key in REALTIME_INPUT_HINT_KEYS:
|
||||
if key in source:
|
||||
target[key] = source[key]
|
||||
|
||||
|
||||
# ---- append decoding ----
|
||||
|
||||
|
||||
def decode_audio_append(
|
||||
event: Mapping[str, object],
|
||||
*,
|
||||
defaults: RealtimeInputDefaults,
|
||||
hints_source: Mapping[str, object] | None = None,
|
||||
) -> RealtimeAudioAppend:
|
||||
"""Validate, convert (to 16 kHz ``pcm_f32le``) and pack one audio append.
|
||||
|
||||
``hints_source`` is the enclosing payload when the audio arrives inside
|
||||
something larger than a bare append --- a ``conversation.item.create``
|
||||
audio part, say --- so its hints apply unless the part overrides them.
|
||||
|
||||
Raises :class:`RealtimeProtocolError` for an unsupported format, undecodable
|
||||
audio or invalid camera frames.
|
||||
"""
|
||||
event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
|
||||
audio = event.get("audio") or event.get("delta")
|
||||
fmt, format_rate = parse_realtime_audio_format(
|
||||
event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
|
||||
)
|
||||
sample_rate_hz = (
|
||||
event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
|
||||
)
|
||||
if not is_supported_realtime_input_format(fmt):
|
||||
raise RealtimeProtocolError(
|
||||
f"Unsupported input_audio_format: {fmt}",
|
||||
code="unsupported_audio_format",
|
||||
event_id=event_id,
|
||||
)
|
||||
try:
|
||||
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
|
||||
audio,
|
||||
fmt,
|
||||
sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise RealtimeProtocolError(str(exc), code="bad_event", event_id=event_id) from exc
|
||||
hints: dict[str, object] = {}
|
||||
if hints_source is not None:
|
||||
copy_realtime_input_hints(hints_source, hints)
|
||||
copy_realtime_input_hints(event, hints)
|
||||
looks_like_speech = input_looks_like_speech(
|
||||
{**(hints_source or {}), **event},
|
||||
audio=audio,
|
||||
fmt=fmt,
|
||||
overlap_silence_rms=defaults.overlap_silence_rms,
|
||||
)
|
||||
video_frames: tuple[str, ...] = ()
|
||||
raw_frames = event.get("video_frames")
|
||||
if raw_frames is not None:
|
||||
frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
|
||||
if frames_error is not None:
|
||||
raise RealtimeProtocolError(frames_error, code="invalid_video_frames", event_id=event_id)
|
||||
if isinstance(raw_frames, list):
|
||||
video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
|
||||
duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
|
||||
audio_end_ms = hints.get("audio_end_ms")
|
||||
try:
|
||||
audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise RealtimeProtocolError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
|
||||
return RealtimeAudioAppend(
|
||||
event_id=event_id,
|
||||
audio=audio_bytes,
|
||||
format=str(fmt),
|
||||
sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
|
||||
is_speech=looks_like_speech,
|
||||
video_frames=video_frames,
|
||||
duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
|
||||
audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
|
||||
hints=hints,
|
||||
)
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""What a Realtime consumer can serve, and the session check that reads it.
|
||||
|
||||
``session.update`` is the one client event whose acceptance depends on who is
|
||||
behind the socket. The audio formats a server can decode, and whether it can do
|
||||
server-side turn detection at all, are properties of the *consumer* --- the
|
||||
duplex engine, or some other runtime --- not of the protocol. The codec still
|
||||
has to do the checking, because it owns the error envelope and the order the
|
||||
checks run in.
|
||||
|
||||
:class:`RealtimeProtocolCapabilities` is how a consumer states those answers.
|
||||
It is the post-#7413 form of the ``supports(turn_detection, formats)`` half of
|
||||
RFC #6592's proposed ``RealtimeModelAdapter``. The other three halves of that
|
||||
proposal --- building a prompt, starting a response, cancelling one --- are not
|
||||
here and must not come here: for a duplex model they are already owned by
|
||||
``DuplexModelPlugin`` together with ``DuplexSessionRunner``, and a second
|
||||
interface over the same responsibility is exactly the fork this layer exists to
|
||||
prevent.
|
||||
|
||||
The duplex binding of this object is
|
||||
``vllm_omni.engine.duplex.realtime_commands.DUPLEX_REALTIME_CAPABILITIES``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from vllm_omni.protocol.realtime.formats import (
|
||||
REALTIME_INPUT_AUDIO_FORMATS,
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS,
|
||||
validate_realtime_session_audio_formats,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"RealtimeProtocolCapabilities",
|
||||
"RealtimeSessionRejection",
|
||||
"validate_session_payload",
|
||||
]
|
||||
|
||||
#: A consumer's answer to "can you do this turn detection?": an error message, or None.
|
||||
TurnDetectionValidator = Callable[[Mapping[str, object]], str | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RealtimeProtocolCapabilities:
|
||||
"""One consumer's answer to what a ``session`` object may ask for.
|
||||
|
||||
The defaults are permissive, not restrictive: every format the codec can
|
||||
decode, and **no turn-detection validation at all**. A consumer that leaves
|
||||
``validate_turn_detection`` at ``None`` therefore accepts whatever
|
||||
``turn_detection`` the session object carries --- including values it cannot
|
||||
actually serve, such as ``semantic_vad``. A consumer that supports only some
|
||||
turn-detection modes (or none) must supply its own validator; see
|
||||
``vllm_omni.engine.duplex.realtime_commands.DUPLEX_REALTIME_CAPABILITIES``.
|
||||
"""
|
||||
|
||||
input_audio_formats: frozenset[str] = field(default_factory=lambda: frozenset(REALTIME_INPUT_AUDIO_FORMATS))
|
||||
output_audio_formats: frozenset[str] = field(default_factory=lambda: frozenset(REALTIME_OUTPUT_AUDIO_FORMATS))
|
||||
#: Consumer-specific ``turn_detection`` validation. ``None`` means the
|
||||
#: consumer takes whatever the session object says without a check of its own.
|
||||
validate_turn_detection: TurnDetectionValidator | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RealtimeSessionRejection:
|
||||
"""Why a session object was refused: the internal error code and the message.
|
||||
|
||||
``param`` names the offending session field where there is one, for the
|
||||
``error.param`` slot of the OpenAI error envelope.
|
||||
"""
|
||||
|
||||
code: str
|
||||
message: str
|
||||
param: str | None = None
|
||||
|
||||
|
||||
def validate_session_payload(
|
||||
session_payload: Mapping[str, object],
|
||||
*,
|
||||
capabilities: RealtimeProtocolCapabilities,
|
||||
) -> RealtimeSessionRejection | None:
|
||||
"""Check a ``session`` object against one consumer; ``None`` when it is acceptable.
|
||||
|
||||
Audio formats are checked before turn detection, so a session object that
|
||||
is wrong in both ways reports the format problem --- keep that order, it is
|
||||
what clients already see.
|
||||
"""
|
||||
format_error = validate_realtime_session_audio_formats(
|
||||
session_payload,
|
||||
input_audio_formats=capabilities.input_audio_formats,
|
||||
output_audio_formats=capabilities.output_audio_formats,
|
||||
)
|
||||
if format_error is not None:
|
||||
return RealtimeSessionRejection(code="unsupported_audio_format", message=format_error)
|
||||
validate_turn_detection = capabilities.validate_turn_detection
|
||||
if validate_turn_detection is not None:
|
||||
turn_detection_error = validate_turn_detection(session_payload)
|
||||
if turn_detection_error is not None:
|
||||
return RealtimeSessionRejection(
|
||||
code="unsupported_turn_detection", message=turn_detection_error, param="turn_detection"
|
||||
)
|
||||
return None
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""OpenAI Realtime client events as typed commands.
|
||||
|
||||
A Realtime server's first job is turning a client JSON event into something
|
||||
typed. These are the 10 client events OpenAI's Realtime API defines, as frozen
|
||||
dataclasses carrying the fields that survive decoding.
|
||||
|
||||
vLLM-Omni's own client events (barge-in, playback acknowledgement, text append,
|
||||
heartbeat, explicit close, generic turn signals) are a separate vocabulary in
|
||||
``vllm_omni.protocol.duplex.commands``.
|
||||
|
||||
Decoding a payload *into* these stays with the consumer, because which command
|
||||
a given event becomes can depend on what that consumer supports --- see
|
||||
``vllm_omni.engine.duplex.realtime_commands.translate_realtime_command``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import ClassVar
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class RealtimeCommand:
|
||||
"""One decoded OpenAI Realtime client event.
|
||||
|
||||
``wire_type`` is the event type the client sent; the typed fields are what
|
||||
survived decoding, so a consumer reads them instead of re-parsing JSON.
|
||||
|
||||
There is deliberately no rendering method here. A server receives commands,
|
||||
it does not emit them, and how a runtime represents one internally is its
|
||||
own business --- the duplex engine renders its mailbox dictionary in
|
||||
``vllm_omni.engine.duplex.commands``.
|
||||
"""
|
||||
|
||||
#: OpenAI Realtime client event this command decodes from.
|
||||
wire_type: ClassVar[str] = ""
|
||||
#: Client correlation id (OpenAI ``event_id``), echoed on error events.
|
||||
event_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class UpdateSession(RealtimeCommand):
|
||||
"""``session.update``: a Realtime ``session`` object patch."""
|
||||
|
||||
wire_type: ClassVar[str] = "session.update"
|
||||
patch: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AppendAudio(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "input_audio_buffer.append"
|
||||
#: Raw audio bytes in ``format`` at ``sample_rate_hz`` (base64 only on the wire).
|
||||
audio: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class Commit(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "input_audio_buffer.commit"
|
||||
final: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ClearInput(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "input_audio_buffer.clear"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ClearOutputAudio(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "output_audio_buffer.clear"
|
||||
#: Explicit response to clear; ``None`` targets the active/last response.
|
||||
response_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CreateResponse(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "response.create"
|
||||
#: Raw Realtime ``response`` object (instructions, voice, modalities, ...).
|
||||
options: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CancelResponse(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "response.cancel"
|
||||
response_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class CreateItem(RealtimeCommand):
|
||||
"""``conversation.item.create`` (history injection or function-call output)."""
|
||||
|
||||
wire_type: ClassVar[str] = "conversation.item.create"
|
||||
item: Mapping[str, object]
|
||||
previous_item_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class DeleteItem(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "conversation.item.delete"
|
||||
item_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TruncateItem(RealtimeCommand):
|
||||
wire_type: ClassVar[str] = "conversation.item.truncate"
|
||||
item_id: str
|
||||
audio_end_ms: int
|
||||
content_index: int = 0
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AppendAudio",
|
||||
"CancelResponse",
|
||||
"ClearInput",
|
||||
"ClearOutputAudio",
|
||||
"Commit",
|
||||
"CreateItem",
|
||||
"CreateResponse",
|
||||
"DeleteItem",
|
||||
"RealtimeCommand",
|
||||
"TruncateItem",
|
||||
"UpdateSession",
|
||||
]
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The OpenAI Realtime error envelope.
|
||||
|
||||
A Realtime server reports a rejected client event as one ``error`` event whose
|
||||
``error`` object carries ``type`` / ``code`` / ``message`` (plus the client's
|
||||
``event_id`` and an optional ``param``). ``type`` is not free-form: it is one
|
||||
of OpenAI's four buckets, derived from our internal code by
|
||||
:data:`REALTIME_ERROR_TYPES_BY_CODE`.
|
||||
|
||||
OpenAI standardises the three ``type`` classes but not the *codes*, so the code
|
||||
vocabulary is a consumer's own. vLLM-Omni's lives in
|
||||
``vllm_omni.protocol.duplex.errors`` (Tier 3 in
|
||||
``docs/serving/realtime_duplex_api.md``); what stays here is the exception type
|
||||
every consumer raises, so the envelope is rendered identically whoever built it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = ["RealtimeProtocolError"]
|
||||
|
||||
|
||||
class RealtimeProtocolError(ValueError):
|
||||
"""A client payload could not be decoded into a valid Realtime intent.
|
||||
|
||||
``code`` is the internal error code that :data:`REALTIME_ERROR_TYPES_BY_CODE`
|
||||
maps to an OpenAI ``error.type``; ``event_id`` is the *client* event id the
|
||||
error answers. ``vllm_omni.engine.duplex.commands.DuplexCommandError`` is
|
||||
the duplex specialization, so a consumer catching either sees the same
|
||||
three attributes.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, *, code: str = "bad_event", event_id: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.event_id = event_id
|
||||
@@ -0,0 +1,467 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""OpenAI Realtime server events: typed classes and their wire rendering.
|
||||
|
||||
Every event is a frozen dataclass with explicit fields, and the Realtime wire
|
||||
JSON is *derived* from those fields by :meth:`RealtimeEvent.to_realtime`. That
|
||||
rendering is pure --- it never consults session state --- which is why it can
|
||||
live here, away from whoever owns the session. The stateful part (response and
|
||||
item ids, content-part bookkeeping) is consumed when the events are
|
||||
*constructed*, by the consumer.
|
||||
|
||||
This module holds the 30 events OpenAI's Realtime API defines. vLLM-Omni's own
|
||||
additions to the wire (listen/speak turn-taking, playback acknowledgement,
|
||||
session resume) are a separate vocabulary in
|
||||
``vllm_omni.protocol.duplex.events``, so a consumer that only speaks GA does
|
||||
not inherit them.
|
||||
|
||||
``session_id`` / ``epoch`` are bound by the consumer when an event leaves it;
|
||||
producers build events with the defaults.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field, fields
|
||||
from typing import ClassVar
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def new_event_id() -> str:
|
||||
return f"event_{uuid4().hex}"
|
||||
|
||||
|
||||
_IDENTITY_FIELDS = frozenset({"session_id", "epoch", "event_id"})
|
||||
|
||||
|
||||
def wire_value(value: object) -> object:
|
||||
if isinstance(value, Mapping):
|
||||
return {str(k): wire_value(v) for k, v in value.items()}
|
||||
if isinstance(value, tuple | list):
|
||||
return [wire_value(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class RealtimeEvent:
|
||||
"""Base of every public session event."""
|
||||
|
||||
#: Realtime ``type`` this class renders to.
|
||||
wire_type: ClassVar[str] = ""
|
||||
#: Fields that are omitted from the wire object when ``None``.
|
||||
optional_wire_fields: ClassVar[frozenset[str]] = frozenset()
|
||||
|
||||
session_id: str = ""
|
||||
epoch: int | None = None
|
||||
#: Server event id (OpenAI ``event_id``); generated at construction.
|
||||
event_id: str = field(default_factory=new_event_id)
|
||||
|
||||
# ---- identity ----
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return self.wire_type
|
||||
|
||||
@property
|
||||
def is_terminal(self) -> bool:
|
||||
return False
|
||||
|
||||
# ---- generic accessors (None when the class has no such field) ----
|
||||
|
||||
@property
|
||||
def response_id(self) -> str | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def item_id(self) -> str | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def audio(self) -> bytes | None:
|
||||
return None
|
||||
|
||||
# ---- wire rendering (pure) ----
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
data: dict[str, object] = {}
|
||||
for f in fields(self):
|
||||
if f.name in _IDENTITY_FIELDS:
|
||||
continue
|
||||
value = getattr(self, f.name)
|
||||
if value is None and f.name in self.optional_wire_fields:
|
||||
continue
|
||||
data[f.name] = wire_value(value)
|
||||
return data
|
||||
|
||||
def to_realtime(self) -> dict[str, object]:
|
||||
"""The Realtime wire JSON object for this event (derived from the fields)."""
|
||||
return {"type": self.wire_type, "event_id": self.event_id, **self._wire_fields()}
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - debugging aid
|
||||
return f"{type(self).__name__}(type={self.wire_type!r}, session_id={self.session_id!r}, epoch={self.epoch!r})"
|
||||
|
||||
|
||||
# ---- shared bases ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseScopedEvent(RealtimeEvent):
|
||||
"""Events addressed to one response (and usually one output item / content part)."""
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
output_index: int = 0
|
||||
content_index: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class _ItemEvent(RealtimeEvent):
|
||||
previous_item_id: str | None = None
|
||||
item: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def item_id(self) -> str | None:
|
||||
value = self.item.get("id")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
# ---- GA server events ----
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionCreated(RealtimeEvent):
|
||||
wire_type = "session.created"
|
||||
|
||||
session: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SessionUpdated(RealtimeEvent):
|
||||
wire_type = "session.updated"
|
||||
|
||||
session: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseCreated(RealtimeEvent):
|
||||
wire_type = "response.created"
|
||||
|
||||
response: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class OutputItemAdded(RealtimeEvent):
|
||||
wire_type = "response.output_item.added"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
output_index: int = 0
|
||||
item: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def item_id(self) -> str | None:
|
||||
value = self.item.get("id")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class OutputItemDone(OutputItemAdded):
|
||||
wire_type = "response.output_item.done"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ContentPartAdded(ResponseScopedEvent):
|
||||
wire_type = "response.content_part.added"
|
||||
|
||||
part: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ContentPartDone(ContentPartAdded):
|
||||
wire_type = "response.content_part.done"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AudioDelta(ResponseScopedEvent):
|
||||
wire_type = "response.output_audio.delta"
|
||||
|
||||
#: Base64 audio in ``format``.
|
||||
delta: str = ""
|
||||
|
||||
@property
|
||||
def audio(self) -> bytes | None:
|
||||
try:
|
||||
return base64.b64decode(self.delta)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class AudioDone(ResponseScopedEvent):
|
||||
wire_type = "response.output_audio.done"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TranscriptDelta(ResponseScopedEvent):
|
||||
wire_type = "response.output_audio_transcript.delta"
|
||||
|
||||
delta: str = ""
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.delta
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TranscriptDone(ResponseScopedEvent):
|
||||
wire_type = "response.output_audio_transcript.done"
|
||||
|
||||
transcript: str = ""
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.transcript
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TextDelta(ResponseScopedEvent):
|
||||
wire_type = "response.output_text.delta"
|
||||
|
||||
delta: str = ""
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.delta
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class TextDone(ResponseScopedEvent):
|
||||
wire_type = "response.output_text.done"
|
||||
|
||||
text: str = "" # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class FunctionCallArgumentsDelta(RealtimeEvent):
|
||||
wire_type = "response.function_call_arguments.delta"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
output_index: int = 0
|
||||
call_id: str = ""
|
||||
delta: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class FunctionCallArgumentsDone(RealtimeEvent):
|
||||
wire_type = "response.function_call_arguments.done"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
output_index: int = 0
|
||||
call_id: str = ""
|
||||
arguments: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseDone(RealtimeEvent):
|
||||
wire_type = "response.done"
|
||||
|
||||
response: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def status(self) -> str | None:
|
||||
value = self.response.get("status")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class RateLimitsUpdated(RealtimeEvent):
|
||||
wire_type = "rate_limits.updated"
|
||||
|
||||
rate_limits: tuple[Mapping[str, object], ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class InputCommitted(RealtimeEvent):
|
||||
wire_type = "input_audio_buffer.committed"
|
||||
|
||||
previous_item_id: str | None = None
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"previous_item_id": self.previous_item_id, "item_id": self.item_id}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class InputCleared(RealtimeEvent):
|
||||
wire_type = "input_audio_buffer.cleared"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SpeechStarted(RealtimeEvent):
|
||||
wire_type = "input_audio_buffer.speech_started"
|
||||
|
||||
audio_start_ms: int = 0
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class SpeechStopped(RealtimeEvent):
|
||||
wire_type = "input_audio_buffer.speech_stopped"
|
||||
|
||||
audio_end_ms: int = 0
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemAdded(_ItemEvent):
|
||||
wire_type = "conversation.item.added"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemCreated(_ItemEvent):
|
||||
wire_type = "conversation.item.created"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemDone(_ItemEvent):
|
||||
wire_type = "conversation.item.done"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemRetrieved(RealtimeEvent):
|
||||
wire_type = "conversation.item.retrieved"
|
||||
|
||||
item: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def item_id(self) -> str | None:
|
||||
value = self.item.get("id")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemDeleted(RealtimeEvent):
|
||||
wire_type = "conversation.item.deleted"
|
||||
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"item_id": self.item_id}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ItemTruncated(RealtimeEvent):
|
||||
wire_type = "conversation.item.truncated"
|
||||
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
content_index: int = 0
|
||||
audio_end_ms: int = 0
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {
|
||||
"item_id": self.item_id,
|
||||
"content_index": self.content_index,
|
||||
"audio_end_ms": self.audio_end_ms,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class InputTranscriptionCompleted(RealtimeEvent):
|
||||
wire_type = "conversation.item.input_audio_transcription.completed"
|
||||
|
||||
item_id: str | None = None # type: ignore[assignment]
|
||||
content_index: int = 0
|
||||
transcript: str = ""
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.transcript
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class OutputAudioCleared(RealtimeEvent):
|
||||
wire_type = "output_audio_buffer.cleared"
|
||||
|
||||
response_id: str | None = None # type: ignore[assignment]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ErrorEvent(RealtimeEvent):
|
||||
wire_type = "error"
|
||||
|
||||
code: str = "internal_error"
|
||||
message: str = ""
|
||||
#: Client ``event_id`` this error answers (OpenAI ``error.event_id``).
|
||||
related_event_id: str | None = None
|
||||
param: str | None = None
|
||||
|
||||
@property
|
||||
def error_type(self) -> str:
|
||||
"""OpenAI's error class for this error.
|
||||
|
||||
OpenAI standardises only the three classes, not the codes, so the base
|
||||
reports the default. A consumer with its own code vocabulary overrides
|
||||
this --- see ``vllm_omni.protocol.duplex.events.ErrorEvent``.
|
||||
"""
|
||||
return "invalid_request_error"
|
||||
|
||||
@property
|
||||
def error(self) -> dict[str, object]:
|
||||
error: dict[str, object] = {"type": self.error_type, "code": self.code, "message": self.message}
|
||||
if self.related_event_id:
|
||||
error["event_id"] = self.related_event_id
|
||||
if self.param:
|
||||
error["param"] = self.param
|
||||
return error
|
||||
|
||||
@property
|
||||
def text(self) -> str | None:
|
||||
return self.message
|
||||
|
||||
def _wire_fields(self) -> dict[str, object]:
|
||||
return {"error": self.error}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AudioDelta",
|
||||
"AudioDone",
|
||||
"ContentPartAdded",
|
||||
"ContentPartDone",
|
||||
"ErrorEvent",
|
||||
"FunctionCallArgumentsDelta",
|
||||
"FunctionCallArgumentsDone",
|
||||
"InputCleared",
|
||||
"InputCommitted",
|
||||
"InputTranscriptionCompleted",
|
||||
"ItemAdded",
|
||||
"ItemCreated",
|
||||
"ItemDeleted",
|
||||
"ItemDone",
|
||||
"ItemRetrieved",
|
||||
"ItemTruncated",
|
||||
"OutputAudioCleared",
|
||||
"OutputItemAdded",
|
||||
"OutputItemDone",
|
||||
"RateLimitsUpdated",
|
||||
"RealtimeEvent",
|
||||
"ResponseCreated",
|
||||
"ResponseDone",
|
||||
"ResponseScopedEvent",
|
||||
"SessionCreated",
|
||||
"SessionUpdated",
|
||||
"SpeechStarted",
|
||||
"SpeechStopped",
|
||||
"TextDelta",
|
||||
"TextDone",
|
||||
"TranscriptDelta",
|
||||
"TranscriptDone",
|
||||
"new_event_id",
|
||||
"wire_value",
|
||||
]
|
||||
@@ -0,0 +1,177 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Realtime audio-format negotiation: what a client may declare and how it is read.
|
||||
|
||||
A Realtime client declares audio formats in three places --- the session object
|
||||
(``session.audio.input.format`` and the older flat ``input_audio_format``), a
|
||||
``response.create`` override, and a ``conversation.item.create`` audio part ---
|
||||
and may spell one format several ways (``audio/pcm``, ``pcm16``, ``s16le``).
|
||||
This module is the single place that normalizes those spellings and says which
|
||||
are supported. Folding a session's declaration into the per-connection append
|
||||
defaults is the job of ``session.py``, which builds on this module.
|
||||
|
||||
Everything here is a pure function of wire payloads: no session state, no model,
|
||||
no engine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Collection, Mapping
|
||||
|
||||
__all__ = [
|
||||
"REALTIME_INPUT_AUDIO_FORMATS",
|
||||
"REALTIME_OUTPUT_AUDIO_FORMATS",
|
||||
"is_supported_realtime_input_format",
|
||||
"parse_realtime_audio_format",
|
||||
"realtime_audio_format_object",
|
||||
"realtime_output_format",
|
||||
"validate_conversation_item_audio_formats",
|
||||
"validate_realtime_response_audio_formats",
|
||||
"validate_realtime_session_audio_formats",
|
||||
]
|
||||
|
||||
REALTIME_INPUT_AUDIO_FORMATS = {
|
||||
"pcm16",
|
||||
"pcm_s16le",
|
||||
"s16le",
|
||||
"pcm_f32le",
|
||||
"g711_ulaw",
|
||||
"g711_alaw",
|
||||
}
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS = {
|
||||
"pcm16",
|
||||
"pcm_s16le",
|
||||
"s16le",
|
||||
"wav",
|
||||
"pcm",
|
||||
"g711_ulaw",
|
||||
"g711_alaw",
|
||||
}
|
||||
|
||||
|
||||
def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]:
|
||||
def normalize_format(fmt: str) -> str:
|
||||
normalized = fmt.lower()
|
||||
if normalized in {"audio/pcm", "pcm"}:
|
||||
return "pcm16"
|
||||
if normalized in {"audio/wav", "wav"}:
|
||||
return "wav"
|
||||
if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}:
|
||||
return "pcm16"
|
||||
if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}:
|
||||
return "pcm_f32le"
|
||||
if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}:
|
||||
return "g711_ulaw"
|
||||
if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}:
|
||||
return "g711_alaw"
|
||||
return fmt
|
||||
|
||||
if isinstance(raw_format, str):
|
||||
return normalize_format(raw_format), None
|
||||
if not isinstance(raw_format, dict):
|
||||
return raw_format, None
|
||||
rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate")
|
||||
sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None
|
||||
fmt = raw_format.get("type") or raw_format.get("format")
|
||||
if not isinstance(fmt, str):
|
||||
return raw_format, sample_rate_hz
|
||||
return normalize_format(fmt), sample_rate_hz
|
||||
|
||||
|
||||
def realtime_output_format(duplex_format: object) -> str:
|
||||
if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}:
|
||||
return duplex_format.lower()
|
||||
if isinstance(duplex_format, str) and duplex_format.lower() == "pcm":
|
||||
return "pcm16"
|
||||
return str(duplex_format or "wav")
|
||||
|
||||
|
||||
def is_supported_realtime_input_format(fmt: object) -> bool:
|
||||
return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS
|
||||
|
||||
|
||||
def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]:
|
||||
if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}:
|
||||
payload: dict[str, object] = {"type": "audio/pcm"}
|
||||
elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le":
|
||||
payload = {"type": "audio/pcm_f32le"}
|
||||
elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw":
|
||||
payload = {"type": "audio/g711_ulaw"}
|
||||
elif isinstance(fmt, str) and fmt.lower() == "g711_alaw":
|
||||
payload = {"type": "audio/g711_alaw"}
|
||||
else:
|
||||
payload = {"type": "audio/wav"}
|
||||
if sample_rate_hz is not None:
|
||||
payload["rate"] = int(sample_rate_hz)
|
||||
return payload
|
||||
|
||||
|
||||
def validate_realtime_session_audio_formats(
|
||||
session_payload: Mapping[str, object],
|
||||
*,
|
||||
input_audio_formats: Collection[str] | None = None,
|
||||
output_audio_formats: Collection[str] | None = None,
|
||||
) -> str | None:
|
||||
"""Reject a session object that declares an audio format we cannot serve.
|
||||
|
||||
The format sets default to everything the codec can decode; a consumer that
|
||||
serves a narrower set passes its own (see
|
||||
``vllm_omni.protocol.realtime.capabilities``).
|
||||
"""
|
||||
supported_input = REALTIME_INPUT_AUDIO_FORMATS if input_audio_formats is None else input_audio_formats
|
||||
supported_output = REALTIME_OUTPUT_AUDIO_FORMATS if output_audio_formats is None else output_audio_formats
|
||||
audio_config = session_payload.get("audio")
|
||||
input_format: object = session_payload.get("input_audio_format")
|
||||
if input_format is None and isinstance(audio_config, dict):
|
||||
audio_input = audio_config.get("input")
|
||||
if isinstance(audio_input, dict):
|
||||
input_format = audio_input.get("format")
|
||||
parsed_input, _ = parse_realtime_audio_format(input_format)
|
||||
if input_format is not None and not (isinstance(parsed_input, str) and parsed_input.lower() in supported_input):
|
||||
return f"Unsupported input_audio_format: {input_format}"
|
||||
|
||||
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
|
||||
if output_format is None and isinstance(audio_config, dict):
|
||||
audio_output = audio_config.get("output")
|
||||
if isinstance(audio_output, dict):
|
||||
output_format = audio_output.get("format")
|
||||
parsed_output, _ = parse_realtime_audio_format(output_format)
|
||||
if output_format is not None and not (isinstance(parsed_output, str) and parsed_output.lower() in supported_output):
|
||||
return f"Unsupported output_audio_format: {output_format}"
|
||||
return None
|
||||
|
||||
|
||||
def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None:
|
||||
output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format")
|
||||
audio_config = response_payload.get("audio")
|
||||
if output_format is None and isinstance(audio_config, dict):
|
||||
audio_output = audio_config.get("output")
|
||||
if isinstance(audio_output, dict):
|
||||
output_format = audio_output.get("format")
|
||||
parsed_output, _ = parse_realtime_audio_format(output_format)
|
||||
if output_format is not None and not (
|
||||
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
|
||||
):
|
||||
return f"Unsupported output_audio_format: {output_format}"
|
||||
return None
|
||||
|
||||
|
||||
def validate_conversation_item_audio_formats(item: object) -> str | None:
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list):
|
||||
return None
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
if part.get("type") not in {"input_audio", "audio"}:
|
||||
continue
|
||||
raw_format = part.get("format")
|
||||
parsed_format, _ = parse_realtime_audio_format(raw_format)
|
||||
if raw_format is not None and not (
|
||||
isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS
|
||||
):
|
||||
return f"Unsupported input_audio format in conversation.item.create: {raw_format}"
|
||||
return None
|
||||
@@ -0,0 +1,197 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Realtime conversation items: shape, truncation and the transcript inside one.
|
||||
|
||||
A Realtime conversation is a list of *items* --- a user message, an assistant
|
||||
message, a function call --- each with a ``content`` list of typed parts. The
|
||||
client creates them (``conversation.item.create``), truncates an assistant
|
||||
item's audio after a barge-in (``conversation.item.truncate``), and the server
|
||||
echoes them back inside ``conversation.item.*`` and ``response.done`` events.
|
||||
|
||||
The rules in this module are the ones both sides have to agree on: which
|
||||
defaults a bare item gets, how far a transcript is cut when the client says it
|
||||
only heard N milliseconds of audio, and what makes a truncate request invalid.
|
||||
Item *storage* --- which items a session currently holds, in what order --- is
|
||||
the consumer's state, not this module's.
|
||||
|
||||
``validate_realtime_video_frames`` sits here too: camera frames ride along an
|
||||
``input_audio_buffer.append`` in the vLLM-Omni extension of the protocol, and
|
||||
validating them is the same kind of wire-shape check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
from collections.abc import Mapping
|
||||
from uuid import uuid4
|
||||
|
||||
__all__ = [
|
||||
"input_transcript_from_item",
|
||||
"normalize_conversation_item",
|
||||
"text_chars_for_audio_ms_from_marks",
|
||||
"truncate_realtime_item_content",
|
||||
"validate_realtime_item_truncate",
|
||||
"validate_realtime_video_frames",
|
||||
]
|
||||
|
||||
|
||||
def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
|
||||
"""Validate omni-duplex camera frames on input_audio_buffer.append.
|
||||
|
||||
Wire contract matches the official MiniCPM-o duplex loop: one base
|
||||
base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
|
||||
stacked composite tiling the sub-frames captured inside it (at most 2
|
||||
images either way). A caller-supplied ``max_slice_nums`` is rejected rather
|
||||
than silently ignored: slicing is Stage 0's decision here, and Stage 0
|
||||
already applies the official HD suggestion for a stacked unit
|
||||
(``max_slice_nums=[2, 1]``). The wire simply does not let the client choose
|
||||
it.
|
||||
"""
|
||||
if max_slice_nums not in (None, 1):
|
||||
return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself"
|
||||
if not isinstance(video_frames, list):
|
||||
return "video_frames must be a list of base64-encoded images"
|
||||
frames = [frame for frame in video_frames if frame is not None]
|
||||
if len(frames) > 2:
|
||||
return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
|
||||
for frame in frames:
|
||||
if not isinstance(frame, str) or not frame:
|
||||
return "video_frames entries must be non-empty base64 strings"
|
||||
if len(frame) > 4_000_000:
|
||||
return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
|
||||
try:
|
||||
header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
|
||||
except (binascii.Error, ValueError):
|
||||
return "video_frames entries must be valid base64"
|
||||
if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
|
||||
return "video_frames entries must be JPEG or PNG images"
|
||||
return None
|
||||
|
||||
|
||||
def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]:
|
||||
normalized = dict(item)
|
||||
normalized.setdefault("id", f"item_{uuid4().hex}")
|
||||
normalized.setdefault("object", "realtime.item")
|
||||
normalized.setdefault("type", "message")
|
||||
normalized.setdefault("status", "completed")
|
||||
if "role" not in normalized and normalized.get("type") == "message":
|
||||
normalized["role"] = "user"
|
||||
if not isinstance(normalized.get("content"), list):
|
||||
normalized["content"] = []
|
||||
return normalized
|
||||
|
||||
|
||||
def text_chars_for_audio_ms_from_marks(
|
||||
audio_end_ms: int,
|
||||
text_len: int,
|
||||
marks: list[object],
|
||||
*,
|
||||
final_ms: object | None = None,
|
||||
) -> int:
|
||||
if text_len <= 0:
|
||||
return 0
|
||||
clean_marks: list[tuple[int, int]] = []
|
||||
for mark in marks:
|
||||
if not isinstance(mark, dict):
|
||||
continue
|
||||
raw_text_chars = mark.get("text_chars")
|
||||
raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
|
||||
if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
|
||||
continue
|
||||
clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
|
||||
if not clean_marks:
|
||||
return 0 if audio_end_ms <= 0 else text_len
|
||||
clean_marks.sort(key=lambda item: item[0])
|
||||
audio_end_ms = max(0, int(audio_end_ms))
|
||||
if audio_end_ms <= 0:
|
||||
return 0
|
||||
previous_ms = 0
|
||||
previous_chars = 0
|
||||
for mark_ms, mark_chars in clean_marks:
|
||||
mark_ms = max(previous_ms, mark_ms)
|
||||
mark_chars = max(previous_chars, min(text_len, mark_chars))
|
||||
if audio_end_ms <= mark_ms:
|
||||
if mark_ms <= previous_ms:
|
||||
return mark_chars
|
||||
ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
|
||||
return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
|
||||
previous_ms = mark_ms
|
||||
previous_chars = mark_chars
|
||||
if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
|
||||
if audio_end_ms >= int(final_ms):
|
||||
return text_len
|
||||
ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
|
||||
return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
|
||||
return text_len if audio_end_ms >= previous_ms else previous_chars
|
||||
|
||||
|
||||
def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None:
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list) or not content:
|
||||
return
|
||||
index = max(0, int(content_index))
|
||||
if index >= len(content):
|
||||
return
|
||||
part = content[index]
|
||||
if not isinstance(part, dict):
|
||||
return
|
||||
transcript = part.get("transcript")
|
||||
if not isinstance(transcript, str) or not transcript:
|
||||
return
|
||||
marks = part.get("audio_text_marks")
|
||||
if isinstance(marks, list):
|
||||
keep_chars = text_chars_for_audio_ms_from_marks(
|
||||
audio_end_ms,
|
||||
len(transcript),
|
||||
marks,
|
||||
final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
|
||||
)
|
||||
part["transcript"] = transcript[:keep_chars].rstrip()
|
||||
return
|
||||
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
|
||||
if isinstance(duration_ms, int | float) and duration_ms > 0:
|
||||
keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
|
||||
part["transcript"] = transcript[:keep_chars].rstrip()
|
||||
elif audio_end_ms <= 0:
|
||||
part["transcript"] = ""
|
||||
|
||||
|
||||
def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None:
|
||||
if item.get("type") != "message" or item.get("role") != "assistant":
|
||||
return "conversation.item.truncate only supports assistant message items"
|
||||
if audio_end_ms < 0:
|
||||
return "conversation.item.truncate requires non-negative audio_end_ms"
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list) or not content:
|
||||
return None
|
||||
index = max(0, int(content_index))
|
||||
if index >= len(content):
|
||||
return f"conversation.item.truncate content_index out of range: {content_index}"
|
||||
part = content[index]
|
||||
if not isinstance(part, dict):
|
||||
return "conversation.item.truncate target content part is invalid"
|
||||
if part.get("type") not in {"audio", "output_audio"}:
|
||||
return "conversation.item.truncate target content part must be audio"
|
||||
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
|
||||
if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
|
||||
return "conversation.item.truncate audio_end_ms exceeds item audio duration"
|
||||
return None
|
||||
|
||||
|
||||
def input_transcript_from_item(item: Mapping[str, object]) -> str:
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list):
|
||||
return ""
|
||||
parts: list[str] = []
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
for key in ("transcript", "text"):
|
||||
value = part.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
|
||||
parts.append(value)
|
||||
break
|
||||
return "".join(parts).strip()
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Reading the Realtime ``session`` object.
|
||||
|
||||
``session.update`` carries one JSON object that a client may spell in either of
|
||||
two shapes: the GA nest (``session.audio.input.format``,
|
||||
``session.audio.output.sample_rate_hz``) or the older flat beta keys
|
||||
(``input_audio_format``, ``sample_rate_hz``). Everything downstream --- the
|
||||
append decoder, the projection that renders ``session.created`` back out, a
|
||||
model's runtime configuration --- has to agree on what a given session object
|
||||
means, so the readers live here once.
|
||||
|
||||
:class:`RealtimeInputDefaults` is the resolved answer for the input path: the
|
||||
format, sample rate and silence threshold an ``input_audio_buffer.append``
|
||||
inherits when it omits them. A connection keeps one of these and re-derives it
|
||||
on every ``session.update``.
|
||||
|
||||
``overlap_policy`` / ``playback_commit_policy`` are vLLM-Omni extension fields
|
||||
on the session object rather than GA OpenAI fields. They are read here because
|
||||
this module owns what the wire may say; what a server *does* with them is the
|
||||
consumer's business.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
|
||||
from vllm_omni.protocol.realtime.formats import (
|
||||
REALTIME_INPUT_AUDIO_FORMATS,
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS,
|
||||
parse_realtime_audio_format,
|
||||
realtime_output_format,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"RealtimeInputDefaults",
|
||||
"apply_realtime_session_defaults",
|
||||
"input_audio_transcription_config",
|
||||
"json_safe_realtime_payload",
|
||||
"realtime_max_output_tokens",
|
||||
"realtime_overlap_fields",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RealtimeInputDefaults:
|
||||
"""Session-level wire defaults an append may omit (derived from the session object)."""
|
||||
|
||||
input_audio_format: str = "pcm16"
|
||||
input_sample_rate_hz: int = 16000
|
||||
output_audio_format: str = "pcm16"
|
||||
output_sample_rate_hz: int | None = None
|
||||
overlap_silence_rms: float = 0.003
|
||||
|
||||
def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults:
|
||||
"""Return defaults updated from a Realtime ``session`` object (session.update)."""
|
||||
values = apply_realtime_session_defaults(self, session_payload)
|
||||
return values
|
||||
|
||||
|
||||
def apply_realtime_session_defaults(
|
||||
defaults: RealtimeInputDefaults,
|
||||
session_payload: Mapping[str, object],
|
||||
) -> RealtimeInputDefaults:
|
||||
"""Derive the wire defaults a Realtime ``session`` object declares."""
|
||||
input_format: object = session_payload.get("input_audio_format")
|
||||
audio_config = session_payload.get("audio")
|
||||
if input_format is None and isinstance(audio_config, dict):
|
||||
audio_input = audio_config.get("input")
|
||||
if isinstance(audio_input, dict):
|
||||
input_format = audio_input.get("format")
|
||||
input_format, input_rate = parse_realtime_audio_format(input_format)
|
||||
# Any: heterogeneous field values splatted into ``dataclasses.replace`` below.
|
||||
updates: dict[str, Any] = {}
|
||||
if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS:
|
||||
updates["input_audio_format"] = input_format
|
||||
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
|
||||
output_rate_raw: object | None = None
|
||||
if output_format is None and isinstance(audio_config, dict):
|
||||
audio_output = audio_config.get("output")
|
||||
if isinstance(audio_output, dict):
|
||||
output_format = audio_output.get("format")
|
||||
output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate")
|
||||
output_format, output_rate = parse_realtime_audio_format(output_format)
|
||||
if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0:
|
||||
output_rate = int(output_rate_raw)
|
||||
if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
|
||||
updates["output_audio_format"] = realtime_output_format(output_format)
|
||||
sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate")
|
||||
if sample_rate is None and isinstance(audio_config, dict):
|
||||
audio_input = audio_config.get("input")
|
||||
if isinstance(audio_input, dict):
|
||||
sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate")
|
||||
if sample_rate is None:
|
||||
sample_rate = input_rate
|
||||
if isinstance(sample_rate, int | float) and sample_rate > 0:
|
||||
updates["input_sample_rate_hz"] = int(sample_rate)
|
||||
if isinstance(output_rate, int | float) and output_rate > 0:
|
||||
updates["output_sample_rate_hz"] = int(output_rate)
|
||||
overlap_fields = realtime_overlap_fields(session_payload)
|
||||
overlap_silence_rms = overlap_fields.get("overlap_silence_rms")
|
||||
if isinstance(overlap_silence_rms, int | float):
|
||||
updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms))
|
||||
return replace(defaults, **updates) if updates else defaults
|
||||
|
||||
|
||||
def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]:
|
||||
fields: dict[str, object] = {}
|
||||
if isinstance(session_payload.get("overlap_policy"), str):
|
||||
fields["overlap_policy"] = session_payload["overlap_policy"]
|
||||
for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"):
|
||||
value = session_payload.get(key)
|
||||
if isinstance(value, int | float):
|
||||
fields[key] = value
|
||||
if isinstance(session_payload.get("playback_commit_policy"), str):
|
||||
fields["playback_commit_policy"] = session_payload["playback_commit_policy"]
|
||||
return fields
|
||||
|
||||
|
||||
def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]:
|
||||
clean: dict[str, object] = {}
|
||||
for key, value in payload.items():
|
||||
if key == "extra_body":
|
||||
continue
|
||||
if isinstance(value, str | int | float | bool) or value is None:
|
||||
clean[key] = value
|
||||
elif isinstance(value, dict):
|
||||
clean[key] = json_safe_realtime_payload(value)
|
||||
elif isinstance(value, list):
|
||||
clean[key] = [
|
||||
(json_safe_realtime_payload(item) if isinstance(item, dict) else item)
|
||||
for item in value
|
||||
if isinstance(item, str | int | float | bool | dict) or item is None
|
||||
]
|
||||
return clean
|
||||
|
||||
|
||||
def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
|
||||
transcription = session_payload.get("input_audio_transcription")
|
||||
if isinstance(transcription, dict):
|
||||
return transcription
|
||||
audio_config = session_payload.get("audio")
|
||||
if not isinstance(audio_config, dict):
|
||||
return None
|
||||
audio_input = audio_config.get("input")
|
||||
if not isinstance(audio_input, dict):
|
||||
return None
|
||||
transcription = audio_input.get("transcription")
|
||||
return transcription if isinstance(transcription, dict) else None
|
||||
|
||||
|
||||
def realtime_max_output_tokens(value: object) -> int | None:
|
||||
"""Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}:
|
||||
return None
|
||||
if isinstance(value, int) and value > 0:
|
||||
return int(value)
|
||||
return None
|
||||
Reference in new issue
Block a user