anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Core] Give the OpenAI Realtime wire codec a reusable home (RFC #6592 P0a) (#7640)

Browse Source 
Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
psv666 authored and GitHub committed 2026-09-17 02:33:40 -07:00
1 parent 825ab20b07
commit e3be42e052
37 files changed
+3792 -1629
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vllm_omni/engine/duplex/audio.py
+38 -212
View File
@@ -1,218 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Compatibility re-export: the Realtime audio codec moved out of the duplex engine.
Decoding PCM / G.711 / WAV, resampling and re-encoding are wire-format work with
no duplex session in them, so the implementation now lives in
``vllm_omni.protocol.realtime.audio`` where a non-duplex Realtime surface can
use it too. Duplex code reaches it through ``vllm_omni.protocol.duplex``, the
one door the engine uses --- which is also where a duplex-specific conversion
would be introduced. Import from there in new code.
"""
from __future__ import annotations
import base64
import binascii
import io
import math
import wave
from vllm_omni.protocol.duplex import (
MAX_INPUT_SAMPLE_RATE_HZ,
MIN_INPUT_SAMPLE_RATE_HZ,
convert_input_audio_with_rate,
convert_output_audio,
decode_g711_alaw,
decode_g711_ulaw,
encode_float32_mono_wav_base64,
encode_g711_alaw,
encode_g711_ulaw,
resample_pcm16_mono,
validate_input_sample_rate_hz,
wav_payload_to_pcm16,
)
import numpy as np
from vllm_omni.utils.audio_resample import StreamingAudioResampler
try:
from audioop import alaw2lin, lin2alaw, lin2ulaw, ulaw2lin
except ImportError: # pragma: no cover - audioop is removed in newer Python.
alaw2lin = lin2alaw = lin2ulaw = ulaw2lin = None
MIN_INPUT_SAMPLE_RATE_HZ = 8_000
MAX_INPUT_SAMPLE_RATE_HZ = 192_000
def validate_input_sample_rate_hz(sample_rate_hz: object) -> int:
if (
isinstance(sample_rate_hz, bool)
or not isinstance(sample_rate_hz, int | float)
or (isinstance(sample_rate_hz, float) and not math.isfinite(sample_rate_hz))
):
raise ValueError("sample_rate_hz must be a finite integer")
rate = int(sample_rate_hz)
if rate != sample_rate_hz:
raise ValueError("sample_rate_hz must be an integer")
if not MIN_INPUT_SAMPLE_RATE_HZ <= rate <= MAX_INPUT_SAMPLE_RATE_HZ:
raise ValueError(f"sample_rate_hz must be between {MIN_INPUT_SAMPLE_RATE_HZ} and {MAX_INPUT_SAMPLE_RATE_HZ}")
return rate
def resample_pcm16_mono(raw: bytes, *, source_rate_hz: int, target_rate_hz: int) -> bytes:
if source_rate_hz <= 0 or target_rate_hz <= 0 or source_rate_hz == target_rate_hz:
return raw
samples = np.frombuffer(raw, dtype="<i2")
rate_gcd = math.gcd(source_rate_hz, target_rate_hz)
reduced_factor = max(source_rate_hz // rate_gcd, target_rate_hz // rate_gcd)
if reduced_factor > StreamingAudioResampler.max_polyphase_factor:
# The FIR size scales with the reduced ratio. Keep unusual client
# rates bounded while common audio rates use the high-quality path.
if samples.size <= 1:
return raw
target_size = max(1, int(round(samples.size * target_rate_hz / source_rate_hz)))
source_x = np.linspace(0.0, 1.0, num=samples.size, endpoint=True)
target_x = np.linspace(0.0, 1.0, num=target_size, endpoint=True)
resampled = np.interp(target_x, source_x, samples)
return np.clip(resampled, -32_768, 32_767).astype("<i2").tobytes()
resampler = StreamingAudioResampler(source_rate_hz, target_rate_hz)
normalized = samples.astype(np.float32) * np.float32(1.0 / 32768.0)
resampled = resampler.process(normalized, final=True)
return np.clip(np.rint(resampled * 32768.0), -32768, 32767).astype("<i2").tobytes()
def decode_g711_ulaw(raw: bytes) -> bytes:
if ulaw2lin is not None:
return ulaw2lin(raw, 2)
data = np.frombuffer(raw, dtype=np.uint8)
value = np.bitwise_not(data).astype(np.int16)
sign = value & 0x80
exponent = (value >> 4) & 0x07
mantissa = value & 0x0F
sample = ((mantissa << 3) + 0x84) << exponent
return np.where(sign != 0, -(sample - 0x84), sample - 0x84).astype("<i2").tobytes()
def decode_g711_alaw(raw: bytes) -> bytes:
if alaw2lin is not None:
return alaw2lin(raw, 2)
data = np.bitwise_xor(np.frombuffer(raw, dtype=np.uint8), 0x55).astype(np.int16)
sign = data & 0x80
exponent = (data >> 4) & 0x07
mantissa = data & 0x0F
sample = np.where(exponent == 0, (mantissa << 4) + 8, ((mantissa << 4) + 0x108) << (exponent - 1))
return np.where(sign != 0, sample, -sample).astype("<i2").tobytes()
def encode_g711_ulaw(raw: bytes) -> bytes:
if lin2ulaw is not None:
return lin2ulaw(raw, 2)
pcm = np.clip(np.frombuffer(raw, dtype="<i2").astype(np.int32), -32635, 32635)
sign = np.where(pcm < 0, 0x80, 0)
magnitude = np.abs(pcm) + 0x84
exponent = np.zeros_like(magnitude)
for exp in range(7):
exponent = np.where(magnitude > (0xFF << exp), exp + 1, exponent)
mantissa = (magnitude >> (exponent + 3)) & 0x0F
return (np.bitwise_not(sign | (exponent << 4) | mantissa) & 0xFF).astype(np.uint8).tobytes()
def encode_g711_alaw(raw: bytes) -> bytes:
if lin2alaw is not None:
return lin2alaw(raw, 2)
pcm = np.frombuffer(raw, dtype="<i2").astype(np.int32)
sign = np.where(pcm >= 0, 0x80, 0x00)
magnitude = np.abs(pcm)
exponent = np.zeros_like(magnitude)
for exp in range(1, 8):
exponent = np.where(magnitude >= (1 << (exp + 7)), exp, exponent)
mantissa = np.where(
exponent == 0,
(magnitude >> 4) & 0x0F,
(magnitude >> (exponent + 3)) & 0x0F,
)
return ((sign | (exponent << 4) | mantissa) ^ 0x55).astype(np.uint8).tobytes()
def wav_payload_to_pcm16(raw: bytes) -> tuple[bytes | None, int | None]:
try:
with wave.open(io.BytesIO(raw), "rb") as wav_file:
channels = wav_file.getnchannels()
sample_width = wav_file.getsampwidth()
sample_rate_hz = wav_file.getframerate()
frames = wav_file.readframes(wav_file.getnframes())
if sample_width != 2:
return None, sample_rate_hz
if channels <= 1:
return frames, sample_rate_hz
pcm = np.frombuffer(frames, dtype="<i2").reshape(-1, channels)
mono = np.mean(pcm.astype(np.float32), axis=1)
return np.clip(mono, -32768, 32767).astype("<i2").tobytes(), sample_rate_hz
except (EOFError, ValueError, wave.Error):
return None, None
def encode_float32_mono_wav_base64(
samples: np.ndarray,
*,
sample_rate_hz: int,
) -> str:
"""Package normalized mono float32 samples as a PCM16 WAV data payload."""
rate = validate_input_sample_rate_hz(sample_rate_hz)
normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1)
pcm16 = np.clip(np.rint(normalized * 32768.0), -32768, 32767).astype("<i2")
with io.BytesIO() as buffer:
with wave.open(buffer, "wb") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(rate)
wav_file.writeframes(pcm16.tobytes())
return base64.b64encode(buffer.getvalue()).decode("ascii")
def convert_input_audio_with_rate(
audio: object,
fmt: object,
*,
sample_rate_hz: int | float | None = None,
target_sample_rate_hz: int = 16_000,
) -> tuple[object, object, int | float | None]:
if not isinstance(audio, str) or not isinstance(fmt, str):
return audio, fmt, sample_rate_hz
normalized = fmt.lower()
if normalized not in {"pcm16", "pcm_s16le", "s16le", "g711_ulaw", "g711_alaw"}:
return audio, fmt, sample_rate_hz
if isinstance(sample_rate_hz, int | float):
sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz)
try:
raw = base64.b64decode(audio.strip(), validate=False)
except (binascii.Error, ValueError):
return audio, fmt, sample_rate_hz
if normalized == "g711_ulaw":
raw = decode_g711_ulaw(raw)
sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
elif normalized == "g711_alaw":
raw = decode_g711_alaw(raw)
sample_rate_hz = sample_rate_hz if isinstance(sample_rate_hz, int | float) else 8_000
elif len(raw) % 2:
return audio, fmt, sample_rate_hz
if isinstance(sample_rate_hz, int | float) and int(sample_rate_hz) != target_sample_rate_hz:
raw = resample_pcm16_mono(
raw,
source_rate_hz=int(sample_rate_hz),
target_rate_hz=target_sample_rate_hz,
)
sample_rate_hz = target_sample_rate_hz
pcm = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
encoded = base64.b64encode(np.ascontiguousarray(pcm, dtype="<f4").tobytes()).decode("ascii")
return encoded, "pcm_f32le", sample_rate_hz
def convert_output_audio(
audio: str,
*,
source_fmt: str,
target_fmt: str,
source_sample_rate_hz: int | None = None,
target_sample_rate_hz: int | None = None,
) -> tuple[str, str, int | None]:
target = target_fmt.lower()
if target not in {"g711_ulaw", "g711_alaw"}:
return audio, source_fmt, source_sample_rate_hz
try:
raw = base64.b64decode(audio, validate=False)
except (binascii.Error, ValueError):
return audio, source_fmt, source_sample_rate_hz
source = source_fmt.lower()
if source == "wav":
pcm_raw, wav_rate = wav_payload_to_pcm16(raw)
if pcm_raw is None:
return audio, source_fmt, source_sample_rate_hz
raw = pcm_raw
source_sample_rate_hz = source_sample_rate_hz or wav_rate
elif source not in {"pcm", "pcm16", "pcm_s16le", "s16le"} or len(raw) % 2:
return audio, source_fmt, source_sample_rate_hz
target_rate = target_sample_rate_hz or 8_000
if source_sample_rate_hz is not None:
raw = resample_pcm16_mono(raw, source_rate_hz=source_sample_rate_hz, target_rate_hz=target_rate)
encoded = encode_g711_ulaw(raw) if target == "g711_ulaw" else encode_g711_alaw(raw)
return base64.b64encode(encoded).decode("ascii"), target, target_rate
__all__ = [
"MAX_INPUT_SAMPLE_RATE_HZ",
"MIN_INPUT_SAMPLE_RATE_HZ",
"convert_input_audio_with_rate",
"convert_output_audio",
"decode_g711_alaw",
"decode_g711_ulaw",
"encode_float32_mono_wav_base64",
"encode_g711_alaw",
"encode_g711_ulaw",
"resample_pcm16_mono",
"validate_input_sample_rate_hz",
"wav_payload_to_pcm16",
]
vllm_omni/engine/duplex/commands.py
+110 -128
View File
@@ -1,41 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Typed commands accepted by a duplex session.
"""The command vocabulary one duplex session accepts, plus its mailbox rendering.
``DuplexSessionHandle.submit()`` takes one of these; the websocket handler and
``InlineDuplexClient`` build them with :func:`command_from_realtime` from the
OpenAI Realtime client event vocabulary. Every command can also render the
session-internal mailbox payload (``payload()``), which is the dictionary
vocabulary the session runner bodies were written against.
``InlineDuplexClient`` build them with :func:`command_from_realtime`.
The *wire* half of each command --- which client event it decodes from and what
fields survive decoding --- now lives in ``vllm_omni.protocol.duplex.commands``,
which carries the whole vocabulary: the eight Tier 1 classes re-exported from
``vllm_omni.protocol.realtime.commands``, the two Tier 2 ones and the seven
Tier 3 ones. The engine imports that module and never the Tier 1 one directly,
so a command that later grows a duplex extension changes one file
(RFC #6592 P0a).
The *engine* half stays here, because it is not wire contract at all:
``payload()`` renders the session-internal mailbox dictionary the runner bodies
were written against, and its ``type`` is the mailbox channel rather than the
client event. Those two genuinely differ --- ``session.update`` and
``conversation.item.create`` / ``.delete`` / ``.truncate`` all travel on the
``turn.signal`` channel --- which is why the halves are separated instead of
the classes being relocated wholesale.
"""
from __future__ import annotations
import base64
from collections.abc import Mapping
from dataclasses import dataclass, field, fields
from dataclasses import dataclass, fields
from typing import TYPE_CHECKING, ClassVar
from vllm_omni.protocol.duplex import RealtimeProtocolError
from vllm_omni.protocol.duplex import commands as _duplex_wire
from vllm_omni.protocol.duplex.commands import RealtimeCommand
if TYPE_CHECKING:
from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
from vllm_omni.protocol.duplex import RealtimeInputDefaults
class DuplexCommandError(ValueError):
"""A client payload could not be turned into a command."""
class DuplexCommandError(RealtimeProtocolError):
"""A client payload could not be turned into a duplex command.
def __init__(self, message: str, *, code: str = "bad_event", event_id: str | None = None) -> None:
super().__init__(message)
self.code = code
self.event_id = event_id
The duplex name for a Realtime protocol error: same ``code`` /
``event_id`` contract, so the error envelope is rendered identically
whichever consumer raised it.
"""
@dataclass(frozen=True, slots=True, kw_only=True)
class DuplexCommand:
class DuplexCommand(RealtimeCommand):
"""A Realtime command as the duplex engine handles it.
Adds the mailbox channel (``type``) and its rendering on top of the wire
command; every concrete class below pairs this with its protocol twin.
"""
#: Mailbox event type this command renders to (see ``payload()``).
type: ClassVar[str] = ""
#: Client correlation id (OpenAI ``event_id``), echoed on error events.
event_id: str | None = None
def payload(self) -> dict[str, object]:
"""Render the session-internal mailbox dictionary."""
@@ -56,19 +77,23 @@ class DuplexCommand:
return data
# ---- GA commands ----
@dataclass(frozen=True, slots=True, kw_only=True)
class AppendAudio(DuplexCommand):
class UpdateSession(DuplexCommand, _duplex_wire.UpdateSession):
type: ClassVar[str] = "turn.signal"
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
data["event"] = "session.update"
data["payload"] = dict(data.pop("patch", {}) or {})
return data
@dataclass(frozen=True, slots=True, kw_only=True)
class AppendAudio(DuplexCommand, _duplex_wire.AppendAudio):
type: ClassVar[str] = "input_audio_buffer.append"
#: Raw audio bytes in ``format`` at ``sample_rate_hz`` (base64 only on the wire).
audio: bytes
format: str = "pcm16"
sample_rate_hz: int | None = None
is_speech: bool | None = None
video_frames: tuple[str, ...] = ()
duration_ms: int | None = None
audio_end_ms: int | None = None
#: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
hints: Mapping[str, object] = field(default_factory=dict)
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
@@ -90,21 +115,8 @@ class AppendAudio(DuplexCommand):
@dataclass(frozen=True, slots=True, kw_only=True)
class AppendText(DuplexCommand):
type: ClassVar[str] = "input.text.append"
text: str
@dataclass(frozen=True, slots=True, kw_only=True)
class Commit(DuplexCommand):
class Commit(DuplexCommand, _duplex_wire.Commit):
type: ClassVar[str] = "input_audio_buffer.commit"
final: bool = True
#: ``None`` means "no explicit request": the runner decides on commit (auto-response
#: sessions answer on their own); ``True`` / ``False`` force it.
create_response: bool | None = None
is_speech: bool | None = None
#: Realtime conversation item created for this commit (wire correlation only).
realtime_item_id: str | None = None
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
@@ -114,10 +126,18 @@ class Commit(DuplexCommand):
@dataclass(frozen=True, slots=True, kw_only=True)
class CreateResponse(DuplexCommand):
class ClearInput(DuplexCommand, _duplex_wire.ClearInput):
type: ClassVar[str] = "input_audio_buffer.clear"
@dataclass(frozen=True, slots=True, kw_only=True)
class ClearOutputAudio(DuplexCommand, _duplex_wire.ClearOutputAudio):
type: ClassVar[str] = "output_audio_buffer.clear"
@dataclass(frozen=True, slots=True, kw_only=True)
class CreateResponse(DuplexCommand, _duplex_wire.CreateResponse):
type: ClassVar[str] = "response.create"
#: Raw Realtime ``response`` object (instructions, voice, modalities, ...).
options: Mapping[str, object] = field(default_factory=dict)
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
@@ -128,84 +148,13 @@ class CreateResponse(DuplexCommand):
@dataclass(frozen=True, slots=True, kw_only=True)
class ClearInput(DuplexCommand):
type: ClassVar[str] = "input_audio_buffer.clear"
@dataclass(frozen=True, slots=True, kw_only=True)
class CancelInput(DuplexCommand):
type: ClassVar[str] = "input.cancel"
@dataclass(frozen=True, slots=True, kw_only=True)
class CancelResponse(DuplexCommand):
class CancelResponse(DuplexCommand, _duplex_wire.CancelResponse):
type: ClassVar[str] = "response.cancel"
response_id: str | None = None
@dataclass(frozen=True, slots=True, kw_only=True)
class BargeIn(DuplexCommand):
type: ClassVar[str] = "barge_in"
@dataclass(frozen=True, slots=True, kw_only=True)
class ClearOutputAudio(DuplexCommand):
type: ClassVar[str] = "output_audio_buffer.clear"
#: Explicit response to clear; ``None`` targets the active/last response.
response_id: str | None = None
@dataclass(frozen=True, slots=True, kw_only=True)
class SignalTurn(DuplexCommand):
"""Generic ``turn.signal`` (local turn transitions such as ``user_started``)."""
class CreateItem(DuplexCommand, _duplex_wire.CreateItem):
type: ClassVar[str] = "turn.signal"
event: str
signal_payload: Mapping[str, object] = field(default_factory=dict)
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
signal_payload = data.pop("signal_payload", None)
if isinstance(signal_payload, Mapping) and signal_payload:
data["payload"] = dict(signal_payload)
return data
@dataclass(frozen=True, slots=True, kw_only=True)
class UpdateSession(DuplexCommand):
"""``session.update``: a Realtime ``session`` object patch."""
type: ClassVar[str] = "turn.signal"
patch: Mapping[str, object] = field(default_factory=dict)
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
data["event"] = "session.update"
data["payload"] = dict(data.pop("patch", {}) or {})
return data
@dataclass(frozen=True, slots=True, kw_only=True)
class AckPlayback(DuplexCommand):
type: ClassVar[str] = "playback.ack"
played_ms: int
committed_ms: int | None = None
response_id: str | None = None
item_id: str | None = None
@dataclass(frozen=True, slots=True, kw_only=True)
class Heartbeat(DuplexCommand):
type: ClassVar[str] = "session.heartbeat"
@dataclass(frozen=True, slots=True, kw_only=True)
class CreateItem(DuplexCommand):
"""``conversation.item.create`` (history injection or function-call output)."""
type: ClassVar[str] = "turn.signal"
item: Mapping[str, object]
previous_item_id: str | None = None
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
@@ -219,9 +168,8 @@ class CreateItem(DuplexCommand):
@dataclass(frozen=True, slots=True, kw_only=True)
class DeleteItem(DuplexCommand):
class DeleteItem(DuplexCommand, _duplex_wire.DeleteItem):
type: ClassVar[str] = "turn.signal"
item_id: str
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
@@ -231,11 +179,8 @@ class DeleteItem(DuplexCommand):
@dataclass(frozen=True, slots=True, kw_only=True)
class TruncateItem(DuplexCommand):
class TruncateItem(DuplexCommand, _duplex_wire.TruncateItem):
type: ClassVar[str] = "turn.signal"
item_id: str
audio_end_ms: int
content_index: int = 0
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
@@ -248,15 +193,51 @@ class TruncateItem(DuplexCommand):
return data
# ---- duplex extension commands ----
@dataclass(frozen=True, slots=True, kw_only=True)
class CloseSession(DuplexCommand):
"""Graceful close requested through the command stream (``session.close``)."""
class AppendText(DuplexCommand, _duplex_wire.AppendText):
type: ClassVar[str] = "input.text.append"
@dataclass(frozen=True, slots=True, kw_only=True)
class CancelInput(DuplexCommand, _duplex_wire.CancelInput):
type: ClassVar[str] = "input.cancel"
@dataclass(frozen=True, slots=True, kw_only=True)
class BargeIn(DuplexCommand, _duplex_wire.BargeIn):
type: ClassVar[str] = "barge_in"
@dataclass(frozen=True, slots=True, kw_only=True)
class SignalTurn(DuplexCommand, _duplex_wire.SignalTurn):
type: ClassVar[str] = "turn.signal"
def payload(self) -> dict[str, object]:
data = DuplexCommand.payload(self)
signal_payload = data.pop("signal_payload", None)
if isinstance(signal_payload, Mapping) and signal_payload:
data["payload"] = dict(signal_payload)
return data
@dataclass(frozen=True, slots=True, kw_only=True)
class AckPlayback(DuplexCommand, _duplex_wire.AckPlayback):
type: ClassVar[str] = "playback.ack"
@dataclass(frozen=True, slots=True, kw_only=True)
class Heartbeat(DuplexCommand, _duplex_wire.Heartbeat):
type: ClassVar[str] = "session.heartbeat"
@dataclass(frozen=True, slots=True, kw_only=True)
class CloseSession(DuplexCommand, _duplex_wire.CloseSession):
type: ClassVar[str] = "session.close"
reason: str = "client_close"
#: Realtime client event types that map onto commands (wire vocabulary).
REALTIME_COMMAND_TYPES: frozenset[str] = frozenset(
{
"input_audio_buffer.append",
@@ -300,7 +281,6 @@ def command_from_realtime(
__all__ = [
"REALTIME_COMMAND_TYPES",
"AckPlayback",
"AppendAudio",
"AppendText",
@@ -317,6 +297,8 @@ __all__ = [
"DuplexCommand",
"DuplexCommandError",
"Heartbeat",
"REALTIME_COMMAND_TYPES",
"RealtimeCommand",
"SignalTurn",
"TruncateItem",
"UpdateSession",
vllm_omni/engine/duplex/config.py
+18 -18
View File
@@ -339,26 +339,26 @@ class DuplexSessionConfig:
``_session_create_from_realtime``.
"""
from vllm_omni.engine.duplex.realtime_commands import (
RealtimeInputDefaults,
DUPLEX_REALTIME_CAPABILITIES,
duplex_response_format,
)
from vllm_omni.engine.duplex.turn_detection import normalize_turn_detection_session_payload
from vllm_omni.protocol.duplex import (
RealtimeInputDefaults,
input_audio_transcription_config,
json_safe_realtime_payload,
realtime_max_output_tokens,
realtime_overlap_fields,
validate_realtime_session_audio_formats,
)
from vllm_omni.engine.duplex.turn_detection import (
normalize_turn_detection_session_payload,
validate_realtime_turn_detection,
validate_session_payload,
)
payload: dict[str, object] = dict(session_payload)
format_error = validate_realtime_session_audio_formats(payload)
if format_error is not None:
raise DuplexConfigError(format_error, code="unsupported_audio_format")
turn_detection_error = validate_realtime_turn_detection(payload)
if turn_detection_error is not None:
raise DuplexConfigError(turn_detection_error, code="unsupported_turn_detection", param="turn_detection")
# One session check for every consumer (ENTRY-INV-002): the same
# capability object ``translate_realtime_command`` uses, so a session
# object is accepted or refused identically whichever door it came in.
rejection = validate_session_payload(payload, capabilities=DUPLEX_REALTIME_CAPABILITIES)
if rejection is not None:
raise DuplexConfigError(rejection.message, code=rejection.code, param=rejection.param)
normalize_turn_detection_session_payload(payload)
defaults = RealtimeInputDefaults().with_session_payload(payload)
payload.update(realtime_overlap_fields(payload))
@@ -446,9 +446,9 @@ class DuplexSessionConfig:
when the patch changes something a live session cannot change.
``audio_started`` is ``playback.generated_ms > 0 or playback.sent_ms > 0``.
"""
from vllm_omni.engine.duplex.realtime_commands import (
from vllm_omni.engine.duplex.realtime_commands import duplex_response_format
from vllm_omni.protocol.duplex import (
REALTIME_OUTPUT_AUDIO_FORMATS,
duplex_response_format,
input_audio_transcription_config,
json_safe_realtime_payload,
parse_realtime_audio_format,
@@ -629,9 +629,9 @@ class ResponseCreateOptions:
for options a model-native duplex session cannot apply per response.
Private runtime keys in ``extra_body`` are dropped.
"""
from vllm_omni.engine.duplex.realtime_commands import (
from vllm_omni.engine.duplex.realtime_commands import duplex_response_format
from vllm_omni.protocol.duplex import (
REALTIME_OUTPUT_AUDIO_FORMATS,
duplex_response_format,
parse_realtime_audio_format,
realtime_max_output_tokens,
)
@@ -782,14 +782,14 @@ def realtime_item_to_history_message(item: object) -> dict[str, object] | None:
def realtime_max_output_tokens(value: object) -> int | None:
"""Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
from vllm_omni.engine.duplex.realtime_commands import realtime_max_output_tokens as _impl
from vllm_omni.protocol.duplex import realtime_max_output_tokens as _impl
return _impl(value)
def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
"""Return the ``input_audio_transcription`` object of a Realtime session payload."""
from vllm_omni.engine.duplex.realtime_commands import input_audio_transcription_config as _impl
from vllm_omni.protocol.duplex import input_audio_transcription_config as _impl
return _impl(session_payload)
vllm_omni/engine/duplex/events.py
+69 -668
View File
@@ -1,677 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Typed events emitted by a duplex session.
"""The event vocabulary one duplex session speaks, assembled from two layers.
Every public event is a frozen dataclass with explicit fields. The Realtime
wire JSON is *derived* from those fields by :meth:`DuplexEvent.to_realtime`,
which is pure: it never consults session state. The stateful part (response /
item ids, content-part bookkeeping) lives in
``vllm_omni.engine.duplex.realtime_events`` on the session runner and is
consumed when the events are *constructed*.
The classes themselves no longer live here. ``vllm_omni.protocol.duplex.events``
carries the whole vocabulary --- 22 Tier 1 classes re-exported from
``vllm_omni.protocol.realtime.events``, 8 Tier 2 ones and 12 Tier 3 ones --- so
a non-duplex Realtime consumer can take the Tier 1 half and its wire rendering
without the duplex session control plane (RFC #6592 P0a). The engine imports
the duplex module and never the Tier 1 one directly.
``session_id`` / ``epoch`` are bound by
``DuplexSessionManager.emit`` when an event leaves the runner; producers
(projector, serving handshake helpers) build events with the defaults.
What is genuinely engine-side stays: :data:`DOMAIN_TERMINAL_EVENTS` and
:data:`MODEL_OUTPUT_EVENTS` are the session runner's epoch-filter policy, not
wire contract --- they name *internal* event types, and which of them may never
be dropped as stale is a runtime decision.
``DuplexEvent`` is re-exported as the name the engine and the clients already
use. It is an alias of
:class:`~vllm_omni.protocol.realtime.events.RealtimeEvent`, not a subclass,
because an event has no engine-internal half --- what the session emits is
exactly what the socket sends. A *command* does have one (``payload()`` renders
the runner's mailbox), which is why ``DuplexCommand`` is a real class.
"""
from __future__ import annotations
import base64
from collections.abc import Mapping
from dataclasses import dataclass, field, fields
from typing import ClassVar, cast
from uuid import uuid4
from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
from vllm_omni.protocol.duplex.events import (
AudioDelta,
AudioDone,
ContentPartAdded,
ContentPartDone,
DuplexEvent,
DuplexRawEvent,
ErrorEvent,
FunctionCallArgumentsDelta,
FunctionCallArgumentsDone,
InputCleared,
InputCommitted,
InputTranscriptionCompleted,
ItemAdded,
ItemCreated,
ItemDeleted,
ItemDone,
ItemRetrieved,
ItemTruncated,
Listen,
OutputAudioCleared,
OutputItemAdded,
OutputItemDone,
OverlapDecision,
PlaybackAcknowledged,
RateLimitsUpdated,
ResponseCreated,
ResponseDone,
SessionClosed,
SessionCreated,
SessionExpired,
SessionHeartbeatAck,
SessionReplaced,
SessionResumed,
SessionResyncRequired,
SessionUpdated,
Speak,
SpeechStarted,
SpeechStopped,
TextDelta,
TextDone,
TranscriptDelta,
TranscriptDone,
TurnEvent,
error_event,
new_event_id,
)
def new_event_id() -> str:
return f"event_{uuid4().hex}"
#: OpenAI Realtime ``error.type`` for each internal error code.
REALTIME_ERROR_TYPES_BY_CODE: dict[str, str] = {
"bad_event": "invalid_request_error",
"bad_audio": "invalid_request_error",
"config_timeout": "invalid_request_error",
"invalid_json": "invalid_request_error",
"event_too_large": "invalid_request_error",
"unknown_event": "invalid_request_error",
"internal_error": "server_error",
"runtime_append_failed": "server_error",
"runtime_append_task_failed": "server_error",
"runtime_signal_failed": "server_error",
"runtime_abort_failed": "server_error",
"runtime_data_plane_stream_failed": "server_error",
"runtime_data_plane_text_without_audio": "server_error",
"resource_exhausted": "rate_limit_error",
"session_exists": "invalid_request_error",
"session_closed": "invalid_request_error",
"unknown_session": "invalid_request_error",
"invalid_duplex_runtime_config": "invalid_request_error",
"instructions_update_unsupported": "invalid_request_error",
"persona_update_unsupported": "invalid_request_error",
"voice_update_unsupported": "invalid_request_error",
"unsupported_nemotron_duplex_mode": "invalid_request_error",
"unsupported_native_response_options": "invalid_request_error",
"runtime_touch_failed": "server_error",
"engine_error": "server_error",
"input_backpressure": "rate_limit_error",
"response_already_active": "invalid_request_error",
"response_not_active": "invalid_request_error",
"response_create_without_input": "invalid_request_error",
"text_only_turn_unsupported": "invalid_request_error",
"commit_aborted": "server_error",
"input_audio_buffer_empty": "invalid_request_error",
"missing_item_id": "invalid_request_error",
"item_not_found": "invalid_request_error",
"playback_item_mismatch": "invalid_request_error",
"playback_item_not_found": "invalid_request_error",
"playback_ack_too_late": "invalid_request_error",
"unsupported_audio_format": "invalid_request_error",
"unsupported_turn_detection": "invalid_request_error",
"unsupported_ref_audio_path": "invalid_request_error",
"ref_audio_required": "invalid_request_error",
"model_update_unsupported": "invalid_request_error",
"voice_update_after_audio_unsupported": "invalid_request_error",
"ref_audio_update_unsupported": "invalid_request_error",
"native_text_append_unsupported": "invalid_request_error",
"invalid_video_frames": "invalid_request_error",
"invalid_function_call_output": "invalid_request_error",
"server_vad_unavailable": "server_error",
}
_IDENTITY_FIELDS = frozenset({"session_id", "epoch", "event_id"})
def _wire_value(value: object) -> object:
if isinstance(value, Mapping):
return {str(k): _wire_value(v) for k, v in value.items()}
if isinstance(value, tuple | list):
return [_wire_value(v) for v in value]
return value
@dataclass(frozen=True, slots=True, kw_only=True)
class DuplexEvent:
"""Base of every public session event."""
#: Realtime ``type`` this class renders to.
wire_type: ClassVar[str] = ""
#: Fields that are omitted from the wire object when ``None``.
optional_wire_fields: ClassVar[frozenset[str]] = frozenset()
session_id: str = ""
epoch: int | None = None
#: Server event id (OpenAI ``event_id``); generated at construction.
event_id: str = field(default_factory=new_event_id)
# ---- identity ----
@property
def type(self) -> str:
return self.wire_type
@property
def is_terminal(self) -> bool:
return False
# ---- generic accessors (None when the class has no such field) ----
@property
def response_id(self) -> str | None:
return None
@property
def item_id(self) -> str | None:
return None
@property
def text(self) -> str | None:
return None
@property
def audio(self) -> bytes | None:
return None
# ---- wire rendering (pure) ----
def _wire_fields(self) -> dict[str, object]:
data: dict[str, object] = {}
for f in fields(self):
if f.name in _IDENTITY_FIELDS:
continue
value = getattr(self, f.name)
if value is None and f.name in self.optional_wire_fields:
continue
data[f.name] = _wire_value(value)
return data
def to_realtime(self) -> dict[str, object]:
"""The Realtime wire JSON object for this event (derived from the fields)."""
return {"type": self.wire_type, "event_id": self.event_id, **self._wire_fields()}
def __repr__(self) -> str: # pragma: no cover - debugging aid
return f"{type(self).__name__}(type={self.wire_type!r}, session_id={self.session_id!r}, epoch={self.epoch!r})"
# ---- session lifecycle ----
@dataclass(frozen=True, slots=True, kw_only=True)
class SessionCreated(DuplexEvent):
wire_type = "session.created"
optional_wire_fields = frozenset({"attachment_generation", "resume_token"})
session: Mapping[str, object] = field(default_factory=dict)
#: Transport resume credentials (set by the websocket handler when resume is supported).
attachment_generation: int | None = None
resume_token: str | None = None
@dataclass(frozen=True, slots=True, kw_only=True)
class SessionUpdated(DuplexEvent):
wire_type = "session.updated"
session: Mapping[str, object] = field(default_factory=dict)
@dataclass(frozen=True, slots=True, kw_only=True)
class SessionResumed(SessionCreated):
wire_type = "session.resumed"
def _wire_fields(self) -> dict[str, object]:
return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
@dataclass(frozen=True, slots=True, kw_only=True)
class SessionReplaced(DuplexEvent):
"""A newer connection took the session over (sent to the replaced socket)."""
wire_type = "session.replaced"
attachment_generation: int = 0
def _wire_fields(self) -> dict[str, object]:
return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
@dataclass(frozen=True, slots=True, kw_only=True)
class SessionResyncRequired(DuplexEvent):
"""The replay journal cannot bridge the gap; the client must start over."""
wire_type = "session.resync_required"
reason: str = "journal_gap"
def _wire_fields(self) -> dict[str, object]:
return {"session_id": self.session_id, **DuplexEvent._wire_fields(self)}
@dataclass(frozen=True, slots=True, kw_only=True)
class SessionClosed(DuplexEvent):
wire_type = "session.closed"
reason: str = "closed"
#: The session-internal close event (kept for clients that inspect it).
details: Mapping[str, object] = field(default_factory=dict)
@property
def is_terminal(self) -> bool:
return True
def _wire_fields(self) -> dict[str, object]:
return {"session_id": self.session_id, "reason": self.reason, "event": _wire_value(self.details)}
@dataclass(frozen=True, slots=True, kw_only=True)
class SessionExpired(SessionClosed):
wire_type = "session.expired"
def _wire_fields(self) -> dict[str, object]:
return {"session_id": self.session_id, "reason": self.reason}
@dataclass(frozen=True, slots=True, kw_only=True)
class SessionHeartbeatAck(DuplexEvent):
wire_type = "session.heartbeat_ack"
def _wire_fields(self) -> dict[str, object]:
return {"session_id": self.session_id}
@dataclass(frozen=True, slots=True, kw_only=True)
class TurnEvent(DuplexEvent):
"""Turn-state transition (``turn.event``)."""
wire_type = "turn.event"
event: str = ""
turn_state: str = ""
def _wire_fields(self) -> dict[str, object]:
return {"session_id": self.session_id, "event": self.event, "turn_state": self.turn_state, "epoch": self.epoch}
# ---- response lifecycle ----
@dataclass(frozen=True, slots=True, kw_only=True)
class _ResponseEvent(DuplexEvent):
"""Events addressed to one response (and usually one output item / content part)."""
response_id: str | None = None # type: ignore[assignment]
item_id: str | None = None # type: ignore[assignment]
output_index: int = 0
content_index: int = 0
@dataclass(frozen=True, slots=True, kw_only=True)
class ResponseCreated(DuplexEvent):
wire_type = "response.created"
response_id: str | None = None # type: ignore[assignment]
response: Mapping[str, object] = field(default_factory=dict)
@dataclass(frozen=True, slots=True, kw_only=True)
class Listen(DuplexEvent):
"""The model decided to keep listening (no spoken response for this turn)."""
wire_type = "response.listen"
optional_wire_fields = frozenset({"response_id"})
response_id: str | None = None # type: ignore[assignment]
details: Mapping[str, object] = field(default_factory=dict)
def _wire_fields(self) -> dict[str, object]:
response: dict[str, object] = {
"object": "realtime.response",
"status": "listening",
"metadata": _wire_value(self.details),
}
data: dict[str, object] = {"session_id": self.session_id, "epoch": self.epoch, "response": response}
if self.response_id:
response["id"] = self.response_id
data["response_id"] = self.response_id
return data
@dataclass(frozen=True, slots=True, kw_only=True)
class Speak(_ResponseEvent):
wire_type = "response.speak"
metadata: Mapping[str, object] = field(default_factory=dict)
@dataclass(frozen=True, slots=True, kw_only=True)
class OverlapDecision(DuplexEvent):
wire_type = "overlap.decision"
policy: str | None = None
action: str | None = None
reason: str | None = None
details: Mapping[str, object] = field(default_factory=dict)
def _wire_fields(self) -> dict[str, object]:
return {
"session_id": self.session_id,
"epoch": self.epoch,
"policy": self.policy,
"action": self.action,
"reason": self.reason,
"metadata": _wire_value(self.details),
}
@dataclass(frozen=True, slots=True, kw_only=True)
class OutputItemAdded(DuplexEvent):
wire_type = "response.output_item.added"
response_id: str | None = None # type: ignore[assignment]
output_index: int = 0
item: Mapping[str, object] = field(default_factory=dict)
@property
def item_id(self) -> str | None:
value = self.item.get("id")
return value if isinstance(value, str) else None
@dataclass(frozen=True, slots=True, kw_only=True)
class OutputItemDone(OutputItemAdded):
wire_type = "response.output_item.done"
@dataclass(frozen=True, slots=True, kw_only=True)
class ContentPartAdded(_ResponseEvent):
wire_type = "response.content_part.added"
part: Mapping[str, object] = field(default_factory=dict)
@dataclass(frozen=True, slots=True, kw_only=True)
class ContentPartDone(ContentPartAdded):
wire_type = "response.content_part.done"
@dataclass(frozen=True, slots=True, kw_only=True)
class AudioDelta(_ResponseEvent):
wire_type = "response.output_audio.delta"
optional_wire_fields = frozenset({"sample_rate_hz", "metadata"})
#: Base64 audio in ``format``.
delta: str = ""
format: str = "pcm16"
sample_rate_hz: int | None = None
metadata: Mapping[str, object] | None = None
@property
def audio(self) -> bytes | None:
try:
return base64.b64decode(self.delta)
except (ValueError, TypeError):
return None
@dataclass(frozen=True, slots=True, kw_only=True)
class AudioDone(_ResponseEvent):
wire_type = "response.output_audio.done"
@dataclass(frozen=True, slots=True, kw_only=True)
class TranscriptDelta(_ResponseEvent):
wire_type = "response.output_audio_transcript.delta"
delta: str = ""
@property
def text(self) -> str | None:
return self.delta
@dataclass(frozen=True, slots=True, kw_only=True)
class TranscriptDone(_ResponseEvent):
wire_type = "response.output_audio_transcript.done"
transcript: str = ""
@property
def text(self) -> str | None:
return self.transcript
@dataclass(frozen=True, slots=True, kw_only=True)
class TextDelta(_ResponseEvent):
wire_type = "response.output_text.delta"
delta: str = ""
@property
def text(self) -> str | None:
return self.delta
@dataclass(frozen=True, slots=True, kw_only=True)
class TextDone(_ResponseEvent):
wire_type = "response.output_text.done"
text: str = "" # type: ignore[assignment]
@dataclass(frozen=True, slots=True, kw_only=True)
class FunctionCallArgumentsDelta(DuplexEvent):
wire_type = "response.function_call_arguments.delta"
response_id: str | None = None # type: ignore[assignment]
item_id: str | None = None # type: ignore[assignment]
output_index: int = 0
call_id: str = ""
delta: str = ""
@dataclass(frozen=True, slots=True, kw_only=True)
class FunctionCallArgumentsDone(DuplexEvent):
wire_type = "response.function_call_arguments.done"
response_id: str | None = None # type: ignore[assignment]
item_id: str | None = None # type: ignore[assignment]
output_index: int = 0
call_id: str = ""
arguments: str = ""
@dataclass(frozen=True, slots=True, kw_only=True)
class ResponseDone(DuplexEvent):
wire_type = "response.done"
response_id: str | None = None # type: ignore[assignment]
response: Mapping[str, object] = field(default_factory=dict)
@property
def status(self) -> str | None:
value = self.response.get("status")
return value if isinstance(value, str) else None
@dataclass(frozen=True, slots=True, kw_only=True)
class RateLimitsUpdated(DuplexEvent):
wire_type = "rate_limits.updated"
rate_limits: tuple[Mapping[str, object], ...] = ()
# ---- input buffer / conversation ----
@dataclass(frozen=True, slots=True, kw_only=True)
class InputCommitted(DuplexEvent):
wire_type = "input_audio_buffer.committed"
previous_item_id: str | None = None
item_id: str | None = None # type: ignore[assignment]
#: The session-internal ``input.committed`` event.
details: Mapping[str, object] = field(default_factory=dict)
def _wire_fields(self) -> dict[str, object]:
return {
"previous_item_id": self.previous_item_id,
"item_id": self.item_id,
"event": _wire_value(self.details),
}
@dataclass(frozen=True, slots=True, kw_only=True)
class InputCleared(DuplexEvent):
wire_type = "input_audio_buffer.cleared"
@dataclass(frozen=True, slots=True, kw_only=True)
class SpeechStarted(DuplexEvent):
wire_type = "input_audio_buffer.speech_started"
audio_start_ms: int = 0
item_id: str | None = None # type: ignore[assignment]
@dataclass(frozen=True, slots=True, kw_only=True)
class SpeechStopped(DuplexEvent):
wire_type = "input_audio_buffer.speech_stopped"
audio_end_ms: int = 0
item_id: str | None = None # type: ignore[assignment]
@dataclass(frozen=True, slots=True, kw_only=True)
class _ItemEvent(DuplexEvent):
previous_item_id: str | None = None
item: Mapping[str, object] = field(default_factory=dict)
@property
def item_id(self) -> str | None:
value = self.item.get("id")
return value if isinstance(value, str) else None
@dataclass(frozen=True, slots=True, kw_only=True)
class ItemAdded(_ItemEvent):
wire_type = "conversation.item.added"
@dataclass(frozen=True, slots=True, kw_only=True)
class ItemCreated(_ItemEvent):
wire_type = "conversation.item.created"
@dataclass(frozen=True, slots=True, kw_only=True)
class ItemDone(_ItemEvent):
wire_type = "conversation.item.done"
@dataclass(frozen=True, slots=True, kw_only=True)
class ItemRetrieved(DuplexEvent):
wire_type = "conversation.item.retrieved"
item: Mapping[str, object] = field(default_factory=dict)
@property
def item_id(self) -> str | None:
value = self.item.get("id")
return value if isinstance(value, str) else None
@dataclass(frozen=True, slots=True, kw_only=True)
class ItemDeleted(DuplexEvent):
wire_type = "conversation.item.deleted"
item_id: str | None = None # type: ignore[assignment]
details: Mapping[str, object] = field(default_factory=dict)
def _wire_fields(self) -> dict[str, object]:
return {"item_id": self.item_id, "event": _wire_value(self.details)}
@dataclass(frozen=True, slots=True, kw_only=True)
class ItemTruncated(DuplexEvent):
wire_type = "conversation.item.truncated"
item_id: str | None = None # type: ignore[assignment]
content_index: int = 0
audio_end_ms: int = 0
details: Mapping[str, object] = field(default_factory=dict)
def _wire_fields(self) -> dict[str, object]:
return {
"item_id": self.item_id,
"content_index": self.content_index,
"audio_end_ms": self.audio_end_ms,
"event": _wire_value(self.details),
}
@dataclass(frozen=True, slots=True, kw_only=True)
class InputTranscriptionCompleted(DuplexEvent):
wire_type = "conversation.item.input_audio_transcription.completed"
item_id: str | None = None # type: ignore[assignment]
content_index: int = 0
transcript: str = ""
@property
def text(self) -> str | None:
return self.transcript
# ---- playback / control ----
@dataclass(frozen=True, slots=True, kw_only=True)
class OutputAudioCleared(DuplexEvent):
wire_type = "output_audio_buffer.cleared"
response_id: str | None = None # type: ignore[assignment]
@dataclass(frozen=True, slots=True, kw_only=True)
class PlaybackAcknowledged(DuplexEvent):
wire_type = "playback.acknowledged"
#: The session-internal ``playback.acknowledged`` event (cursor, committed_ms ...).
details: Mapping[str, object] = field(default_factory=dict)
def _wire_fields(self) -> dict[str, object]:
return {"event": _wire_value(self.details)}
@dataclass(frozen=True, slots=True, kw_only=True)
class ErrorEvent(DuplexEvent):
wire_type = "error"
code: str = "internal_error"
message: str = ""
#: Client ``event_id`` this error answers (OpenAI ``error.event_id``).
related_event_id: str | None = None
param: str | None = None
#: Extra keys merged into the wire ``error`` object (``retryable`` ...).
extra: Mapping[str, object] = field(default_factory=dict)
@property
def error_type(self) -> str:
return REALTIME_ERROR_TYPES_BY_CODE.get(self.code, "invalid_request_error")
@property
def error(self) -> dict[str, object]:
error: dict[str, object] = {"type": self.error_type, "code": self.code, "message": self.message}
if self.related_event_id:
error["event_id"] = self.related_event_id
if self.param:
error["param"] = self.param
error.update(cast("Mapping[str, object]", _wire_value(self.extra)))
return error
@property
def text(self) -> str | None:
return self.message
def _wire_fields(self) -> dict[str, object]:
return {"error": self.error}
@dataclass(frozen=True, slots=True, kw_only=True)
class DuplexRawEvent(DuplexEvent):
"""A session-internal event with no dedicated Realtime type (``duplex.<internal type>``)."""
wire_type = "duplex.raw"
internal_type: str = ""
details: Mapping[str, object] = field(default_factory=dict)
@property
def type(self) -> str:
return f"duplex.{self.internal_type}"
def to_realtime(self) -> dict[str, object]:
return {"type": self.type, "event_id": self.event_id, "event": _wire_value(self.details)}
def error_event(
code: str,
message: str,
*,
event_id: object | None = None,
param: object | None = None,
extra: Mapping[str, object] | None = None,
) -> ErrorEvent:
"""Build the error event for an internal code; ``event_id`` is the *client* event id."""
return ErrorEvent(
code=code,
message=message,
related_event_id=event_id if isinstance(event_id, str) and event_id else None,
param=param if isinstance(param, str) and param else None,
extra=dict(extra or {}),
)
#: Internal events that terminate a response/session and must never be dropped as stale.
DOMAIN_TERMINAL_EVENTS = frozenset(
{
"response.done",
@@ -682,7 +84,6 @@ DOMAIN_TERMINAL_EVENTS = frozenset(
}
)
#: Internal streaming model-output events subject to epoch/close stale filtering.
MODEL_OUTPUT_EVENTS = frozenset(
{
"response.created",
@@ -705,13 +106,11 @@ MODEL_OUTPUT_EVENTS = frozenset(
__all__ = [
"DOMAIN_TERMINAL_EVENTS",
"MODEL_OUTPUT_EVENTS",
"REALTIME_ERROR_TYPES_BY_CODE",
"AudioDelta",
"AudioDone",
"ContentPartAdded",
"ContentPartDone",
"DOMAIN_TERMINAL_EVENTS",
"DuplexEvent",
"DuplexRawEvent",
"ErrorEvent",
@@ -727,11 +126,13 @@ __all__ = [
"ItemRetrieved",
"ItemTruncated",
"Listen",
"MODEL_OUTPUT_EVENTS",
"OutputAudioCleared",
"OutputItemAdded",
"OutputItemDone",
"OverlapDecision",
"PlaybackAcknowledged",
"REALTIME_ERROR_TYPES_BY_CODE",
"RateLimitsUpdated",
"ResponseCreated",
"ResponseDone",
vllm_omni/engine/duplex/realtime_commands.py
+79 -584
View File
@@ -3,26 +3,31 @@
"""OpenAI Realtime client events -> :class:`DuplexCommand` (stateless).
This is the mapping half of the former ``entrypoints/duplex/realtime_input.py``
translator. Everything that needed per-session state in the old translator
(input-buffer emptiness for commits, response-id fallbacks for cancels,
conversation-item lookups, VAD) is resolved by the session runner through the
helpers on :class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
This module is the *duplex binding* of the shared Realtime codec. Parsing,
validating and decoding a client event is not duplex-specific and lives in
``vllm_omni.protocol.realtime``, reached through ``vllm_omni.protocol.duplex``;
what lives here is the part that is: which
:class:`~vllm_omni.engine.duplex.commands.DuplexCommand` a decoded event
becomes, what the duplex engine can serve
(:data:`DUPLEX_REALTIME_CAPABILITIES`), and the mapping between a Realtime
output format and the duplex ``response_format`` vocabulary.
Everything that needs per-session state (input-buffer emptiness for commits,
response-id fallbacks for cancels, conversation-item lookups, VAD) is resolved
by the session runner through the helpers on
:class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
the commands produced here carry the raw client intent only.
The shared names re-exported at the bottom are a compatibility surface for
existing importers. The canonical definitions are in
``vllm_omni.protocol.realtime`` and there is exactly one of each.
"""
from __future__ import annotations
import base64
import binascii
from collections.abc import Mapping
from dataclasses import dataclass, replace
from typing import Any, cast
from uuid import uuid4
from typing import cast
import numpy as np
from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate
from vllm_omni.engine.duplex.commands import (
AckPlayback,
AppendAudio,
@@ -44,93 +49,58 @@ from vllm_omni.engine.duplex.commands import (
TruncateItem,
UpdateSession,
)
REALTIME_INPUT_AUDIO_FORMATS = {
"pcm16",
"pcm_s16le",
"s16le",
"pcm_f32le",
"g711_ulaw",
"g711_alaw",
}
REALTIME_OUTPUT_AUDIO_FORMATS = {
"pcm16",
"pcm_s16le",
"s16le",
"wav",
"pcm",
"g711_ulaw",
"g711_alaw",
}
#: Wire hint keys copied verbatim from a client append onto the internal payload.
REALTIME_INPUT_HINT_KEYS = (
"duration_ms",
"audio_duration_ms",
"audio_start_ms",
"audio_end_ms",
"is_speech",
"speech",
"speech_probability",
"vad",
"overlap_action",
"overlap",
"force_barge_in",
"force_listen",
"text",
"transcript",
from vllm_omni.protocol.duplex import (
REALTIME_INPUT_AUDIO_FORMATS,
REALTIME_INPUT_HINT_KEYS,
REALTIME_OUTPUT_AUDIO_FORMATS,
RealtimeInputDefaults,
RealtimeProtocolCapabilities,
RealtimeProtocolError,
apply_realtime_session_defaults,
copy_realtime_input_hints,
decode_audio_append,
input_audio_transcription_config,
input_explicitly_non_speech,
input_looks_like_speech,
input_transcript_from_item,
is_supported_realtime_input_format,
json_safe_realtime_payload,
normalize_conversation_item,
parse_realtime_audio_format,
realtime_audio_format_object,
realtime_max_output_tokens,
realtime_output_format,
realtime_overlap_fields,
text_chars_for_audio_ms_from_marks,
truncate_realtime_item_content,
validate_conversation_item_audio_formats,
validate_realtime_item_truncate,
validate_realtime_response_audio_formats,
validate_realtime_session_audio_formats,
validate_realtime_video_frames,
validate_session_payload,
)
@dataclass(frozen=True, slots=True)
class RealtimeInputDefaults:
"""Session-level wire defaults an append may omit (derived from the session object)."""
def _validate_duplex_turn_detection(session_payload: Mapping[str, object]) -> str | None:
"""The duplex engine's answer for ``turn_detection`` on a session object.
input_audio_format: str = "pcm16"
input_sample_rate_hz: int = 16000
output_audio_format: str = "pcm16"
output_sample_rate_hz: int | None = None
overlap_silence_rms: float = 0.003
Imported lazily: the validator lives with the Silero VAD backend, and a
connection that never sends a session object should not pay for loading it.
"""
from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults:
"""Return defaults updated from a Realtime ``session`` object (session.update)."""
values = apply_realtime_session_defaults(self, session_payload)
return values
return validate_realtime_turn_detection(session_payload)
# ---- audio format helpers ----
def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]:
def normalize_format(fmt: str) -> str:
normalized = fmt.lower()
if normalized in {"audio/pcm", "pcm"}:
return "pcm16"
if normalized in {"audio/wav", "wav"}:
return "wav"
if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}:
return "pcm16"
if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}:
return "pcm_f32le"
if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}:
return "g711_ulaw"
if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}:
return "g711_alaw"
return fmt
if isinstance(raw_format, str):
return normalize_format(raw_format), None
if not isinstance(raw_format, dict):
return raw_format, None
rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate")
sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None
fmt = raw_format.get("type") or raw_format.get("format")
if not isinstance(fmt, str):
return raw_format, sample_rate_hz
return normalize_format(fmt), sample_rate_hz
#: What the duplex engine can serve, for the shared session-object validator.
DUPLEX_REALTIME_CAPABILITIES = RealtimeProtocolCapabilities(
validate_turn_detection=_validate_duplex_turn_detection,
)
def duplex_response_format(realtime_format: str) -> str:
"""A Realtime output format -> the duplex ``response_format`` vocabulary."""
normalized = realtime_format.lower()
if normalized in {"pcm16", "pcm_s16le", "s16le"}:
return "pcm"
@@ -141,433 +111,6 @@ def duplex_response_format(realtime_format: str) -> str:
return "wav"
def realtime_output_format(duplex_format: object) -> str:
if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}:
return duplex_format.lower()
if isinstance(duplex_format, str) and duplex_format.lower() == "pcm":
return "pcm16"
return str(duplex_format or "wav")
def is_supported_realtime_input_format(fmt: object) -> bool:
return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS
def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]:
if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}:
payload: dict[str, object] = {"type": "audio/pcm"}
elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le":
payload = {"type": "audio/pcm_f32le"}
elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw":
payload = {"type": "audio/g711_ulaw"}
elif isinstance(fmt, str) and fmt.lower() == "g711_alaw":
payload = {"type": "audio/g711_alaw"}
else:
payload = {"type": "audio/wav"}
if sample_rate_hz is not None:
payload["rate"] = int(sample_rate_hz)
return payload
def validate_realtime_session_audio_formats(session_payload: Mapping[str, object]) -> str | None:
audio_config = session_payload.get("audio")
input_format: object = session_payload.get("input_audio_format")
if input_format is None and isinstance(audio_config, dict):
audio_input = audio_config.get("input")
if isinstance(audio_input, dict):
input_format = audio_input.get("format")
parsed_input, _ = parse_realtime_audio_format(input_format)
if input_format is not None and not (
isinstance(parsed_input, str) and parsed_input.lower() in REALTIME_INPUT_AUDIO_FORMATS
):
return f"Unsupported input_audio_format: {input_format}"
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
if output_format is None and isinstance(audio_config, dict):
audio_output = audio_config.get("output")
if isinstance(audio_output, dict):
output_format = audio_output.get("format")
parsed_output, _ = parse_realtime_audio_format(output_format)
if output_format is not None and not (
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
):
return f"Unsupported output_audio_format: {output_format}"
return None
def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None:
output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format")
audio_config = response_payload.get("audio")
if output_format is None and isinstance(audio_config, dict):
audio_output = audio_config.get("output")
if isinstance(audio_output, dict):
output_format = audio_output.get("format")
parsed_output, _ = parse_realtime_audio_format(output_format)
if output_format is not None and not (
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
):
return f"Unsupported output_audio_format: {output_format}"
return None
def validate_conversation_item_audio_formats(item: object) -> str | None:
if not isinstance(item, dict):
return None
content = item.get("content")
if not isinstance(content, list):
return None
for part in content:
if not isinstance(part, dict):
continue
if part.get("type") not in {"input_audio", "audio"}:
continue
raw_format = part.get("format")
parsed_format, _ = parse_realtime_audio_format(raw_format)
if raw_format is not None and not (
isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS
):
return f"Unsupported input_audio format in conversation.item.create: {raw_format}"
return None
def apply_realtime_session_defaults(
defaults: RealtimeInputDefaults,
session_payload: Mapping[str, object],
) -> RealtimeInputDefaults:
"""Derive the wire defaults a Realtime ``session`` object declares."""
input_format: object = session_payload.get("input_audio_format")
audio_config = session_payload.get("audio")
if input_format is None and isinstance(audio_config, dict):
audio_input = audio_config.get("input")
if isinstance(audio_input, dict):
input_format = audio_input.get("format")
input_format, input_rate = parse_realtime_audio_format(input_format)
# Any: heterogeneous field values splatted into ``dataclasses.replace`` below.
updates: dict[str, Any] = {}
if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS:
updates["input_audio_format"] = input_format
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
output_rate_raw: object | None = None
if output_format is None and isinstance(audio_config, dict):
audio_output = audio_config.get("output")
if isinstance(audio_output, dict):
output_format = audio_output.get("format")
output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate")
output_format, output_rate = parse_realtime_audio_format(output_format)
if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0:
output_rate = int(output_rate_raw)
if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
updates["output_audio_format"] = realtime_output_format(output_format)
sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate")
if sample_rate is None and isinstance(audio_config, dict):
audio_input = audio_config.get("input")
if isinstance(audio_input, dict):
sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate")
if sample_rate is None:
sample_rate = input_rate
if isinstance(sample_rate, int | float) and sample_rate > 0:
updates["input_sample_rate_hz"] = int(sample_rate)
if isinstance(output_rate, int | float) and output_rate > 0:
updates["output_sample_rate_hz"] = int(output_rate)
overlap_fields = realtime_overlap_fields(session_payload)
overlap_silence_rms = overlap_fields.get("overlap_silence_rms")
if isinstance(overlap_silence_rms, int | float):
updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms))
return replace(defaults, **updates) if updates else defaults
def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]:
fields: dict[str, object] = {}
if isinstance(session_payload.get("overlap_policy"), str):
fields["overlap_policy"] = session_payload["overlap_policy"]
for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"):
value = session_payload.get(key)
if isinstance(value, int | float):
fields[key] = value
if isinstance(session_payload.get("playback_commit_policy"), str):
fields["playback_commit_policy"] = session_payload["playback_commit_policy"]
return fields
def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]:
clean: dict[str, object] = {}
for key, value in payload.items():
if key == "extra_body":
continue
if isinstance(value, str | int | float | bool) or value is None:
clean[key] = value
elif isinstance(value, dict):
clean[key] = json_safe_realtime_payload(value)
elif isinstance(value, list):
clean[key] = [
(json_safe_realtime_payload(item) if isinstance(item, dict) else item)
for item in value
if isinstance(item, str | int | float | bool | dict) or item is None
]
return clean
def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
transcription = session_payload.get("input_audio_transcription")
if isinstance(transcription, dict):
return transcription
audio_config = session_payload.get("audio")
if not isinstance(audio_config, dict):
return None
audio_input = audio_config.get("input")
if not isinstance(audio_input, dict):
return None
transcription = audio_input.get("transcription")
return transcription if isinstance(transcription, dict) else None
def realtime_max_output_tokens(value: object) -> int | None:
"""Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
if value is None:
return None
if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}:
return None
if isinstance(value, int) and value > 0:
return int(value)
return None
# ---- speech hints ----
def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
for key in ("is_speech", "speech"):
value = event.get(key)
if isinstance(value, bool):
return not value
vad = event.get("vad")
if isinstance(vad, dict):
value = vad.get("is_speech")
if isinstance(value, bool):
return not value
probability = vad.get("speech_probability", vad.get("probability"))
if isinstance(probability, int | float):
return float(probability) < 0.5
probability = event.get("speech_probability")
return isinstance(probability, int | float) and float(probability) < 0.5
def input_looks_like_speech(
event: Mapping[str, object],
*,
audio: object,
fmt: object,
overlap_silence_rms: float,
) -> bool:
if input_explicitly_non_speech(event):
return False
for key in ("is_speech", "speech"):
value = event.get(key)
if isinstance(value, bool):
return value
vad = event.get("vad")
if isinstance(vad, dict):
probability = vad.get("speech_probability", vad.get("probability"))
if isinstance(probability, int | float):
return float(probability) >= 0.5
probability = event.get("speech_probability")
if isinstance(probability, int | float):
return float(probability) >= 0.5
if fmt != "pcm_f32le":
return True
if isinstance(audio, bytes | bytearray):
raw = bytes(audio)
elif isinstance(audio, str):
try:
raw = base64.b64decode(audio, validate=True)
except (binascii.Error, ValueError):
return True
else:
return True
if len(raw) < 4 or len(raw) % 4 != 0:
return True
samples = np.frombuffer(raw, dtype=np.float32)
if samples.size == 0:
return False
rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
threshold = event.get("overlap_silence_rms")
if not isinstance(threshold, int | float):
vad = event.get("vad")
if isinstance(vad, dict):
threshold = vad.get("silence_rms")
silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
return rms >= max(0.0, silence_rms)
def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
for key in REALTIME_INPUT_HINT_KEYS:
if key in source:
target[key] = source[key]
def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
"""Validate omni-duplex camera frames on input_audio_buffer.append.
Wire contract matches the official MiniCPM-o duplex loop: one base
base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
stacked composite tiling the sub-frames captured inside it (at most 2
images either way). A caller-supplied ``max_slice_nums`` is rejected rather
than silently ignored: slicing is Stage 0's decision here, and Stage 0
already applies the official HD suggestion for a stacked unit
(``max_slice_nums=[2, 1]``). The wire simply does not let the client choose
it.
"""
if max_slice_nums not in (None, 1):
return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself"
if not isinstance(video_frames, list):
return "video_frames must be a list of base64-encoded images"
frames = [frame for frame in video_frames if frame is not None]
if len(frames) > 2:
return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
for frame in frames:
if not isinstance(frame, str) or not frame:
return "video_frames entries must be non-empty base64 strings"
if len(frame) > 4_000_000:
return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
try:
header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
except (binascii.Error, ValueError):
return "video_frames entries must be valid base64"
if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
return "video_frames entries must be JPEG or PNG images"
return None
# ---- conversation items ----
def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]:
normalized = dict(item)
normalized.setdefault("id", f"item_{uuid4().hex}")
normalized.setdefault("object", "realtime.item")
normalized.setdefault("type", "message")
normalized.setdefault("status", "completed")
if "role" not in normalized and normalized.get("type") == "message":
normalized["role"] = "user"
if not isinstance(normalized.get("content"), list):
normalized["content"] = []
return normalized
def text_chars_for_audio_ms_from_marks(
audio_end_ms: int,
text_len: int,
marks: list[object],
*,
final_ms: object | None = None,
) -> int:
if text_len <= 0:
return 0
clean_marks: list[tuple[int, int]] = []
for mark in marks:
if not isinstance(mark, dict):
continue
raw_text_chars = mark.get("text_chars")
raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
continue
clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
if not clean_marks:
return 0 if audio_end_ms <= 0 else text_len
clean_marks.sort(key=lambda item: item[0])
audio_end_ms = max(0, int(audio_end_ms))
if audio_end_ms <= 0:
return 0
previous_ms = 0
previous_chars = 0
for mark_ms, mark_chars in clean_marks:
mark_ms = max(previous_ms, mark_ms)
mark_chars = max(previous_chars, min(text_len, mark_chars))
if audio_end_ms <= mark_ms:
if mark_ms <= previous_ms:
return mark_chars
ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
previous_ms = mark_ms
previous_chars = mark_chars
if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
if audio_end_ms >= int(final_ms):
return text_len
ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
return text_len if audio_end_ms >= previous_ms else previous_chars
def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None:
content = item.get("content")
if not isinstance(content, list) or not content:
return
index = max(0, int(content_index))
if index >= len(content):
return
part = content[index]
if not isinstance(part, dict):
return
transcript = part.get("transcript")
if not isinstance(transcript, str) or not transcript:
return
marks = part.get("audio_text_marks")
if isinstance(marks, list):
keep_chars = text_chars_for_audio_ms_from_marks(
audio_end_ms,
len(transcript),
marks,
final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
)
part["transcript"] = transcript[:keep_chars].rstrip()
return
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
if isinstance(duration_ms, int | float) and duration_ms > 0:
keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
part["transcript"] = transcript[:keep_chars].rstrip()
elif audio_end_ms <= 0:
part["transcript"] = ""
def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None:
if item.get("type") != "message" or item.get("role") != "assistant":
return "conversation.item.truncate only supports assistant message items"
if audio_end_ms < 0:
return "conversation.item.truncate requires non-negative audio_end_ms"
content = item.get("content")
if not isinstance(content, list) or not content:
return None
index = max(0, int(content_index))
if index >= len(content):
return f"conversation.item.truncate content_index out of range: {content_index}"
part = content[index]
if not isinstance(part, dict):
return "conversation.item.truncate target content part is invalid"
if part.get("type") not in {"audio", "output_audio"}:
return "conversation.item.truncate target content part must be audio"
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
return "conversation.item.truncate audio_end_ms exceeds item audio duration"
return None
def input_transcript_from_item(item: Mapping[str, object]) -> str:
content = item.get("content")
if not isinstance(content, list):
return ""
parts: list[str] = []
for part in content:
if not isinstance(part, dict):
continue
for key in ("transcript", "text"):
value = part.get(key)
if isinstance(value, str) and value:
if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
parts.append(value)
break
return "".join(parts).strip()
# ---- append / item audio conversion ----
@@ -577,67 +120,21 @@ def build_append_audio(
defaults: RealtimeInputDefaults,
hints_source: Mapping[str, object] | None = None,
) -> AppendAudio:
"""Validate, convert (to 16 kHz ``pcm_f32le`` base64) and pack one audio append.
Speech classification without VAD (``is_speech`` on the command) uses the
explicit wire hints and the RMS fallback exactly like the old translator.
"""
event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
audio = event.get("audio") or event.get("delta")
fmt, format_rate = parse_realtime_audio_format(
event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
)
sample_rate_hz = (
event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
)
if not is_supported_realtime_input_format(fmt):
raise DuplexCommandError(
f"Unsupported input_audio_format: {fmt}",
code="unsupported_audio_format",
event_id=event_id,
)
"""Decode one audio append (shared codec) and pack it as the duplex command."""
try:
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
audio,
fmt,
sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
)
except ValueError as exc:
raise DuplexCommandError(str(exc), code="bad_event", event_id=event_id) from exc
hints: dict[str, object] = {}
if hints_source is not None:
copy_realtime_input_hints(hints_source, hints)
copy_realtime_input_hints(event, hints)
looks_like_speech = input_looks_like_speech(
{**(hints_source or {}), **event},
audio=audio,
fmt=fmt,
overlap_silence_rms=defaults.overlap_silence_rms,
)
video_frames: tuple[str, ...] = ()
raw_frames = event.get("video_frames")
if raw_frames is not None:
frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
if frames_error is not None:
raise DuplexCommandError(frames_error, code="invalid_video_frames", event_id=event_id)
if isinstance(raw_frames, list):
video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
audio_end_ms = hints.get("audio_end_ms")
try:
audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
except (binascii.Error, ValueError) as exc:
raise DuplexCommandError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
decoded = decode_audio_append(event, defaults=defaults, hints_source=hints_source)
except RealtimeProtocolError as exc:
raise DuplexCommandError(str(exc), code=exc.code, event_id=exc.event_id) from exc
return AppendAudio(
event_id=event_id,
audio=audio_bytes,
format=str(fmt),
sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
is_speech=looks_like_speech,
video_frames=video_frames,
duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
hints=hints,
event_id=decoded.event_id,
audio=decoded.audio,
format=decoded.format,
sample_rate_hz=decoded.sample_rate_hz,
is_speech=decoded.is_speech,
video_frames=decoded.video_frames,
duration_ms=decoded.duration_ms,
audio_end_ms=decoded.audio_end_ms,
hints=decoded.hints,
)
@@ -664,14 +161,9 @@ def translate_realtime_command(
if event_type == "session.update":
session = payload.get("session")
session_payload: Mapping[str, object] = session if isinstance(session, dict) else payload
format_error = validate_realtime_session_audio_formats(session_payload)
if format_error is not None:
raise DuplexCommandError(format_error, code="unsupported_audio_format", event_id=event_id)
from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
turn_detection_error = validate_realtime_turn_detection(session_payload)
if turn_detection_error is not None:
raise DuplexCommandError(turn_detection_error, code="unsupported_turn_detection", event_id=event_id)
rejection = validate_session_payload(session_payload, capabilities=DUPLEX_REALTIME_CAPABILITIES)
if rejection is not None:
raise DuplexCommandError(rejection.message, code=rejection.code, event_id=event_id)
return UpdateSession(event_id=event_id, patch=dict(session_payload))
if event_type == "conversation.item.create":
@@ -855,7 +347,10 @@ def translate_realtime_command(
raise DuplexCommandError(f"Unknown duplex event type: {event_type}", code="unknown_event", event_id=event_id)
#: Names this module used to define, re-exported so existing importers keep
#: working. The canonical definitions live in ``vllm_omni.protocol.realtime``.
__all__ = [
"DUPLEX_REALTIME_CAPABILITIES",
"REALTIME_INPUT_AUDIO_FORMATS",
"REALTIME_INPUT_HINT_KEYS",
"REALTIME_OUTPUT_AUDIO_FORMATS",
vllm_omni/engine/duplex/realtime_events.py
+9 -3
View File
@@ -15,6 +15,12 @@ carries the input-side bookkeeping the old input translator kept (input buffer
flags, conversation items, response-id fallbacks); the ``resolve_*`` /
``note_*`` helpers give the runner the same behaviour for the corresponding
commands.
What is *not* here is the model- and runtime-agnostic half of the codec ---
audio format negotiation, conversation-item shape and truncation, transcript
extraction, audio conversion. That lives in ``vllm_omni.protocol.realtime`` so
a non-duplex Realtime surface can use it without the duplex session; this
module is the duplex consumer of it (RFC #6592 P0a).
"""
from __future__ import annotations
@@ -24,7 +30,6 @@ from dataclasses import dataclass, field
from typing import TYPE_CHECKING, cast
from uuid import uuid4
from vllm_omni.engine.duplex.audio import convert_output_audio
from vllm_omni.engine.duplex.commands import (
AppendAudio,
CancelResponse,
@@ -74,10 +79,11 @@ from vllm_omni.engine.duplex.events import (
TranscriptDone,
error_event,
)
from vllm_omni.engine.duplex.realtime_commands import (
from vllm_omni.engine.duplex.realtime_commands import build_append_audio
from vllm_omni.protocol.duplex import (
RealtimeInputDefaults,
apply_realtime_session_defaults,
build_append_audio,
convert_output_audio,
copy_realtime_input_hints,
input_looks_like_speech,
input_transcript_from_item,
vllm_omni/engine/duplex/session/runner.py
+1 -1
View File
@@ -28,7 +28,6 @@ from typing import TYPE_CHECKING, TypeVar
from vllm.logger import init_logger
from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate
from vllm_omni.engine.duplex.commands import (
AckPlayback,
AppendAudio,
@@ -103,6 +102,7 @@ from vllm_omni.engine.duplex.turn_detection import (
TurnDetectionResult,
)
from vllm_omni.metrics.stats import StageRequestStats
from vllm_omni.protocol.duplex import convert_input_audio_with_rate
if TYPE_CHECKING:
from vllm.config import ModelConfig