anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Core] Give the OpenAI Realtime wire codec a reusable home (RFC #6592 P0a) (#7640)

Browse Source 
Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
psv666 authored and GitHub committed 2026-09-17 02:33:40 -07:00
1 parent 825ab20b07
commit e3be42e052
37 files changed
+3792 -1629
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vllm_omni/engine/duplex/realtime_commands.py
+79 -584
View File
@@ -3,26 +3,31 @@
"""OpenAI Realtime client events -> :class:`DuplexCommand` (stateless).
This is the mapping half of the former ``entrypoints/duplex/realtime_input.py``
translator. Everything that needed per-session state in the old translator
(input-buffer emptiness for commits, response-id fallbacks for cancels,
conversation-item lookups, VAD) is resolved by the session runner through the
helpers on :class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
This module is the *duplex binding* of the shared Realtime codec. Parsing,
validating and decoding a client event is not duplex-specific and lives in
``vllm_omni.protocol.realtime``, reached through ``vllm_omni.protocol.duplex``;
what lives here is the part that is: which
:class:`~vllm_omni.engine.duplex.commands.DuplexCommand` a decoded event
becomes, what the duplex engine can serve
(:data:`DUPLEX_REALTIME_CAPABILITIES`), and the mapping between a Realtime
output format and the duplex ``response_format`` vocabulary.
Everything that needs per-session state (input-buffer emptiness for commits,
response-id fallbacks for cancels, conversation-item lookups, VAD) is resolved
by the session runner through the helpers on
:class:`~vllm_omni.engine.duplex.realtime_events.RealtimeProjectionState`;
the commands produced here carry the raw client intent only.
The shared names re-exported at the bottom are a compatibility surface for
existing importers. The canonical definitions are in
``vllm_omni.protocol.realtime`` and there is exactly one of each.
"""
from __future__ import annotations
import base64
import binascii
from collections.abc import Mapping
from dataclasses import dataclass, replace
from typing import Any, cast
from uuid import uuid4
from typing import cast
import numpy as np
from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate
from vllm_omni.engine.duplex.commands import (
AckPlayback,
AppendAudio,
@@ -44,93 +49,58 @@ from vllm_omni.engine.duplex.commands import (
TruncateItem,
UpdateSession,
)
REALTIME_INPUT_AUDIO_FORMATS = {
"pcm16",
"pcm_s16le",
"s16le",
"pcm_f32le",
"g711_ulaw",
"g711_alaw",
}
REALTIME_OUTPUT_AUDIO_FORMATS = {
"pcm16",
"pcm_s16le",
"s16le",
"wav",
"pcm",
"g711_ulaw",
"g711_alaw",
}
#: Wire hint keys copied verbatim from a client append onto the internal payload.
REALTIME_INPUT_HINT_KEYS = (
"duration_ms",
"audio_duration_ms",
"audio_start_ms",
"audio_end_ms",
"is_speech",
"speech",
"speech_probability",
"vad",
"overlap_action",
"overlap",
"force_barge_in",
"force_listen",
"text",
"transcript",
from vllm_omni.protocol.duplex import (
REALTIME_INPUT_AUDIO_FORMATS,
REALTIME_INPUT_HINT_KEYS,
REALTIME_OUTPUT_AUDIO_FORMATS,
RealtimeInputDefaults,
RealtimeProtocolCapabilities,
RealtimeProtocolError,
apply_realtime_session_defaults,
copy_realtime_input_hints,
decode_audio_append,
input_audio_transcription_config,
input_explicitly_non_speech,
input_looks_like_speech,
input_transcript_from_item,
is_supported_realtime_input_format,
json_safe_realtime_payload,
normalize_conversation_item,
parse_realtime_audio_format,
realtime_audio_format_object,
realtime_max_output_tokens,
realtime_output_format,
realtime_overlap_fields,
text_chars_for_audio_ms_from_marks,
truncate_realtime_item_content,
validate_conversation_item_audio_formats,
validate_realtime_item_truncate,
validate_realtime_response_audio_formats,
validate_realtime_session_audio_formats,
validate_realtime_video_frames,
validate_session_payload,
)
@dataclass(frozen=True, slots=True)
class RealtimeInputDefaults:
"""Session-level wire defaults an append may omit (derived from the session object)."""
def _validate_duplex_turn_detection(session_payload: Mapping[str, object]) -> str | None:
"""The duplex engine's answer for ``turn_detection`` on a session object.
input_audio_format: str = "pcm16"
input_sample_rate_hz: int = 16000
output_audio_format: str = "pcm16"
output_sample_rate_hz: int | None = None
overlap_silence_rms: float = 0.003
Imported lazily: the validator lives with the Silero VAD backend, and a
connection that never sends a session object should not pay for loading it.
"""
from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
def with_session_payload(self, session_payload: Mapping[str, object]) -> RealtimeInputDefaults:
"""Return defaults updated from a Realtime ``session`` object (session.update)."""
values = apply_realtime_session_defaults(self, session_payload)
return values
return validate_realtime_turn_detection(session_payload)
# ---- audio format helpers ----
def parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]:
def normalize_format(fmt: str) -> str:
normalized = fmt.lower()
if normalized in {"audio/pcm", "pcm"}:
return "pcm16"
if normalized in {"audio/wav", "wav"}:
return "wav"
if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}:
return "pcm16"
if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}:
return "pcm_f32le"
if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}:
return "g711_ulaw"
if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}:
return "g711_alaw"
return fmt
if isinstance(raw_format, str):
return normalize_format(raw_format), None
if not isinstance(raw_format, dict):
return raw_format, None
rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate")
sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None
fmt = raw_format.get("type") or raw_format.get("format")
if not isinstance(fmt, str):
return raw_format, sample_rate_hz
return normalize_format(fmt), sample_rate_hz
#: What the duplex engine can serve, for the shared session-object validator.
DUPLEX_REALTIME_CAPABILITIES = RealtimeProtocolCapabilities(
validate_turn_detection=_validate_duplex_turn_detection,
)
def duplex_response_format(realtime_format: str) -> str:
"""A Realtime output format -> the duplex ``response_format`` vocabulary."""
normalized = realtime_format.lower()
if normalized in {"pcm16", "pcm_s16le", "s16le"}:
return "pcm"
@@ -141,433 +111,6 @@ def duplex_response_format(realtime_format: str) -> str:
return "wav"
def realtime_output_format(duplex_format: object) -> str:
if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}:
return duplex_format.lower()
if isinstance(duplex_format, str) and duplex_format.lower() == "pcm":
return "pcm16"
return str(duplex_format or "wav")
def is_supported_realtime_input_format(fmt: object) -> bool:
return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS
def realtime_audio_format_object(fmt: object, *, sample_rate_hz: int | None = None) -> dict[str, object]:
if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le", "pcm"}:
payload: dict[str, object] = {"type": "audio/pcm"}
elif isinstance(fmt, str) and fmt.lower() == "pcm_f32le":
payload = {"type": "audio/pcm_f32le"}
elif isinstance(fmt, str) and fmt.lower() == "g711_ulaw":
payload = {"type": "audio/g711_ulaw"}
elif isinstance(fmt, str) and fmt.lower() == "g711_alaw":
payload = {"type": "audio/g711_alaw"}
else:
payload = {"type": "audio/wav"}
if sample_rate_hz is not None:
payload["rate"] = int(sample_rate_hz)
return payload
def validate_realtime_session_audio_formats(session_payload: Mapping[str, object]) -> str | None:
audio_config = session_payload.get("audio")
input_format: object = session_payload.get("input_audio_format")
if input_format is None and isinstance(audio_config, dict):
audio_input = audio_config.get("input")
if isinstance(audio_input, dict):
input_format = audio_input.get("format")
parsed_input, _ = parse_realtime_audio_format(input_format)
if input_format is not None and not (
isinstance(parsed_input, str) and parsed_input.lower() in REALTIME_INPUT_AUDIO_FORMATS
):
return f"Unsupported input_audio_format: {input_format}"
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
if output_format is None and isinstance(audio_config, dict):
audio_output = audio_config.get("output")
if isinstance(audio_output, dict):
output_format = audio_output.get("format")
parsed_output, _ = parse_realtime_audio_format(output_format)
if output_format is not None and not (
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
):
return f"Unsupported output_audio_format: {output_format}"
return None
def validate_realtime_response_audio_formats(response_payload: Mapping[str, object]) -> str | None:
output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format")
audio_config = response_payload.get("audio")
if output_format is None and isinstance(audio_config, dict):
audio_output = audio_config.get("output")
if isinstance(audio_output, dict):
output_format = audio_output.get("format")
parsed_output, _ = parse_realtime_audio_format(output_format)
if output_format is not None and not (
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
):
return f"Unsupported output_audio_format: {output_format}"
return None
def validate_conversation_item_audio_formats(item: object) -> str | None:
if not isinstance(item, dict):
return None
content = item.get("content")
if not isinstance(content, list):
return None
for part in content:
if not isinstance(part, dict):
continue
if part.get("type") not in {"input_audio", "audio"}:
continue
raw_format = part.get("format")
parsed_format, _ = parse_realtime_audio_format(raw_format)
if raw_format is not None and not (
isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS
):
return f"Unsupported input_audio format in conversation.item.create: {raw_format}"
return None
def apply_realtime_session_defaults(
defaults: RealtimeInputDefaults,
session_payload: Mapping[str, object],
) -> RealtimeInputDefaults:
"""Derive the wire defaults a Realtime ``session`` object declares."""
input_format: object = session_payload.get("input_audio_format")
audio_config = session_payload.get("audio")
if input_format is None and isinstance(audio_config, dict):
audio_input = audio_config.get("input")
if isinstance(audio_input, dict):
input_format = audio_input.get("format")
input_format, input_rate = parse_realtime_audio_format(input_format)
# Any: heterogeneous field values splatted into ``dataclasses.replace`` below.
updates: dict[str, Any] = {}
if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS:
updates["input_audio_format"] = input_format
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
output_rate_raw: object | None = None
if output_format is None and isinstance(audio_config, dict):
audio_output = audio_config.get("output")
if isinstance(audio_output, dict):
output_format = audio_output.get("format")
output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate")
output_format, output_rate = parse_realtime_audio_format(output_format)
if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0:
output_rate = int(output_rate_raw)
if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
updates["output_audio_format"] = realtime_output_format(output_format)
sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate")
if sample_rate is None and isinstance(audio_config, dict):
audio_input = audio_config.get("input")
if isinstance(audio_input, dict):
sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate")
if sample_rate is None:
sample_rate = input_rate
if isinstance(sample_rate, int | float) and sample_rate > 0:
updates["input_sample_rate_hz"] = int(sample_rate)
if isinstance(output_rate, int | float) and output_rate > 0:
updates["output_sample_rate_hz"] = int(output_rate)
overlap_fields = realtime_overlap_fields(session_payload)
overlap_silence_rms = overlap_fields.get("overlap_silence_rms")
if isinstance(overlap_silence_rms, int | float):
updates["overlap_silence_rms"] = max(0.0, float(overlap_silence_rms))
return replace(defaults, **updates) if updates else defaults
def realtime_overlap_fields(session_payload: Mapping[str, object]) -> dict[str, object]:
fields: dict[str, object] = {}
if isinstance(session_payload.get("overlap_policy"), str):
fields["overlap_policy"] = session_payload["overlap_policy"]
for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"):
value = session_payload.get(key)
if isinstance(value, int | float):
fields[key] = value
if isinstance(session_payload.get("playback_commit_policy"), str):
fields["playback_commit_policy"] = session_payload["playback_commit_policy"]
return fields
def json_safe_realtime_payload(payload: Mapping[str, object]) -> dict[str, object]:
clean: dict[str, object] = {}
for key, value in payload.items():
if key == "extra_body":
continue
if isinstance(value, str | int | float | bool) or value is None:
clean[key] = value
elif isinstance(value, dict):
clean[key] = json_safe_realtime_payload(value)
elif isinstance(value, list):
clean[key] = [
(json_safe_realtime_payload(item) if isinstance(item, dict) else item)
for item in value
if isinstance(item, str | int | float | bool | dict) or item is None
]
return clean
def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None:
transcription = session_payload.get("input_audio_transcription")
if isinstance(transcription, dict):
return transcription
audio_config = session_payload.get("audio")
if not isinstance(audio_config, dict):
return None
audio_input = audio_config.get("input")
if not isinstance(audio_input, dict):
return None
transcription = audio_input.get("transcription")
return transcription if isinstance(transcription, dict) else None
def realtime_max_output_tokens(value: object) -> int | None:
"""Normalize Realtime max output tokens (``"inf"`` -> ``None``)."""
if value is None:
return None
if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}:
return None
if isinstance(value, int) and value > 0:
return int(value)
return None
# ---- speech hints ----
def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
for key in ("is_speech", "speech"):
value = event.get(key)
if isinstance(value, bool):
return not value
vad = event.get("vad")
if isinstance(vad, dict):
value = vad.get("is_speech")
if isinstance(value, bool):
return not value
probability = vad.get("speech_probability", vad.get("probability"))
if isinstance(probability, int | float):
return float(probability) < 0.5
probability = event.get("speech_probability")
return isinstance(probability, int | float) and float(probability) < 0.5
def input_looks_like_speech(
event: Mapping[str, object],
*,
audio: object,
fmt: object,
overlap_silence_rms: float,
) -> bool:
if input_explicitly_non_speech(event):
return False
for key in ("is_speech", "speech"):
value = event.get(key)
if isinstance(value, bool):
return value
vad = event.get("vad")
if isinstance(vad, dict):
probability = vad.get("speech_probability", vad.get("probability"))
if isinstance(probability, int | float):
return float(probability) >= 0.5
probability = event.get("speech_probability")
if isinstance(probability, int | float):
return float(probability) >= 0.5
if fmt != "pcm_f32le":
return True
if isinstance(audio, bytes | bytearray):
raw = bytes(audio)
elif isinstance(audio, str):
try:
raw = base64.b64decode(audio, validate=True)
except (binascii.Error, ValueError):
return True
else:
return True
if len(raw) < 4 or len(raw) % 4 != 0:
return True
samples = np.frombuffer(raw, dtype=np.float32)
if samples.size == 0:
return False
rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
threshold = event.get("overlap_silence_rms")
if not isinstance(threshold, int | float):
vad = event.get("vad")
if isinstance(vad, dict):
threshold = vad.get("silence_rms")
silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
return rms >= max(0.0, silence_rms)
def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
for key in REALTIME_INPUT_HINT_KEYS:
if key in source:
target[key] = source[key]
def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
"""Validate omni-duplex camera frames on input_audio_buffer.append.
Wire contract matches the official MiniCPM-o duplex loop: one base
base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
stacked composite tiling the sub-frames captured inside it (at most 2
images either way). A caller-supplied ``max_slice_nums`` is rejected rather
than silently ignored: slicing is Stage 0's decision here, and Stage 0
already applies the official HD suggestion for a stacked unit
(``max_slice_nums=[2, 1]``). The wire simply does not let the client choose
it.
"""
if max_slice_nums not in (None, 1):
return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself"
if not isinstance(video_frames, list):
return "video_frames must be a list of base64-encoded images"
frames = [frame for frame in video_frames if frame is not None]
if len(frames) > 2:
return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
for frame in frames:
if not isinstance(frame, str) or not frame:
return "video_frames entries must be non-empty base64 strings"
if len(frame) > 4_000_000:
return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
try:
header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
except (binascii.Error, ValueError):
return "video_frames entries must be valid base64"
if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
return "video_frames entries must be JPEG or PNG images"
return None
# ---- conversation items ----
def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]:
normalized = dict(item)
normalized.setdefault("id", f"item_{uuid4().hex}")
normalized.setdefault("object", "realtime.item")
normalized.setdefault("type", "message")
normalized.setdefault("status", "completed")
if "role" not in normalized and normalized.get("type") == "message":
normalized["role"] = "user"
if not isinstance(normalized.get("content"), list):
normalized["content"] = []
return normalized
def text_chars_for_audio_ms_from_marks(
audio_end_ms: int,
text_len: int,
marks: list[object],
*,
final_ms: object | None = None,
) -> int:
if text_len <= 0:
return 0
clean_marks: list[tuple[int, int]] = []
for mark in marks:
if not isinstance(mark, dict):
continue
raw_text_chars = mark.get("text_chars")
raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
continue
clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
if not clean_marks:
return 0 if audio_end_ms <= 0 else text_len
clean_marks.sort(key=lambda item: item[0])
audio_end_ms = max(0, int(audio_end_ms))
if audio_end_ms <= 0:
return 0
previous_ms = 0
previous_chars = 0
for mark_ms, mark_chars in clean_marks:
mark_ms = max(previous_ms, mark_ms)
mark_chars = max(previous_chars, min(text_len, mark_chars))
if audio_end_ms <= mark_ms:
if mark_ms <= previous_ms:
return mark_chars
ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
previous_ms = mark_ms
previous_chars = mark_chars
if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
if audio_end_ms >= int(final_ms):
return text_len
ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
return text_len if audio_end_ms >= previous_ms else previous_chars
def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None:
content = item.get("content")
if not isinstance(content, list) or not content:
return
index = max(0, int(content_index))
if index >= len(content):
return
part = content[index]
if not isinstance(part, dict):
return
transcript = part.get("transcript")
if not isinstance(transcript, str) or not transcript:
return
marks = part.get("audio_text_marks")
if isinstance(marks, list):
keep_chars = text_chars_for_audio_ms_from_marks(
audio_end_ms,
len(transcript),
marks,
final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
)
part["transcript"] = transcript[:keep_chars].rstrip()
return
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
if isinstance(duration_ms, int | float) and duration_ms > 0:
keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
part["transcript"] = transcript[:keep_chars].rstrip()
elif audio_end_ms <= 0:
part["transcript"] = ""
def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None:
if item.get("type") != "message" or item.get("role") != "assistant":
return "conversation.item.truncate only supports assistant message items"
if audio_end_ms < 0:
return "conversation.item.truncate requires non-negative audio_end_ms"
content = item.get("content")
if not isinstance(content, list) or not content:
return None
index = max(0, int(content_index))
if index >= len(content):
return f"conversation.item.truncate content_index out of range: {content_index}"
part = content[index]
if not isinstance(part, dict):
return "conversation.item.truncate target content part is invalid"
if part.get("type") not in {"audio", "output_audio"}:
return "conversation.item.truncate target content part must be audio"
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
return "conversation.item.truncate audio_end_ms exceeds item audio duration"
return None
def input_transcript_from_item(item: Mapping[str, object]) -> str:
content = item.get("content")
if not isinstance(content, list):
return ""
parts: list[str] = []
for part in content:
if not isinstance(part, dict):
continue
for key in ("transcript", "text"):
value = part.get(key)
if isinstance(value, str) and value:
if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
parts.append(value)
break
return "".join(parts).strip()
# ---- append / item audio conversion ----
@@ -577,67 +120,21 @@ def build_append_audio(
defaults: RealtimeInputDefaults,
hints_source: Mapping[str, object] | None = None,
) -> AppendAudio:
"""Validate, convert (to 16 kHz ``pcm_f32le`` base64) and pack one audio append.
Speech classification without VAD (``is_speech`` on the command) uses the
explicit wire hints and the RMS fallback exactly like the old translator.
"""
event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
audio = event.get("audio") or event.get("delta")
fmt, format_rate = parse_realtime_audio_format(
event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
)
sample_rate_hz = (
event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
)
if not is_supported_realtime_input_format(fmt):
raise DuplexCommandError(
f"Unsupported input_audio_format: {fmt}",
code="unsupported_audio_format",
event_id=event_id,
)
"""Decode one audio append (shared codec) and pack it as the duplex command."""
try:
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
audio,
fmt,
sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
)
except ValueError as exc:
raise DuplexCommandError(str(exc), code="bad_event", event_id=event_id) from exc
hints: dict[str, object] = {}
if hints_source is not None:
copy_realtime_input_hints(hints_source, hints)
copy_realtime_input_hints(event, hints)
looks_like_speech = input_looks_like_speech(
{**(hints_source or {}), **event},
audio=audio,
fmt=fmt,
overlap_silence_rms=defaults.overlap_silence_rms,
)
video_frames: tuple[str, ...] = ()
raw_frames = event.get("video_frames")
if raw_frames is not None:
frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
if frames_error is not None:
raise DuplexCommandError(frames_error, code="invalid_video_frames", event_id=event_id)
if isinstance(raw_frames, list):
video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
audio_end_ms = hints.get("audio_end_ms")
try:
audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
except (binascii.Error, ValueError) as exc:
raise DuplexCommandError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
decoded = decode_audio_append(event, defaults=defaults, hints_source=hints_source)
except RealtimeProtocolError as exc:
raise DuplexCommandError(str(exc), code=exc.code, event_id=exc.event_id) from exc
return AppendAudio(
event_id=event_id,
audio=audio_bytes,
format=str(fmt),
sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
is_speech=looks_like_speech,
video_frames=video_frames,
duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
hints=hints,
event_id=decoded.event_id,
audio=decoded.audio,
format=decoded.format,
sample_rate_hz=decoded.sample_rate_hz,
is_speech=decoded.is_speech,
video_frames=decoded.video_frames,
duration_ms=decoded.duration_ms,
audio_end_ms=decoded.audio_end_ms,
hints=decoded.hints,
)
@@ -664,14 +161,9 @@ def translate_realtime_command(
if event_type == "session.update":
session = payload.get("session")
session_payload: Mapping[str, object] = session if isinstance(session, dict) else payload
format_error = validate_realtime_session_audio_formats(session_payload)
if format_error is not None:
raise DuplexCommandError(format_error, code="unsupported_audio_format", event_id=event_id)
from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
turn_detection_error = validate_realtime_turn_detection(session_payload)
if turn_detection_error is not None:
raise DuplexCommandError(turn_detection_error, code="unsupported_turn_detection", event_id=event_id)
rejection = validate_session_payload(session_payload, capabilities=DUPLEX_REALTIME_CAPABILITIES)
if rejection is not None:
raise DuplexCommandError(rejection.message, code=rejection.code, event_id=event_id)
return UpdateSession(event_id=event_id, patch=dict(session_payload))
if event_type == "conversation.item.create":
@@ -855,7 +347,10 @@ def translate_realtime_command(
raise DuplexCommandError(f"Unknown duplex event type: {event_type}", code="unknown_event", event_id=event_id)
#: Names this module used to define, re-exported so existing importers keep
#: working. The canonical definitions live in ``vllm_omni.protocol.realtime``.
__all__ = [
"DUPLEX_REALTIME_CAPABILITIES",
"REALTIME_INPUT_AUDIO_FORMATS",
"REALTIME_INPUT_HINT_KEYS",
"REALTIME_OUTPUT_AUDIO_FORMATS",