244 lines
8.9 KiB
Python
244 lines
8.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
"""Decoding one ``input_audio_buffer.append``.
|
|
|
|
An append is the hottest client event on a Realtime connection and the one with
|
|
the most wire surface: base64 audio in any accepted format, an optional rate,
|
|
optional camera frames, and a set of optional hints (``duration_ms``,
|
|
``is_speech``, a VAD probability, a transcript) that clients attach and servers
|
|
are free to use or ignore.
|
|
|
|
:func:`decode_audio_append` turns that into :class:`RealtimeAudioAppend`, a
|
|
value object: bytes at 16 kHz ``pcm_f32le``, the surviving hints, and a speech
|
|
verdict. It is a pure function of the event plus the session's
|
|
:class:`~vllm_omni.protocol.realtime.session.RealtimeInputDefaults`, so any
|
|
consumer decodes an append the same way. Building a consumer's own command
|
|
object out of it is one constructor call --- see
|
|
``vllm_omni.engine.duplex.realtime_commands.build_append_audio``, which wraps it
|
|
for the duplex ``AppendAudio``.
|
|
|
|
The speech verdict here is the *client-declared* one (explicit flags, a VAD
|
|
probability the client sent, or an RMS floor as a last resort). It is not
|
|
server-side VAD; that is a runtime concern and lives with whoever owns the
|
|
session.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import binascii
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass, field
|
|
from typing import cast
|
|
|
|
import numpy as np
|
|
|
|
from vllm_omni.protocol.realtime.audio import convert_input_audio_with_rate
|
|
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
|
|
from vllm_omni.protocol.realtime.formats import (
|
|
is_supported_realtime_input_format,
|
|
parse_realtime_audio_format,
|
|
)
|
|
from vllm_omni.protocol.realtime.items import validate_realtime_video_frames
|
|
from vllm_omni.protocol.realtime.session import RealtimeInputDefaults
|
|
|
|
__all__ = [
|
|
"REALTIME_INPUT_HINT_KEYS",
|
|
"RealtimeAudioAppend",
|
|
"copy_realtime_input_hints",
|
|
"decode_audio_append",
|
|
"input_explicitly_non_speech",
|
|
"input_looks_like_speech",
|
|
]
|
|
|
|
#: Wire hint keys copied verbatim from a client append onto the internal payload.
|
|
REALTIME_INPUT_HINT_KEYS = (
|
|
"duration_ms",
|
|
"audio_duration_ms",
|
|
"audio_start_ms",
|
|
"audio_end_ms",
|
|
"is_speech",
|
|
"speech",
|
|
"speech_probability",
|
|
"vad",
|
|
"overlap_action",
|
|
"overlap",
|
|
"force_barge_in",
|
|
"force_listen",
|
|
"text",
|
|
"transcript",
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class RealtimeAudioAppend:
|
|
"""One decoded ``input_audio_buffer.append``.
|
|
|
|
``audio`` is raw bytes in ``format`` at ``sample_rate_hz`` --- after
|
|
:func:`decode_audio_append` that is 16 kHz ``pcm_f32le`` for every input
|
|
format the codec converts.
|
|
"""
|
|
|
|
audio: bytes
|
|
format: str
|
|
sample_rate_hz: int | None = None
|
|
is_speech: bool | None = None
|
|
video_frames: tuple[str, ...] = ()
|
|
duration_ms: int | None = None
|
|
audio_end_ms: int | None = None
|
|
#: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
|
|
hints: dict[str, object] = field(default_factory=dict)
|
|
#: Client correlation id (OpenAI ``event_id``), echoed on error events.
|
|
event_id: str | None = None
|
|
|
|
|
|
# ---- speech hints ----
|
|
|
|
|
|
def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
|
|
for key in ("is_speech", "speech"):
|
|
value = event.get(key)
|
|
if isinstance(value, bool):
|
|
return not value
|
|
vad = event.get("vad")
|
|
if isinstance(vad, dict):
|
|
value = vad.get("is_speech")
|
|
if isinstance(value, bool):
|
|
return not value
|
|
probability = vad.get("speech_probability", vad.get("probability"))
|
|
if isinstance(probability, int | float):
|
|
return float(probability) < 0.5
|
|
probability = event.get("speech_probability")
|
|
return isinstance(probability, int | float) and float(probability) < 0.5
|
|
|
|
|
|
def input_looks_like_speech(
|
|
event: Mapping[str, object],
|
|
*,
|
|
audio: object,
|
|
fmt: object,
|
|
overlap_silence_rms: float,
|
|
) -> bool:
|
|
if input_explicitly_non_speech(event):
|
|
return False
|
|
for key in ("is_speech", "speech"):
|
|
value = event.get(key)
|
|
if isinstance(value, bool):
|
|
return value
|
|
vad = event.get("vad")
|
|
if isinstance(vad, dict):
|
|
probability = vad.get("speech_probability", vad.get("probability"))
|
|
if isinstance(probability, int | float):
|
|
return float(probability) >= 0.5
|
|
probability = event.get("speech_probability")
|
|
if isinstance(probability, int | float):
|
|
return float(probability) >= 0.5
|
|
if fmt != "pcm_f32le":
|
|
return True
|
|
if isinstance(audio, bytes | bytearray):
|
|
raw = bytes(audio)
|
|
elif isinstance(audio, str):
|
|
try:
|
|
raw = base64.b64decode(audio, validate=True)
|
|
except (binascii.Error, ValueError):
|
|
return True
|
|
else:
|
|
return True
|
|
if len(raw) < 4 or len(raw) % 4 != 0:
|
|
return True
|
|
samples = np.frombuffer(raw, dtype=np.float32)
|
|
if samples.size == 0:
|
|
return False
|
|
rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
|
|
threshold = event.get("overlap_silence_rms")
|
|
if not isinstance(threshold, int | float):
|
|
vad = event.get("vad")
|
|
if isinstance(vad, dict):
|
|
threshold = vad.get("silence_rms")
|
|
silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
|
|
return rms >= max(0.0, silence_rms)
|
|
|
|
|
|
def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
|
|
for key in REALTIME_INPUT_HINT_KEYS:
|
|
if key in source:
|
|
target[key] = source[key]
|
|
|
|
|
|
# ---- append decoding ----
|
|
|
|
|
|
def decode_audio_append(
|
|
event: Mapping[str, object],
|
|
*,
|
|
defaults: RealtimeInputDefaults,
|
|
hints_source: Mapping[str, object] | None = None,
|
|
) -> RealtimeAudioAppend:
|
|
"""Validate, convert (to 16 kHz ``pcm_f32le``) and pack one audio append.
|
|
|
|
``hints_source`` is the enclosing payload when the audio arrives inside
|
|
something larger than a bare append --- a ``conversation.item.create``
|
|
audio part, say --- so its hints apply unless the part overrides them.
|
|
|
|
Raises :class:`RealtimeProtocolError` for an unsupported format, undecodable
|
|
audio or invalid camera frames.
|
|
"""
|
|
event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
|
|
audio = event.get("audio") or event.get("delta")
|
|
fmt, format_rate = parse_realtime_audio_format(
|
|
event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
|
|
)
|
|
sample_rate_hz = (
|
|
event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
|
|
)
|
|
if not is_supported_realtime_input_format(fmt):
|
|
raise RealtimeProtocolError(
|
|
f"Unsupported input_audio_format: {fmt}",
|
|
code="unsupported_audio_format",
|
|
event_id=event_id,
|
|
)
|
|
try:
|
|
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
|
|
audio,
|
|
fmt,
|
|
sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
|
|
)
|
|
except ValueError as exc:
|
|
raise RealtimeProtocolError(str(exc), code="bad_event", event_id=event_id) from exc
|
|
hints: dict[str, object] = {}
|
|
if hints_source is not None:
|
|
copy_realtime_input_hints(hints_source, hints)
|
|
copy_realtime_input_hints(event, hints)
|
|
looks_like_speech = input_looks_like_speech(
|
|
{**(hints_source or {}), **event},
|
|
audio=audio,
|
|
fmt=fmt,
|
|
overlap_silence_rms=defaults.overlap_silence_rms,
|
|
)
|
|
video_frames: tuple[str, ...] = ()
|
|
raw_frames = event.get("video_frames")
|
|
if raw_frames is not None:
|
|
frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
|
|
if frames_error is not None:
|
|
raise RealtimeProtocolError(frames_error, code="invalid_video_frames", event_id=event_id)
|
|
if isinstance(raw_frames, list):
|
|
video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
|
|
duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
|
|
audio_end_ms = hints.get("audio_end_ms")
|
|
try:
|
|
audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
|
|
except (binascii.Error, ValueError) as exc:
|
|
raise RealtimeProtocolError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
|
|
return RealtimeAudioAppend(
|
|
event_id=event_id,
|
|
audio=audio_bytes,
|
|
format=str(fmt),
|
|
sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
|
|
is_speech=looks_like_speech,
|
|
video_frames=video_frames,
|
|
duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
|
|
audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
|
|
hints=hints,
|
|
)
|