Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
psv666 authored and GitHub committed
1 parent
825ab20b07
commit
e3be42e052
37 files changed
+3792
-1629
No files matched your search
@@ -0,0 +1,243 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Decoding one ``input_audio_buffer.append``.
|
||||
|
||||
An append is the hottest client event on a Realtime connection and the one with
|
||||
the most wire surface: base64 audio in any accepted format, an optional rate,
|
||||
optional camera frames, and a set of optional hints (``duration_ms``,
|
||||
``is_speech``, a VAD probability, a transcript) that clients attach and servers
|
||||
are free to use or ignore.
|
||||
|
||||
:func:`decode_audio_append` turns that into :class:`RealtimeAudioAppend`, a
|
||||
value object: bytes at 16 kHz ``pcm_f32le``, the surviving hints, and a speech
|
||||
verdict. It is a pure function of the event plus the session's
|
||||
:class:`~vllm_omni.protocol.realtime.session.RealtimeInputDefaults`, so any
|
||||
consumer decodes an append the same way. Building a consumer's own command
|
||||
object out of it is one constructor call --- see
|
||||
``vllm_omni.engine.duplex.realtime_commands.build_append_audio``, which wraps it
|
||||
for the duplex ``AppendAudio``.
|
||||
|
||||
The speech verdict here is the *client-declared* one (explicit flags, a VAD
|
||||
probability the client sent, or an RMS floor as a last resort). It is not
|
||||
server-side VAD; that is a runtime concern and lives with whoever owns the
|
||||
session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm_omni.protocol.realtime.audio import convert_input_audio_with_rate
|
||||
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
|
||||
from vllm_omni.protocol.realtime.formats import (
|
||||
is_supported_realtime_input_format,
|
||||
parse_realtime_audio_format,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.items import validate_realtime_video_frames
|
||||
from vllm_omni.protocol.realtime.session import RealtimeInputDefaults
|
||||
|
||||
__all__ = [
|
||||
"REALTIME_INPUT_HINT_KEYS",
|
||||
"RealtimeAudioAppend",
|
||||
"copy_realtime_input_hints",
|
||||
"decode_audio_append",
|
||||
"input_explicitly_non_speech",
|
||||
"input_looks_like_speech",
|
||||
]
|
||||
|
||||
#: Wire hint keys copied verbatim from a client append onto the internal payload.
|
||||
REALTIME_INPUT_HINT_KEYS = (
|
||||
"duration_ms",
|
||||
"audio_duration_ms",
|
||||
"audio_start_ms",
|
||||
"audio_end_ms",
|
||||
"is_speech",
|
||||
"speech",
|
||||
"speech_probability",
|
||||
"vad",
|
||||
"overlap_action",
|
||||
"overlap",
|
||||
"force_barge_in",
|
||||
"force_listen",
|
||||
"text",
|
||||
"transcript",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RealtimeAudioAppend:
|
||||
"""One decoded ``input_audio_buffer.append``.
|
||||
|
||||
``audio`` is raw bytes in ``format`` at ``sample_rate_hz`` --- after
|
||||
:func:`decode_audio_append` that is 16 kHz ``pcm_f32le`` for every input
|
||||
format the codec converts.
|
||||
"""
|
||||
|
||||
audio: bytes
|
||||
format: str
|
||||
sample_rate_hz: int | None = None
|
||||
is_speech: bool | None = None
|
||||
video_frames: tuple[str, ...] = ()
|
||||
duration_ms: int | None = None
|
||||
audio_end_ms: int | None = None
|
||||
#: Model-neutral hints carried through from the wire (rms, vad, transcript hints ...).
|
||||
hints: dict[str, object] = field(default_factory=dict)
|
||||
#: Client correlation id (OpenAI ``event_id``), echoed on error events.
|
||||
event_id: str | None = None
|
||||
|
||||
|
||||
# ---- speech hints ----
|
||||
|
||||
|
||||
def input_explicitly_non_speech(event: Mapping[str, object]) -> bool:
|
||||
for key in ("is_speech", "speech"):
|
||||
value = event.get(key)
|
||||
if isinstance(value, bool):
|
||||
return not value
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
value = vad.get("is_speech")
|
||||
if isinstance(value, bool):
|
||||
return not value
|
||||
probability = vad.get("speech_probability", vad.get("probability"))
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) < 0.5
|
||||
probability = event.get("speech_probability")
|
||||
return isinstance(probability, int | float) and float(probability) < 0.5
|
||||
|
||||
|
||||
def input_looks_like_speech(
|
||||
event: Mapping[str, object],
|
||||
*,
|
||||
audio: object,
|
||||
fmt: object,
|
||||
overlap_silence_rms: float,
|
||||
) -> bool:
|
||||
if input_explicitly_non_speech(event):
|
||||
return False
|
||||
for key in ("is_speech", "speech"):
|
||||
value = event.get(key)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
probability = vad.get("speech_probability", vad.get("probability"))
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) >= 0.5
|
||||
probability = event.get("speech_probability")
|
||||
if isinstance(probability, int | float):
|
||||
return float(probability) >= 0.5
|
||||
if fmt != "pcm_f32le":
|
||||
return True
|
||||
if isinstance(audio, bytes | bytearray):
|
||||
raw = bytes(audio)
|
||||
elif isinstance(audio, str):
|
||||
try:
|
||||
raw = base64.b64decode(audio, validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
if len(raw) < 4 or len(raw) % 4 != 0:
|
||||
return True
|
||||
samples = np.frombuffer(raw, dtype=np.float32)
|
||||
if samples.size == 0:
|
||||
return False
|
||||
rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
|
||||
threshold = event.get("overlap_silence_rms")
|
||||
if not isinstance(threshold, int | float):
|
||||
vad = event.get("vad")
|
||||
if isinstance(vad, dict):
|
||||
threshold = vad.get("silence_rms")
|
||||
silence_rms = float(threshold) if isinstance(threshold, int | float) else overlap_silence_rms
|
||||
return rms >= max(0.0, silence_rms)
|
||||
|
||||
|
||||
def copy_realtime_input_hints(source: Mapping[str, object], target: dict[str, object]) -> None:
|
||||
for key in REALTIME_INPUT_HINT_KEYS:
|
||||
if key in source:
|
||||
target[key] = source[key]
|
||||
|
||||
|
||||
# ---- append decoding ----
|
||||
|
||||
|
||||
def decode_audio_append(
|
||||
event: Mapping[str, object],
|
||||
*,
|
||||
defaults: RealtimeInputDefaults,
|
||||
hints_source: Mapping[str, object] | None = None,
|
||||
) -> RealtimeAudioAppend:
|
||||
"""Validate, convert (to 16 kHz ``pcm_f32le``) and pack one audio append.
|
||||
|
||||
``hints_source`` is the enclosing payload when the audio arrives inside
|
||||
something larger than a bare append --- a ``conversation.item.create``
|
||||
audio part, say --- so its hints apply unless the part overrides them.
|
||||
|
||||
Raises :class:`RealtimeProtocolError` for an unsupported format, undecodable
|
||||
audio or invalid camera frames.
|
||||
"""
|
||||
event_id = cast("str", event.get("event_id")) if isinstance(event.get("event_id"), str) else None
|
||||
audio = event.get("audio") or event.get("delta")
|
||||
fmt, format_rate = parse_realtime_audio_format(
|
||||
event.get("format") or event.get("input_audio_format") or defaults.input_audio_format
|
||||
)
|
||||
sample_rate_hz = (
|
||||
event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or defaults.input_sample_rate_hz
|
||||
)
|
||||
if not is_supported_realtime_input_format(fmt):
|
||||
raise RealtimeProtocolError(
|
||||
f"Unsupported input_audio_format: {fmt}",
|
||||
code="unsupported_audio_format",
|
||||
event_id=event_id,
|
||||
)
|
||||
try:
|
||||
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
|
||||
audio,
|
||||
fmt,
|
||||
sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise RealtimeProtocolError(str(exc), code="bad_event", event_id=event_id) from exc
|
||||
hints: dict[str, object] = {}
|
||||
if hints_source is not None:
|
||||
copy_realtime_input_hints(hints_source, hints)
|
||||
copy_realtime_input_hints(event, hints)
|
||||
looks_like_speech = input_looks_like_speech(
|
||||
{**(hints_source or {}), **event},
|
||||
audio=audio,
|
||||
fmt=fmt,
|
||||
overlap_silence_rms=defaults.overlap_silence_rms,
|
||||
)
|
||||
video_frames: tuple[str, ...] = ()
|
||||
raw_frames = event.get("video_frames")
|
||||
if raw_frames is not None:
|
||||
frames_error = validate_realtime_video_frames(raw_frames, event.get("max_slice_nums"))
|
||||
if frames_error is not None:
|
||||
raise RealtimeProtocolError(frames_error, code="invalid_video_frames", event_id=event_id)
|
||||
if isinstance(raw_frames, list):
|
||||
video_frames = tuple(frame for frame in raw_frames if isinstance(frame, str) and frame)
|
||||
duration_ms = hints.get("duration_ms", hints.get("audio_duration_ms"))
|
||||
audio_end_ms = hints.get("audio_end_ms")
|
||||
try:
|
||||
audio_bytes = base64.b64decode(audio, validate=True) if isinstance(audio, str) and audio else b""
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise RealtimeProtocolError("input audio is not valid base64", code="bad_audio", event_id=event_id) from exc
|
||||
return RealtimeAudioAppend(
|
||||
event_id=event_id,
|
||||
audio=audio_bytes,
|
||||
format=str(fmt),
|
||||
sample_rate_hz=int(sample_rate_hz) if isinstance(sample_rate_hz, int | float) else None,
|
||||
is_speech=looks_like_speech,
|
||||
video_frames=video_frames,
|
||||
duration_ms=int(duration_ms) if isinstance(duration_ms, int | float) else None,
|
||||
audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else None,
|
||||
hints=hints,
|
||||
)
|
||||
Reference in new issue
Block a user