198 lines
8.6 KiB
Python
198 lines
8.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
"""Realtime conversation items: shape, truncation and the transcript inside one.
|
|
|
|
A Realtime conversation is a list of *items* --- a user message, an assistant
|
|
message, a function call --- each with a ``content`` list of typed parts. The
|
|
client creates them (``conversation.item.create``), truncates an assistant
|
|
item's audio after a barge-in (``conversation.item.truncate``), and the server
|
|
echoes them back inside ``conversation.item.*`` and ``response.done`` events.
|
|
|
|
The rules in this module are the ones both sides have to agree on: which
|
|
defaults a bare item gets, how far a transcript is cut when the client says it
|
|
only heard N milliseconds of audio, and what makes a truncate request invalid.
|
|
Item *storage* --- which items a session currently holds, in what order --- is
|
|
the consumer's state, not this module's.
|
|
|
|
``validate_realtime_video_frames`` sits here too: camera frames ride along an
|
|
``input_audio_buffer.append`` in the vLLM-Omni extension of the protocol, and
|
|
validating them is the same kind of wire-shape check.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import binascii
|
|
from collections.abc import Mapping
|
|
from uuid import uuid4
|
|
|
|
__all__ = [
|
|
"input_transcript_from_item",
|
|
"normalize_conversation_item",
|
|
"text_chars_for_audio_ms_from_marks",
|
|
"truncate_realtime_item_content",
|
|
"validate_realtime_item_truncate",
|
|
"validate_realtime_video_frames",
|
|
]
|
|
|
|
|
|
def validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
|
|
"""Validate omni-duplex camera frames on input_audio_buffer.append.
|
|
|
|
Wire contract matches the official MiniCPM-o duplex loop: one base
|
|
base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
|
|
stacked composite tiling the sub-frames captured inside it (at most 2
|
|
images either way). A caller-supplied ``max_slice_nums`` is rejected rather
|
|
than silently ignored: slicing is Stage 0's decision here, and Stage 0
|
|
already applies the official HD suggestion for a stacked unit
|
|
(``max_slice_nums=[2, 1]``). The wire simply does not let the client choose
|
|
it.
|
|
"""
|
|
if max_slice_nums not in (None, 1):
|
|
return "max_slice_nums is not selectable on the wire; Stage0 slices stacked units itself"
|
|
if not isinstance(video_frames, list):
|
|
return "video_frames must be a list of base64-encoded images"
|
|
frames = [frame for frame in video_frames if frame is not None]
|
|
if len(frames) > 2:
|
|
return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
|
|
for frame in frames:
|
|
if not isinstance(frame, str) or not frame:
|
|
return "video_frames entries must be non-empty base64 strings"
|
|
if len(frame) > 4_000_000:
|
|
return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
|
|
try:
|
|
header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
|
|
except (binascii.Error, ValueError):
|
|
return "video_frames entries must be valid base64"
|
|
if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
|
|
return "video_frames entries must be JPEG or PNG images"
|
|
return None
|
|
|
|
|
|
def normalize_conversation_item(item: Mapping[str, object]) -> dict[str, object]:
|
|
normalized = dict(item)
|
|
normalized.setdefault("id", f"item_{uuid4().hex}")
|
|
normalized.setdefault("object", "realtime.item")
|
|
normalized.setdefault("type", "message")
|
|
normalized.setdefault("status", "completed")
|
|
if "role" not in normalized and normalized.get("type") == "message":
|
|
normalized["role"] = "user"
|
|
if not isinstance(normalized.get("content"), list):
|
|
normalized["content"] = []
|
|
return normalized
|
|
|
|
|
|
def text_chars_for_audio_ms_from_marks(
|
|
audio_end_ms: int,
|
|
text_len: int,
|
|
marks: list[object],
|
|
*,
|
|
final_ms: object | None = None,
|
|
) -> int:
|
|
if text_len <= 0:
|
|
return 0
|
|
clean_marks: list[tuple[int, int]] = []
|
|
for mark in marks:
|
|
if not isinstance(mark, dict):
|
|
continue
|
|
raw_text_chars = mark.get("text_chars")
|
|
raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
|
|
if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
|
|
continue
|
|
clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
|
|
if not clean_marks:
|
|
return 0 if audio_end_ms <= 0 else text_len
|
|
clean_marks.sort(key=lambda item: item[0])
|
|
audio_end_ms = max(0, int(audio_end_ms))
|
|
if audio_end_ms <= 0:
|
|
return 0
|
|
previous_ms = 0
|
|
previous_chars = 0
|
|
for mark_ms, mark_chars in clean_marks:
|
|
mark_ms = max(previous_ms, mark_ms)
|
|
mark_chars = max(previous_chars, min(text_len, mark_chars))
|
|
if audio_end_ms <= mark_ms:
|
|
if mark_ms <= previous_ms:
|
|
return mark_chars
|
|
ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
|
|
return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
|
|
previous_ms = mark_ms
|
|
previous_chars = mark_chars
|
|
if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
|
|
if audio_end_ms >= int(final_ms):
|
|
return text_len
|
|
ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
|
|
return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
|
|
return text_len if audio_end_ms >= previous_ms else previous_chars
|
|
|
|
|
|
def truncate_realtime_item_content(item: dict[str, object], *, content_index: int, audio_end_ms: int) -> None:
|
|
content = item.get("content")
|
|
if not isinstance(content, list) or not content:
|
|
return
|
|
index = max(0, int(content_index))
|
|
if index >= len(content):
|
|
return
|
|
part = content[index]
|
|
if not isinstance(part, dict):
|
|
return
|
|
transcript = part.get("transcript")
|
|
if not isinstance(transcript, str) or not transcript:
|
|
return
|
|
marks = part.get("audio_text_marks")
|
|
if isinstance(marks, list):
|
|
keep_chars = text_chars_for_audio_ms_from_marks(
|
|
audio_end_ms,
|
|
len(transcript),
|
|
marks,
|
|
final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
|
|
)
|
|
part["transcript"] = transcript[:keep_chars].rstrip()
|
|
return
|
|
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
|
|
if isinstance(duration_ms, int | float) and duration_ms > 0:
|
|
keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
|
|
part["transcript"] = transcript[:keep_chars].rstrip()
|
|
elif audio_end_ms <= 0:
|
|
part["transcript"] = ""
|
|
|
|
|
|
def validate_realtime_item_truncate(item: Mapping[str, object], *, content_index: int, audio_end_ms: int) -> str | None:
|
|
if item.get("type") != "message" or item.get("role") != "assistant":
|
|
return "conversation.item.truncate only supports assistant message items"
|
|
if audio_end_ms < 0:
|
|
return "conversation.item.truncate requires non-negative audio_end_ms"
|
|
content = item.get("content")
|
|
if not isinstance(content, list) or not content:
|
|
return None
|
|
index = max(0, int(content_index))
|
|
if index >= len(content):
|
|
return f"conversation.item.truncate content_index out of range: {content_index}"
|
|
part = content[index]
|
|
if not isinstance(part, dict):
|
|
return "conversation.item.truncate target content part is invalid"
|
|
if part.get("type") not in {"audio", "output_audio"}:
|
|
return "conversation.item.truncate target content part must be audio"
|
|
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
|
|
if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
|
|
return "conversation.item.truncate audio_end_ms exceeds item audio duration"
|
|
return None
|
|
|
|
|
|
def input_transcript_from_item(item: Mapping[str, object]) -> str:
|
|
content = item.get("content")
|
|
if not isinstance(content, list):
|
|
return ""
|
|
parts: list[str] = []
|
|
for part in content:
|
|
if not isinstance(part, dict):
|
|
continue
|
|
for key in ("transcript", "text"):
|
|
value = part.get(key)
|
|
if isinstance(value, str) and value:
|
|
if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
|
|
parts.append(value)
|
|
break
|
|
return "".join(parts).strip()
|