164 lines
6.1 KiB
Python
164 lines
6.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
"""The OpenAI Realtime wire codec, shared by every vLLM-Omni Realtime surface.
|
|
|
|
What this is
|
|
------------
|
|
The model-agnostic half of ``WS /v1/realtime``: how a client event is parsed,
|
|
which audio formats and session fields are accepted, what a conversation item
|
|
means, how audio is decoded and resampled, and the shape of the error envelope
|
|
(the *codes* inside it are a consumer's own --- ours are in
|
|
``vllm_omni.protocol.duplex.errors``). It is a library of pure functions and value objects.
|
|
|
|
What it is not
|
|
--------------
|
|
It owns no session. It does not decide when a response starts, what a model is
|
|
prompted with, whether a turn is over, or where committed audio goes. Those are
|
|
runtime decisions, and after PR #7413 a duplex session's runtime decisions all
|
|
belong to ``vllm_omni.engine.duplex`` --- the session runner, the typed
|
|
``DuplexCommand`` / ``DuplexEvent`` boundary and the ``DuplexModelPlugin`` seam.
|
|
This package sits *under* that: the duplex engine binds the codec to its own
|
|
command and event vocabulary in
|
|
``vllm_omni.engine.duplex.realtime_commands`` and
|
|
``vllm_omni.engine.duplex.realtime_events``.
|
|
|
|
Why it is separate (RFC #6592, P0a)
|
|
-----------------------------------
|
|
So that a second Realtime surface --- a Qwen3-Omni GA profile, say --- can reuse
|
|
the parsing, validation, format negotiation and error envelope without adopting
|
|
the duplex session control plane (lease manager, commit policy, model channel),
|
|
and without a second copy of the codec drifting away from this one. A consumer
|
|
states what it can serve through
|
|
:class:`~vllm_omni.protocol.realtime.capabilities.RealtimeProtocolCapabilities`
|
|
and keeps its own session state.
|
|
|
|
Known duplex-shaped defaults
|
|
----------------------------
|
|
This package is model-agnostic in its *dependencies*, not yet in every
|
|
*default*. Two constants still carry the assumptions of its first consumer and
|
|
a second one will have to parameterize them:
|
|
|
|
* :func:`~vllm_omni.protocol.realtime.audio.convert_input_audio_with_rate`
|
|
resamples to 16 kHz, which is MiniCPM-o's feature-extractor rate. A GA client
|
|
sending 24 kHz is converted down rather than served at its own rate.
|
|
* :func:`~vllm_omni.protocol.realtime.items.validate_realtime_video_frames`
|
|
encodes the MiniCPM-o duplex camera contract (at most two frames per append,
|
|
``max_slice_nums`` not selectable on the wire).
|
|
|
|
Both are behaviour this extraction deliberately did **not** change (RFC #6592
|
|
P0a is behaviour-preserving); making them per-consumer is P0b/P1 work.
|
|
|
|
Dependency rule
|
|
---------------
|
|
Nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``,
|
|
``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``;
|
|
``tests/protocol/realtime/test_protocol_import_boundary.py`` asserts it.
|
|
"""
|
|
|
|
from vllm_omni.protocol.realtime.audio import (
|
|
MAX_INPUT_SAMPLE_RATE_HZ,
|
|
MIN_INPUT_SAMPLE_RATE_HZ,
|
|
convert_input_audio_with_rate,
|
|
convert_output_audio,
|
|
encode_float32_mono_wav_base64,
|
|
resample_pcm16_mono,
|
|
validate_input_sample_rate_hz,
|
|
wav_payload_to_pcm16,
|
|
)
|
|
from vllm_omni.protocol.realtime.audio_input import (
|
|
REALTIME_INPUT_HINT_KEYS,
|
|
RealtimeAudioAppend,
|
|
copy_realtime_input_hints,
|
|
decode_audio_append,
|
|
input_explicitly_non_speech,
|
|
input_looks_like_speech,
|
|
)
|
|
from vllm_omni.protocol.realtime.capabilities import (
|
|
RealtimeProtocolCapabilities,
|
|
RealtimeSessionRejection,
|
|
validate_session_payload,
|
|
)
|
|
from vllm_omni.protocol.realtime.commands import RealtimeCommand
|
|
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
|
|
from vllm_omni.protocol.realtime.events import (
|
|
RealtimeEvent,
|
|
ResponseScopedEvent,
|
|
new_event_id,
|
|
wire_value,
|
|
)
|
|
from vllm_omni.protocol.realtime.formats import (
|
|
REALTIME_INPUT_AUDIO_FORMATS,
|
|
REALTIME_OUTPUT_AUDIO_FORMATS,
|
|
is_supported_realtime_input_format,
|
|
parse_realtime_audio_format,
|
|
realtime_audio_format_object,
|
|
realtime_output_format,
|
|
validate_conversation_item_audio_formats,
|
|
validate_realtime_response_audio_formats,
|
|
validate_realtime_session_audio_formats,
|
|
)
|
|
from vllm_omni.protocol.realtime.items import (
|
|
input_transcript_from_item,
|
|
normalize_conversation_item,
|
|
text_chars_for_audio_ms_from_marks,
|
|
truncate_realtime_item_content,
|
|
validate_realtime_item_truncate,
|
|
validate_realtime_video_frames,
|
|
)
|
|
from vllm_omni.protocol.realtime.session import (
|
|
RealtimeInputDefaults,
|
|
apply_realtime_session_defaults,
|
|
input_audio_transcription_config,
|
|
json_safe_realtime_payload,
|
|
realtime_max_output_tokens,
|
|
realtime_overlap_fields,
|
|
)
|
|
|
|
__all__ = [
|
|
"MAX_INPUT_SAMPLE_RATE_HZ",
|
|
"MIN_INPUT_SAMPLE_RATE_HZ",
|
|
"REALTIME_INPUT_AUDIO_FORMATS",
|
|
"REALTIME_INPUT_HINT_KEYS",
|
|
"REALTIME_OUTPUT_AUDIO_FORMATS",
|
|
"RealtimeAudioAppend",
|
|
"RealtimeCommand",
|
|
"RealtimeEvent",
|
|
"RealtimeInputDefaults",
|
|
"RealtimeProtocolCapabilities",
|
|
"RealtimeProtocolError",
|
|
"RealtimeSessionRejection",
|
|
"ResponseScopedEvent",
|
|
"apply_realtime_session_defaults",
|
|
"convert_input_audio_with_rate",
|
|
"convert_output_audio",
|
|
"copy_realtime_input_hints",
|
|
"decode_audio_append",
|
|
"encode_float32_mono_wav_base64",
|
|
"input_audio_transcription_config",
|
|
"input_explicitly_non_speech",
|
|
"input_looks_like_speech",
|
|
"input_transcript_from_item",
|
|
"is_supported_realtime_input_format",
|
|
"json_safe_realtime_payload",
|
|
"new_event_id",
|
|
"normalize_conversation_item",
|
|
"parse_realtime_audio_format",
|
|
"realtime_audio_format_object",
|
|
"realtime_max_output_tokens",
|
|
"realtime_output_format",
|
|
"realtime_overlap_fields",
|
|
"resample_pcm16_mono",
|
|
"text_chars_for_audio_ms_from_marks",
|
|
"truncate_realtime_item_content",
|
|
"validate_conversation_item_audio_formats",
|
|
"validate_input_sample_rate_hz",
|
|
"validate_realtime_item_truncate",
|
|
"validate_realtime_response_audio_formats",
|
|
"validate_realtime_session_audio_formats",
|
|
"validate_realtime_video_frames",
|
|
"validate_session_payload",
|
|
"wav_payload_to_pcm16",
|
|
"wire_value",
|
|
]
|