Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
psv666 authored and GitHub committed
1 parent
825ab20b07
commit
e3be42e052
37 files changed
+3792
-1629
No files matched your search
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The OpenAI Realtime wire codec, shared by every vLLM-Omni Realtime surface.
|
||||
|
||||
What this is
|
||||
------------
|
||||
The model-agnostic half of ``WS /v1/realtime``: how a client event is parsed,
|
||||
which audio formats and session fields are accepted, what a conversation item
|
||||
means, how audio is decoded and resampled, and the shape of the error envelope
|
||||
(the *codes* inside it are a consumer's own --- ours are in
|
||||
``vllm_omni.protocol.duplex.errors``). It is a library of pure functions and value objects.
|
||||
|
||||
What it is not
|
||||
--------------
|
||||
It owns no session. It does not decide when a response starts, what a model is
|
||||
prompted with, whether a turn is over, or where committed audio goes. Those are
|
||||
runtime decisions, and after PR #7413 a duplex session's runtime decisions all
|
||||
belong to ``vllm_omni.engine.duplex`` --- the session runner, the typed
|
||||
``DuplexCommand`` / ``DuplexEvent`` boundary and the ``DuplexModelPlugin`` seam.
|
||||
This package sits *under* that: the duplex engine binds the codec to its own
|
||||
command and event vocabulary in
|
||||
``vllm_omni.engine.duplex.realtime_commands`` and
|
||||
``vllm_omni.engine.duplex.realtime_events``.
|
||||
|
||||
Why it is separate (RFC #6592, P0a)
|
||||
-----------------------------------
|
||||
So that a second Realtime surface --- a Qwen3-Omni GA profile, say --- can reuse
|
||||
the parsing, validation, format negotiation and error envelope without adopting
|
||||
the duplex session control plane (lease manager, commit policy, model channel),
|
||||
and without a second copy of the codec drifting away from this one. A consumer
|
||||
states what it can serve through
|
||||
:class:`~vllm_omni.protocol.realtime.capabilities.RealtimeProtocolCapabilities`
|
||||
and keeps its own session state.
|
||||
|
||||
Known duplex-shaped defaults
|
||||
----------------------------
|
||||
This package is model-agnostic in its *dependencies*, not yet in every
|
||||
*default*. Two constants still carry the assumptions of its first consumer and
|
||||
a second one will have to parameterize them:
|
||||
|
||||
* :func:`~vllm_omni.protocol.realtime.audio.convert_input_audio_with_rate`
|
||||
resamples to 16 kHz, which is MiniCPM-o's feature-extractor rate. A GA client
|
||||
sending 24 kHz is converted down rather than served at its own rate.
|
||||
* :func:`~vllm_omni.protocol.realtime.items.validate_realtime_video_frames`
|
||||
encodes the MiniCPM-o duplex camera contract (at most two frames per append,
|
||||
``max_slice_nums`` not selectable on the wire).
|
||||
|
||||
Both are behaviour this extraction deliberately did **not** change (RFC #6592
|
||||
P0a is behaviour-preserving); making them per-consumer is P0b/P1 work.
|
||||
|
||||
Dependency rule
|
||||
---------------
|
||||
Nothing here may import ``vllm_omni.engine``, ``vllm_omni.entrypoints``,
|
||||
``vllm_omni.model_executor``, ``vllm_omni.worker`` or ``vllm_omni.clients``;
|
||||
``tests/protocol/realtime/test_protocol_import_boundary.py`` asserts it.
|
||||
"""
|
||||
|
||||
from vllm_omni.protocol.realtime.audio import (
|
||||
MAX_INPUT_SAMPLE_RATE_HZ,
|
||||
MIN_INPUT_SAMPLE_RATE_HZ,
|
||||
convert_input_audio_with_rate,
|
||||
convert_output_audio,
|
||||
encode_float32_mono_wav_base64,
|
||||
resample_pcm16_mono,
|
||||
validate_input_sample_rate_hz,
|
||||
wav_payload_to_pcm16,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.audio_input import (
|
||||
REALTIME_INPUT_HINT_KEYS,
|
||||
RealtimeAudioAppend,
|
||||
copy_realtime_input_hints,
|
||||
decode_audio_append,
|
||||
input_explicitly_non_speech,
|
||||
input_looks_like_speech,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.capabilities import (
|
||||
RealtimeProtocolCapabilities,
|
||||
RealtimeSessionRejection,
|
||||
validate_session_payload,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.commands import RealtimeCommand
|
||||
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
|
||||
from vllm_omni.protocol.realtime.events import (
|
||||
RealtimeEvent,
|
||||
ResponseScopedEvent,
|
||||
new_event_id,
|
||||
wire_value,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.formats import (
|
||||
REALTIME_INPUT_AUDIO_FORMATS,
|
||||
REALTIME_OUTPUT_AUDIO_FORMATS,
|
||||
is_supported_realtime_input_format,
|
||||
parse_realtime_audio_format,
|
||||
realtime_audio_format_object,
|
||||
realtime_output_format,
|
||||
validate_conversation_item_audio_formats,
|
||||
validate_realtime_response_audio_formats,
|
||||
validate_realtime_session_audio_formats,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.items import (
|
||||
input_transcript_from_item,
|
||||
normalize_conversation_item,
|
||||
text_chars_for_audio_ms_from_marks,
|
||||
truncate_realtime_item_content,
|
||||
validate_realtime_item_truncate,
|
||||
validate_realtime_video_frames,
|
||||
)
|
||||
from vllm_omni.protocol.realtime.session import (
|
||||
RealtimeInputDefaults,
|
||||
apply_realtime_session_defaults,
|
||||
input_audio_transcription_config,
|
||||
json_safe_realtime_payload,
|
||||
realtime_max_output_tokens,
|
||||
realtime_overlap_fields,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MAX_INPUT_SAMPLE_RATE_HZ",
|
||||
"MIN_INPUT_SAMPLE_RATE_HZ",
|
||||
"REALTIME_INPUT_AUDIO_FORMATS",
|
||||
"REALTIME_INPUT_HINT_KEYS",
|
||||
"REALTIME_OUTPUT_AUDIO_FORMATS",
|
||||
"RealtimeAudioAppend",
|
||||
"RealtimeCommand",
|
||||
"RealtimeEvent",
|
||||
"RealtimeInputDefaults",
|
||||
"RealtimeProtocolCapabilities",
|
||||
"RealtimeProtocolError",
|
||||
"RealtimeSessionRejection",
|
||||
"ResponseScopedEvent",
|
||||
"apply_realtime_session_defaults",
|
||||
"convert_input_audio_with_rate",
|
||||
"convert_output_audio",
|
||||
"copy_realtime_input_hints",
|
||||
"decode_audio_append",
|
||||
"encode_float32_mono_wav_base64",
|
||||
"input_audio_transcription_config",
|
||||
"input_explicitly_non_speech",
|
||||
"input_looks_like_speech",
|
||||
"input_transcript_from_item",
|
||||
"is_supported_realtime_input_format",
|
||||
"json_safe_realtime_payload",
|
||||
"new_event_id",
|
||||
"normalize_conversation_item",
|
||||
"parse_realtime_audio_format",
|
||||
"realtime_audio_format_object",
|
||||
"realtime_max_output_tokens",
|
||||
"realtime_output_format",
|
||||
"realtime_overlap_fields",
|
||||
"resample_pcm16_mono",
|
||||
"text_chars_for_audio_ms_from_marks",
|
||||
"truncate_realtime_item_content",
|
||||
"validate_conversation_item_audio_formats",
|
||||
"validate_input_sample_rate_hz",
|
||||
"validate_realtime_item_truncate",
|
||||
"validate_realtime_response_audio_formats",
|
||||
"validate_realtime_session_audio_formats",
|
||||
"validate_realtime_video_frames",
|
||||
"validate_session_payload",
|
||||
"wav_payload_to_pcm16",
|
||||
"wire_value",
|
||||
]
|
||||
Reference in new issue
Block a user