[Core] Give the OpenAI Realtime wire codec a reusable home (RFC #6592 P0a) (#7640)

Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
psv666 authored and GitHub committed 2026-09-17 02:33:40 -07:00
1 parent 825ab20b07
commit e3be42e052
37 files changed
+3792 -1629

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
@@ -0,0 +1,144 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""``protocol.duplex`` is the only door a duplex consumer uses.
The layering is a chain::
protocol.realtime <- protocol.duplex <- engine / entrypoints / clients
The middle link only earns its keep if the last arrow is the *only* one. If the
engine may also reach past it straight into ``protocol.realtime``, then giving a
Tier 1 helper a duplex-specific version later means touching every call site
instead of one file --- which is exactly the change
``convert_input_audio_with_rate`` is expected to need (it resamples to
MiniCPM-o's 16 kHz, not the client's rate).
So this asserts two things: nobody skips the middle link, and the middle link is
complete enough that nobody has to.
"""
from __future__ import annotations
import ast
from pathlib import Path
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
REPO_ROOT = Path(__file__).resolve().parents[3]
#: Every module that speaks the duplex protocol. These import
#: ``vllm_omni.protocol.duplex``; reaching into ``vllm_omni.protocol.realtime``
#: from here skips the extension point.
CONSUMER_PATHS = (
"vllm_omni/engine/duplex",
"vllm_omni/engine/duplex_omni_engine.py",
"vllm_omni/entrypoints/duplex",
"vllm_omni/entrypoints/duplex_omni.py",
"vllm_omni/clients/duplex.py",
"vllm_omni/clients/inline_duplex.py",
)
def _consumer_modules() -> list[Path]:
found: list[Path] = []
for entry in CONSUMER_PATHS:
path = REPO_ROOT / entry
found.extend(sorted(path.rglob("*.py")) if path.is_dir() else [path])
assert found, "consumer list is stale"
return found
@pytest.mark.parametrize("module", _consumer_modules(), ids=lambda p: p.name)
def test_a_duplex_consumer_does_not_skip_the_duplex_protocol_package(module: Path) -> None:
# Source-level, so a lazy import inside a function body is caught too.
tree = ast.parse(module.read_text())
offenders = []
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and (node.module or "").startswith("vllm_omni.protocol.realtime"):
offenders.append(f"line {node.lineno}: from {node.module} import ...")
elif isinstance(node, ast.Import):
offenders.extend(
f"line {node.lineno}: import {alias.name}"
for alias in node.names
if alias.name.startswith("vllm_omni.protocol.realtime")
)
assert not offenders, (
f"{module.relative_to(REPO_ROOT)} imports the Tier 1 package directly; "
f"import from vllm_omni.protocol.duplex instead:\n " + "\n ".join(offenders)
)
def test_the_duplex_package_carries_the_whole_command_vocabulary() -> None:
from vllm_omni.engine.duplex import commands as engine_commands
from vllm_omni.protocol.duplex import commands as duplex_wire
# Every command the engine handles is nameable through the one door.
engine_names = {
name
for name in engine_commands.__all__
if isinstance(getattr(engine_commands, name), type)
and issubclass(getattr(engine_commands, name), engine_commands.DuplexCommand)
and getattr(engine_commands, name) is not engine_commands.DuplexCommand
}
assert engine_names <= set(duplex_wire.__all__)
def test_the_duplex_package_carries_the_whole_event_vocabulary() -> None:
from vllm_omni.engine.duplex import events as engine_events
from vllm_omni.protocol.duplex import events as duplex_wire
engine_names = {
name
for name in engine_events.__all__
if isinstance(getattr(engine_events, name), type)
and issubclass(getattr(engine_events, name), engine_events.DuplexEvent)
}
assert engine_names <= set(duplex_wire.__all__)
@pytest.mark.parametrize(
"module_pair",
[("events", "events"), ("commands", "commands")],
ids=["events", "commands"],
)
def test_a_reexported_tier1_name_is_the_tier1_object(module_pair: tuple[str, str]) -> None:
"""Re-export, not re-declaration: the Tier 1 class is one object."""
import importlib
duplex_mod = importlib.import_module(f"vllm_omni.protocol.duplex.{module_pair[0]}")
realtime_mod = importlib.import_module(f"vllm_omni.protocol.realtime.{module_pair[1]}")
shared = [n for n in duplex_mod.__all__ if hasattr(realtime_mod, n)]
assert shared, "nothing re-exported --- the facade is not doing its job"
for name in shared:
duplex_obj = getattr(duplex_mod, name)
realtime_obj = getattr(realtime_mod, name)
if duplex_obj is realtime_obj:
continue
# The only permitted divergence is a Tier 2 class extending its twin.
assert isinstance(duplex_obj, type) and issubclass(duplex_obj, realtime_obj), (
f"{name} in protocol.duplex is neither the Tier 1 object nor a subclass of it"
)
def test_duplex_event_is_an_alias_but_duplex_command_is_not() -> None:
"""The asymmetry is deliberate, and a later reader should not 'fix' it.
An event has no engine-internal half --- what the session emits is what the
socket sends --- so ``DuplexEvent`` is simply ``RealtimeEvent``. A command
does: ``payload()`` renders the session runner's mailbox dictionary, and for
four of the seventeen the mailbox channel is not even the client event type.
"""
from vllm_omni.engine.duplex.commands import DuplexCommand
from vllm_omni.protocol.duplex.events import DuplexEvent
from vllm_omni.protocol.realtime.commands import RealtimeCommand
from vllm_omni.protocol.realtime.events import RealtimeEvent
assert DuplexEvent is RealtimeEvent
assert DuplexCommand is not RealtimeCommand
assert issubclass(DuplexCommand, RealtimeCommand)
assert hasattr(DuplexCommand, "payload") and not hasattr(RealtimeCommand, "payload")
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The shared Realtime codec must not depend on any runtime that uses it.
``vllm_omni.protocol.realtime`` exists so a second Realtime surface can reuse
the wire codec without adopting the duplex session control plane. That only
holds while the arrows point one way: the engine, the entrypoints and the model
code import the protocol, never the reverse. Nothing in the package's own code
enforces that, so it is asserted here --- once on the source (which catches a
lazy import inside a function body) and once on a real import (which catches a
dependency arriving through a third module).
"""
from __future__ import annotations
import ast
import os
import subprocess
import sys
from pathlib import Path
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
REPO_ROOT = Path(__file__).resolve().parents[3]
PROTOCOL_DIR = REPO_ROOT / "vllm_omni" / "protocol"
#: Packages that consume the protocol. A dependency on any of them would make
#: the codec unusable outside the runtime it accidentally reached into.
#: ``vllm_omni.utils`` is deliberately absent: it is leaf helpers (the audio
#: resampler) that consume nothing, so depending on it costs a reuser nothing.
FORBIDDEN_PREFIXES = (
"vllm_omni.engine",
"vllm_omni.entrypoints",
"vllm_omni.model_executor",
"vllm_omni.worker",
"vllm_omni.clients",
)
def _protocol_sources() -> list[Path]:
sources = sorted(PROTOCOL_DIR.rglob("*.py"))
assert sources, f"no python sources under {PROTOCOL_DIR}"
return sources
def _imported_modules(source: Path) -> set[str]:
"""Every module name imported anywhere in the file, lazy imports included."""
names: set[str] = set()
for node in ast.walk(ast.parse(source.read_text())):
if isinstance(node, ast.Import):
names.update(alias.name for alias in node.names)
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
names.add(node.module)
return names
@pytest.mark.parametrize("source", _protocol_sources(), ids=lambda path: path.name)
def test_protocol_sources_do_not_import_a_consumer(source: Path) -> None:
offenders = sorted(
name
for name in _imported_modules(source)
if any(name == prefix or name.startswith(prefix + ".") for prefix in FORBIDDEN_PREFIXES)
)
assert not offenders, f"{source.relative_to(REPO_ROOT)} imports {', '.join(offenders)}"
def test_importing_the_protocol_package_loads_no_consumer() -> None:
# A child interpreter, because this test session has already imported the
# duplex engine: only a fresh process can tell what the package pulls in.
# ``vllm_omni`` itself is imported first, because its ``__init__`` loads a
# baseline graph (patches, the model registry) that has nothing to do with
# this package; only what the protocol import *adds* is measured.
env = os.environ.copy()
env.pop("CUDA_VISIBLE_DEVICES", None)
env.pop("HIP_VISIBLE_DEVICES", None)
script = f"""
import sys
import vllm_omni # noqa: F401
baseline = set(sys.modules)
import vllm_omni.protocol.realtime # noqa: F401
forbidden_prefixes = {FORBIDDEN_PREFIXES!r}
added = sorted(
name
for name in set(sys.modules) - baseline
if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes)
)
if added:
raise SystemExit("the realtime protocol package loaded consumers: " + ", ".join(added))
"""
result = subprocess.run(
[sys.executable, "-c", script],
cwd=REPO_ROOT,
env=env,
capture_output=True,
text=True,
timeout=180,
check=False,
)
assert result.returncode == 0, result.stderr
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""``RealtimeProtocolCapabilities``: one consumer's answers, the codec's checking.
This is the seam that lets a non-duplex Realtime surface validate a
``session.update`` without a ``DuplexModelPlugin``. What it has to get right is
narrow: the order the checks run in (formats before turn detection, because
that is the order clients already see), the error codes, and that a consumer
which declares a narrower format set is actually narrowed.
"""
from __future__ import annotations
from collections.abc import Mapping
import pytest
from vllm_omni.protocol.realtime.capabilities import (
RealtimeProtocolCapabilities,
RealtimeSessionRejection,
validate_session_payload,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_DEFAULT = RealtimeProtocolCapabilities()
def test_a_consumer_without_a_turn_detection_check_accepts_a_plain_session():
assert (
validate_session_payload({"model": "m", "audio": {"input": {"format": "pcm16"}}}, capabilities=_DEFAULT) is None
)
def test_a_consumer_without_a_turn_detection_check_ignores_turn_detection():
payload = {"turn_detection": {"type": "semantic_vad"}}
assert validate_session_payload(payload, capabilities=_DEFAULT) is None
def test_an_unsupported_input_format_is_rejected_with_the_realtime_code():
rejection = validate_session_payload({"input_audio_format": "opus"}, capabilities=_DEFAULT)
assert rejection == RealtimeSessionRejection(
code="unsupported_audio_format", message="Unsupported input_audio_format: opus"
)
def test_a_consumer_may_declare_a_narrower_format_set():
pcm_only = RealtimeProtocolCapabilities(input_audio_formats=frozenset({"pcm16"}))
assert validate_session_payload({"input_audio_format": "pcm16"}, capabilities=pcm_only) is None
rejection = validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=pcm_only)
assert rejection is not None
assert rejection.code == "unsupported_audio_format"
# The default capabilities still accept it: the narrowing is per consumer.
assert validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=_DEFAULT) is None
def test_the_turn_detection_check_runs_and_reports_its_own_code():
def reject_everything(session_payload: Mapping[str, object]) -> str | None:
del session_payload
return "server_vad is not available"
capabilities = RealtimeProtocolCapabilities(validate_turn_detection=reject_everything)
rejection = validate_session_payload({"turn_detection": {"type": "server_vad"}}, capabilities=capabilities)
assert rejection == RealtimeSessionRejection(
code="unsupported_turn_detection", message="server_vad is not available", param="turn_detection"
)
def test_a_bad_format_is_reported_before_a_bad_turn_detection():
calls: list[Mapping[str, object]] = []
def record(session_payload: Mapping[str, object]) -> str | None:
calls.append(session_payload)
return "turn detection is wrong too"
capabilities = RealtimeProtocolCapabilities(validate_turn_detection=record)
rejection = validate_session_payload(
{"input_audio_format": "opus", "turn_detection": {"type": "semantic_vad"}},
capabilities=capabilities,
)
assert rejection is not None
assert rejection.code == "unsupported_audio_format"
assert calls == []
@@ -0,0 +1,256 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""There is one Realtime codec, and the duplex engine uses it.
RFC #6592 P0a rejects the "copy the codec and fix it up" option because the two
copies drift. The extraction is only worth anything while the duplex names are
*the same objects* as the protocol ones, not lookalikes --- so assert identity,
which a re-implementation cannot satisfy.
"""
from __future__ import annotations
import dataclasses
import pytest
from vllm_omni.engine.duplex import audio as duplex_audio
from vllm_omni.engine.duplex import events as duplex_events
from vllm_omni.engine.duplex import realtime_commands as duplex_codec
from vllm_omni.engine.duplex.commands import DuplexCommandError
from vllm_omni.protocol import realtime as protocol
from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
#: Names ``engine.duplex.realtime_commands`` re-exports for its existing importers.
_REEXPORTED_FROM_PROTOCOL = (
"REALTIME_INPUT_AUDIO_FORMATS",
"REALTIME_INPUT_HINT_KEYS",
"REALTIME_OUTPUT_AUDIO_FORMATS",
"RealtimeInputDefaults",
"apply_realtime_session_defaults",
"copy_realtime_input_hints",
"input_audio_transcription_config",
"input_explicitly_non_speech",
"input_looks_like_speech",
"input_transcript_from_item",
"is_supported_realtime_input_format",
"json_safe_realtime_payload",
"normalize_conversation_item",
"parse_realtime_audio_format",
"realtime_audio_format_object",
"realtime_max_output_tokens",
"realtime_output_format",
"realtime_overlap_fields",
"text_chars_for_audio_ms_from_marks",
"truncate_realtime_item_content",
"validate_conversation_item_audio_formats",
"validate_realtime_item_truncate",
"validate_realtime_response_audio_formats",
"validate_realtime_session_audio_formats",
"validate_realtime_video_frames",
)
@pytest.mark.parametrize("name", _REEXPORTED_FROM_PROTOCOL)
def test_the_duplex_codec_name_is_the_protocol_object(name: str) -> None:
assert getattr(duplex_codec, name) is getattr(protocol, name)
@pytest.mark.parametrize(
"name",
("convert_input_audio_with_rate", "convert_output_audio", "resample_pcm16_mono", "wav_payload_to_pcm16"),
)
def test_the_duplex_audio_shim_is_the_protocol_object(name: str) -> None:
assert getattr(duplex_audio, name) is getattr(protocol, name)
def test_a_duplex_command_error_is_a_realtime_protocol_error() -> None:
# The duplex binding converts the codec's error into its own type; both
# carry the same code/event_id, which is what the error envelope renders.
error = DuplexCommandError("bad", code="bad_audio", event_id="event_1")
assert isinstance(error, RealtimeProtocolError)
assert (error.code, error.event_id) == ("bad_audio", "event_1")
def test_the_duplex_append_command_is_built_from_the_shared_decoder() -> None:
defaults = protocol.RealtimeInputDefaults()
event = {"event_id": "event_1", "audio": "", "format": "pcm16", "duration_ms": 40}
decoded = protocol.decode_audio_append(event, defaults=defaults)
command = duplex_codec.build_append_audio(event, defaults=defaults)
assert (command.audio, command.format, command.sample_rate_hz) == (
decoded.audio,
decoded.format,
decoded.sample_rate_hz,
)
assert (command.is_speech, command.video_frames, command.duration_ms) == (
decoded.is_speech,
decoded.video_frames,
decoded.duration_ms,
)
assert (command.audio_end_ms, dict(command.hints), command.event_id) == (
decoded.audio_end_ms,
decoded.hints,
decoded.event_id,
)
def test_the_shared_decoder_error_reaches_the_client_as_a_duplex_command_error() -> None:
defaults = protocol.RealtimeInputDefaults()
event = {"event_id": "event_1", "audio": "AAAA", "format": "opus"}
with pytest.raises(RealtimeProtocolError) as shared:
protocol.decode_audio_append(event, defaults=defaults)
with pytest.raises(DuplexCommandError) as duplex:
duplex_codec.build_append_audio(event, defaults=defaults)
assert (duplex.value.code, str(duplex.value)) == (shared.value.code, str(shared.value))
assert duplex.value.event_id == shared.value.event_id == "event_1"
def test_the_duplex_capabilities_reject_unimplemented_turn_detection() -> None:
# The duplex answer comes from engine.duplex.turn_detection, reached through
# the capability object rather than imported by the codec.
rejection = protocol.validate_session_payload(
{"turn_detection": {"type": "semantic_vad"}},
capabilities=duplex_codec.DUPLEX_REALTIME_CAPABILITIES,
)
assert rejection is not None
assert rejection.code == "unsupported_turn_detection"
# ---- the class split (RFC #6592 P0a, second half) ----
#
# The event and command classes moved out of the engine too. The same identity
# rule applies: the duplex names must BE the protocol objects, and the mailbox
# half must have stayed behind.
# `docs/serving/realtime_duplex_api.md` sorts every message into three tiers.
# Tier 1 is pure OpenAI and lives in protocol/realtime; Tier 2 (OpenAI names
# carrying our extensions) and Tier 3 (ours alone) live in protocol/duplex.
_TIER1_EVENTS = ("SessionUpdated", "ItemAdded", "ItemCreated", "TextDelta", "OutputItemAdded")
_TIER2_EVENTS = (
"SessionCreated",
"ResponseCreated",
"ResponseDone",
"AudioDelta",
"InputCommitted",
"ItemDeleted",
"ItemTruncated",
"ErrorEvent",
)
_TIER3_EVENTS = ("Listen", "Speak", "OverlapDecision", "PlaybackAcknowledged", "SessionResumed")
_TIER1_COMMANDS = ("UpdateSession", "CreateItem", "CancelResponse", "ClearInput")
_TIER2_COMMANDS = ("AppendAudio", "Commit")
_TIER3_COMMANDS = ("BargeIn", "AckPlayback", "Heartbeat", "SignalTurn", "CloseSession")
@pytest.mark.parametrize("name", _TIER1_EVENTS)
def test_tier1_events_are_the_pure_openai_objects(name: str) -> None:
from vllm_omni.protocol.realtime import events as realtime_events
assert duplex_events.__dict__[name] is getattr(realtime_events, name)
@pytest.mark.parametrize("name", _TIER2_EVENTS + _TIER3_EVENTS)
def test_tier2_and_tier3_events_come_from_protocol_duplex(name: str) -> None:
from vllm_omni.protocol.duplex import events as duplex_wire_events
assert duplex_events.__dict__[name] is getattr(duplex_wire_events, name)
@pytest.mark.parametrize("name", _TIER2_EVENTS)
def test_tier2_extends_its_tier1_twin_rather_than_replacing_it(name: str) -> None:
"""A Tier 2 class must be its Tier 1 class plus fields, never a fork of it."""
from vllm_omni.protocol.duplex import events as duplex_wire_events
from vllm_omni.protocol.realtime import events as realtime_events
tier1 = getattr(realtime_events, name)
tier2 = getattr(duplex_wire_events, name)
assert issubclass(tier2, tier1)
assert tier2.wire_type == tier1.wire_type
tier1_fields = {f.name for f in dataclasses.fields(tier1)}
tier2_fields = {f.name for f in dataclasses.fields(tier2)}
# Purely additive: Tier 2 adds, never drops or renames.
assert tier1_fields < tier2_fields
@pytest.mark.parametrize("name", _TIER1_EVENTS + _TIER1_COMMANDS)
def test_tier1_carries_no_vllm_omni_extension_fields(name: str) -> None:
"""The point of the split: protocol/realtime must be honestly pure OpenAI."""
from vllm_omni.protocol.realtime import commands as realtime_commands
from vllm_omni.protocol.realtime import events as realtime_events
cls = getattr(realtime_events, name, None) or getattr(realtime_commands, name)
fields = {f.name for f in dataclasses.fields(cls)}
# Extension fields catalogued as Tier 2 in the serving doc.
assert not (
fields
& {
"attachment_generation",
"resume_token",
"details",
"extra",
"is_speech",
"video_frames",
"hints",
"realtime_item_id",
}
)
def test_the_error_code_vocabulary_is_tier3() -> None:
"""OpenAI standardises the error classes, not our codes (doc: Tier 3)."""
from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
from vllm_omni.protocol.realtime import errors as realtime_errors
assert duplex_events.REALTIME_ERROR_TYPES_BY_CODE is REALTIME_ERROR_TYPES_BY_CODE
assert not hasattr(realtime_errors, "REALTIME_ERROR_TYPES_BY_CODE")
def test_the_tier1_error_event_reports_only_openai_classes() -> None:
from vllm_omni.protocol.duplex import events as duplex_wire_events
from vllm_omni.protocol.realtime import events as realtime_events
# Tier 1 knows the envelope shape but not our code vocabulary.
assert realtime_events.ErrorEvent(code="resource_exhausted").error_type == "invalid_request_error"
# Tier 2 resolves it through our table.
assert duplex_wire_events.ErrorEvent(code="resource_exhausted").error_type == "rate_limit_error"
@pytest.mark.parametrize("name", _TIER1_COMMANDS + _TIER2_COMMANDS + _TIER3_COMMANDS)
def test_engine_commands_subclass_their_protocol_wire_type(name: str) -> None:
from vllm_omni.engine.duplex import commands as engine_commands
from vllm_omni.protocol.duplex import commands as duplex_wire_commands
from vllm_omni.protocol.realtime import commands as realtime_commands
engine_cls = getattr(engine_commands, name)
wire_cls = getattr(duplex_wire_commands, name, None) or getattr(realtime_commands, name)
assert issubclass(engine_cls, wire_cls)
assert engine_cls.wire_type == wire_cls.wire_type
def test_the_mailbox_channel_stayed_engine_side() -> None:
"""``payload()``'s ``type`` is the runner's channel, not the client event.
These four are the proof that the two halves are genuinely different, and
so the reason the classes were split rather than relocated wholesale.
"""
from vllm_omni.engine.duplex import commands as engine_commands
from vllm_omni.protocol.realtime import commands as realtime_commands
for name in ("UpdateSession", "CreateItem", "DeleteItem", "TruncateItem"):
engine_cls = getattr(engine_commands, name)
wire_cls = getattr(realtime_commands, name)
assert engine_cls.type == "turn.signal"
assert wire_cls.wire_type != "turn.signal"
# The wire class carries no mailbox vocabulary at all.
assert "type" not in vars(wire_cls)