[Core] Give the OpenAI Realtime wire codec a reusable home (RFC #6592 P0a) (#7640)

Signed-off-by: psv666 <2693925048@qq.com>

tests/protocol/realtime/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

tests/protocol/realtime/test_protocol_import_boundary.py

@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""The shared Realtime codec must not depend on any runtime that uses it.
+
+``vllm_omni.protocol.realtime`` exists so a second Realtime surface can reuse
+the wire codec without adopting the duplex session control plane. That only
+holds while the arrows point one way: the engine, the entrypoints and the model
+code import the protocol, never the reverse. Nothing in the package's own code
+enforces that, so it is asserted here --- once on the source (which catches a
+lazy import inside a function body) and once on a real import (which catches a
+dependency arriving through a third module).
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+PROTOCOL_DIR = REPO_ROOT / "vllm_omni" / "protocol"
+
+#: Packages that consume the protocol. A dependency on any of them would make
+#: the codec unusable outside the runtime it accidentally reached into.
+#: ``vllm_omni.utils`` is deliberately absent: it is leaf helpers (the audio
+#: resampler) that consume nothing, so depending on it costs a reuser nothing.
+FORBIDDEN_PREFIXES = (
+    "vllm_omni.engine",
+    "vllm_omni.entrypoints",
+    "vllm_omni.model_executor",
+    "vllm_omni.worker",
+    "vllm_omni.clients",
+)
+
+
+def _protocol_sources() -> list[Path]:
+    sources = sorted(PROTOCOL_DIR.rglob("*.py"))
+    assert sources, f"no python sources under {PROTOCOL_DIR}"
+    return sources
+
+
+def _imported_modules(source: Path) -> set[str]:
+    """Every module name imported anywhere in the file, lazy imports included."""
+    names: set[str] = set()
+    for node in ast.walk(ast.parse(source.read_text())):
+        if isinstance(node, ast.Import):
+            names.update(alias.name for alias in node.names)
+        elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
+            names.add(node.module)
+    return names
+
+
+@pytest.mark.parametrize("source", _protocol_sources(), ids=lambda path: path.name)
+def test_protocol_sources_do_not_import_a_consumer(source: Path) -> None:
+    offenders = sorted(
+        name
+        for name in _imported_modules(source)
+        if any(name == prefix or name.startswith(prefix + ".") for prefix in FORBIDDEN_PREFIXES)
+    )
+    assert not offenders, f"{source.relative_to(REPO_ROOT)} imports {', '.join(offenders)}"
+
+
+def test_importing_the_protocol_package_loads_no_consumer() -> None:
+    # A child interpreter, because this test session has already imported the
+    # duplex engine: only a fresh process can tell what the package pulls in.
+    # ``vllm_omni`` itself is imported first, because its ``__init__`` loads a
+    # baseline graph (patches, the model registry) that has nothing to do with
+    # this package; only what the protocol import *adds* is measured.
+    env = os.environ.copy()
+    env.pop("CUDA_VISIBLE_DEVICES", None)
+    env.pop("HIP_VISIBLE_DEVICES", None)
+    script = f"""
+import sys
+
+import vllm_omni  # noqa: F401
+
+baseline = set(sys.modules)
+
+import vllm_omni.protocol.realtime  # noqa: F401
+
+forbidden_prefixes = {FORBIDDEN_PREFIXES!r}
+added = sorted(
+    name
+    for name in set(sys.modules) - baseline
+    if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes)
+)
+if added:
+    raise SystemExit("the realtime protocol package loaded consumers: " + ", ".join(added))
+"""
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        cwd=REPO_ROOT,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=180,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr

tests/protocol/realtime/test_realtime_capabilities.py

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""``RealtimeProtocolCapabilities``: one consumer's answers, the codec's checking.
+
+This is the seam that lets a non-duplex Realtime surface validate a
+``session.update`` without a ``DuplexModelPlugin``. What it has to get right is
+narrow: the order the checks run in (formats before turn detection, because
+that is the order clients already see), the error codes, and that a consumer
+which declares a narrower format set is actually narrowed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+import pytest
+
+from vllm_omni.protocol.realtime.capabilities import (
+    RealtimeProtocolCapabilities,
+    RealtimeSessionRejection,
+    validate_session_payload,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+_DEFAULT = RealtimeProtocolCapabilities()
+
+
+def test_a_consumer_without_a_turn_detection_check_accepts_a_plain_session():
+    assert (
+        validate_session_payload({"model": "m", "audio": {"input": {"format": "pcm16"}}}, capabilities=_DEFAULT) is None
+    )
+
+
+def test_a_consumer_without_a_turn_detection_check_ignores_turn_detection():
+    payload = {"turn_detection": {"type": "semantic_vad"}}
+
+    assert validate_session_payload(payload, capabilities=_DEFAULT) is None
+
+
+def test_an_unsupported_input_format_is_rejected_with_the_realtime_code():
+    rejection = validate_session_payload({"input_audio_format": "opus"}, capabilities=_DEFAULT)
+
+    assert rejection == RealtimeSessionRejection(
+        code="unsupported_audio_format", message="Unsupported input_audio_format: opus"
+    )
+
+
+def test_a_consumer_may_declare_a_narrower_format_set():
+    pcm_only = RealtimeProtocolCapabilities(input_audio_formats=frozenset({"pcm16"}))
+
+    assert validate_session_payload({"input_audio_format": "pcm16"}, capabilities=pcm_only) is None
+    rejection = validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=pcm_only)
+    assert rejection is not None
+    assert rejection.code == "unsupported_audio_format"
+    # The default capabilities still accept it: the narrowing is per consumer.
+    assert validate_session_payload({"input_audio_format": "g711_ulaw"}, capabilities=_DEFAULT) is None
+
+
+def test_the_turn_detection_check_runs_and_reports_its_own_code():
+    def reject_everything(session_payload: Mapping[str, object]) -> str | None:
+        del session_payload
+        return "server_vad is not available"
+
+    capabilities = RealtimeProtocolCapabilities(validate_turn_detection=reject_everything)
+
+    rejection = validate_session_payload({"turn_detection": {"type": "server_vad"}}, capabilities=capabilities)
+
+    assert rejection == RealtimeSessionRejection(
+        code="unsupported_turn_detection", message="server_vad is not available", param="turn_detection"
+    )
+
+
+def test_a_bad_format_is_reported_before_a_bad_turn_detection():
+    calls: list[Mapping[str, object]] = []
+
+    def record(session_payload: Mapping[str, object]) -> str | None:
+        calls.append(session_payload)
+        return "turn detection is wrong too"
+
+    capabilities = RealtimeProtocolCapabilities(validate_turn_detection=record)
+
+    rejection = validate_session_payload(
+        {"input_audio_format": "opus", "turn_detection": {"type": "semantic_vad"}},
+        capabilities=capabilities,
+    )
+
+    assert rejection is not None
+    assert rejection.code == "unsupported_audio_format"
+    assert calls == []

tests/protocol/realtime/test_realtime_codec_single_source.py

@@ -0,0 +1,256 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""There is one Realtime codec, and the duplex engine uses it.
+
+RFC #6592 P0a rejects the "copy the codec and fix it up" option because the two
+copies drift. The extraction is only worth anything while the duplex names are
+*the same objects* as the protocol ones, not lookalikes --- so assert identity,
+which a re-implementation cannot satisfy.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+import pytest
+
+from vllm_omni.engine.duplex import audio as duplex_audio
+from vllm_omni.engine.duplex import events as duplex_events
+from vllm_omni.engine.duplex import realtime_commands as duplex_codec
+from vllm_omni.engine.duplex.commands import DuplexCommandError
+from vllm_omni.protocol import realtime as protocol
+from vllm_omni.protocol.realtime.errors import RealtimeProtocolError
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+#: Names ``engine.duplex.realtime_commands`` re-exports for its existing importers.
+_REEXPORTED_FROM_PROTOCOL = (
+    "REALTIME_INPUT_AUDIO_FORMATS",
+    "REALTIME_INPUT_HINT_KEYS",
+    "REALTIME_OUTPUT_AUDIO_FORMATS",
+    "RealtimeInputDefaults",
+    "apply_realtime_session_defaults",
+    "copy_realtime_input_hints",
+    "input_audio_transcription_config",
+    "input_explicitly_non_speech",
+    "input_looks_like_speech",
+    "input_transcript_from_item",
+    "is_supported_realtime_input_format",
+    "json_safe_realtime_payload",
+    "normalize_conversation_item",
+    "parse_realtime_audio_format",
+    "realtime_audio_format_object",
+    "realtime_max_output_tokens",
+    "realtime_output_format",
+    "realtime_overlap_fields",
+    "text_chars_for_audio_ms_from_marks",
+    "truncate_realtime_item_content",
+    "validate_conversation_item_audio_formats",
+    "validate_realtime_item_truncate",
+    "validate_realtime_response_audio_formats",
+    "validate_realtime_session_audio_formats",
+    "validate_realtime_video_frames",
+)
+
+
+@pytest.mark.parametrize("name", _REEXPORTED_FROM_PROTOCOL)
+def test_the_duplex_codec_name_is_the_protocol_object(name: str) -> None:
+    assert getattr(duplex_codec, name) is getattr(protocol, name)
+
+
+@pytest.mark.parametrize(
+    "name",
+    ("convert_input_audio_with_rate", "convert_output_audio", "resample_pcm16_mono", "wav_payload_to_pcm16"),
+)
+def test_the_duplex_audio_shim_is_the_protocol_object(name: str) -> None:
+    assert getattr(duplex_audio, name) is getattr(protocol, name)
+
+
+def test_a_duplex_command_error_is_a_realtime_protocol_error() -> None:
+    # The duplex binding converts the codec's error into its own type; both
+    # carry the same code/event_id, which is what the error envelope renders.
+    error = DuplexCommandError("bad", code="bad_audio", event_id="event_1")
+
+    assert isinstance(error, RealtimeProtocolError)
+    assert (error.code, error.event_id) == ("bad_audio", "event_1")
+
+
+def test_the_duplex_append_command_is_built_from_the_shared_decoder() -> None:
+    defaults = protocol.RealtimeInputDefaults()
+    event = {"event_id": "event_1", "audio": "", "format": "pcm16", "duration_ms": 40}
+
+    decoded = protocol.decode_audio_append(event, defaults=defaults)
+    command = duplex_codec.build_append_audio(event, defaults=defaults)
+
+    assert (command.audio, command.format, command.sample_rate_hz) == (
+        decoded.audio,
+        decoded.format,
+        decoded.sample_rate_hz,
+    )
+    assert (command.is_speech, command.video_frames, command.duration_ms) == (
+        decoded.is_speech,
+        decoded.video_frames,
+        decoded.duration_ms,
+    )
+    assert (command.audio_end_ms, dict(command.hints), command.event_id) == (
+        decoded.audio_end_ms,
+        decoded.hints,
+        decoded.event_id,
+    )
+
+
+def test_the_shared_decoder_error_reaches_the_client_as_a_duplex_command_error() -> None:
+    defaults = protocol.RealtimeInputDefaults()
+    event = {"event_id": "event_1", "audio": "AAAA", "format": "opus"}
+
+    with pytest.raises(RealtimeProtocolError) as shared:
+        protocol.decode_audio_append(event, defaults=defaults)
+    with pytest.raises(DuplexCommandError) as duplex:
+        duplex_codec.build_append_audio(event, defaults=defaults)
+
+    assert (duplex.value.code, str(duplex.value)) == (shared.value.code, str(shared.value))
+    assert duplex.value.event_id == shared.value.event_id == "event_1"
+
+
+def test_the_duplex_capabilities_reject_unimplemented_turn_detection() -> None:
+    # The duplex answer comes from engine.duplex.turn_detection, reached through
+    # the capability object rather than imported by the codec.
+    rejection = protocol.validate_session_payload(
+        {"turn_detection": {"type": "semantic_vad"}},
+        capabilities=duplex_codec.DUPLEX_REALTIME_CAPABILITIES,
+    )
+
+    assert rejection is not None
+    assert rejection.code == "unsupported_turn_detection"
+
+
+# ---- the class split (RFC #6592 P0a, second half) ----
+#
+# The event and command classes moved out of the engine too. The same identity
+# rule applies: the duplex names must BE the protocol objects, and the mailbox
+# half must have stayed behind.
+
+# `docs/serving/realtime_duplex_api.md` sorts every message into three tiers.
+# Tier 1 is pure OpenAI and lives in protocol/realtime; Tier 2 (OpenAI names
+# carrying our extensions) and Tier 3 (ours alone) live in protocol/duplex.
+_TIER1_EVENTS = ("SessionUpdated", "ItemAdded", "ItemCreated", "TextDelta", "OutputItemAdded")
+_TIER2_EVENTS = (
+    "SessionCreated",
+    "ResponseCreated",
+    "ResponseDone",
+    "AudioDelta",
+    "InputCommitted",
+    "ItemDeleted",
+    "ItemTruncated",
+    "ErrorEvent",
+)
+_TIER3_EVENTS = ("Listen", "Speak", "OverlapDecision", "PlaybackAcknowledged", "SessionResumed")
+_TIER1_COMMANDS = ("UpdateSession", "CreateItem", "CancelResponse", "ClearInput")
+_TIER2_COMMANDS = ("AppendAudio", "Commit")
+_TIER3_COMMANDS = ("BargeIn", "AckPlayback", "Heartbeat", "SignalTurn", "CloseSession")
+
+
+@pytest.mark.parametrize("name", _TIER1_EVENTS)
+def test_tier1_events_are_the_pure_openai_objects(name: str) -> None:
+    from vllm_omni.protocol.realtime import events as realtime_events
+
+    assert duplex_events.__dict__[name] is getattr(realtime_events, name)
+
+
+@pytest.mark.parametrize("name", _TIER2_EVENTS + _TIER3_EVENTS)
+def test_tier2_and_tier3_events_come_from_protocol_duplex(name: str) -> None:
+    from vllm_omni.protocol.duplex import events as duplex_wire_events
+
+    assert duplex_events.__dict__[name] is getattr(duplex_wire_events, name)
+
+
+@pytest.mark.parametrize("name", _TIER2_EVENTS)
+def test_tier2_extends_its_tier1_twin_rather_than_replacing_it(name: str) -> None:
+    """A Tier 2 class must be its Tier 1 class plus fields, never a fork of it."""
+    from vllm_omni.protocol.duplex import events as duplex_wire_events
+    from vllm_omni.protocol.realtime import events as realtime_events
+
+    tier1 = getattr(realtime_events, name)
+    tier2 = getattr(duplex_wire_events, name)
+
+    assert issubclass(tier2, tier1)
+    assert tier2.wire_type == tier1.wire_type
+    tier1_fields = {f.name for f in dataclasses.fields(tier1)}
+    tier2_fields = {f.name for f in dataclasses.fields(tier2)}
+    # Purely additive: Tier 2 adds, never drops or renames.
+    assert tier1_fields < tier2_fields
+
+
+@pytest.mark.parametrize("name", _TIER1_EVENTS + _TIER1_COMMANDS)
+def test_tier1_carries_no_vllm_omni_extension_fields(name: str) -> None:
+    """The point of the split: protocol/realtime must be honestly pure OpenAI."""
+    from vllm_omni.protocol.realtime import commands as realtime_commands
+    from vllm_omni.protocol.realtime import events as realtime_events
+
+    cls = getattr(realtime_events, name, None) or getattr(realtime_commands, name)
+    fields = {f.name for f in dataclasses.fields(cls)}
+    # Extension fields catalogued as Tier 2 in the serving doc.
+    assert not (
+        fields
+        & {
+            "attachment_generation",
+            "resume_token",
+            "details",
+            "extra",
+            "is_speech",
+            "video_frames",
+            "hints",
+            "realtime_item_id",
+        }
+    )
+
+
+def test_the_error_code_vocabulary_is_tier3() -> None:
+    """OpenAI standardises the error classes, not our codes (doc: Tier 3)."""
+    from vllm_omni.protocol.duplex.errors import REALTIME_ERROR_TYPES_BY_CODE
+    from vllm_omni.protocol.realtime import errors as realtime_errors
+
+    assert duplex_events.REALTIME_ERROR_TYPES_BY_CODE is REALTIME_ERROR_TYPES_BY_CODE
+    assert not hasattr(realtime_errors, "REALTIME_ERROR_TYPES_BY_CODE")
+
+
+def test_the_tier1_error_event_reports_only_openai_classes() -> None:
+    from vllm_omni.protocol.duplex import events as duplex_wire_events
+    from vllm_omni.protocol.realtime import events as realtime_events
+
+    # Tier 1 knows the envelope shape but not our code vocabulary.
+    assert realtime_events.ErrorEvent(code="resource_exhausted").error_type == "invalid_request_error"
+    # Tier 2 resolves it through our table.
+    assert duplex_wire_events.ErrorEvent(code="resource_exhausted").error_type == "rate_limit_error"
+
+
+@pytest.mark.parametrize("name", _TIER1_COMMANDS + _TIER2_COMMANDS + _TIER3_COMMANDS)
+def test_engine_commands_subclass_their_protocol_wire_type(name: str) -> None:
+    from vllm_omni.engine.duplex import commands as engine_commands
+    from vllm_omni.protocol.duplex import commands as duplex_wire_commands
+    from vllm_omni.protocol.realtime import commands as realtime_commands
+
+    engine_cls = getattr(engine_commands, name)
+    wire_cls = getattr(duplex_wire_commands, name, None) or getattr(realtime_commands, name)
+
+    assert issubclass(engine_cls, wire_cls)
+    assert engine_cls.wire_type == wire_cls.wire_type
+
+
+def test_the_mailbox_channel_stayed_engine_side() -> None:
+    """``payload()``'s ``type`` is the runner's channel, not the client event.
+
+    These four are the proof that the two halves are genuinely different, and
+    so the reason the classes were split rather than relocated wholesale.
+    """
+    from vllm_omni.engine.duplex import commands as engine_commands
+    from vllm_omni.protocol.realtime import commands as realtime_commands
+
+    for name in ("UpdateSession", "CreateItem", "DeleteItem", "TruncateItem"):
+        engine_cls = getattr(engine_commands, name)
+        wire_cls = getattr(realtime_commands, name)
+        assert engine_cls.type == "turn.signal"
+        assert wire_cls.wire_type != "turn.signal"
+        # The wire class carries no mailbox vocabulary at all.
+        assert "type" not in vars(wire_cls)