# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Session configuration, capabilities and value types of the duplex session model. These classes are the typed half of the public contract: ``DuplexOmni`` accepts a ``DuplexSessionConfig`` (or the equivalent OpenAI Realtime ``session`` object via :meth:`DuplexSessionConfig.from_realtime`) and reports ``DuplexCapabilities``. They used to live in ``entrypoints/duplex/protocol.py``. """ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass, field from enum import Enum from typing import Any, cast class DuplexConfigError(ValueError): """A Realtime session/response object was rejected while mapping it onto the duplex config.""" def __init__(self, message: str, *, code: str = "bad_event", param: str | None = None) -> None: super().__init__(message) self.code = code self.param = param class DuplexOverlapPolicy(str, Enum): LISTEN_ONLY = "listen_only" BARGE_IN_ON_SPEECH = "barge_in_on_speech" class DuplexPlaybackCommitPolicy(str, Enum): COMMIT_ALL_ON_DONE = "commit_all_on_done" ACK_ONLY = "ack_only" class DuplexSessionState(str, Enum): OPEN = "open" CLOSING = "closing" CLOSED = "closed" class DuplexTurnState(str, Enum): IDLE = "idle" USER_SPEAKING = "user_speaking" USER_COMMITTED = "user_committed" ASSISTANT_GENERATING = "assistant_generating" ASSISTANT_PLAYING = "assistant_playing" BARGE_IN = "barge_in" class DuplexTurnEventType(str, Enum): USER_STARTED = "user_started" USER_COMMITTED = "user_committed" ASSISTANT_STARTED = "assistant_started" ASSISTANT_DONE = "assistant_done" BARGE_IN = "barge_in" PLAYBACK_ACK = "playback_ack" TIMEOUT = "timeout" CLOSE = "close" @dataclass class DuplexCapabilities: """Runtime/model capabilities exposed by the duplex serving protocol. These are intentionally explicit so the serving layer does not assume all duplex models support the same input append, rollback, or turn policy. ``supports_core_kv_lease`` is reserved for scheduler-owned KV lifecycle; model-owned decoder/TTS state must use ``supports_model_internal_state``. ``supports_core_resumable_request`` means the scheduler can resume the same request id across streaming updates, but it is not a KV lease by itself. Realtime support means this endpoint can speak the native Realtime event schema for the supported audio duplex paths while keeping model- or scheduler-specific limits explicit in the capability payload. """ supports_session_adapter: bool = True supports_model_native_turn_policy: bool = False supports_external_turn_signal: bool = True supports_client_commit: bool = True supports_barge_in: bool = True supports_playback_ack: bool = True supports_input_append: bool = False supports_replace_latest_chunk: bool = True supports_reencode_context: bool = True supports_rollback_to_checkpoint: bool = False supports_turn_commit_only: bool = True supports_kv_lease: bool = False supports_core_kv_lease: bool = False supports_model_internal_state: bool = False supports_stage_resumption: bool = False supports_scheduler_native_append: bool = False supports_core_resumable_request: bool = False supports_stage_connector_handoff: bool = False supports_independent_io_streams: bool = False supports_realtime_endpoint: bool = False supports_multi_session: bool = False supports_multi_session_same_replica: bool = False supports_session_lease: bool = False supports_session_resume: bool = False session_admission_mode: str = "serving_managed" supports_audio_truncate: bool = False #: Whether this model can serve ``/v1/chat/completions`` on a duplex #: session. That needs one thing the protocol cannot assume: the model #: honours ``DuplexSessionConfig.initial_user_text``, so a turn can be #: seeded from text. A model-native model decides to speak from the audio #: it hears, and silence is its signal *not* to take a turn, so without #: seeding a text prompt has no representation and the turn never #: completes. ``text_turn_priming_units`` is how many silence units such a #: seeded turn must be given to generate on -- it still speaks per unit. supports_image_input: bool = False supports_text_only_turn: bool = False supports_chat_completions: bool = False text_turn_priming_units: int = 0 requires_model_runner_kv: bool = False requires_native_stage_role: bool = False adapter_patterns: list[str] = field(default_factory=lambda: ["chunk_group_append"]) signal_sources: list[str] = field(default_factory=lambda: ["client_event", "server_policy", "model_native"]) stage_handoff_transport: str | None = None chunk_period_ms: int | None = 1000 target_barge_in_latency_ms: int | None = 1000 def as_dict(self) -> dict[str, object]: return { "supports_session_adapter": self.supports_session_adapter, "supports_model_native_turn_policy": self.supports_model_native_turn_policy, "supports_external_turn_signal": self.supports_external_turn_signal, "supports_client_commit": self.supports_client_commit, "supports_barge_in": self.supports_barge_in, "supports_playback_ack": self.supports_playback_ack, "supports_input_append": self.supports_input_append, "supports_replace_latest_chunk": self.supports_replace_latest_chunk, "supports_reencode_context": self.supports_reencode_context, "supports_rollback_to_checkpoint": self.supports_rollback_to_checkpoint, "supports_turn_commit_only": self.supports_turn_commit_only, "supports_kv_lease": self.supports_kv_lease, "supports_core_kv_lease": self.supports_core_kv_lease, "supports_model_internal_state": self.supports_model_internal_state, "supports_stage_resumption": self.supports_stage_resumption, "supports_scheduler_native_append": self.supports_scheduler_native_append, "supports_core_resumable_request": self.supports_core_resumable_request, "supports_stage_connector_handoff": self.supports_stage_connector_handoff, "supports_independent_io_streams": self.supports_independent_io_streams, "supports_realtime_endpoint": self.supports_realtime_endpoint, "supports_multi_session": self.supports_multi_session, "supports_multi_session_same_replica": self.supports_multi_session_same_replica, "supports_session_lease": self.supports_session_lease, "supports_session_resume": self.supports_session_resume, "session_admission_mode": self.session_admission_mode, "supports_audio_truncate": self.supports_audio_truncate, "supports_image_input": self.supports_image_input, "supports_text_only_turn": self.supports_text_only_turn, "requires_model_runner_kv": self.requires_model_runner_kv, "requires_native_stage_role": self.requires_native_stage_role, "implementation_level": ( "model_native_duplex" if self.supports_model_native_turn_policy else "turn_based_duplex" ), "adapter_patterns": self.adapter_patterns, "input_modes": ["append_audio_chunk"], "signal_sources": self.signal_sources, "stage_handoff_transport": self.stage_handoff_transport, "chunk_period_ms": self.chunk_period_ms, "target_barge_in_latency_ms": self.target_barge_in_latency_ms, } @dataclass class DuplexPlaybackCursor: generated_ms: int = 0 sent_ms: int = 0 played_ms: int = 0 committed_ms: int = 0 text_requires_complete_audio: bool = False audio_complete: bool = False def acknowledge(self, played_ms: int, committed_ms: int | None = None) -> None: self.played_ms = max(self.played_ms, max(0, int(played_ms))) if committed_ms is None: committed_ms = self.played_ms self.committed_ms = max(self.committed_ms, max(0, int(committed_ms))) def truncate_committed(self, committed_ms: int) -> None: self.committed_ms = max(0, min(max(self.sent_ms, self.generated_ms), int(committed_ms))) def as_dict(self) -> dict[str, int]: return { "generated_ms": self.generated_ms, "sent_ms": self.sent_ms, "played_ms": self.played_ms, "committed_ms": self.committed_ms, } def snapshot(self) -> DuplexPlaybackView: return DuplexPlaybackView( **self.as_dict(), text_requires_complete_audio=self.text_requires_complete_audio, audio_complete=self.audio_complete, ) @dataclass(frozen=True, slots=True) class DuplexPlaybackView: generated_ms: int = 0 sent_ms: int = 0 played_ms: int = 0 committed_ms: int = 0 text_requires_complete_audio: bool = False audio_complete: bool = False def as_dict(self) -> dict[str, int]: return { "generated_ms": self.generated_ms, "sent_ms": self.sent_ms, "played_ms": self.played_ms, "committed_ms": self.committed_ms, } @dataclass class DuplexAudioChunk: data: str format: str = "wav" sample_rate_hz: int | None = None @dataclass class DuplexSessionConfig: model: str | None = None modalities: list[str] = field(default_factory=lambda: ["text", "audio"]) instructions: str | None = None #: Text the session treats as the user's opening turn. A caller that cannot #: reach the model by speech (an HTTP request, a batch job) puts its prompt #: here; ``supports_text_only_turn`` says whether the model can answer it. initial_user_text: str | None = None voice: str | None = None ref_audio: str | None = None response_format: str = "wav" temperature: float | None = None max_tokens: int | None = None speed: float | None = None use_tts_template: bool = True idle_timeout_s: float = 300.0 overlap_policy: str = DuplexOverlapPolicy.LISTEN_ONLY.value overlap_short_ack_ms: int = 700 overlap_barge_in_ms: int = 1200 overlap_silence_rms: float = 0.003 playback_commit_policy: str = DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value extra_body: dict[str, object] = field(default_factory=dict) def as_dict(self) -> dict[str, object]: return { "model": self.model, "modalities": list(self.modalities), "instructions": self.instructions, "initial_user_text": self.initial_user_text, "voice": self.voice, "ref_audio": self.ref_audio, "response_format": self.response_format, "temperature": self.temperature, "max_tokens": self.max_tokens, "speed": self.speed, "use_tts_template": self.use_tts_template, "idle_timeout_s": self.idle_timeout_s, "overlap_policy": self.overlap_policy, "overlap_short_ack_ms": self.overlap_short_ack_ms, "overlap_barge_in_ms": self.overlap_barge_in_ms, "overlap_silence_rms": self.overlap_silence_rms, "playback_commit_policy": self.playback_commit_policy, "extra_body": dict(self.extra_body), } def normalized(self) -> DuplexSessionConfig: """Apply the clamps / defaults ``from_event`` applies to a wire object (typed configs skip them).""" return DuplexSessionConfig.from_event({"session": self.as_dict()}) @classmethod def from_event(cls, event: dict[str, object]) -> DuplexSessionConfig: payload = event.get("session") if isinstance(payload, dict): source = payload else: source = event config = cls() if isinstance(source.get("model"), str): config.model = source["model"] if isinstance(source.get("instructions"), str): config.instructions = source["instructions"] if isinstance(source.get("initial_user_text"), str): config.initial_user_text = source["initial_user_text"] if isinstance(source.get("voice"), str): config.voice = source["voice"] if isinstance(source.get("ref_audio"), str): config.ref_audio = source["ref_audio"] if isinstance(source.get("response_format"), str): config.response_format = source["response_format"] if isinstance(source.get("use_tts_template"), bool): config.use_tts_template = bool(source["use_tts_template"]) if isinstance(source.get("temperature"), int | float): config.temperature = float(source["temperature"]) if isinstance(source.get("max_tokens"), int): config.max_tokens = int(source["max_tokens"]) if isinstance(source.get("speed"), int | float): config.speed = float(source["speed"]) if isinstance(source.get("idle_timeout_s"), int | float): config.idle_timeout_s = float(source["idle_timeout_s"]) if isinstance(source.get("overlap_policy"), str): config.overlap_policy = cls._normalize_overlap_policy(source["overlap_policy"]) if isinstance(source.get("overlap_short_ack_ms"), int | float): config.overlap_short_ack_ms = max(0, int(source["overlap_short_ack_ms"])) if isinstance(source.get("overlap_barge_in_ms"), int | float): config.overlap_barge_in_ms = max(0, int(source["overlap_barge_in_ms"])) if isinstance(source.get("overlap_silence_rms"), int | float): config.overlap_silence_rms = max(0.0, float(source["overlap_silence_rms"])) if isinstance(source.get("playback_commit_policy"), str): config.playback_commit_policy = cls._normalize_playback_commit_policy(source["playback_commit_policy"]) if isinstance(source.get("modalities"), list) and all(isinstance(x, str) for x in source["modalities"]): config.modalities = list(source["modalities"]) if isinstance(source.get("extra_body"), dict): config.extra_body = dict(source["extra_body"]) extra = config.extra_body overlap_policy = extra.get("overlap_policy") if isinstance(overlap_policy, str): config.overlap_policy = cls._normalize_overlap_policy(overlap_policy) short_ack_ms = extra.get("overlap_short_ack_ms") if isinstance(short_ack_ms, int | float): config.overlap_short_ack_ms = max(0, int(short_ack_ms)) barge_in_ms = extra.get("overlap_barge_in_ms") if isinstance(barge_in_ms, int | float): config.overlap_barge_in_ms = max(0, int(barge_in_ms)) silence_rms = extra.get("overlap_silence_rms") if isinstance(silence_rms, int | float): config.overlap_silence_rms = max(0.0, float(silence_rms)) playback_commit_policy = extra.get("playback_commit_policy") if isinstance(playback_commit_policy, str): config.playback_commit_policy = cls._normalize_playback_commit_policy(playback_commit_policy) return config @classmethod def from_realtime( cls, session_payload: Mapping[str, object], *, model: str | None = None, ) -> DuplexSessionConfig: """Build a config from an OpenAI Realtime ``session`` object (session.update / open_session). Applies the wire defaults, validates audio formats and turn detection (raising :class:`DuplexConfigError` with the same codes the Realtime translator used), fills ``turn_detection`` defaults / derives ``overlap_policy``, and stores the Realtime-only fields under the ``realtime_*`` keys of ``extra_body`` exactly like the old ``_session_create_from_realtime``. """ from vllm_omni.engine.duplex.realtime_commands import ( DUPLEX_REALTIME_CAPABILITIES, duplex_response_format, ) from vllm_omni.engine.duplex.turn_detection import normalize_turn_detection_session_payload from vllm_omni.protocol.duplex import ( RealtimeInputDefaults, input_audio_transcription_config, json_safe_realtime_payload, realtime_max_output_tokens, realtime_overlap_fields, validate_session_payload, ) payload: dict[str, object] = dict(session_payload) # One session check for every consumer (ENTRY-INV-002): the same # capability object ``translate_realtime_command`` uses, so a session # object is accepted or refused identically whichever door it came in. rejection = validate_session_payload(payload, capabilities=DUPLEX_REALTIME_CAPABILITIES) if rejection is not None: raise DuplexConfigError(rejection.message, code=rejection.code, param=rejection.param) normalize_turn_detection_session_payload(payload) defaults = RealtimeInputDefaults().with_session_payload(payload) payload.update(realtime_overlap_fields(payload)) audio_config = payload.get("audio") audio_input = audio_config.get("input") if isinstance(audio_config, dict) else None audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None extra_body_payload = payload.get("extra_body") extra_body: dict[str, object] = dict(extra_body_payload) if isinstance(extra_body_payload, dict) else {} extra_body["realtime_session_payload"] = json_safe_realtime_payload(payload) if isinstance(payload.get("tools"), list): extra_body["realtime_tools"] = payload["tools"] if isinstance(payload.get("tool_choice"), str | dict): extra_body["realtime_tool_choice"] = payload["tool_choice"] metadata = payload.get("metadata") if isinstance(metadata, dict): extra_body["realtime_metadata"] = dict(metadata) include = payload.get("include") if isinstance(include, list): extra_body["realtime_include"] = list(include) prompt = payload.get("prompt") if isinstance(prompt, dict): extra_body["realtime_prompt"] = dict(prompt) transcription = input_audio_transcription_config(payload) if isinstance(transcription, dict): extra_body["realtime_input_audio_transcription"] = dict(transcription) noise_reduction = payload.get("input_audio_noise_reduction") if isinstance(noise_reduction, dict): extra_body["realtime_input_audio_noise_reduction"] = dict(noise_reduction) audio_noise_reduction = audio_input.get("noise_reduction") if isinstance(audio_input, dict) else None if isinstance(audio_noise_reduction, dict): extra_body["realtime_input_audio_noise_reduction"] = dict(audio_noise_reduction) if isinstance(audio_config, dict): extra_body["realtime_audio"] = dict(audio_config) if isinstance(payload.get("tracing"), str | dict): extra_body["realtime_tracing"] = payload["tracing"] if isinstance(payload.get("turn_detection"), dict): extra_body["realtime_turn_detection"] = dict(cast("Mapping[str, object]", payload["turn_detection"])) elif "turn_detection" in payload and payload.get("turn_detection") is None: extra_body["realtime_turn_detection"] = None extra_body.setdefault("realtime_output_audio_format", defaults.output_audio_format) voice = payload.get("voice") if not isinstance(voice, str) and isinstance(audio_output, dict): voice = audio_output.get("voice") speed = payload.get("speed") if not isinstance(speed, int | float) and isinstance(audio_output, dict): speed = audio_output.get("speed") resolved_model = payload.get("model") if isinstance(payload.get("model"), str) else model session: dict[str, object] = { "model": resolved_model, "modalities": payload.get("modalities") or payload.get("output_modalities") or ["text", "audio"], "instructions": payload.get("instructions"), # Not a Realtime field, but a session one: a client that cannot # reach the model by speech seeds its opening turn here. Without # this a websocket client could only use the older # ``extra_body.duplex_initial_user_text`` spelling. "initial_user_text": payload.get("initial_user_text"), "voice": voice, "ref_audio": payload.get("ref_audio"), "response_format": duplex_response_format(defaults.output_audio_format), "temperature": payload.get("temperature"), "max_tokens": realtime_max_output_tokens( payload.get("max_response_output_tokens") or payload.get("max_output_tokens") or payload.get("max_tokens") ), "speed": speed, "idle_timeout_s": payload.get("idle_timeout_s") or 300.0, **realtime_overlap_fields(payload), "extra_body": extra_body, } return cls.from_event({"session": session}) def apply_realtime_update( self, patch: Mapping[str, object], *, session_id: str | None = None, audio_started: bool = False, ) -> None: """Apply a Realtime ``session.update`` patch in place. Raises :class:`DuplexConfigError` (``code`` in ``model_update_unsupported``, ``voice_update_after_audio_unsupported``, ``ref_audio_update_unsupported``) when the patch changes something a live session cannot change. ``audio_started`` is ``playback.generated_ms > 0 or playback.sent_ms > 0``. """ from vllm_omni.engine.duplex.realtime_commands import duplex_response_format from vllm_omni.protocol.duplex import ( REALTIME_OUTPUT_AUDIO_FORMATS, input_audio_transcription_config, json_safe_realtime_payload, parse_realtime_audio_format, realtime_max_output_tokens, ) # Any: the patch is read with ``isinstance(payload.get(...))`` guards and consumed # through ``payload[...]``; typing the values ``object`` would need a cast per read. payload: dict[str, Any] = dict(patch) model = payload.get("model") audio_config = payload.get("audio") audio_input = audio_config.get("input") if isinstance(audio_config, dict) else None audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None voice = payload.get("voice") if not isinstance(voice, str) and isinstance(audio_output, dict): voice = audio_output.get("voice") if isinstance(model, str) and self.model is not None and model != self.model: raise DuplexConfigError( "session.update cannot change model for an open realtime duplex session", code="model_update_unsupported", ) if isinstance(model, str) and self.model is None: self.model = model if isinstance(voice, str) and audio_started: raise DuplexConfigError( "session.update cannot change voice after audio output has started", code="voice_update_after_audio_unsupported", ) if isinstance(payload.get("ref_audio"), str): raise DuplexConfigError( "session.update cannot change ref_audio after the session is open", code="ref_audio_update_unsupported" ) if isinstance(payload.get("instructions"), str): self.instructions = str(payload["instructions"]) elif "instructions" in payload and payload.get("instructions") is None: self.instructions = None if isinstance(voice, str): self.voice = str(voice) elif "voice" in payload and payload.get("voice") is None: self.voice = None response_format = payload.get("output_audio_format") or payload.get("response_format") if response_format is None and isinstance(audio_output, dict): response_format = audio_output.get("format") response_format, _ = parse_realtime_audio_format(response_format) if isinstance(response_format, str) and response_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: self.response_format = duplex_response_format(response_format) if isinstance(payload.get("temperature"), int | float): self.temperature = float(payload["temperature"]) speed = payload.get("speed") if not isinstance(speed, int | float) and isinstance(audio_output, dict): speed = audio_output.get("speed") if isinstance(speed, int | float): self.speed = float(speed) max_tokens = ( payload.get("max_response_output_tokens") if "max_response_output_tokens" in payload else payload.get("max_output_tokens") if "max_output_tokens" in payload else payload.get("max_tokens") ) if "max_response_output_tokens" in payload or "max_output_tokens" in payload or "max_tokens" in payload: self.max_tokens = realtime_max_output_tokens(max_tokens) if isinstance(payload.get("overlap_policy"), str): self.overlap_policy = self._normalize_overlap_policy(str(payload["overlap_policy"])) if isinstance(payload.get("overlap_short_ack_ms"), int | float): self.overlap_short_ack_ms = max(0, int(payload["overlap_short_ack_ms"])) if isinstance(payload.get("overlap_barge_in_ms"), int | float): self.overlap_barge_in_ms = max(0, int(payload["overlap_barge_in_ms"])) if isinstance(payload.get("overlap_silence_rms"), int | float): self.overlap_silence_rms = max(0.0, float(payload["overlap_silence_rms"])) if isinstance(payload.get("playback_commit_policy"), str): self.playback_commit_policy = self._normalize_playback_commit_policy(str(payload["playback_commit_policy"])) modalities = payload.get("modalities") or payload.get("output_modalities") if isinstance(modalities, list) and all(isinstance(item, str) for item in modalities): self.modalities = list(modalities) if isinstance(payload.get("extra_body"), dict): self.extra_body.update(payload["extra_body"]) extra = payload["extra_body"] if isinstance(extra.get("overlap_policy"), str): self.overlap_policy = self._normalize_overlap_policy(str(extra["overlap_policy"])) if isinstance(extra.get("playback_commit_policy"), str): self.playback_commit_policy = self._normalize_playback_commit_policy( str(extra["playback_commit_policy"]) ) if isinstance(payload.get("tools"), list): self.extra_body["realtime_tools"] = payload["tools"] elif "tools" in payload and payload.get("tools") is None: self.extra_body.pop("realtime_tools", None) if isinstance(payload.get("tool_choice"), str | dict): self.extra_body["realtime_tool_choice"] = payload["tool_choice"] elif "tool_choice" in payload and payload.get("tool_choice") is None: self.extra_body.pop("realtime_tool_choice", None) if isinstance(payload.get("metadata"), dict): self.extra_body["realtime_metadata"] = dict(payload["metadata"]) elif "metadata" in payload and payload.get("metadata") is None: self.extra_body.pop("realtime_metadata", None) if isinstance(payload.get("include"), list): self.extra_body["realtime_include"] = list(payload["include"]) elif "include" in payload and payload.get("include") is None: self.extra_body.pop("realtime_include", None) if isinstance(payload.get("prompt"), dict): self.extra_body["realtime_prompt"] = dict(payload["prompt"]) elif "prompt" in payload and payload.get("prompt") is None: self.extra_body.pop("realtime_prompt", None) transcription = input_audio_transcription_config(payload) if isinstance(transcription, dict): self.extra_body["realtime_input_audio_transcription"] = dict(transcription) elif "input_audio_transcription" in payload and payload.get("input_audio_transcription") is None: self.extra_body.pop("realtime_input_audio_transcription", None) if isinstance(payload.get("input_audio_noise_reduction"), dict): self.extra_body["realtime_input_audio_noise_reduction"] = dict(payload["input_audio_noise_reduction"]) elif "input_audio_noise_reduction" in payload and payload.get("input_audio_noise_reduction") is None: self.extra_body.pop("realtime_input_audio_noise_reduction", None) if isinstance(audio_input, dict) and isinstance(audio_input.get("noise_reduction"), dict): self.extra_body["realtime_input_audio_noise_reduction"] = dict(audio_input["noise_reduction"]) elif isinstance(audio_input, dict) and audio_input.get("noise_reduction") is None: self.extra_body.pop("realtime_input_audio_noise_reduction", None) if isinstance(payload.get("audio"), dict): self.extra_body["realtime_audio"] = dict(payload["audio"]) elif "audio" in payload and payload.get("audio") is None: self.extra_body.pop("realtime_audio", None) if isinstance(payload.get("tracing"), str | dict): self.extra_body["realtime_tracing"] = payload["tracing"] elif "tracing" in payload and payload.get("tracing") is None: self.extra_body.pop("realtime_tracing", None) if isinstance(payload.get("turn_detection"), dict): self.extra_body["realtime_turn_detection"] = dict(payload["turn_detection"]) elif "turn_detection" in payload and payload.get("turn_detection") is None: self.extra_body["realtime_turn_detection"] = None self.extra_body["realtime_session_payload"] = json_safe_realtime_payload(payload) return None @staticmethod def _normalize_overlap_policy(value: str) -> str: normalized = value.strip().lower() if normalized in {policy.value for policy in DuplexOverlapPolicy}: return normalized return DuplexOverlapPolicy.LISTEN_ONLY.value @staticmethod def _normalize_playback_commit_policy(value: str) -> str: normalized = value.strip().lower() if normalized in {policy.value for policy in DuplexPlaybackCommitPolicy}: return normalized return DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value @dataclass(frozen=True) class ResponseCreateOptions: instructions: str | None = None voice: str | None = None response_format: str | None = None temperature: float | None = None max_tokens: int | None = None speed: float | None = None modalities: tuple[str, ...] | None = None extra_body: Mapping[str, object] = field(default_factory=dict) def apply_to(self, config: DuplexSessionConfig) -> None: for field_name in ("instructions", "voice", "response_format", "temperature", "max_tokens", "speed"): value = getattr(self, field_name) if value is not None: setattr(config, field_name, value) if self.modalities is not None: config.modalities = list(self.modalities) config.extra_body.update(self.extra_body) @classmethod def from_realtime( cls, response_payload: Mapping[str, object], *, private_runtime_config_keys: frozenset[str] = frozenset(), ) -> ResponseCreateOptions: """Parse an OpenAI Realtime ``response`` object into response-scoped options. Raises :class:`DuplexConfigError` (``code="unsupported_native_response_options"``) for options a model-native duplex session cannot apply per response. Private runtime keys in ``extra_body`` are dropped. """ from vllm_omni.engine.duplex.realtime_commands import duplex_response_format from vllm_omni.protocol.duplex import ( REALTIME_OUTPUT_AUDIO_FORMATS, parse_realtime_audio_format, ) payload: dict[str, object] = dict(response_payload) audio_config = payload.get("audio") audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None nested_voice = audio_output.get("voice") if isinstance(audio_output, dict) else None unsupported = ( payload.get("instructions") is not None or payload.get("voice") is not None or nested_voice is not None or payload.get("temperature") is not None or any( payload.get(field_name) is not None for field_name in ("max_response_output_tokens", "max_output_tokens", "max_tokens") ) or payload.get("tools") is not None or payload.get("tool_choice") is not None ) if unsupported: raise DuplexConfigError( "response.create options are not supported by a model-native duplex session", code="unsupported_native_response_options", ) instructions = str(payload["instructions"]) if isinstance(payload.get("instructions"), str) else None voice = payload.get("voice") if not isinstance(voice, str) and isinstance(audio_output, dict): voice = audio_output.get("voice") voice = str(voice) if isinstance(voice, str) else None response_format = payload.get("output_audio_format") or payload.get("response_format") if response_format is None and isinstance(audio_output, dict): response_format = audio_output.get("format") parsed_format, _ = parse_realtime_audio_format(response_format) output_audio_format: str | None if isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: output_audio_format = str(duplex_response_format(parsed_format)) else: output_audio_format = None temperature = ( float(cast("int | float", payload["temperature"])) if isinstance(payload.get("temperature"), int | float) else None ) speed = payload.get("speed") if not isinstance(speed, int | float) and isinstance(audio_output, dict): speed = audio_output.get("speed") speed = float(speed) if isinstance(speed, int | float) else None max_tokens = ( payload.get("max_response_output_tokens") if "max_response_output_tokens" in payload else payload.get("max_output_tokens") if "max_output_tokens" in payload else payload.get("max_tokens") ) if "max_response_output_tokens" in payload or "max_output_tokens" in payload or "max_tokens" in payload: token_limit = realtime_max_output_tokens(max_tokens) else: token_limit = None modalities = payload.get("modalities") or payload.get("output_modalities") if isinstance(modalities, list) and all(isinstance(item, str) for item in modalities): modalities = tuple(modalities) else: modalities = None response_extra: dict[str, object] = {} conversation = payload.get("conversation") if isinstance(conversation, str): response_extra["realtime_response_conversation"] = conversation metadata = payload.get("metadata") if isinstance(metadata, dict): response_extra["realtime_response_metadata"] = dict(metadata) prompt = payload.get("prompt") if isinstance(prompt, dict): response_extra["realtime_response_prompt"] = dict(prompt) if isinstance(payload.get("tools"), list): response_extra["realtime_response_tools"] = payload["tools"] if isinstance(payload.get("tool_choice"), str | dict): response_extra["realtime_response_tool_choice"] = payload["tool_choice"] extra_body = payload.get("extra_body") if isinstance(extra_body, dict): response_extra.update( (key, value) for key, value in extra_body.items() if key not in private_runtime_config_keys ) return cls( instructions=instructions, voice=voice, response_format=output_audio_format, temperature=temperature, max_tokens=token_limit, speed=speed, modalities=modalities, extra_body=response_extra, ) @dataclass class DuplexCommittedInput: message: dict[str, object] turn_id: int epoch: int input_commit_seq: int @dataclass class DuplexAssistantAudioTextMark: text_chars: int audio_end_ms: int def realtime_item_to_history_message(item: object) -> dict[str, object] | None: """Convert a Realtime conversation item into a chat history message (None when empty).""" if not isinstance(item, dict): return None role = item.get("role") if role not in {"system", "user", "assistant"}: return None content = item.get("content") if isinstance(content, str): text = content.strip() return {"role": role, "content": text} if text else None if not isinstance(content, list): return None text_chunks: list[str] = [] audio_chunks: list[dict[str, object]] = [] for part in content: if not isinstance(part, dict): continue part_type = part.get("type") if part_type in {"input_text", "text", "output_text"} and isinstance(part.get("text"), str): text_chunks.append(str(part["text"])) elif part_type == "input_image" and isinstance(part.get("image_url"), str): audio_chunks.append({"type": "image_url", "image_url": {"url": part["image_url"]}}) elif part_type in {"input_audio", "audio"}: audio = part.get("audio") or part.get("data") fmt = part.get("format") if isinstance(part.get("format"), str) else "wav" if isinstance(audio, str) and audio: audio_chunks.append({"type": "audio_url", "audio_url": {"url": f"data:audio/{fmt};base64,{audio}"}}) elif part_type in {"audio_transcript", "transcript"} and isinstance(part.get("text"), str): text_chunks.append(str(part["text"])) text = "".join(text_chunks).strip() if audio_chunks: content_items: list[dict[str, object]] = [] if text: content_items.append({"type": "text", "text": text}) content_items.extend(audio_chunks) return {"role": role, "content": content_items} if text: return {"role": role, "content": text} return None def realtime_max_output_tokens(value: object) -> int | None: """Normalize Realtime max output tokens (``"inf"`` -> ``None``).""" from vllm_omni.protocol.duplex import realtime_max_output_tokens as _impl return _impl(value) def input_audio_transcription_config(session_payload: Mapping[str, object]) -> dict[str, object] | None: """Return the ``input_audio_transcription`` object of a Realtime session payload.""" from vllm_omni.protocol.duplex import input_audio_transcription_config as _impl return _impl(session_payload) __all__ = [ "DuplexAssistantAudioTextMark", "DuplexAudioChunk", "DuplexCapabilities", "DuplexCommittedInput", "DuplexConfigError", "DuplexOverlapPolicy", "DuplexPlaybackCommitPolicy", "DuplexPlaybackCursor", "DuplexPlaybackView", "DuplexSessionConfig", "DuplexSessionState", "DuplexTurnEventType", "DuplexTurnState", "ResponseCreateOptions", "input_audio_transcription_config", "realtime_item_to_history_message", "realtime_max_output_tokens", ]