# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Control of one duplex session: server VAD, and the events that reconfigure it. Server turn detection and ``session.update`` are the same concern seen from two sides. The detector is per-session mutable state, and ``session.update`` is the event that replaces it mid-flight -- along with the session config, the runtime config and the sampling policy, each of which has to be validated against a candidate before the live session adopts it. The rest of ``turn.signal`` (conversation items, local turn markers) travels the same path and is small, so it lives here rather than in a module of its own. The one thing this does not own is ordering against the data plane: a ``session.update`` must not overtake an append still in flight, so the runner supplies ``wait_for_append_tail``. """ from __future__ import annotations from collections.abc import Awaitable, Callable from copy import deepcopy from vllm.logger import init_logger from vllm_omni.engine.duplex.config import DuplexConfigError, realtime_item_to_history_message from vllm_omni.engine.duplex.plugin import DuplexRuntimeConfigError from vllm_omni.engine.duplex.session import helpers from vllm_omni.engine.duplex.session.context import DuplexSessionContext from vllm_omni.engine.duplex.session.emitter import SessionEmitter from vllm_omni.engine.duplex.session.lease import DuplexLeaseActivity from vllm_omni.engine.duplex.session.model_channel import ModelChannel from vllm_omni.engine.duplex.turn_detection import ( PendingTurnDetectionUpdate, ServerTurnDetector, ServerVADUnavailableError, SileroVADBackendProvider, TurnDetectionConfig, TurnDetectionResult, apply_turn_detection_result, ) logger = init_logger(__name__) class SessionControl: """Server VAD and the control events that reconfigure one session.""" def __init__( self, ctx: DuplexSessionContext, out: SessionEmitter, model: ModelChannel, *, wait_for_append_tail: Callable[[], Awaitable[bool]], ) -> None: self._ctx = ctx self._out = out self._model = model self._wait_for_append_tail = wait_for_append_tail self._config: TurnDetectionConfig | None = None self._detector: ServerTurnDetector | None = None def init_turn_detection(self) -> None: turn_detection = self._ctx.session.config.extra_body.get("realtime_turn_detection") if not isinstance(turn_detection, dict) or turn_detection.get("type") != "server_vad": self._config = None self._detector = None return try: config = TurnDetectionConfig.from_realtime(turn_detection) self._config = config self._detector = config.build_detector(self._vad_backend_provider()) except Exception as exc: logger.warning("Duplex session %s: turn detection disabled: %s", self._ctx.session.session_id, exc) self._config = None self._detector = None def _vad_backend_provider(self) -> SileroVADBackendProvider | None: """The engine-wide Silero backend, so one model serves every session. Held by the manager rather than built here: the ONNX backend is shared, and ``duplex_session.server_vad_model_path`` is a deploy-level setting. """ return getattr(self._ctx.manager, "vad_backend_provider", None) def reset_vad(self) -> None: """Drop the detector's speech state (a barge-in or a clear starts a new turn).""" if self._detector is not None: self._detector.reset() async def run_turn_detection(self, event: dict[str, object]) -> TurnDetectionResult | None: detector = self._detector if detector is None: return None audio = event.get("audio") if not isinstance(audio, str) or not audio: return None fmt = event.get("format") if isinstance(event.get("format"), str) else "pcm_f32le" sample_rate_hz = event.get("sample_rate_hz") try: result = await self._ctx.services.offload( detector.process, audio, fmt=fmt, sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int) else None, audio_end_ms=event.get("audio_end_ms") if isinstance(event.get("audio_end_ms"), int) else None, ) except ServerVADUnavailableError as exc: self._out.emit_error("server_vad_unavailable", str(exc)) self._detector = None return None except ValueError as exc: self._out.emit_error("bad_audio", str(exc)) return None if result.speech_started or result.speech_stopped: logger.info( "Duplex VAD session=%s started=%s stopped=%s active=%s probability=%.3f " "start_ms=%s end_ms=%s commit=%s", self._ctx.session.session_id, result.speech_started, result.speech_stopped, result.speech_active, result.speech_probability, result.audio_start_ms, result.audio_end_ms, result.should_commit, ) apply_turn_detection_result(event, result) return result # ------------------------------------------------------------------ # # Control events # # ------------------------------------------------------------------ # async def on_turn_signal(self, event: dict[str, object]) -> None: session = self._ctx.session turn_event = event.get("event") realtime_event_id = event.get("realtime_event_id") if not isinstance(turn_event, str): self._out.emit_error("bad_event", "turn.signal requires event") return if turn_event == "barge_in" and not session.capabilities.supports_barge_in: self._out.emit_events([helpers.barge_in_unsupported_error()]) return if turn_event == "session.update": payload = event.get("payload") if not isinstance(payload, dict): self._out.emit_error( "bad_event", "session.update requires a session payload", event_id=realtime_event_id ) return await self.on_session_update( payload, realtime_event_id=realtime_event_id if isinstance(realtime_event_id, str) else None, ) return if turn_event == "conversation.item.create": await self._on_conversation_item_create(event) return if turn_event == "conversation.item.delete": payload = event.get("payload") item_id = payload.get("item_id") if isinstance(payload, dict) else None deleted = session.delete_history_item(item_id) if isinstance(item_id, str) else False self._out.emit( { "type": "conversation.item.deleted", "session_id": session.session_id, "item_id": item_id, "deleted": deleted, } ) return if turn_event == "conversation.item.truncate": payload = event.get("payload") item_id = payload.get("item_id") if isinstance(payload, dict) else None audio_end_ms = payload.get("audio_end_ms") if isinstance(payload, dict) else None truncated = ( session.truncate_history_item( item_id, audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else 0, hard=True, ) if isinstance(item_id, str) else False ) self._out.emit( { "type": "conversation.item.truncated", "session_id": session.session_id, "item_id": item_id, "content_index": (payload.get("content_index", 0) if isinstance(payload, dict) else 0), "audio_end_ms": audio_end_ms, "truncated": truncated, } ) return self._out.emit_events([session.signal_turn(turn_event, event)]) async def on_session_update(self, payload: dict[str, object], *, realtime_event_id: str | None) -> None: session = self._ctx.session model_state = self._ctx.model_state pending_turn_detection: PendingTurnDetectionUpdate | None = None def reject_update() -> None: if pending_turn_detection is not None: pending_turn_detection.reject() try: pending_turn_detection = PendingTurnDetectionUpdate.prepare( payload, backend_provider=self._vad_backend_provider() ) except Exception as exc: self._out.emit_error("unsupported_turn_detection", str(exc), event_id=realtime_event_id) return if not await self._wait_for_append_tail(): self._out.emit_error( "session_update_aborted", "session.update was not applied because the preceding append failed", event_id=realtime_event_id, ) reject_update() return try: self._ctx.plugin.validate_client_extra_body(payload.get("extra_body")) except DuplexRuntimeConfigError as exc: self._out.emit_error(exc.code, str(exc), event_id=realtime_event_id) reject_update() return candidate_config = deepcopy(session.config) audio_started = session.playback.generated_ms > 0 or session.playback.sent_ms > 0 try: candidate_config.apply_realtime_update( payload, session_id=session.session_id, audio_started=audio_started, ) except DuplexConfigError as exc: self._out.emit_error(exc.code, str(exc) or "session.update was rejected", event_id=realtime_event_id) reject_update() return if candidate_config.instructions != session.config.instructions and model_state.context_locked: self._out.emit_error( "instructions_update_unsupported", "session.update cannot change instructions after the native duplex context is initialized", event_id=realtime_event_id, ) reject_update() return requests_audio = any(str(modality).lower() == "audio" for modality in candidate_config.modalities) if ( requests_audio and "ref_audio_data" not in session.runtime_config and getattr(self._ctx.plugin, "requires_ref_audio", False) ): self._out.emit_error( "ref_audio_required", "Native duplex audio output requires ref_audio", event_id=realtime_event_id ) reject_update() return try: candidate_runtime_config = self._ctx.plugin.runtime_config_for_update( candidate_config, dict(session.runtime_config), ) # Validate the sampling policy for the candidate before adopting it. self._ctx.plugin.configure_sampling_params( runtime_config=dict(candidate_runtime_config), defaults=tuple(self._ctx.stage_port.sampling_defaults()), ) except (DuplexRuntimeConfigError, DuplexConfigError) as exc: self._out.emit_error(exc.code, str(exc), event_id=realtime_event_id) reject_update() return except Exception as exc: self._out.emit_error("runtime_signal_failed", str(exc), event_id=realtime_event_id) reject_update() return session.replace_config(candidate_config) session.replace_runtime_config(candidate_runtime_config) try: session.touch_lease(DuplexLeaseActivity.SIGNAL) except Exception: pass if pending_turn_detection is not None: self._config, self._detector = pending_turn_detection.commit(self._detector) projector = self._out.require_projector() projector.apply_session_defaults(payload) self._out.emit({"type": "session.updated", "session": session.as_public_dict()}) async def _on_conversation_item_create(self, event: dict[str, object]) -> None: session = self._ctx.session payload = event.get("payload") item = payload.get("item") if isinstance(payload, dict) else None item_type = item.get("type") if isinstance(item, dict) else None if item_type == "function_call_output" and isinstance(item, dict): if not await self._wait_for_append_tail(): return try: candidate_runtime_config = self._ctx.plugin.runtime_config_for_function_output( session.config, dict(session.runtime_config), item, ) except DuplexRuntimeConfigError as exc: self._out.emit_error(exc.code, str(exc)) return if candidate_runtime_config is not None: session.replace_runtime_config(candidate_runtime_config) self._out.emit( { "type": "conversation.item.created", "session_id": session.session_id, "item": item, "created": True, } ) self._ctx.services.spawn( self._model.maybe_continue_response( expected_epoch=session.epoch, expected_model_turn_id=session.turn_id, ), name="duplex-continue", ) return item_payload = item if isinstance(item, dict) else None raw_parts = item_payload.get("content", []) if item_payload is not None else [] parts: list[object] = [part for part in raw_parts] if isinstance(raw_parts, list) else [] images = [p for p in parts if isinstance(p, dict) and p.get("type") == "input_image"] if images: if item_payload is None or item_payload.get("role") != "user": self._out.emit_error("invalid_image", "input_image is supported only in user messages") return if not session.capabilities.supports_image_input: self._out.emit_error("unsupported", "This model does not support input_image conversation items") return from vllm_omni.engine.duplex.realtime_commands import validate_realtime_video_frames urls: list[str] = [] for part in images: url = part.get("image_url") if not isinstance(url, str) or not url.startswith( ("data:image/jpeg;base64,", "data:image/png;base64,") ): self._out.emit_error("invalid_image", "input_image requires a JPEG or PNG base64 data URL") return error = validate_realtime_video_frames([url.split(",", 1)[1]], None) if error: self._out.emit_error("invalid_image", error) return urls.append(url) existing: list[str] = [] for history_message in session.history: content = history_message.get("content") if not isinstance(content, list): continue for part in content: if not isinstance(part, dict) or part.get("type") != "image_url": continue image_url = part.get("image_url") nested_url = image_url.get("url") if isinstance(image_url, dict) else None if isinstance(nested_url, str): existing.append(nested_url) if len(existing) + len(urls) > 8 or sum(map(len, existing + urls)) > 4 * 1024 * 1024: self._out.emit_error("input_backpressure", "Image context exceeds 8 images or 4 MiB; delete old items") return message = realtime_item_to_history_message(item) item_id = item.get("id") if isinstance(item, dict) else None if message is not None: session.append_history_message(message) session.register_history_item(item_id if isinstance(item_id, str) else None, message) if message.get("role") == "user": # A later response.create may answer this without any audio. session.notify_new_user_item() self._out.emit( { "type": "conversation.item.created", "session_id": session.session_id, "item": item, "created": message is not None, } )