# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import asyncio import hashlib import threading import time from collections.abc import Mapping from dataclasses import dataclass, replace from pathlib import Path from typing import TYPE_CHECKING, Literal, Protocol import numpy as np from vllm.transformers_utils.repo_utils import try_get_local_file if TYPE_CHECKING: from vllm_omni.entrypoints.openai.audio_utils_mixin import StreamingAudioResampler SILERO_VAD_REPO_ID = "istupakov/silero-vad-onnx" SILERO_VAD_REVISION = "8b14476858ef240c50b3884bb38cc67290c1cc70" SILERO_VAD_FILENAME = "silero_vad.onnx" SILERO_VAD_SHA256 = "1a153a22f4509e292a94e67d6f9b85e8deb25b4988682b7e174c65279d8788e3" SILERO_VAD_MIN_THRESHOLD = 0.15 SILERO_VAD_DEFAULT_MIN_SPEECH_DURATION_MS = 96 _SERVER_VAD_FIELDS = { "type", "threshold", "prefix_padding_ms", "silence_duration_ms", "create_response", "interrupt_response", "min_speech_duration_ms", } @dataclass(frozen=True, slots=True) class ServerVADConfig: """Validated OpenAI-compatible ``server_vad`` session configuration.""" type: Literal["server_vad"] = "server_vad" threshold: float = 0.5 prefix_padding_ms: int = 300 silence_duration_ms: int = 500 create_response: bool = True # Omission is resolved from runtime capabilities before the session is opened. interrupt_response: bool | None = None min_speech_duration_ms: int | None = None @classmethod def from_value(cls, value: object) -> ServerVADConfig: if not isinstance(value, dict): raise ValueError("turn_detection must be null or an object") unknown = sorted(set(value) - _SERVER_VAD_FIELDS) if unknown: raise ValueError(f"Unknown server_vad field(s): {', '.join(unknown)}") vad_type = value.get("type") if vad_type != "server_vad": raise ValueError("turn_detection.type must be 'server_vad'") threshold = value.get("threshold", 0.5) if isinstance(threshold, bool) or not isinstance(threshold, int | float) or not 0 <= threshold <= 1: raise ValueError("server_vad.threshold must be a number between 0 and 1") prefix_padding_ms = value.get("prefix_padding_ms", 300) if isinstance(prefix_padding_ms, bool) or not isinstance(prefix_padding_ms, int) or prefix_padding_ms < 0: raise ValueError("server_vad.prefix_padding_ms must be a non-negative integer") silence_duration_ms = value.get("silence_duration_ms", 500) if ( isinstance(silence_duration_ms, bool) or not isinstance(silence_duration_ms, int) or silence_duration_ms <= 0 ): raise ValueError("server_vad.silence_duration_ms must be a positive integer") create_response = value.get("create_response", True) if not isinstance(create_response, bool): raise ValueError("server_vad.create_response must be a boolean") interrupt_response = value.get("interrupt_response") if "interrupt_response" in value and not isinstance(interrupt_response, bool): raise ValueError("server_vad.interrupt_response must be a boolean") min_speech_duration_ms = value.get("min_speech_duration_ms") if min_speech_duration_ms is not None and ( isinstance(min_speech_duration_ms, bool) or not isinstance(min_speech_duration_ms, int | float) or not np.isfinite(float(min_speech_duration_ms)) or min_speech_duration_ms < 0 ): raise ValueError("server_vad.min_speech_duration_ms must be a non-negative number") return cls( type=vad_type, threshold=float(threshold), prefix_padding_ms=prefix_padding_ms, silence_duration_ms=silence_duration_ms, create_response=create_response, interrupt_response=interrupt_response, min_speech_duration_ms=(int(min_speech_duration_ms) if min_speech_duration_ms is not None else None), ) def as_dict(self) -> dict[str, object]: result: dict[str, object] = { "type": self.type, "threshold": self.threshold, "prefix_padding_ms": self.prefix_padding_ms, "silence_duration_ms": self.silence_duration_ms, "create_response": self.create_response, } if self.interrupt_response is not None: result["interrupt_response"] = self.interrupt_response if self.min_speech_duration_ms is not None: result["min_speech_duration_ms"] = self.min_speech_duration_ms return result def parse_session_turn_detection( payload: Mapping[str, object], ) -> tuple[bool, ServerVADConfig | None]: """Extract and validate OpenAI-compatible turn detection aliases.""" configured_values: list[tuple[str, object]] = [] if "turn_detection" in payload: configured_values.append(("turn_detection", payload["turn_detection"])) audio = payload.get("audio") if isinstance(audio, Mapping): audio_input = audio.get("input") if isinstance(audio_input, Mapping) and "turn_detection" in audio_input: configured_values.append(("audio.input.turn_detection", audio_input["turn_detection"])) if not configured_values: return False, None first_path, first_value = configured_values[0] first_config = None if first_value is None else ServerVADConfig.from_value(first_value) for field_path, value in configured_values[1:]: config = None if value is None else ServerVADConfig.from_value(value) if first_config is not None and config is not None: # An omitted option must not conflict with an explicit alias value. if first_config.interrupt_response is None: first_config = replace(first_config, interrupt_response=config.interrupt_response) if config.interrupt_response is None: config = replace(config, interrupt_response=first_config.interrupt_response) if config != first_config: raise ValueError(f"{first_path} and {field_path} must not specify conflicting values") return True, first_config class SpeechDetectorBackend(Protocol): """Detector contract; per-stream state is owned by the pipeline.""" frame_samples: int def new_state(self) -> object: ... def infer(self, frame: np.ndarray, state: object) -> tuple[float, object]: ... class SpeechDetectorBackendProvider(Protocol): def get(self) -> SpeechDetectorBackend: ... @dataclass(frozen=True, slots=True) class SpeechEndpointDecision: speech_started: bool = False speech_stopped: bool = False audio_start_ms: int | None = None audio_end_ms: int | None = None endpoint_delay_ms: int | None = None class SpeechEndpointPolicy(Protocol): @property def speech_active(self) -> bool: ... def update( self, probability: float, *, frame_start_sample: int, frame_samples: int, ) -> SpeechEndpointDecision: ... def reset(self) -> None: ... class ThresholdEndpointPolicy: """Apply threshold and trailing-silence endpoint rules.""" def __init__(self, config: ServerVADConfig, *, sample_rate_hz: int) -> None: self.config = config self.sample_rate_hz = sample_rate_hz self._speech_active = False self._candidate_samples = 0 self._candidate_start_sample: int | None = None self._silence_samples = 0 @property def speech_active(self) -> bool: return self._speech_active def update( self, probability: float, *, frame_start_sample: int, frame_samples: int, ) -> SpeechEndpointDecision: if probability >= self.config.threshold: self._silence_samples = 0 if not self._speech_active: if self._candidate_samples == 0: self._candidate_start_sample = frame_start_sample self._candidate_samples += frame_samples min_speech_duration_ms = self.config.min_speech_duration_ms or 0 min_speech_samples = max(1, round(min_speech_duration_ms * self.sample_rate_hz / 1000)) if self._candidate_samples < min_speech_samples: return SpeechEndpointDecision() self._speech_active = True detected_start_sample = ( self._candidate_start_sample if self._candidate_start_sample is not None else frame_start_sample ) detected_start_ms = round(detected_start_sample * 1000 / self.sample_rate_hz) self._candidate_samples = 0 self._candidate_start_sample = None return SpeechEndpointDecision( speech_started=True, audio_start_ms=max(0, detected_start_ms - self.config.prefix_padding_ms), ) return SpeechEndpointDecision() if not self._speech_active: self._candidate_samples = 0 self._candidate_start_sample = None return SpeechEndpointDecision() # Match Silero v6.2's streaming VAD hysteresis: activation uses the # configured threshold, while potential speech end starts only below # ``max(threshold - 0.15, 0.01)``. Once a silence candidate exists, # intermediate frames keep elapsed time moving but cannot themselves # close the turn. negative_threshold = max(self.config.threshold - SILERO_VAD_MIN_THRESHOLD, 0.01) below_negative_threshold = probability < negative_threshold if not below_negative_threshold and self._silence_samples == 0: return SpeechEndpointDecision() self._silence_samples += frame_samples if not below_negative_threshold: return SpeechEndpointDecision() silence_limit = max(1, round(self.config.silence_duration_ms * self.sample_rate_hz / 1000)) if self._silence_samples < silence_limit: return SpeechEndpointDecision() # OpenAI defines ``audio_end_ms`` as the end of the audio sent to the # model, including the trailing silence used for endpoint detection. audio_end_sample = frame_start_sample + frame_samples endpoint_delay_ms = round(self._silence_samples * 1000 / self.sample_rate_hz) self.reset() return SpeechEndpointDecision( speech_stopped=True, audio_end_ms=max(0, round(audio_end_sample * 1000 / self.sample_rate_hz)), endpoint_delay_ms=endpoint_delay_ms, ) def reset(self) -> None: self._speech_active = False self._candidate_samples = 0 self._candidate_start_sample = None self._silence_samples = 0 @dataclass(frozen=True, slots=True) class ServerVADFrame: samples: np.ndarray decision: SpeechEndpointDecision @dataclass(frozen=True, slots=True) class ServerVADBatch: frames: tuple[ServerVADFrame, ...] inference_ms: float class ServerVADPipeline: """Frame continuous 16 kHz audio and apply acoustic endpointing. The pipeline retains only input-resampling residuals, a partial frame, detector state, and timing counters. Commit-eligible prefix and utterance audio remain session-owned. """ sample_rate_hz = 16_000 def __init__( self, backend: SpeechDetectorBackend, config: ServerVADConfig, *, endpoint_policy: SpeechEndpointPolicy | None = None, ) -> None: self.backend = backend self.config = config self.endpoint_policy = endpoint_policy or ThresholdEndpointPolicy( config, sample_rate_hz=self.sample_rate_hz, ) self._input_resampler: StreamingAudioResampler | None = None self._source_sample_rate_hz: int | None = None self._scratch = np.empty(0, dtype=np.float32) self._detector_state = backend.new_state() self._processed_samples = 0 self._stream_start_ms = 0 @property def speech_active(self) -> bool: return self.endpoint_policy.speech_active @property def source_sample_rate_hz(self) -> int | None: return self._source_sample_rate_hz @property def scratch_bytes(self) -> int: resampler_bytes = self._input_resampler.scratch_bytes if self._input_resampler is not None else 0 return int(self._scratch.nbytes) + resampler_bytes async def push_pcm16( self, samples: bytes, *, source_sample_rate_hz: int, ) -> ServerVADBatch: """Normalize chunked mono PCM16 input and run endpoint detection.""" if isinstance(source_sample_rate_hz, bool) or source_sample_rate_hz not in {16_000, 24_000}: raise ValueError("server_vad PCM16 input sample rate must be 16000 or 24000 Hz") if len(samples) % np.dtype(" ServerVADBatch: normalized = np.ascontiguousarray(samples, dtype=np.float32).reshape(-1) return await asyncio.to_thread(self._push_sync, normalized) def _push_sync(self, samples: np.ndarray) -> ServerVADBatch: started_at = time.perf_counter() if self._scratch.size: samples = np.concatenate((self._scratch, samples)) frame_samples = int(self.backend.frame_samples) complete_samples = samples.size - samples.size % frame_samples # Retained state must not keep the whole input chunk alive through a view. self._scratch = samples[complete_samples:].copy() results: list[ServerVADFrame] = [] for offset in range(0, complete_samples, frame_samples): frame = np.ascontiguousarray(samples[offset : offset + frame_samples], dtype=np.float32) frame_start = self._processed_samples probability, self._detector_state = self.backend.infer(frame, self._detector_state) probability = min(1.0, max(0.0, float(probability))) decision = self.endpoint_policy.update( probability, frame_start_sample=frame_start, frame_samples=frame_samples, ) if decision.speech_started and decision.audio_start_ms is not None: # Prefix audio before the most recent reset is no longer available. decision = replace(decision, audio_start_ms=max(self._stream_start_ms, decision.audio_start_ms)) self._processed_samples += frame_samples results.append( ServerVADFrame( samples=frame, decision=decision, ) ) return ServerVADBatch( frames=tuple(results), inference_ms=(time.perf_counter() - started_at) * 1000, ) def reset(self) -> None: """Discard stream state while preserving the session-relative audio clock.""" self._stream_start_ms = round(self._processed_samples * 1000 / self.sample_rate_hz) self._input_resampler = None self._source_sample_rate_hz = None self._scratch = np.empty(0, dtype=np.float32) self._detector_state = self.backend.new_state() self.endpoint_policy.reset() @dataclass(frozen=True, slots=True) class _SileroVADState: model_state: np.ndarray context: np.ndarray class SileroVADBackend: """Shared ONNX Runtime Silero v6.2 detector running on CPU.""" sample_rate_hz = 16_000 frame_samples = 512 context_samples = 64 model_state_shape = (2, 1, 128) def __init__(self, model_path: str | Path) -> None: try: import onnxruntime as ort except ImportError as exc: # pragma: no cover - platform packaging supplies ORT. raise RuntimeError("server_vad requires ONNX Runtime") from exc session_options = ort.SessionOptions() session_options.inter_op_num_threads = 1 session_options.intra_op_num_threads = 1 self.model_path = Path(model_path) self._session = ort.InferenceSession( str(self.model_path), providers=["CPUExecutionProvider"], sess_options=session_options, ) input_names = {item.name for item in self._session.get_inputs()} if "input" not in input_names or "sr" not in input_names or "state" not in input_names: raise RuntimeError(f"Unsupported Silero ONNX input contract: {sorted(input_names)}") self._warm_up() def _warm_up(self) -> None: self.infer(np.zeros(self.frame_samples, dtype=np.float32), self.new_state()) def new_state(self) -> _SileroVADState: return _SileroVADState( model_state=np.zeros(self.model_state_shape, dtype=np.float32), context=np.zeros((1, self.context_samples), dtype=np.float32), ) def infer(self, frame: np.ndarray, state: object) -> tuple[float, object]: if not isinstance(state, _SileroVADState): raise TypeError("Silero detector state must be created by SileroVADBackend.new_state()") model_state = np.ascontiguousarray(state.model_state, dtype=np.float32) context = np.ascontiguousarray(state.context, dtype=np.float32) if model_state.shape != self.model_state_shape: raise ValueError(f"Silero model state must have shape {self.model_state_shape}, got {model_state.shape}") expected_context_shape = (1, self.context_samples) if context.shape != expected_context_shape: raise ValueError(f"Silero context must have shape {expected_context_shape}, got {context.shape}") audio = np.ascontiguousarray(frame, dtype=np.float32).reshape(1, -1) if audio.shape[1] != self.frame_samples: raise ValueError( f"Silero detector frame must contain exactly {self.frame_samples} samples, got {audio.shape[1]}" ) model_input = np.concatenate((context, audio), axis=1) inputs = { "input": model_input, "state": model_state, "sr": np.asarray(self.sample_rate_hz, dtype=np.int64), } # ONNX Runtime permits concurrent Run calls on one CPU session. Stream # state is explicit, so independent pipelines can safely share the session. output = self._session.run(None, inputs) if len(output) < 2: raise RuntimeError("Silero ONNX model did not return probability and model state") probability = float(np.asarray(output[0]).reshape(-1)[0]) next_state = _SileroVADState( model_state=np.ascontiguousarray(output[1], dtype=np.float32), context=np.ascontiguousarray(model_input[:, -self.context_samples :], dtype=np.float32), ) return probability, next_state class SileroVADBackendProvider: """Resolve, verify, and load one Silero model instance per process.""" def __init__(self, *, model_path: str | None = None) -> None: self.model_path = model_path self._backend: SileroVADBackend | None = None self._lock = threading.Lock() def get(self) -> SileroVADBackend: if self._backend is not None: return self._backend with self._lock: if self._backend is None: path = self._resolve_local_artifact() self._verify_checksum(path) self._backend = SileroVADBackend(path) return self._backend def _resolve_local_artifact(self) -> Path: if self.model_path: path = Path(self.model_path).expanduser() if path.is_file(): return path raise RuntimeError(f"Configured Silero VAD model does not exist: {path}") cached = try_get_local_file( model=SILERO_VAD_REPO_ID, file_name=SILERO_VAD_FILENAME, revision=SILERO_VAD_REVISION, ) if isinstance(cached, Path) and cached.is_file(): return cached raise RuntimeError( "Silero VAD artifact is not available locally. Pre-download the pinned artifact " "or configure duplex_session.server_vad_model_path." ) def _verify_checksum(self, path: Path) -> None: digest = hashlib.sha256(path.read_bytes()).hexdigest() if digest != SILERO_VAD_SHA256: raise RuntimeError(f"Silero VAD checksum mismatch for {path}: expected {SILERO_VAD_SHA256}, got {digest}")