vllm-omni/vllm_omni/engine/duplex/turn_detection.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

"""Server-side turn detection (``turn_detection: server_vad``) for the session runner.

Moved from the Realtime input translator: the runner owns one
``ServerTurnDetector`` per session, feeds each appended chunk to it and emits
``input_audio_buffer.speech_started`` / ``speech_stopped`` from the result.
The two-phase ``session.update`` commit (prepare / commit / reject) that used
to live on the translator is expressed by :class:`PendingTurnDetectionUpdate`.
"""

from __future__ import annotations

from collections.abc import Mapping
from dataclasses import dataclass

import numpy as np

from vllm_omni.engine.duplex.vad import (
    SILERO_VAD_MIN_THRESHOLD,
    ServerVADUnavailableError,
    SileroStreamingVAD,
    SileroVADBackendProvider,
    SileroVADConfig,
    StreamingVADResult,
)

__all__ = [
    "PendingTurnDetectionUpdate",
    "SileroVADBackendProvider",
    "ServerTurnDetector",
    "ServerVADUnavailableError",
    "TurnDetectionConfig",
    "TurnDetectionResult",
    "apply_turn_detection_result",
    "configured_realtime_turn_detection",
    "normalize_turn_detection_session_payload",
    "validate_realtime_turn_detection",
]


def configured_realtime_turn_detection(session_payload: Mapping[str, object]) -> tuple[str | None, object]:
    """Return ``(field_path, value)`` for the turn detection setting present in the payload."""
    field = "turn_detection" if "turn_detection" in session_payload else None
    selected = session_payload.get("turn_detection")
    audio_config = session_payload.get("audio")
    if isinstance(audio_config, dict):
        audio_input = audio_config.get("input")
        if isinstance(audio_input, dict) and "turn_detection" in audio_input:
            field, selected = "audio.input.turn_detection", audio_input["turn_detection"]
    return field, selected


def _valid_vad_number(value: object, *, minimum: float, maximum: float | None = None, strict: bool = False) -> bool:
    return (
        isinstance(value, int | float)
        and not isinstance(value, bool)
        and np.isfinite(float(value))
        and (float(value) > minimum if strict else float(value) >= minimum)
        and (maximum is None or float(value) <= maximum)
    )


#: Every field a ``server_vad`` object may carry. Unknown keys are refused
#: rather than ignored: a misspelled tuning knob that silently does nothing is
#: worse than a rejected session, because the endpointing still *looks* applied.
_SERVER_VAD_FIELDS = frozenset(
    {
        "type",
        "threshold",
        "prefix_padding_ms",
        "silence_duration_ms",
        "create_response",
        "interrupt_response",
        "min_speech_duration_ms",
    }
)


def validate_realtime_turn_detection(session_payload: Mapping[str, object]) -> str | None:
    """Validate the ``turn_detection`` object of a Realtime session payload (None when valid)."""
    field, turn_detection = configured_realtime_turn_detection(session_payload)
    if field is None:
        if session_payload.get("overlap_policy") == "barge_in_on_speech":
            return (
                "overlap_policy='barge_in_on_speech' requires turn_detection.type='server_vad' on the Realtime endpoint"
            )
        return None
    if turn_detection is not None and not isinstance(turn_detection, dict):
        return f"{field} must be null or an object with type='server_vad'"
    if isinstance(turn_detection, dict):
        if turn_detection.get("type") != "server_vad":
            return f"{field}.type must be 'server_vad'"
        unknown = sorted(set(turn_detection) - _SERVER_VAD_FIELDS)
        if unknown:
            return f"Unknown {field} field(s): {', '.join(unknown)}"
        threshold = turn_detection.get("threshold", 0.5)
        if not _valid_vad_number(threshold, minimum=SILERO_VAD_MIN_THRESHOLD, strict=True, maximum=1):
            return f"{field}.threshold must be greater than {SILERO_VAD_MIN_THRESHOLD} and at most 1"
        for name in ("prefix_padding_ms", "silence_duration_ms", "min_speech_duration_ms"):
            value = turn_detection.get(name)
            if value is not None and not _valid_vad_number(value, minimum=0):
                return f"{field}.{name} must be a non-negative number"
        if turn_detection.get("interrupt_response", True) is not True:
            return (
                f"{field}.interrupt_response=false is unsupported; use turn_detection=null for model-owned listen/speak"
            )
    desired_policy = "barge_in_on_speech" if turn_detection is not None else "listen_only"
    overlap_policy = session_payload.get("overlap_policy")
    if isinstance(overlap_policy, str) and overlap_policy != desired_policy:
        return f"overlap_policy={overlap_policy!r} conflicts with turn_detection; expected {desired_policy!r}"
    return None


@dataclass(frozen=True, slots=True)
class TurnDetectionConfig:
    """Normalized ``server_vad`` configuration (all fields filled with defaults)."""

    threshold: float = 0.5
    prefix_padding_ms: int = 300
    silence_duration_ms: int = 500
    min_speech_duration_ms: int = 96
    create_response: bool = True
    interrupt_response: bool = True

    @classmethod
    def from_realtime(cls, turn_detection: Mapping[str, object]) -> TurnDetectionConfig:
        return cls(
            threshold=float(turn_detection.get("threshold", 0.5)),
            prefix_padding_ms=int(turn_detection.get("prefix_padding_ms", 300)),
            silence_duration_ms=int(turn_detection.get("silence_duration_ms", 500)),
            min_speech_duration_ms=int(turn_detection.get("min_speech_duration_ms", 96)),
            create_response=bool(turn_detection.get("create_response", True)),
            interrupt_response=bool(turn_detection.get("interrupt_response", True)),
        )

    @property
    def overlap_policy(self) -> str:
        return "barge_in_on_speech"

    def build_detector(self, backend_provider: SileroVADBackendProvider | None = None) -> ServerTurnDetector:
        return ServerTurnDetector(self, backend_provider=backend_provider)


def normalize_turn_detection_session_payload(
    session_payload: dict[str, object],
) -> tuple[bool, TurnDetectionConfig | None]:
    """Fill ``turn_detection`` defaults and derive ``overlap_policy`` in place.

    Returns ``(configured, config)`` where ``configured`` is False when the
    payload does not mention turn detection at all (nothing changes), and
    ``config`` is ``None`` for ``turn_detection: null`` (model-owned turns,
    ``overlap_policy`` forced to ``listen_only``).
    """
    field, configured = configured_realtime_turn_detection(session_payload)
    if field is None:
        return False, None
    if configured is None:
        session_payload["overlap_policy"] = "listen_only"
        return True, None
    assert isinstance(configured, dict)
    merged = {
        "type": "server_vad",
        "interrupt_response": True,
        "threshold": 0.5,
        "prefix_padding_ms": 300,
        "silence_duration_ms": 500,
        "min_speech_duration_ms": 96,
        **configured,
    }
    config = TurnDetectionConfig.from_realtime(merged)
    session_payload["turn_detection"] = dict(merged)
    session_payload["overlap_policy"] = config.overlap_policy
    return True, config


@dataclass(frozen=True, slots=True)
class TurnDetectionResult:
    is_speech: bool
    speech_active: bool
    speech_started: bool = False
    speech_stopped: bool = False
    speech_probability: float = 0.0
    audio_start_ms: int | None = None
    audio_end_ms: int | None = None
    #: A ``speech_stopped`` under ``server_vad`` ends the user turn.
    should_commit: bool = False
    #: Whether the auto-commit should also request a response (``create_response``).
    create_response: bool = True

    def as_payload(self) -> dict[str, object]:
        return {
            "backend": "silero",
            "is_speech": self.is_speech,
            "speech_active": self.speech_active,
            "speech_started": self.speech_started,
            "speech_stopped": self.speech_stopped,
            "speech_probability": self.speech_probability,
        }


class ServerTurnDetector:
    """One Silero-backed streaming detector per session."""

    def __init__(
        self,
        config: TurnDetectionConfig,
        *,
        backend_provider: SileroVADBackendProvider | None = None,
    ) -> None:
        self.config = config
        self._vad = SileroStreamingVAD(
            SileroVADConfig(
                threshold=float(config.threshold),
                prefix_padding_ms=int(config.prefix_padding_ms),
                silence_duration_ms=int(config.silence_duration_ms),
                min_speech_duration_ms=max(32, int(config.min_speech_duration_ms)),
            ),
            backend_provider=backend_provider,
        )

    @property
    def vad(self) -> SileroStreamingVAD:
        return self._vad

    def reset(self) -> None:
        self._vad.reset()

    def process(
        self,
        base64_audio: str,
        *,
        fmt: str,
        sample_rate_hz: int | None,
        audio_end_ms: int | None = None,
    ) -> TurnDetectionResult:
        """Score one appended chunk. Raises ``ServerVADUnavailableError`` / ``ValueError`` like the VAD."""
        result: StreamingVADResult = self._vad.process_base64(base64_audio, fmt=fmt, sample_rate_hz=sample_rate_hz)
        stop_ms = result.speech_end_ms if result.speech_end_ms is not None else audio_end_ms
        return TurnDetectionResult(
            is_speech=result.is_speech,
            speech_active=result.speech_active,
            speech_started=result.speech_started,
            speech_stopped=result.speech_stopped,
            speech_probability=result.speech_probability,
            audio_start_ms=result.speech_start_ms,
            audio_end_ms=stop_ms,
            should_commit=bool(result.speech_stopped and not result.speech_active),
            create_response=self.config.create_response,
        )


def apply_turn_detection_result(payload: dict[str, object], result: TurnDetectionResult) -> None:
    """Merge a detector result onto an internal ``input_audio_buffer.append`` payload.

    Mirrors what the old translator attached: ``is_speech`` from the detector,
    the ``vad`` hint block, and ``force_listen`` while speech is active.
    """
    payload["is_speech"] = result.is_speech
    payload["vad"] = result.as_payload()
    if result.speech_active:
        payload["force_listen"] = True
    if result.audio_start_ms is not None and result.speech_started:
        payload["audio_start_ms"] = result.audio_start_ms
    if result.audio_end_ms is not None and result.speech_stopped:
        payload["audio_end_ms"] = result.audio_end_ms


@dataclass
class PendingTurnDetectionUpdate:
    """A ``session.update`` turn-detection change awaiting the engine ACK (two-phase)."""

    config: TurnDetectionConfig | None
    detector: ServerTurnDetector | None

    @classmethod
    def prepare(
        cls,
        session_payload: dict[str, object],
        *,
        backend_provider: SileroVADBackendProvider | None = None,
    ) -> PendingTurnDetectionUpdate | None:
        """Normalize the payload in place and stage the new detector; None when not configured."""
        configured, config = normalize_turn_detection_session_payload(session_payload)
        if not configured:
            return None
        detector = config.build_detector(backend_provider) if config is not None else None
        return cls(config=config, detector=detector)

    def commit(
        self, current: ServerTurnDetector | None
    ) -> tuple[TurnDetectionConfig | None, ServerTurnDetector | None]:
        """Adopt the staged detector; the replaced one is reset. Returns the new (config, detector)."""
        if current is not None:
            current.reset()
        return self.config, self.detector

    def reject(self) -> None:
        if self.detector is not None:
            self.detector.reset()