Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
49 files changed
+3056
-1715
No files matched your search
@@ -16,9 +16,12 @@ import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from importlib import import_module
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pybase64 as base64
|
||||
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.contracts import (
|
||||
DuplexAppendPlan,
|
||||
@@ -219,6 +222,20 @@ class DefaultDuplexModelSessionState(DuplexModelSessionState):
|
||||
self.silence_deadline_monotonic = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DuplexDataPlaneContext:
|
||||
"""Session state the runner hands a data plane to project one stage output."""
|
||||
|
||||
epoch: int = 0
|
||||
turn_id: int = 0
|
||||
active_response_turn_id: int | None = None
|
||||
active_response_id: str | None = None
|
||||
auto_responds: bool = False
|
||||
response_format: str = "wav"
|
||||
speed: float | None = None
|
||||
modalities: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class DuplexDataPlane(ABC):
|
||||
"""Projects raw stage outputs of one model into internal duplex events."""
|
||||
|
||||
@@ -277,12 +294,29 @@ class DuplexModelPlugin(ABC):
|
||||
private_runtime_config_keys: frozenset[str] = frozenset()
|
||||
#: Samples per silence unit the runner appends to keep a model turn going.
|
||||
silence_continuation_samples: int = 16000
|
||||
#: Sample rate of that unit: the runner submits it through ``plan_append``
|
||||
#: exactly like client audio, so it must be a unit the model accepts.
|
||||
silence_continuation_sample_rate_hz: int = 16000
|
||||
data_plane: DuplexDataPlane
|
||||
|
||||
def __init__(self, encode_audio: EncodeAudio) -> None:
|
||||
# Constructor-only: concrete plugins hand the encoder to their data plane.
|
||||
del encode_audio
|
||||
|
||||
def silence_unit_payload(self) -> dict[str, object]:
|
||||
"""One silence unit as an append payload (``pcm_f32le`` zeros).
|
||||
|
||||
Used by the runner's turn continuation and by the startup warmup; a
|
||||
model whose unit is not plain zero PCM overrides it.
|
||||
"""
|
||||
samples = int(self.silence_continuation_samples)
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": _silence_pcm_f32le_base64(samples),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": int(self.silence_continuation_sample_rate_hz),
|
||||
}
|
||||
|
||||
# ---- engine policy (was DuplexRuntimeExtension) ----
|
||||
|
||||
@abstractmethod
|
||||
@@ -430,8 +464,15 @@ class DuplexModelPlugin(ABC):
|
||||
@abstractmethod
|
||||
def capabilities(self, *, max_sessions: int) -> DuplexCapabilities: ...
|
||||
|
||||
@abstractmethod
|
||||
def validate_client_extra_body(self, extra_body: object) -> None: ...
|
||||
def validate_client_extra_body(self, extra_body: object) -> None:
|
||||
"""Refuse client ``extra_body`` keys the server owns (``private_runtime_config_keys``)."""
|
||||
if not isinstance(extra_body, Mapping):
|
||||
return
|
||||
private_keys = sorted(self.private_runtime_config_keys.intersection(extra_body))
|
||||
if private_keys:
|
||||
raise DuplexRuntimeConfigError(
|
||||
f"{self.plugin_id} runtime configuration is server-owned: " + ", ".join(private_keys)
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
async def prepare_runtime_config(
|
||||
@@ -445,7 +486,6 @@ class DuplexModelPlugin(ABC):
|
||||
current: Mapping[str, object],
|
||||
) -> dict[str, object]: ...
|
||||
|
||||
@abstractmethod
|
||||
def data_plane_context(
|
||||
self,
|
||||
*,
|
||||
@@ -457,7 +497,18 @@ class DuplexModelPlugin(ABC):
|
||||
response_format: str,
|
||||
speed: float | None,
|
||||
modalities: tuple[str, ...],
|
||||
) -> object: ...
|
||||
) -> object:
|
||||
"""The context handed to ``data_plane.project``; the default is the generic dataclass."""
|
||||
return DuplexDataPlaneContext(
|
||||
epoch=epoch,
|
||||
turn_id=turn_id,
|
||||
active_response_turn_id=active_response_turn_id,
|
||||
active_response_id=active_response_id,
|
||||
auto_responds=auto_responds,
|
||||
response_format=response_format,
|
||||
speed=speed,
|
||||
modalities=modalities,
|
||||
)
|
||||
|
||||
# Optional hook: build the runtime config patch for a function-call output
|
||||
# item. Plugins without tools keep the default (no change).
|
||||
@@ -515,6 +566,11 @@ def validate_duplex_plugin_sampling(plugin: DuplexModelPlugin, *, sampling_defau
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _silence_pcm_f32le_base64(samples: int) -> str:
|
||||
return base64.b64encode(bytes(max(0, samples) * 4)).decode("ascii")
|
||||
|
||||
|
||||
def payload_turn_id(payload: object) -> int | None:
|
||||
if not isinstance(payload, Mapping):
|
||||
return None
|
||||
@@ -537,6 +593,7 @@ def coerce_int(value: object) -> int | None:
|
||||
__all__ = [
|
||||
"DefaultDuplexModelSessionState",
|
||||
"DuplexDataPlane",
|
||||
"DuplexDataPlaneContext",
|
||||
"DuplexModelPlugin",
|
||||
"DuplexModelSessionState",
|
||||
"DuplexRuntimeConfigError",
|
||||
|
||||
Reference in new issue
Block a user