[Frontend] Add server-side VAD for turn-based omni models (#6618)
Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com>
This commit is contained in:
1 parent
031612f250
commit
c9c25b33c3
36 files changed
+4446
-940
No files matched your search
@@ -568,6 +568,48 @@ def test_websocket_routes_emit_stable_unavailable_frames_and_close(path: str, pa
|
||||
websocket.receive_text()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("duplex_query", "expected_handler"),
|
||||
[
|
||||
(None, "duplex"),
|
||||
("1", "duplex"),
|
||||
("true", "duplex"),
|
||||
("on", "duplex"),
|
||||
("0", "legacy"),
|
||||
("false", "legacy"),
|
||||
],
|
||||
)
|
||||
async def test_realtime_route_defaults_to_configured_duplex_handler(
|
||||
monkeypatch, duplex_query: str | None, expected_handler: str
|
||||
) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
class _DuplexHandler:
|
||||
async def handle_realtime_session(self, _websocket) -> None:
|
||||
calls.append("duplex")
|
||||
|
||||
class _LegacyConnection:
|
||||
async def handle_connection(self) -> None:
|
||||
calls.append("legacy")
|
||||
|
||||
monkeypatch.setattr(api_server, "RealtimeConnection", lambda _websocket, _serving: _LegacyConnection())
|
||||
query_params = {} if duplex_query is None else {"duplex": duplex_query}
|
||||
websocket = SimpleNamespace(
|
||||
app=SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
openai_serving_duplex=_DuplexHandler(),
|
||||
openai_serving_realtime=object(),
|
||||
)
|
||||
),
|
||||
query_params=query_params,
|
||||
)
|
||||
|
||||
await api_server.realtime_websocket(websocket)
|
||||
|
||||
assert calls == [expected_handler]
|
||||
|
||||
|
||||
def test_health_without_engine_returns_stable_unhealthy_response() -> None:
|
||||
"""Lock ``/health`` with no engine initialized.
|
||||
|
||||
|
||||
@@ -163,6 +163,8 @@ class TestStreamingAudioResampler:
|
||||
|
||||
chunked = StreamingAudioResampler(24000, 8000)
|
||||
pieces = [chunked.process(chunk) for chunk in np.split(waveform, [137, 2048, 9001, 17003])]
|
||||
# Bounded FIR history must own its data, not retain the last large chunk.
|
||||
assert chunked._history.base is None
|
||||
pieces.append(chunked.process(np.empty(0), final=True))
|
||||
actual = np.concatenate(pieces)
|
||||
|
||||
@@ -193,9 +195,15 @@ class TestStreamingAudioResampler:
|
||||
output_rms = np.sqrt(np.mean(output[100:-100] ** 2))
|
||||
assert output_rms < input_rms * 0.01
|
||||
|
||||
def test_rejects_non_integer_downsampling_ratio(self):
|
||||
with pytest.raises(ValueError, match="integer downsampling ratio"):
|
||||
StreamingAudioResampler(24000, 16000)
|
||||
def test_supports_rational_downsampling_ratio(self):
|
||||
waveform = np.sin(np.linspace(0, 200 * np.pi, 24000, endpoint=False)).astype(np.float32)
|
||||
resampler = StreamingAudioResampler(24000, 16000)
|
||||
|
||||
output = resampler.process(waveform, final=True)
|
||||
reference = torchaudio.functional.resample(torch.from_numpy(waveform), 24000, 16000).numpy()
|
||||
|
||||
assert output.shape == (16000,)
|
||||
np.testing.assert_allclose(output[100:-100], reference[100:-100], atol=2e-3, rtol=1e-3)
|
||||
|
||||
def test_rejects_multichannel_audio(self):
|
||||
resampler = StreamingAudioResampler(24000, 8000)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -3,10 +3,11 @@
|
||||
"""
|
||||
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
||||
|
||||
Three scenarios:
|
||||
Four scenarios:
|
||||
- Ready CI: async_chunk on, smoke only (no send delay, no accuracy check).
|
||||
- Merge CI: async_chunk on + send delay, full accuracy check.
|
||||
- Merge CI: async_chunk off, no send delay, full accuracy check.
|
||||
- Server VAD: two turns without client commits.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -28,11 +29,18 @@ from tests.helpers.media import (
|
||||
generate_synthetic_audio,
|
||||
)
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.config.stage_config import load_deploy_config
|
||||
from vllm_omni.entrypoints.duplex.server_vad import (
|
||||
SILERO_VAD_FILENAME,
|
||||
SILERO_VAD_REPO_ID,
|
||||
SILERO_VAD_REVISION,
|
||||
)
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
MODEL = os.environ.get("VLLM_OMNI_TEST_QWEN3_OMNI_MODEL", "Qwen/Qwen3-Omni-30B-A3B-Instruct")
|
||||
SERVER_VAD_MODEL_PATH = os.environ.get("VLLM_OMNI_TEST_SILERO_VAD_MODEL_PATH")
|
||||
|
||||
# Synthetic input for realtime E2E (``generate_synthetic_audio``); distinct cache file per phrase.
|
||||
REALTIME_SYNTH_PHRASE_TEXT = (
|
||||
@@ -48,6 +56,15 @@ SEND_DELAY_MS = 200
|
||||
|
||||
# CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU via ``platforms:``.
|
||||
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||
server_vad_stage_config = modify_stage_config(
|
||||
default_stage_config,
|
||||
{
|
||||
"async_chunk": True,
|
||||
"duplex_session": (
|
||||
{"server_vad_model_path": SERVER_VAD_MODEL_PATH} if SERVER_VAD_MODEL_PATH is not None else {}
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
realtime_sync_server_params = [
|
||||
pytest.param(
|
||||
@@ -72,6 +89,17 @@ realtime_async_chunk_server_params = [
|
||||
),
|
||||
]
|
||||
|
||||
realtime_server_vad_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=server_vad_stage_config,
|
||||
use_stage_cli=True,
|
||||
),
|
||||
id="server_vad",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _pcm16_mono_16k_from_wav_bytes(wav_bytes: bytes) -> bytes:
|
||||
with wave.open(io.BytesIO(wav_bytes), "rb") as wf:
|
||||
@@ -183,6 +211,86 @@ async def _run_realtime_audio_roundtrip(
|
||||
}
|
||||
|
||||
|
||||
async def _append_pcm16_chunks(ws, pcm16: bytes, chunk_bytes: int) -> None:
|
||||
for offset in range(0, len(pcm16), chunk_bytes):
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(pcm16[offset : offset + chunk_bytes]).decode("utf-8"),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _receive_server_vad_turn(ws) -> list[dict]:
|
||||
events: list[dict] = []
|
||||
while True:
|
||||
message = await asyncio.wait_for(ws.recv(), timeout=600)
|
||||
if isinstance(message, bytes):
|
||||
continue
|
||||
event = json.loads(message)
|
||||
events.append(event)
|
||||
if event.get("type") == "error":
|
||||
raise AssertionError(f"WebSocket error: {event}")
|
||||
if event.get("type") == "response.done":
|
||||
return events
|
||||
|
||||
|
||||
async def _run_server_vad_audio_roundtrips(
|
||||
host: str,
|
||||
port: int,
|
||||
model: str,
|
||||
pcm16: bytes,
|
||||
*,
|
||||
chunk_ms: int = 100,
|
||||
turns: int = 2,
|
||||
) -> list[list[dict]]:
|
||||
chunk_bytes = max(16_000 * 2 // 1000 * chunk_ms, 2)
|
||||
turn_events: list[list[dict]] = []
|
||||
|
||||
async with websockets.connect(f"ws://{host}:{port}/v1/realtime", max_size=64 * 1024 * 1024) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"model": model,
|
||||
"audio": {
|
||||
"input": {
|
||||
"format": {"type": "audio/pcm", "rate": 16_000},
|
||||
"turn_detection": {"type": "server_vad"},
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
silence = bytes(16_000 * 2 * 3 // 2)
|
||||
for _ in range(turns):
|
||||
await _append_pcm16_chunks(ws, pcm16, chunk_bytes)
|
||||
await _append_pcm16_chunks(ws, silence, chunk_bytes)
|
||||
turn_events.append(await _receive_server_vad_turn(ws))
|
||||
|
||||
return turn_events
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def cached_silero_vad_artifact() -> str:
|
||||
"""Prepare the pinned artifact before the serving subprocess starts."""
|
||||
if SERVER_VAD_MODEL_PATH is not None:
|
||||
return SERVER_VAD_MODEL_PATH
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
return hf_hub_download(
|
||||
repo_id=SILERO_VAD_REPO_ID,
|
||||
filename=SILERO_VAD_FILENAME,
|
||||
revision=SILERO_VAD_REVISION,
|
||||
)
|
||||
|
||||
|
||||
def _synthetic_pcm16_input(
|
||||
*,
|
||||
phrase_text: str = REALTIME_SYNTH_PHRASE_TEXT,
|
||||
@@ -296,6 +404,66 @@ class TestQwen3OmniRealtimeWebSocket:
|
||||
f"Expected an issue-like audio response longer than 5 seconds, got {output_duration_s:.2f}s"
|
||||
)
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", realtime_server_vad_server_params, indirect=True)
|
||||
def test_server_vad_multi_turn_without_client_commit(
|
||||
self,
|
||||
cached_silero_vad_artifact: str,
|
||||
omni_server,
|
||||
) -> None:
|
||||
"""Two Qwen turns are endpointed without client commits."""
|
||||
assert cached_silero_vad_artifact
|
||||
deploy = load_deploy_config(server_vad_stage_config)
|
||||
assert deploy.session_mode == "turn"
|
||||
assert deploy.async_chunk is True
|
||||
assert deploy.duplex_session.server_vad_model_path == SERVER_VAD_MODEL_PATH
|
||||
pcm16 = _synthetic_pcm16_input()
|
||||
|
||||
turns = asyncio.run(
|
||||
_run_server_vad_audio_roundtrips(
|
||||
omni_server.host,
|
||||
omni_server.port,
|
||||
omni_server.model,
|
||||
pcm16,
|
||||
chunk_ms=100,
|
||||
turns=2,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(turns) == 2
|
||||
required_sequence = [
|
||||
"input_audio_buffer.speech_started",
|
||||
"input_audio_buffer.speech_stopped",
|
||||
"input_audio_buffer.committed",
|
||||
"response.created",
|
||||
"response.audio.delta",
|
||||
"response.audio.done",
|
||||
"response.done",
|
||||
]
|
||||
input_item_ids: list[str] = []
|
||||
response_ids: list[str] = []
|
||||
for events in turns:
|
||||
event_types = [event["type"] for event in events]
|
||||
positions = [event_types.index(event_type) for event_type in required_sequence]
|
||||
assert positions == sorted(positions)
|
||||
|
||||
started = next(event for event in events if event["type"] == "input_audio_buffer.speech_started")
|
||||
stopped = next(event for event in events if event["type"] == "input_audio_buffer.speech_stopped")
|
||||
committed = next(event for event in events if event["type"] == "input_audio_buffer.committed")
|
||||
created = next(event for event in events if event["type"] == "response.created")["response"]
|
||||
done = next(event for event in events if event["type"] == "response.done")["response"]
|
||||
|
||||
assert started["item_id"] == stopped["item_id"] == committed["item_id"]
|
||||
assert created["id"] == done["id"]
|
||||
assert done["status"] == "completed"
|
||||
input_item_ids.append(committed["item_id"])
|
||||
response_ids.append(created["id"])
|
||||
|
||||
assert len(set(input_item_ids)) == 2
|
||||
assert len(set(response_ids)) == 2
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
import threading
|
||||
import wave
|
||||
import weakref
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexSession,
|
||||
DuplexSessionConfig,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.server_vad import (
|
||||
ServerVADConfig,
|
||||
ServerVADFrame,
|
||||
ServerVADPipeline,
|
||||
SileroVADBackend,
|
||||
SileroVADBackendProvider,
|
||||
SpeechEndpointDecision,
|
||||
ThresholdEndpointPolicy,
|
||||
parse_session_turn_detection,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class SequenceDetector:
|
||||
frame_samples = 160
|
||||
|
||||
def __init__(self, probabilities: list[float]) -> None:
|
||||
self.probabilities = probabilities
|
||||
|
||||
def new_state(self) -> int:
|
||||
return 0
|
||||
|
||||
def infer(self, frame: np.ndarray, state: object) -> tuple[float, object]:
|
||||
index = int(state)
|
||||
probability = self.probabilities[index] if index < len(self.probabilities) else 0.0
|
||||
return probability, index + 1
|
||||
|
||||
|
||||
def test_server_vad_config_defaults_and_validation():
|
||||
config = ServerVADConfig.from_value({"type": "server_vad"})
|
||||
|
||||
assert config.type == "server_vad"
|
||||
assert config.threshold == 0.5
|
||||
assert config.prefix_padding_ms == 300
|
||||
assert config.silence_duration_ms == 500
|
||||
assert config.create_response is True
|
||||
assert config.interrupt_response is None
|
||||
assert "interrupt_response" not in config.as_dict()
|
||||
assert config.min_speech_duration_ms is None
|
||||
assert config.as_dict()["type"] == config.type
|
||||
|
||||
interrupting = ServerVADConfig.from_value(
|
||||
{
|
||||
"type": "server_vad",
|
||||
"interrupt_response": True,
|
||||
"min_speech_duration_ms": 96,
|
||||
}
|
||||
)
|
||||
assert interrupting.interrupt_response is True
|
||||
assert interrupting.min_speech_duration_ms == 96
|
||||
for value in (None, 0, "true"):
|
||||
with pytest.raises(ValueError, match="interrupt_response must be a boolean"):
|
||||
ServerVADConfig.from_value({"type": "server_vad", "interrupt_response": value})
|
||||
for enabled in (False, True):
|
||||
_, aliased = parse_session_turn_detection(
|
||||
{
|
||||
"turn_detection": {"type": "server_vad"},
|
||||
"audio": {"input": {"turn_detection": {"type": "server_vad", "interrupt_response": enabled}}},
|
||||
}
|
||||
)
|
||||
assert aliased is not None and aliased.interrupt_response is enabled
|
||||
with pytest.raises(ValueError, match="Unknown"):
|
||||
ServerVADConfig.from_value({"type": "server_vad", "semantic_eagerness": "high"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prefix_padding_ms", [0, 10, 30])
|
||||
async def test_server_vad_pipeline_handles_arbitrary_chunk_boundaries(prefix_padding_ms: int):
|
||||
detector = SequenceDetector([0.0, 0.9, 0.8, 0.0, 0.0])
|
||||
pipeline = ServerVADPipeline(
|
||||
detector,
|
||||
ServerVADConfig(prefix_padding_ms=prefix_padding_ms, silence_duration_ms=20),
|
||||
)
|
||||
samples = np.zeros(detector.frame_samples * 5, dtype=np.float32)
|
||||
|
||||
batches = [
|
||||
await pipeline.push(samples[:73]),
|
||||
await pipeline.push(samples[73:431]),
|
||||
await pipeline.push(samples[431:]),
|
||||
]
|
||||
frames = [frame for batch in batches for frame in batch.frames]
|
||||
|
||||
assert len(frames) == 5
|
||||
assert [index for index, frame in enumerate(frames) if frame.decision.speech_started] == [1]
|
||||
assert [index for index, frame in enumerate(frames) if frame.decision.speech_stopped] == [4]
|
||||
assert frames[1].decision.audio_start_ms == max(0, 10 - prefix_padding_ms)
|
||||
assert frames[4].decision.audio_end_ms == 50
|
||||
|
||||
pipeline.reset()
|
||||
reset_batch = await pipeline.push(samples)
|
||||
assert [index for index, frame in enumerate(reset_batch.frames) if frame.decision.speech_started] == [1]
|
||||
assert [index for index, frame in enumerate(reset_batch.frames) if frame.decision.speech_stopped] == [4]
|
||||
# Keep session-relative timestamps, but never reference prefix audio
|
||||
# discarded by reset at 50 ms.
|
||||
assert reset_batch.frames[1].decision.audio_start_ms == max(50, 60 - prefix_padding_ms)
|
||||
assert reset_batch.frames[4].decision.audio_end_ms == 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_retained_audio_does_not_keep_input_chunk():
|
||||
pipeline = ServerVADPipeline(SequenceDetector([]), ServerVADConfig())
|
||||
session = DuplexSession("server-vad-memory", DuplexSessionConfig())
|
||||
samples = np.zeros(160 * 100 + 1, dtype=np.float32)
|
||||
source_ref = weakref.ref(samples)
|
||||
batch = await pipeline.push(samples)
|
||||
session.append_server_vad_frame(
|
||||
batch.frames[-1].samples,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=80,
|
||||
)
|
||||
session.append_server_vad_frame(
|
||||
batch.frames[-1].samples,
|
||||
speech_started=True,
|
||||
speech_stopped=False,
|
||||
prefix_samples=80,
|
||||
)
|
||||
del batch, samples
|
||||
# Allow the completed to_thread callback to release its result, too.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert source_ref() is None
|
||||
assert pipeline.scratch_bytes == 4
|
||||
assert session.server_vad_utterance_bytes == (80 + 160) * 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_pipeline_pcm16_resampling_is_split_invariant():
|
||||
sample_index = np.arange(2_400, dtype=np.int32)
|
||||
source = ((sample_index * 7919) % 65_536 - 32_768).astype("<i2")
|
||||
# Supply ample silence after the signal so the streaming resampler can
|
||||
# emit a complete set of VAD frames without depending on its FIR width.
|
||||
right_context = np.zeros(240, dtype="<i2")
|
||||
continuous_source = np.concatenate((source, right_context))
|
||||
|
||||
async def process(chunk_sizes: list[int]) -> tuple[np.ndarray, list[SpeechEndpointDecision]]:
|
||||
pipeline = ServerVADPipeline(
|
||||
SequenceDetector([0.0] * 10),
|
||||
ServerVADConfig(),
|
||||
)
|
||||
frames: list[ServerVADFrame] = []
|
||||
offset = 0
|
||||
for chunk_size in chunk_sizes:
|
||||
chunk = continuous_source[offset : offset + chunk_size]
|
||||
batch = await pipeline.push_pcm16(chunk.tobytes(), source_sample_rate_hz=24_000)
|
||||
frames.extend(batch.frames)
|
||||
offset += chunk_size
|
||||
assert offset == continuous_source.size
|
||||
return np.concatenate([frame.samples for frame in frames]), [frame.decision for frame in frames]
|
||||
|
||||
whole_samples, whole_decisions = await process([continuous_source.size])
|
||||
split_samples, split_decisions = await process([7, 13, 511, 512, 513, continuous_source.size - 1_556])
|
||||
|
||||
np.testing.assert_array_equal(split_samples, whole_samples)
|
||||
assert whole_samples.size == 1_600
|
||||
assert split_decisions == whole_decisions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_pipeline_reset_clears_resampler_and_source_rate_lock():
|
||||
pipeline = ServerVADPipeline(
|
||||
SequenceDetector([0.0]),
|
||||
ServerVADConfig(),
|
||||
)
|
||||
|
||||
batch = await pipeline.push_pcm16(
|
||||
np.asarray([30_000, -30_000], dtype="<i2").tobytes(),
|
||||
source_sample_rate_hz=24_000,
|
||||
)
|
||||
assert not batch.frames
|
||||
|
||||
pipeline.reset()
|
||||
|
||||
source = np.zeros(160, dtype="<i2")
|
||||
source[:3] = [-32768, 0, 32767]
|
||||
batch = await pipeline.push_pcm16(source.tobytes(), source_sample_rate_hz=16_000)
|
||||
|
||||
assert len(batch.frames) == 1
|
||||
np.testing.assert_array_equal(
|
||||
batch.frames[0].samples[:3],
|
||||
np.asarray([-1.0, 0.0, 32767 / 32768], dtype=np.float32),
|
||||
)
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_uses_silero_v62_exit_hysteresis():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(threshold=0.5, silence_duration_ms=20),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
started = policy.update(0.8, frame_start_sample=0, frame_samples=frame_samples)
|
||||
assert started.speech_started is True
|
||||
|
||||
# Values below the activation threshold but above threshold - 0.15 must
|
||||
# not begin a silence candidate or close an active speech segment.
|
||||
for frame_index in range(1, 5):
|
||||
decision = policy.update(
|
||||
0.4,
|
||||
frame_start_sample=frame_index * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
assert decision.speech_stopped is False
|
||||
|
||||
first_silence = policy.update(
|
||||
0.2,
|
||||
frame_start_sample=5 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
stopped = policy.update(
|
||||
0.2,
|
||||
frame_start_sample=6 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
|
||||
assert first_silence.speech_stopped is False
|
||||
assert stopped.speech_stopped is True
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_honors_minimum_speech_duration():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(prefix_padding_ms=0, min_speech_duration_ms=25),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
assert not policy.update(0.9, frame_start_sample=0, frame_samples=frame_samples).speech_started
|
||||
assert not policy.update(0.9, frame_start_sample=160, frame_samples=frame_samples).speech_started
|
||||
started = policy.update(0.9, frame_start_sample=320, frame_samples=frame_samples)
|
||||
|
||||
assert started.speech_started is True
|
||||
assert started.audio_start_ms == 0
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_clamps_silero_exit_threshold():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(threshold=0.1, silence_duration_ms=20),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
started = policy.update(0.9, frame_start_sample=0, frame_samples=frame_samples)
|
||||
assert started.speech_started is True
|
||||
|
||||
first_silence = policy.update(
|
||||
0.0,
|
||||
frame_start_sample=frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
stopped = policy.update(
|
||||
0.0,
|
||||
frame_start_sample=2 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
|
||||
assert first_silence.speech_stopped is False
|
||||
assert stopped.speech_stopped is True
|
||||
# The committed audio ends at 30 ms: 10 ms of speech followed by the
|
||||
# 20 ms of silence required to detect the endpoint.
|
||||
assert stopped.audio_end_ms == 30
|
||||
assert stopped.endpoint_delay_ms == 20
|
||||
|
||||
|
||||
def test_silero_backend_matches_upstream_v62_streaming_contract(monkeypatch, tmp_path):
|
||||
class FakeSessionOptions:
|
||||
inter_op_num_threads = 0
|
||||
intra_op_num_threads = 0
|
||||
|
||||
class FakeInferenceSession:
|
||||
def __init__(self, path: str, *, providers: list[str], sess_options: object) -> None:
|
||||
self.providers = providers
|
||||
self.sess_options = sess_options
|
||||
self.calls: list[dict[str, np.ndarray]] = []
|
||||
self.run_barrier: threading.Barrier | None = None
|
||||
sessions.append(self)
|
||||
|
||||
def get_inputs(self) -> list[SimpleNamespace]:
|
||||
return [SimpleNamespace(name=name) for name in ("input", "state", "sr")]
|
||||
|
||||
def run(self, _outputs: object, inputs: dict[str, np.ndarray]) -> list[np.ndarray]:
|
||||
self.calls.append({name: np.array(value, copy=True) for name, value in inputs.items()})
|
||||
if self.run_barrier is not None:
|
||||
self.run_barrier.wait(timeout=5)
|
||||
return [
|
||||
np.asarray([[0.75]], dtype=np.float32),
|
||||
np.full((2, 1, 128), len(self.calls), dtype=np.float32),
|
||||
]
|
||||
|
||||
sessions: list[FakeInferenceSession] = []
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"onnxruntime",
|
||||
SimpleNamespace(
|
||||
SessionOptions=FakeSessionOptions,
|
||||
InferenceSession=FakeInferenceSession,
|
||||
),
|
||||
)
|
||||
|
||||
model_path = tmp_path / "silero_vad.onnx"
|
||||
model_path.write_bytes(b"fake-model")
|
||||
backend = SileroVADBackend(model_path)
|
||||
session = sessions[0]
|
||||
|
||||
assert session.providers == ["CPUExecutionProvider"]
|
||||
assert session.sess_options.inter_op_num_threads == 1
|
||||
assert session.sess_options.intra_op_num_threads == 1
|
||||
assert session.calls[0]["input"].shape == (1, 576)
|
||||
assert session.calls[0]["state"].shape == (2, 1, 128)
|
||||
np.testing.assert_array_equal(session.calls[0]["state"], np.zeros((2, 1, 128), dtype=np.float32))
|
||||
assert session.calls[0]["sr"].shape == ()
|
||||
assert session.calls[0]["sr"].item() == 16_000
|
||||
|
||||
state = backend.new_state()
|
||||
first_frame = np.arange(backend.frame_samples, dtype=np.float32)
|
||||
probability, state = backend.infer(first_frame, state)
|
||||
first_call = session.calls[1]
|
||||
|
||||
assert probability == pytest.approx(0.75)
|
||||
np.testing.assert_array_equal(
|
||||
first_call["input"][:, : backend.context_samples],
|
||||
np.zeros((1, backend.context_samples), dtype=np.float32),
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
first_call["input"][:, backend.context_samples :],
|
||||
first_frame[None, :],
|
||||
)
|
||||
np.testing.assert_array_equal(first_call["state"], np.zeros((2, 1, 128), dtype=np.float32))
|
||||
|
||||
second_frame = -first_frame
|
||||
backend.infer(second_frame, state)
|
||||
second_call = session.calls[2]
|
||||
np.testing.assert_array_equal(
|
||||
second_call["input"][:, : backend.context_samples],
|
||||
first_frame[None, -backend.context_samples :],
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
second_call["input"][:, backend.context_samples :],
|
||||
second_frame[None, :],
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
second_call["state"],
|
||||
np.full((2, 1, 128), 2, dtype=np.float32),
|
||||
)
|
||||
|
||||
# Independent pipeline states may enter the shared ORT session concurrently.
|
||||
session.run_barrier = threading.Barrier(2)
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = [
|
||||
executor.submit(backend.infer, first_frame, backend.new_state()),
|
||||
executor.submit(backend.infer, second_frame, backend.new_state()),
|
||||
]
|
||||
assert [future.result()[0] for future in futures] == pytest.approx([0.75, 0.75])
|
||||
|
||||
|
||||
def test_duplex_session_owns_server_vad_prefix_and_utterance_audio():
|
||||
session = DuplexSession("server-vad-session", DuplexSessionConfig())
|
||||
frame = np.zeros(160, dtype=np.float32)
|
||||
|
||||
session.reserve_input_bytes(frame.nbytes * 3, limit=frame.nbytes * 4)
|
||||
released = session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
assert released == 0
|
||||
released = session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
assert released == frame.nbytes
|
||||
session.release_input_bytes(released)
|
||||
session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=True,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
|
||||
assert session.server_vad_utterance_bytes == frame.nbytes * 2
|
||||
assert session.stage_server_vad_audio_for_commit() is True
|
||||
committed = session.commit_user_input()
|
||||
assert committed is not None
|
||||
assert committed.message["role"] == "user"
|
||||
audio_url = committed.message["content"][0]["audio_url"]["url"]
|
||||
assert audio_url.startswith("data:audio/wav;base64,")
|
||||
wav_bytes = base64.b64decode(audio_url.partition(",")[2])
|
||||
with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file:
|
||||
assert wav_file.getnchannels() == 1
|
||||
assert wav_file.getsampwidth() == 2
|
||||
assert wav_file.getframerate() == 16_000
|
||||
assert wav_file.getnframes() == frame.size * 2
|
||||
|
||||
|
||||
def test_silero_provider_rejects_missing_or_invalid_local_artifact(tmp_path):
|
||||
missing = SileroVADBackendProvider(model_path=str(tmp_path / "missing.onnx"))
|
||||
with pytest.raises(RuntimeError, match="does not exist"):
|
||||
missing.get()
|
||||
|
||||
invalid_path = tmp_path / "silero_vad.onnx"
|
||||
invalid_path.write_bytes(b"not-the-pinned-model")
|
||||
invalid = SileroVADBackendProvider(model_path=str(invalid_path))
|
||||
with pytest.raises(RuntimeError, match="checksum mismatch"):
|
||||
invalid.get()
|
||||
Reference in new issue
Block a user