927 lines
37 KiB
Python
927 lines
37 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
"""
|
|
E2E online tests for Qwen3-Omni /v1/realtime WebSocket (streaming PCM in, audio out).
|
|
|
|
Four scenarios:
|
|
- Ready CI: async_chunk on, smoke only (no send delay, no accuracy check).
|
|
- Merge CI: async_chunk on + send delay, full accuracy check.
|
|
- Merge CI: async_chunk off, no send delay, full accuracy check.
|
|
- Server VAD: two turns without client commits.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import io
|
|
import json
|
|
import os
|
|
import wave
|
|
from contextlib import asynccontextmanager, suppress
|
|
from pathlib import Path
|
|
from typing import Any, NamedTuple
|
|
|
|
import numpy as np
|
|
import pytest
|
|
import websockets
|
|
import yaml
|
|
|
|
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import validated_input_wav, validated_soft_interrupt_wav
|
|
from tests.helpers.mark import hardware_test
|
|
from tests.helpers.media import (
|
|
convert_audio_bytes_to_text,
|
|
cosine_similarity_text,
|
|
generate_synthetic_audio,
|
|
)
|
|
from tests.helpers.runtime import OmniServer, OmniServerParams, get_model_prefix
|
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
|
from vllm_omni.config.stage_config import load_deploy_config
|
|
from vllm_omni.engine.duplex.contracts import duplex_resource_request_belongs_to_session
|
|
from vllm_omni.engine.duplex.vad import (
|
|
SILERO_VAD_FILENAME,
|
|
SILERO_VAD_REPO_ID,
|
|
SILERO_VAD_REVISION,
|
|
)
|
|
|
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
|
|
MODEL = os.environ.get("VLLM_OMNI_TEST_QWEN3_OMNI_MODEL", "Qwen/Qwen3-Omni-30B-A3B-Instruct")
|
|
SERVER_VAD_MODEL_PATH = os.environ.get("VLLM_OMNI_TEST_SILERO_VAD_MODEL_PATH")
|
|
|
|
# Synthetic input for realtime E2E (``generate_synthetic_audio``); distinct cache file per phrase.
|
|
REALTIME_SYNTH_PHRASE_TEXT = (
|
|
"Translate into Chinese: Beijing is the Capital of China. It is the center of culture and politics"
|
|
)
|
|
ISSUE_6474_SYNTH_PHRASE_TEXT = (
|
|
"Can you tell me the current temperature and weather conditions in New York City? "
|
|
"What about Los Angeles? Please compare them in detail using at least eight complete sentences."
|
|
)
|
|
|
|
# Simulate realtime upload pacing (``openai_realtime_client.py --send-delay-ms``).
|
|
SEND_DELAY_MS = 200
|
|
|
|
# CI overlay bakes in async_chunk: False and covers CUDA/ROCm/XPU via ``platforms:``.
|
|
default_stage_config = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
|
server_vad_stage_config = modify_stage_config(
|
|
default_stage_config,
|
|
{
|
|
"async_chunk": True,
|
|
"duplex_session": (
|
|
{"server_vad_model_path": SERVER_VAD_MODEL_PATH} if SERVER_VAD_MODEL_PATH is not None else {}
|
|
),
|
|
},
|
|
)
|
|
|
|
realtime_sync_server_params = [
|
|
pytest.param(
|
|
OmniServerParams(
|
|
model=MODEL,
|
|
stage_config_path=default_stage_config,
|
|
use_stage_cli=True,
|
|
),
|
|
id="sync",
|
|
),
|
|
]
|
|
|
|
realtime_async_chunk_server_params = [
|
|
pytest.param(
|
|
OmniServerParams(
|
|
model=MODEL,
|
|
stage_config_path=default_stage_config,
|
|
use_stage_cli=True,
|
|
server_args=["--async-chunk"],
|
|
),
|
|
id="async_chunk",
|
|
),
|
|
]
|
|
|
|
realtime_server_vad_server_params = [
|
|
pytest.param(
|
|
OmniServerParams(
|
|
model=MODEL,
|
|
stage_config_path=server_vad_stage_config,
|
|
use_stage_cli=True,
|
|
),
|
|
id="server_vad",
|
|
),
|
|
]
|
|
|
|
|
|
def _pcm16_mono_16k_from_wav_bytes(wav_bytes: bytes) -> bytes:
|
|
with wave.open(io.BytesIO(wav_bytes), "rb") as wf:
|
|
if wf.getnchannels() != 1:
|
|
raise ValueError(f"Expected mono WAV, got {wf.getnchannels()} channels")
|
|
if wf.getsampwidth() != 2:
|
|
raise ValueError(f"Expected 16-bit PCM, sampwidth={wf.getsampwidth()}")
|
|
if wf.getframerate() != 16000:
|
|
raise ValueError(f"Expected 16 kHz input for /v1/realtime, got {wf.getframerate()} Hz")
|
|
if wf.getcomptype() != "NONE":
|
|
raise ValueError(f"Expected uncompressed PCM, comptype={wf.getcomptype()!r}")
|
|
return wf.readframes(wf.getnframes())
|
|
|
|
|
|
def _wav_bytes_from_pcm16(pcm: bytes, sample_rate_hz: int) -> bytes:
|
|
buf = io.BytesIO()
|
|
with wave.open(buf, "wb") as wf:
|
|
wf.setnchannels(1)
|
|
wf.setsampwidth(2)
|
|
wf.setframerate(sample_rate_hz)
|
|
wf.writeframes(pcm)
|
|
return buf.getvalue()
|
|
|
|
|
|
async def _run_realtime_audio_roundtrip(
|
|
host: str,
|
|
port: int,
|
|
model: str,
|
|
pcm16: bytes,
|
|
*,
|
|
chunk_ms: int = 100,
|
|
send_delay_ms: int = 0,
|
|
completion_timeout_s: float = 600,
|
|
) -> dict:
|
|
uri = f"ws://{host}:{port}/v1/realtime"
|
|
incremental: list[bytes] = []
|
|
output_sr = 24000
|
|
text_chunks: list[str] = []
|
|
final_text = ""
|
|
delta_events = 0
|
|
|
|
bytes_per_ms = 16000 * 2 // 1000
|
|
chunk_bytes = max(bytes_per_ms * chunk_ms, 2)
|
|
|
|
async with websockets.connect(uri, max_size=64 * 1024 * 1024) as ws:
|
|
await ws.send(json.dumps({"type": "session.update", "model": model}))
|
|
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
|
|
|
|
for i in range(0, len(pcm16), chunk_bytes):
|
|
chunk = pcm16[i : i + chunk_bytes]
|
|
await ws.send(
|
|
json.dumps(
|
|
{
|
|
"type": "input_audio_buffer.append",
|
|
"audio": base64.b64encode(chunk).decode("utf-8"),
|
|
}
|
|
)
|
|
)
|
|
if send_delay_ms > 0:
|
|
await asyncio.sleep(send_delay_ms / 1000.0)
|
|
|
|
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
|
|
|
while True:
|
|
message = await asyncio.wait_for(ws.recv(), timeout=completion_timeout_s)
|
|
if isinstance(message, bytes):
|
|
continue
|
|
|
|
event = json.loads(message)
|
|
event_type = event.get("type")
|
|
|
|
if event_type == "session.created":
|
|
continue
|
|
|
|
if event_type == "response.output_audio.delta":
|
|
delta_events += 1
|
|
sr = event.get("sample_rate_hz")
|
|
if isinstance(sr, int) and sr > 0:
|
|
output_sr = sr
|
|
audio_b64 = event.get("audio", "")
|
|
if audio_b64:
|
|
incremental.append(base64.b64decode(audio_b64))
|
|
continue
|
|
|
|
if event_type == "transcription.delta":
|
|
d = event.get("delta", "")
|
|
if d:
|
|
text_chunks.append(d)
|
|
continue
|
|
|
|
if event_type == "transcription.done":
|
|
final_text = event.get("text", "") or "".join(text_chunks)
|
|
continue
|
|
|
|
if event_type == "response.output_audio.done":
|
|
break
|
|
|
|
if event_type == "error":
|
|
raise AssertionError(f"WebSocket error: {event}")
|
|
|
|
raise AssertionError(f"Unexpected WebSocket event: {event}")
|
|
|
|
out_pcm = b"".join(incremental)
|
|
return {
|
|
"output_pcm": out_pcm,
|
|
"output_sample_rate": output_sr,
|
|
"transcription_text": final_text if final_text else "".join(text_chunks),
|
|
"delta_events": delta_events,
|
|
}
|
|
|
|
|
|
async def _append_pcm16_chunks(ws, pcm16: bytes, chunk_bytes: int) -> None:
|
|
for offset in range(0, len(pcm16), chunk_bytes):
|
|
await ws.send(
|
|
json.dumps(
|
|
{
|
|
"type": "input_audio_buffer.append",
|
|
"audio": base64.b64encode(pcm16[offset : offset + chunk_bytes]).decode("utf-8"),
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
async def _receive_server_vad_turn(ws) -> list[dict]:
|
|
events: list[dict] = []
|
|
while True:
|
|
message = await asyncio.wait_for(ws.recv(), timeout=600)
|
|
if isinstance(message, bytes):
|
|
continue
|
|
event = json.loads(message)
|
|
events.append(event)
|
|
if event.get("type") == "error":
|
|
raise AssertionError(f"WebSocket error: {event}")
|
|
if event.get("type") == "response.done":
|
|
return events
|
|
|
|
|
|
async def _run_server_vad_audio_roundtrips(
|
|
host: str,
|
|
port: int,
|
|
model: str,
|
|
pcm16: bytes,
|
|
*,
|
|
chunk_ms: int = 100,
|
|
turns: int = 2,
|
|
) -> list[list[dict]]:
|
|
chunk_bytes = max(16_000 * 2 // 1000 * chunk_ms, 2)
|
|
turn_events: list[list[dict]] = []
|
|
|
|
async with websockets.connect(f"ws://{host}:{port}/v1/realtime", max_size=64 * 1024 * 1024) as ws:
|
|
await ws.send(
|
|
json.dumps(
|
|
{
|
|
"type": "session.update",
|
|
"session": {
|
|
"model": model,
|
|
"audio": {
|
|
"input": {
|
|
"format": {"type": "audio/pcm", "rate": 16_000},
|
|
"turn_detection": {"type": "server_vad"},
|
|
}
|
|
},
|
|
},
|
|
}
|
|
)
|
|
)
|
|
|
|
silence = bytes(16_000 * 2 * 3 // 2)
|
|
for _ in range(turns):
|
|
await _append_pcm16_chunks(ws, pcm16, chunk_bytes)
|
|
await _append_pcm16_chunks(ws, silence, chunk_bytes)
|
|
turn_events.append(await _receive_server_vad_turn(ws))
|
|
|
|
return turn_events
|
|
|
|
|
|
@pytest.fixture(scope="class")
|
|
def cached_silero_vad_artifact() -> str:
|
|
"""Prepare the pinned artifact before the serving subprocess starts."""
|
|
if SERVER_VAD_MODEL_PATH is not None:
|
|
return SERVER_VAD_MODEL_PATH
|
|
|
|
from huggingface_hub import hf_hub_download
|
|
|
|
return hf_hub_download(
|
|
repo_id=SILERO_VAD_REPO_ID,
|
|
filename=SILERO_VAD_FILENAME,
|
|
revision=SILERO_VAD_REVISION,
|
|
)
|
|
|
|
|
|
def _synthetic_pcm16_input(
|
|
*,
|
|
phrase_text: str = REALTIME_SYNTH_PHRASE_TEXT,
|
|
duration_s: int = 10,
|
|
) -> bytes:
|
|
syn = generate_synthetic_audio(
|
|
duration_s,
|
|
1,
|
|
sample_rate=16000,
|
|
phrase_text=phrase_text,
|
|
)
|
|
wav_bytes = base64.b64decode(syn["base64"])
|
|
return _pcm16_mono_16k_from_wav_bytes(wav_bytes)
|
|
|
|
|
|
def _server_vad_pcm16_input() -> bytes:
|
|
"""Load the fixed single-turn speech fixture used by the Server VAD E2E."""
|
|
return _pcm16_mono_16k_from_wav_bytes(validated_input_wav().read_bytes())
|
|
|
|
|
|
def _assert_realtime_smoke(result: dict) -> None:
|
|
out_pcm = result["output_pcm"]
|
|
assert result["delta_events"] >= 1
|
|
assert out_pcm, "No output PCM from response.output_audio.delta"
|
|
assert len(out_pcm) % 2 == 0
|
|
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
|
assert result["output_sample_rate"] > 0
|
|
|
|
|
|
def _assert_realtime_accuracy(
|
|
result: dict,
|
|
whisper_model_size: str = "large-v3",
|
|
threshold: float = 0.8,
|
|
) -> None:
|
|
"""Assert that whisper transcription of audio output matches model text.
|
|
|
|
Args:
|
|
result: Roundtrip result dict from ``_run_realtime_audio_roundtrip``.
|
|
whisper_model_size: Whisper model used to transcribe the generated audio
|
|
for the accuracy check. Defaults to ``large-v3``: the default
|
|
``small`` model mishears short Chinese TTS clips (observed:
|
|
北京→韦京 and a dropped leading sentence, sim=0.443), which
|
|
caused spurious sim<0.8 failures under async_chunk codec
|
|
variability even though audio generation was correct. large-v3
|
|
transcribes these clips reliably, so a failure here now points
|
|
at the model, not the ASR grader.
|
|
threshold: Minimum cosine similarity (with length penalty) required to
|
|
pass. Default 0.8. Do not lower per-callsite without data:
|
|
at 0.35 the assertion no longer detects real audio
|
|
regressions. If a variant genuinely needs a different gate
|
|
(e.g. whisper partial transcripts under async_chunk), propose
|
|
it in its own PR with measurements.
|
|
"""
|
|
final_text = (result["transcription_text"] or "").strip()
|
|
assert final_text, "Expected non-empty transcription (model text stream)"
|
|
|
|
wav_out = _wav_bytes_from_pcm16(result["output_pcm"], result["output_sample_rate"])
|
|
whisper_text = convert_audio_bytes_to_text(wav_out, model_size=whisper_model_size).strip()
|
|
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
|
|
|
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
|
assert sim > threshold, (
|
|
f"Output audio transcript should match model text (sim={sim:.3f}, "
|
|
f"threshold={threshold}): "
|
|
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
|
)
|
|
|
|
|
|
class TestQwen3OmniRealtimeWebSocket:
|
|
@pytest.fixture(scope="class")
|
|
def omni_server(self, request, run_level):
|
|
# Release the legacy turn deployment before the duplex class below
|
|
# starts another model server on the same pair of GPUs.
|
|
from tests.helpers.fixtures.runtime import omni_fixture_lock
|
|
from tests.helpers.runtime import iter_omni_server
|
|
|
|
yield from iter_omni_server(request, run_level, omni_fixture_lock)
|
|
|
|
@pytest.mark.advanced_model
|
|
@pytest.mark.omni
|
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
|
def test_streaming_audio_input_pcm_output_async_chunk(self, omni_server) -> None:
|
|
"""Merge CI: async_chunk on, paced upload, full accuracy check."""
|
|
pcm16 = _synthetic_pcm16_input()
|
|
|
|
result = asyncio.run(
|
|
_run_realtime_audio_roundtrip(
|
|
omni_server.host,
|
|
omni_server.port,
|
|
omni_server.model,
|
|
pcm16,
|
|
chunk_ms=100,
|
|
send_delay_ms=SEND_DELAY_MS,
|
|
)
|
|
)
|
|
|
|
_assert_realtime_smoke(result)
|
|
_assert_realtime_accuracy(result)
|
|
|
|
@pytest.mark.advanced_model
|
|
@pytest.mark.omni
|
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
|
def test_long_audio_response_completes_async_chunk(self, omni_server) -> None:
|
|
"""Regression for #6474: long async-chunk audio must emit response.output_audio.done."""
|
|
issue_6474_pcm16 = _synthetic_pcm16_input(
|
|
phrase_text=ISSUE_6474_SYNTH_PHRASE_TEXT,
|
|
)
|
|
issue_6474_result = asyncio.run(
|
|
_run_realtime_audio_roundtrip(
|
|
omni_server.host,
|
|
omni_server.port,
|
|
omni_server.model,
|
|
issue_6474_pcm16,
|
|
chunk_ms=100,
|
|
send_delay_ms=SEND_DELAY_MS,
|
|
completion_timeout_s=180,
|
|
)
|
|
)
|
|
|
|
_assert_realtime_smoke(issue_6474_result)
|
|
output_duration_s = len(issue_6474_result["output_pcm"]) / (2 * issue_6474_result["output_sample_rate"])
|
|
assert output_duration_s > 5, (
|
|
f"Expected an issue-like audio response longer than 5 seconds, got {output_duration_s:.2f}s"
|
|
)
|
|
|
|
@pytest.mark.advanced_model
|
|
@pytest.mark.omni
|
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
@pytest.mark.parametrize("omni_server", realtime_server_vad_server_params, indirect=True)
|
|
def test_server_vad_multi_turn_without_client_commit(
|
|
self,
|
|
cached_silero_vad_artifact: str,
|
|
omni_server,
|
|
) -> None:
|
|
"""Two Qwen turns are endpointed without client commits."""
|
|
assert cached_silero_vad_artifact
|
|
deploy = load_deploy_config(server_vad_stage_config)
|
|
assert deploy.session_mode == "turn"
|
|
assert deploy.async_chunk is True
|
|
assert deploy.duplex_session.server_vad_model_path == SERVER_VAD_MODEL_PATH
|
|
pcm16 = _server_vad_pcm16_input()
|
|
|
|
turns = asyncio.run(
|
|
_run_server_vad_audio_roundtrips(
|
|
omni_server.host,
|
|
omni_server.port,
|
|
omni_server.model,
|
|
pcm16,
|
|
chunk_ms=100,
|
|
turns=2,
|
|
)
|
|
)
|
|
|
|
assert len(turns) == 2
|
|
updated_session = next(event["session"] for event in turns[0] if event["type"] == "session.updated")
|
|
effective_turn_detection = updated_session["audio"]["input"]["turn_detection"]
|
|
assert effective_turn_detection["type"] == "server_vad"
|
|
assert effective_turn_detection["silence_duration_ms"] == 500
|
|
assert effective_turn_detection["create_response"] is True
|
|
assert effective_turn_detection["interrupt_response"] is False
|
|
required_sequence = [
|
|
"input_audio_buffer.speech_started",
|
|
"input_audio_buffer.speech_stopped",
|
|
"input_audio_buffer.committed",
|
|
"response.created",
|
|
"response.output_audio.delta",
|
|
"response.output_audio.done",
|
|
"response.done",
|
|
]
|
|
input_item_ids: list[str] = []
|
|
response_ids: list[str] = []
|
|
for events in turns:
|
|
event_types = [event["type"] for event in events]
|
|
for event_type in required_sequence:
|
|
if event_type != "response.output_audio.delta":
|
|
assert event_types.count(event_type) == 1, event_types
|
|
positions = [event_types.index(event_type) for event_type in required_sequence]
|
|
assert positions == sorted(positions)
|
|
|
|
started = next(event for event in events if event["type"] == "input_audio_buffer.speech_started")
|
|
stopped = next(event for event in events if event["type"] == "input_audio_buffer.speech_stopped")
|
|
committed = next(event for event in events if event["type"] == "input_audio_buffer.committed")
|
|
created = next(event for event in events if event["type"] == "response.created")["response"]
|
|
done = next(event for event in events if event["type"] == "response.done")["response"]
|
|
|
|
assert started["item_id"] == stopped["item_id"] == committed["item_id"]
|
|
input_item_id = committed["item_id"]
|
|
history_events = [
|
|
event
|
|
for event in events
|
|
if event["type"] in {"conversation.item.added", "conversation.item.done"}
|
|
and event["item"]["id"] == input_item_id
|
|
]
|
|
assert [event["type"] for event in history_events] == [
|
|
"conversation.item.added",
|
|
"conversation.item.done",
|
|
]
|
|
assert all(event["item"]["role"] == "user" for event in history_events)
|
|
output_pcm = b"".join(
|
|
base64.b64decode(event["delta"])
|
|
for event in events
|
|
if event["type"] == "response.output_audio.delta" and event.get("delta")
|
|
)
|
|
assert output_pcm
|
|
assert created["id"] == done["id"]
|
|
assert done["status"] == "completed"
|
|
input_item_ids.append(input_item_id)
|
|
response_ids.append(created["id"])
|
|
|
|
assert len(set(input_item_ids)) == 2
|
|
assert len(set(response_ids)) == 2
|
|
|
|
@pytest.mark.advanced_model
|
|
@pytest.mark.omni
|
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
|
@pytest.mark.parametrize("omni_server", realtime_sync_server_params, indirect=True)
|
|
def test_streaming_audio_input_pcm_output(self, omni_server) -> None:
|
|
"""Merge CI: async_chunk off, no send delay, full accuracy check."""
|
|
pcm16 = _synthetic_pcm16_input()
|
|
|
|
result = asyncio.run(
|
|
_run_realtime_audio_roundtrip(
|
|
omni_server.host,
|
|
omni_server.port,
|
|
omni_server.model,
|
|
pcm16,
|
|
chunk_ms=100,
|
|
send_delay_ms=0,
|
|
)
|
|
)
|
|
|
|
_assert_realtime_smoke(result)
|
|
_assert_realtime_accuracy(result)
|
|
|
|
|
|
# These regressions use the engine-owned Qwen duplex plugin. The older class
|
|
# above intentionally exercises the separate turn/STT deployment.
|
|
_DUPLEX_REPEATS = int(os.environ.get("VLLM_OMNI_TEST_QWEN_DUPLEX_REPEATS", "20"))
|
|
_DUPLEX_LONG_INSTRUCTIONS = "Answer the user's question in English using at least eight complete sentences."
|
|
_DUPLEX_SHORT_INSTRUCTIONS = "Answer the user's question in one short English sentence."
|
|
|
|
|
|
class _DuplexTestRuntime(NamedTuple):
|
|
server: OmniServer
|
|
processor: Any
|
|
artifacts: Path
|
|
|
|
|
|
@pytest.fixture(scope="class")
|
|
def qwen_duplex_server(tmp_path_factory, cached_silero_vad_artifact, run_level):
|
|
"""Load real weights once for all interruption repetitions and history checks."""
|
|
from transformers import AutoProcessor
|
|
|
|
assert run_level in {"advanced_model", "full_model"}, "Duplex playback acceptance requires real weights"
|
|
artifacts = tmp_path_factory.mktemp("qwen_duplex_playback")
|
|
deploy = artifacts / "deploy.yaml"
|
|
# An absolute base keeps the production YAML's relative inheritance intact
|
|
# when the test's Silero override lives in a temporary directory.
|
|
deploy.write_text(
|
|
yaml.safe_dump(
|
|
{
|
|
"base_config": get_deploy_config_path("qwen3_omni_duplex.yaml"),
|
|
"duplex_session": {"server_vad_model_path": cached_silero_vad_artifact},
|
|
}
|
|
)
|
|
)
|
|
assert load_deploy_config(deploy).session_mode == "duplex"
|
|
model = get_model_prefix() + MODEL
|
|
processor = AutoProcessor.from_pretrained(model, trust_remote_code=True)
|
|
with OmniServer(
|
|
model,
|
|
["--deploy-config", str(deploy), "--init-timeout", "900", "--stage-init-timeout", "600"],
|
|
env_dict={"VLLM_OMNI_QWEN_VISUAL_DEBUG_DIR": str(artifacts)},
|
|
) as server:
|
|
yield _DuplexTestRuntime(server=server, processor=processor, artifacts=artifacts)
|
|
|
|
|
|
def _duplex_fixture_pcm(path: Path, *, duration_s: float | None = None) -> bytes:
|
|
"""Replay checked-in speech at the web client's negotiated 24 kHz PCM16 rate."""
|
|
pcm = np.frombuffer(_pcm16_mono_16k_from_wav_bytes(path.read_bytes()), dtype="<i2")
|
|
if duration_s is not None:
|
|
pcm = pcm[: int(duration_s * 16000)]
|
|
positions = np.arange(len(pcm) * 3 // 2) * (2.0 / 3.0)
|
|
return np.interp(positions, np.arange(len(pcm)), pcm).astype("<i2").tobytes()
|
|
|
|
|
|
def _duplex_response_id(event: dict) -> str | None:
|
|
return event.get("response_id") or event.get("response", {}).get("id")
|
|
|
|
|
|
def _duplex_response_events(events: list[dict], response_id: str, event_type: str) -> list[dict]:
|
|
return [event for event in events if event["type"] == event_type and _duplex_response_id(event) == response_id]
|
|
|
|
|
|
async def _duplex_receive_until(ws, events: list[dict], predicate, *, timeout_s: float = 180) -> None:
|
|
# One deadline covers the whole operation, including an endless stream that
|
|
# would otherwise reset a per-recv timeout forever.
|
|
async def receive() -> None:
|
|
while not predicate():
|
|
raw = await ws.recv()
|
|
assert isinstance(raw, str), "Expected JSON Realtime events"
|
|
event = json.loads(raw)
|
|
events.append(event)
|
|
assert event.get("type") != "error", event
|
|
|
|
await asyncio.wait_for(receive(), timeout=timeout_s)
|
|
|
|
|
|
async def _duplex_send_pcm(ws, pcm: bytes, *, paced: bool = False, trailing_silence: bool = True) -> None:
|
|
if trailing_silence:
|
|
pcm += bytes(24000 * 2) # One second exceeds the configured 500 ms endpoint.
|
|
chunk_bytes = 24000 * 2 // 5
|
|
for offset in range(0, len(pcm), chunk_bytes):
|
|
await ws.send(
|
|
json.dumps(
|
|
{
|
|
"type": "input_audio_buffer.append",
|
|
"audio": base64.b64encode(pcm[offset : offset + chunk_bytes]).decode(),
|
|
}
|
|
)
|
|
)
|
|
if paced:
|
|
await asyncio.sleep(0.2)
|
|
|
|
|
|
async def _duplex_add_image(ws) -> None:
|
|
from PIL import Image
|
|
|
|
image = io.BytesIO()
|
|
Image.new("RGB", (448, 336), (32, 96, 160)).save(image, format="JPEG")
|
|
await ws.send(
|
|
json.dumps(
|
|
{
|
|
"type": "conversation.item.create",
|
|
"item": {
|
|
"id": "camera_1",
|
|
"type": "message",
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "input_image",
|
|
"image_url": "data:image/jpeg;base64," + base64.b64encode(image.getvalue()).decode(),
|
|
}
|
|
],
|
|
},
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def _duplex_recorded_session(runtime, log_path: Path, *, instructions: str, vad: bool = True):
|
|
events: list[dict] = []
|
|
server = runtime.server
|
|
async with websockets.connect(
|
|
f"ws://{server.host}:{server.port}/v1/realtime?duplex=1", max_size=64 * 1024 * 1024
|
|
) as ws:
|
|
failed = True
|
|
try:
|
|
await ws.send(
|
|
json.dumps(
|
|
{
|
|
"type": "session.update",
|
|
"session": {
|
|
"model": server.model,
|
|
"instructions": instructions,
|
|
"temperature": 0,
|
|
"max_output_tokens": 384 if vad else 64,
|
|
"audio": {
|
|
"input": {
|
|
"format": {"type": "audio/pcm", "rate": 24000},
|
|
"turn_detection": {
|
|
"type": "server_vad",
|
|
"threshold": 0.5,
|
|
"prefix_padding_ms": 300,
|
|
"silence_duration_ms": 500,
|
|
"create_response": True,
|
|
"interrupt_response": True,
|
|
}
|
|
if vad
|
|
else None,
|
|
},
|
|
"output": {"format": {"type": "audio/pcm", "rate": 24000}},
|
|
},
|
|
},
|
|
}
|
|
)
|
|
)
|
|
await _duplex_receive_until(ws, events, lambda: any(e["type"] == "session.updated" for e in events))
|
|
session = next(e["session"] for e in events if e["type"] == "session.updated")
|
|
if vad:
|
|
accepted = session["audio"]["input"]["turn_detection"]
|
|
assert accepted["interrupt_response"] is True
|
|
assert accepted["create_response"] is True
|
|
await _duplex_add_image(ws)
|
|
yield ws, events, session["id"]
|
|
failed = False
|
|
finally:
|
|
# Closing the transport alone leaves a resumable session occupying
|
|
# the deployment's single session slot. Always release the session.
|
|
try:
|
|
await ws.send(json.dumps({"type": "session.close"}))
|
|
await _duplex_receive_until(
|
|
ws, events, lambda: any(e["type"] == "session.closed" for e in events), timeout_s=15
|
|
)
|
|
except Exception:
|
|
if not failed:
|
|
raise
|
|
finally:
|
|
log_path.write_text(json.dumps(events, ensure_ascii=False, indent=2))
|
|
|
|
|
|
def _duplex_assert_prompt(
|
|
runtime,
|
|
session_id: str,
|
|
*,
|
|
audio_count: int,
|
|
instructions: str,
|
|
request_count: int,
|
|
assistant_texts: list[str] | None = None,
|
|
):
|
|
captures = []
|
|
for path in runtime.artifacts.glob("*/input.json"):
|
|
capture = json.loads(path.read_text())
|
|
if duplex_resource_request_belongs_to_session(capture["request_id"], session_id):
|
|
captures.append((path.stat().st_mtime_ns, capture))
|
|
assert len(captures) == request_count, captures
|
|
capture = max(captures, key=lambda item: item[0])[1]
|
|
assert capture["audio_count"] == audio_count
|
|
# The camera belongs to the first user turn and leaves with its audio.
|
|
image_retained = request_count <= 4
|
|
assert len(capture["images"]) == int(image_retained)
|
|
messages: list[dict[str, Any]] = [{"role": "system", "content": instructions}]
|
|
if assistant_texts is None:
|
|
assistant_texts = [""] * (audio_count - 1)
|
|
assert len(assistant_texts) == audio_count - 1
|
|
for index in range(audio_count):
|
|
if index:
|
|
messages.append({"role": "assistant", "content": assistant_texts[index - 1]})
|
|
parts = [{"type": "image"}] if index == 0 and image_retained else []
|
|
parts.append({"type": "audio"})
|
|
messages.append({"role": "user", "content": parts})
|
|
expected = runtime.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
|
assert capture["prompt"] == expected
|
|
# Use the checkpoint tokenizer too: a repr(messages) processor stub cannot
|
|
# establish that an empty assistant survives as a real turn separator.
|
|
tokenizer = runtime.processor.tokenizer
|
|
actual_ids = tokenizer.encode(capture["prompt"], add_special_tokens=False)
|
|
separator_ids = tokenizer.encode("<|im_start|>assistant\n<|im_end|>\n", add_special_tokens=False)
|
|
assert sum(
|
|
actual_ids[index : index + len(separator_ids)] == separator_ids for index in range(len(actual_ids))
|
|
) == assistant_texts.count("")
|
|
|
|
|
|
def _duplex_assert_completed_audio(events: list[dict], response_id: str) -> None:
|
|
terminal = _duplex_response_events(events, response_id, "response.done")
|
|
assert len(terminal) == 1, terminal
|
|
assert terminal[0]["response"]["status"] == "completed", terminal
|
|
audio = _duplex_response_events(events, response_id, "response.output_audio.delta")
|
|
assert sum(len(base64.b64decode(event["delta"])) for event in audio) > 4800
|
|
assert any(
|
|
event.get("delta")
|
|
for event in events
|
|
if _duplex_response_id(event) == response_id
|
|
and event["type"] in {"response.output_text.delta", "response.output_audio_transcript.delta"}
|
|
), "The follow-up must produce text as well as audio"
|
|
|
|
|
|
async def _run_duplex_playback_interruption(runtime, log_path: Path, *, generation_finished: bool) -> None:
|
|
async with _duplex_recorded_session(runtime, log_path, instructions=_DUPLEX_LONG_INSTRUCTIONS) as (
|
|
ws,
|
|
events,
|
|
session_id,
|
|
):
|
|
await _duplex_send_pcm(ws, _duplex_fixture_pcm(validated_input_wav()))
|
|
await _duplex_receive_until(ws, events, lambda: any(e["type"] == "response.created" for e in events))
|
|
old_id = next(_duplex_response_id(e) for e in events if e["type"] == "response.created")
|
|
assert old_id
|
|
await _duplex_receive_until(
|
|
ws,
|
|
events,
|
|
lambda: (
|
|
sum(
|
|
len(base64.b64decode(e["delta"]))
|
|
for e in _duplex_response_events(events, old_id, "response.output_audio.delta")
|
|
)
|
|
> 4800
|
|
),
|
|
)
|
|
if generation_finished:
|
|
await _duplex_receive_until(ws, events, lambda: _duplex_response_events(events, old_id, "response.done"))
|
|
await ws.send(
|
|
json.dumps({"type": "playback.ack", "response_id": old_id, "played_ms": 100, "committed_ms": 100})
|
|
)
|
|
await _duplex_receive_until(ws, events, lambda: any(e["type"] == "playback.acknowledged" for e in events))
|
|
ack = next(e["event"] for e in events if e["type"] == "playback.acknowledged")
|
|
assert ack["item_id"] == f"item_{old_id}"
|
|
assert ack["committed_ms"] == 100
|
|
assert ack["playback"]["sent_ms"] > 100
|
|
cursor = len(events)
|
|
sender = asyncio.create_task(
|
|
_duplex_send_pcm(ws, _duplex_fixture_pcm(validated_soft_interrupt_wav(), duration_s=4.7), paced=True)
|
|
)
|
|
try:
|
|
await _duplex_receive_until(
|
|
ws, events, lambda: any(e["type"] == "input_audio_buffer.speech_started" for e in events[cursor:])
|
|
)
|
|
speech_index = next(
|
|
i for i in range(cursor, len(events)) if events[i]["type"] == "input_audio_buffer.speech_started"
|
|
)
|
|
if not generation_finished:
|
|
assert not _duplex_response_events(events[:speech_index], old_id, "response.done"), (
|
|
"Generation completed before speech interruption; the in-progress scenario was not exercised"
|
|
)
|
|
await _duplex_receive_until(
|
|
ws,
|
|
events,
|
|
lambda: any(e["type"] == "response.done" and _duplex_response_id(e) != old_id for e in events[cursor:]),
|
|
)
|
|
await sender
|
|
finally:
|
|
sender.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await sender
|
|
new_id = next(
|
|
_duplex_response_id(e)
|
|
for e in events[cursor:]
|
|
if e["type"] == "response.created" and _duplex_response_id(e) != old_id
|
|
)
|
|
# Include everything observed through session.closed when checking duplicates.
|
|
terminals = _duplex_response_events(events, old_id, "response.done")
|
|
assert len(terminals) == 1, terminals
|
|
assert terminals[0]["response"]["status"] == ("completed" if generation_finished else "cancelled")
|
|
if generation_finished:
|
|
cleared = _duplex_response_events(events, old_id, "output_audio_buffer.cleared")
|
|
assert len(cleared) == 1, cleared
|
|
boundary = events.index(cleared[0])
|
|
else:
|
|
boundary = events.index(terminals[0])
|
|
assert not _duplex_response_events(events[boundary + 1 :], old_id, "response.output_audio.delta")
|
|
assert new_id and new_id != old_id
|
|
_duplex_assert_completed_audio(events, new_id)
|
|
_duplex_assert_prompt(runtime, session_id, audio_count=2, instructions=_DUPLEX_LONG_INSTRUCTIONS, request_count=2)
|
|
|
|
|
|
async def _run_duplex_history_pruning(runtime, log_path: Path) -> None:
|
|
async with _duplex_recorded_session(runtime, log_path, instructions=_DUPLEX_SHORT_INSTRUCTIONS, vad=False) as (
|
|
ws,
|
|
events,
|
|
session_id,
|
|
):
|
|
response_ids = []
|
|
heard_texts: list[str] = []
|
|
pcm = _duplex_fixture_pcm(validated_input_wav())
|
|
for turn in range(6):
|
|
cursor = len(events)
|
|
await _duplex_send_pcm(ws, pcm, trailing_silence=False)
|
|
await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
|
|
await ws.send(json.dumps({"type": "response.create"}))
|
|
await _duplex_receive_until(ws, events, lambda: any(e["type"] == "response.done" for e in events[cursor:]))
|
|
response_id = next(_duplex_response_id(e) for e in events[cursor:] if e["type"] == "response.done")
|
|
assert response_id and response_id not in response_ids
|
|
response_ids.append(response_id)
|
|
_duplex_assert_completed_audio(events, response_id)
|
|
_duplex_assert_prompt(
|
|
runtime,
|
|
session_id,
|
|
audio_count=min(turn + 1, 4),
|
|
instructions=_DUPLEX_SHORT_INSTRUCTIONS,
|
|
request_count=turn + 1,
|
|
assistant_texts=heard_texts[-3:],
|
|
)
|
|
# Complete playback so pruning is checked against real assistant
|
|
# answers paired with retained user audio. Repeated empty assistant
|
|
# turns can make Qwen emit EOS; interruption tests cover those slots.
|
|
done = _duplex_response_events(events, response_id, "response.done")[0]["response"]
|
|
played_ms = done["metadata"]["playback"]["sent_ms"]
|
|
heard_texts.append(
|
|
"".join(
|
|
e["delta"]
|
|
for e in _duplex_response_events(events, response_id, "response.output_audio_transcript.delta")
|
|
).strip()
|
|
)
|
|
assert heard_texts[-1]
|
|
cursor = len(events)
|
|
await ws.send(
|
|
json.dumps(
|
|
{
|
|
"type": "playback.ack",
|
|
"response_id": response_id,
|
|
"played_ms": played_ms,
|
|
"committed_ms": played_ms,
|
|
}
|
|
)
|
|
)
|
|
await _duplex_receive_until(
|
|
ws, events, lambda: any(e["type"] == "playback.acknowledged" for e in events[cursor:])
|
|
)
|
|
ack = next(e["event"] for e in events[cursor:] if e["type"] == "playback.acknowledged")
|
|
assert ack["item_id"] == f"item_{response_id}"
|
|
assert ack["committed_ms"] == ack["playback"]["sent_ms"] == played_ms
|
|
assert ack["history_committed"] is True
|
|
for response_id in response_ids:
|
|
_duplex_assert_completed_audio(events, response_id)
|
|
|
|
|
|
@pytest.mark.advanced_model
|
|
@pytest.mark.omni
|
|
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
|
class TestQwen3OmniDuplexPlayback:
|
|
@pytest.mark.parametrize("generation_finished", [False, True], ids=["generating", "queued-playback"])
|
|
@pytest.mark.parametrize("repetition", range(_DUPLEX_REPEATS))
|
|
def test_speech_interrupts_playback(
|
|
self, qwen_duplex_server, tmp_path: Path, generation_finished: bool, repetition: int
|
|
):
|
|
"""Reuse one live deployment; every repetition opens an independent call."""
|
|
asyncio.run(
|
|
_run_duplex_playback_interruption(
|
|
qwen_duplex_server, tmp_path / f"events-{repetition}.json", generation_finished=generation_finished
|
|
)
|
|
)
|
|
|
|
def test_mixed_image_audio_history_pruning(self, qwen_duplex_server, tmp_path: Path):
|
|
asyncio.run(_run_duplex_history_pruning(qwen_duplex_server, tmp_path / "events.json"))
|