[Frontend] Add a shared realtime web UI for MiniCPM-o and Qwen3-Omni (#7222) (#7585)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-09-17 23:32:31 +08:00
1 parent eb1abbadf8
commit 882bf8bc95
53 files changed
+3438 -633

No files matched your search

+2 -2
View File
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
EnvironmentVariableCategory.PUBLIC_OMNI: 28,
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
EnvironmentVariableCategory.MODEL_SPECIFIC: 64,
EnvironmentVariableCategory.MODEL_SPECIFIC: 65,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
EnvironmentVariableCategory.INTERNAL: 2,
}
@@ -181,7 +181,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
ModelEnvironmentVariableDisposition.PROMOTE: 38,
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
ModelEnvironmentVariableDisposition.INTERNALIZE: 15,
ModelEnvironmentVariableDisposition.INTERNALIZE: 16,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
}
+2 -2
View File
@@ -567,10 +567,10 @@ def test_overlap_policy_defaults_and_invalid_values_to_listen_only():
assert DuplexSessionConfig.from_event({"session": {"overlap_policy": "bogus"}}).overlap_policy == "listen_only"
def test_capabilities_as_dict_reports_model_native_duplex_constants():
def test_capabilities_as_dict_reports_declared_implementation():
caps = DuplexCapabilities().as_dict()
assert caps["implementation_level"] == "model_native_duplex"
assert caps["implementation_level"] == "turn_based_duplex"
assert caps["input_modes"] == ["append_audio_chunk"]
assert caps["supports_kv_lease"] is False
assert caps["supports_core_kv_lease"] is False
+4 -2
View File
@@ -196,7 +196,7 @@ class FakePlugin(DuplexModelPlugin):
def capabilities(self, *, max_sessions: int) -> DuplexCapabilities:
del max_sessions
return DuplexCapabilities(supports_input_append=True)
return DuplexCapabilities(supports_input_append=True, supports_core_resumable_request=True)
def validate_client_extra_body(self, extra_body: object) -> None:
pass
@@ -408,7 +408,9 @@ async def test_open_answers_with_capabilities_and_emits_session_created() -> Non
assert result.control_id == "open-sid-open"
assert result.session_id == "sid-open"
assert result.lease_generation == 0
assert result.capabilities == DuplexCapabilities(supports_input_append=True)
assert result.capabilities == DuplexCapabilities(
supports_input_append=True, supports_core_resumable_request=True
)
assert result.public_session is not None
assert result.public_session["id"] == "sid-open"
assert result.public_session["voice"] == "test"
+25 -6
View File
@@ -107,8 +107,8 @@ def test_endpoint_decisions_match_the_serving_side_policy(
# --------------------------------------------------------------------------- #
def test_a_loud_frame_does_not_restart_the_silence_timer() -> None:
"""Silero v6.2 hysteresis: once silence is running, a loud frame only delays it.
def test_hysteresis_band_does_not_restart_the_silence_timer() -> None:
"""Scores between the negative and activation thresholds preserve the timer.
The frame at 0.45 sits above the negative threshold (0.35) but below the
activation threshold. It must not cancel the pending endpoint, or a speaker
@@ -164,9 +164,14 @@ def _pcm16_b64(samples: np.ndarray) -> str:
def test_pcm16_input_is_accepted_alongside_float32() -> None:
config = SileroVADConfig(threshold=0.5)
tone = np.zeros(FRAME * 2, dtype=np.float32)
tone: np.ndarray = np.zeros(FRAME * 2, dtype=np.float32)
seen: list[int] = []
vad = SileroStreamingVAD(config, frame_scorer=lambda frame: seen.append(frame.size) or 0.0)
def _score_frame(frame: np.ndarray) -> float:
seen.append(frame.size)
return 0.0
vad = SileroStreamingVAD(config, frame_scorer=_score_frame)
vad.process_base64(_pcm16_b64(tone), fmt="pcm16", sample_rate_hz=SAMPLE_RATE_HZ)
assert seen == [FRAME, FRAME]
@@ -272,7 +277,7 @@ def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tm
intra_op_num_threads = 0
class FakeInferenceSession:
def __init__(self, path: str, *, providers: list[str], sess_options: object) -> None:
def __init__(self, path: str, *, providers: list[str], sess_options: FakeSessionOptions) -> None:
self.providers = providers
self.sess_options = sess_options
self.calls: list[dict[str, np.ndarray]] = []
@@ -313,7 +318,7 @@ def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tm
assert session.calls[0]["sr"].item() == SAMPLE_RATE_HZ
state = backend.new_state()
first_frame = np.arange(backend.frame_samples, dtype=np.float32)
first_frame: np.ndarray = np.arange(backend.frame_samples, dtype=np.float32)
probability, state = backend.infer(first_frame, state)
first_call = session.calls[1]
@@ -377,3 +382,17 @@ def test_the_documented_server_vad_fields_are_all_accepted() -> None:
)
is None
)
@pytest.mark.parametrize("speech_probability", [0.5, 0.9])
def test_resumed_speech_cancels_pending_silence(speech_probability):
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=96, min_speech_duration_ms=32)
# 32 ms pause, resumed speech, then a full 96 ms pause: one utterance.
scores = [0.9, 0.1, speech_probability, speech_probability, 0.1, 0.1, 0.1]
assert _drive(config, scores) == [("start", 0), ("stop", 224)]
def test_short_pauses_do_not_split_continuous_speech_into_repeated_turns():
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=500, min_speech_duration_ms=32)
scores = [0.9] + [0.1, 0.9, 0.9, 0.9] * 20
assert _drive(config, scores) == [("start", 0)]
+45
View File
@@ -515,3 +515,48 @@ async def test_a_dead_replica_closes_the_sessions_it_was_serving() -> None:
messages = [output_q.get_nowait() for _ in range(output_q.qsize())]
types = [getattr(getattr(m, "event", None), "type", type(m).__name__) for m in messages]
assert types[-1] in {"session.expired", "session.closed"}, f"the client needs a terminal event, got {types}"
@pytest.mark.asyncio
async def test_turn_plugin_processes_multimodal_prompt_before_stage_submission(monkeypatch):
from vllm_omni.engine.orchestrator import build_engine_core_request_from_tokens
from vllm_omni.model_executor.models.qwen3_omni.duplex.plugin import Qwen3OmniDuplexPlugin
orchestrator, clients, rpc_q, _ = _build()
plugin = Qwen3OmniDuplexPlugin(_encode_audio)
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: "audio prompt")
orchestrator.plugin = orchestrator.session_manager.plugin = plugin
seen = []
def process_inputs(**kwargs):
seen.append(kwargs)
return build_engine_core_request_from_tokens(
request_id=kwargs["request_id"],
prompt={"prompt_token_ids": [1, 2]},
params=kwargs["params"],
model_config=orchestrator.stage_pools[0].stage_vllm_config.model_config,
resumable=kwargs["resumable"],
)
monkeypatch.setattr(
orchestrator, "_get_stage_input_processor", lambda stage_id: SimpleNamespace(process_inputs=process_inputs)
)
await orchestrator._dispatch_message(
OpenDuplexSessionMessage(
control_id="open-qwen",
session_id=SESSION_ID,
session_config=DuplexSessionConfig(model="qwen", modalities=["text", "audio"]),
)
)
assert (await rpc_q.get()).ok
try:
await _submit(orchestrator, _append_audio())
assert not clients[0].add_request_calls
await _submit(orchestrator, commands.Commit(final=True, create_response=True))
assert len(seen) == 1
assert seen[0]["prompt"]["multi_modal_data"]["audio"][0][0].shape == (16000,)
submitted = clients[0].add_request_calls[0][0]
assert submitted.resumable is False
assert not orchestrator.request_states[submitted.request_id].streaming.enabled
finally:
await _close(orchestrator, rpc_q)
@@ -47,6 +47,7 @@ from argparse import Namespace
from contextlib import asynccontextmanager
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
import pytest
from fastapi import FastAPI, HTTPException
@@ -564,7 +565,7 @@ async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch
Fails if HTTP requests lose ``request_timestamp``, or WebSocket scopes
start getting stamped / mutated incorrectly.
"""
captured: dict[str, object] = {}
captured: dict[str, Any] = {}
def fake_build_openai_app(args, supported_tasks):
return FastAPI()
@@ -592,8 +593,8 @@ async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch
await api_server.omni_run_server_worker("127.0.0.1:8000", _FakeSocket(), _minimal_args())
middleware = captured["served_app"]
http_scope = {"type": "http", "state": {}}
ws_scope = {"type": "websocket"}
http_scope: dict[str, Any] = {"type": "http", "state": {}}
ws_scope: dict[str, Any] = {"type": "websocket"}
async def _receive():
return {"type": "http.disconnect"}
@@ -920,11 +921,10 @@ async def test_pure_diffusion_app_state_key_snapshot(monkeypatch) -> None:
engine = _FakeEngineClient(stage_configs=[stage])
def _for_diffusion_factory(label: str):
@classmethod
def _factory(cls, *args, **kwargs):
return _marker(label)
return _factory
return classmethod(_factory)
monkeypatch.setattr(api_server.OmniOpenAIServingChat, "for_diffusion", _for_diffusion_factory("chat"))
monkeypatch.setattr(api_server.OmniOpenAIServingChatBatch, "for_diffusion", _for_diffusion_factory("chat_batch"))
@@ -965,13 +965,11 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
speech_kwargs = {}
def _for_diffusion_factory(label: str):
@classmethod
def _factory(cls, *args, **kwargs):
return _marker(label)
return _factory
return classmethod(_factory)
@classmethod
def _speech_factory(cls, *args, **kwargs):
speech_kwargs.update(kwargs)
return _marker("speech")
@@ -985,7 +983,7 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
)
monkeypatch.setattr(api_server.OmniOpenAIServingVideo, "for_diffusion", _for_diffusion_factory("video"))
monkeypatch.setattr(api_server.OmniStreamingVideoOutputHandler, "__init__", lambda self, *a, **k: None)
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", _speech_factory)
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", classmethod(_speech_factory))
monkeypatch.setattr(
api_server.ServingRealtimeRobotOpenPI,
"create_policy_server",
@@ -490,3 +490,31 @@ async def test_a_model_that_does_not_declare_chat_completions_does_not_get_the_r
assert state.openai_serving_chat is None
assert isinstance(state.openai_serving_duplex, OmniDuplexSessionHandler)
def test_qwen_plugin_preserves_custom_turn_deployment_default(monkeypatch, tmp_path):
from vllm_omni.model_executor.models.qwen3_omni.pipeline import QWEN3_OMNI_PIPELINE
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **kwargs: QWEN3_OMNI_PIPELINE)
deploy = tmp_path / "qwen.yaml"
deploy.write_text("stages: []\n")
assert not api_server._should_serve_duplex("qwen", {"deploy_config": str(deploy)})
deploy.write_text("session_mode: duplex\nstages: []\n")
assert api_server._should_serve_duplex("qwen", {"deploy_config": str(deploy)})
@pytest.mark.parametrize("flag", ["1", "true", "on"])
def test_turn_deployment_rejects_explicit_duplex_without_falling_back_to_stt(flag: str) -> None:
app = _duplex_app(None)
# Even an available STT service must not silently accept a duplex request.
app.state.openai_serving_realtime = object()
with TestClient(app) as client:
with client.websocket_connect(f"/v1/realtime?duplex={flag}") as websocket:
assert websocket.receive_json() == {
"type": "error",
"code": "unsupported",
"error": "VAD realtime is not enabled",
}
with pytest.raises(WebSocketDisconnect) as exc:
websocket.receive_text()
assert exc.value.code == 1008
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import hashlib
import importlib.util
import sys
@@ -70,7 +73,12 @@ def test_build_app_injects_public_realtime_url():
def test_build_app_versions_client_bundle_and_disables_index_cache():
client = TestClient(build_app())
expected_version_hash = hashlib.sha256()
for asset_name in ("app.js", "pcm_worklet.js", "playback_worklet.js"):
for asset_name in (
"app.js",
"pcm_worklet.js",
"playback_worklet.js",
*[str(p.relative_to(STATIC_DIR)) for p in sorted((STATIC_DIR / "profiles").glob("*.js"))],
):
expected_version_hash.update((STATIC_DIR / asset_name).read_bytes())
expected_version = expected_version_hash.hexdigest()[:12]
@@ -97,3 +105,41 @@ def test_build_app_exposes_realtime_websocket_and_static_assets():
assert client.get("/static/app.js").status_code == 200
assert client.get("/static/pcm_worklet.js").status_code == 200
assert client.get("/static/playback_worklet.js").status_code == 200
def test_qwen_config_does_not_inject_reference_voice():
client = TestClient(build_app(profile="qwen3-turn", adapter="vad", model="Qwen/Qwen3-Omni-30B-A3B-Instruct"))
page = client.get("/").text
assert '"profile": "qwen3-turn"' in page
assert '"adapter": "vad"' in page
assert '"refAudio": null' in page
for name in ("common", "minicpm_native", "qwen3_turn"):
assert client.get(f"/static/profiles/{name}.js").status_code == 200
with pytest.raises(ValueError, match="ref-audio"):
build_app(profile="qwen3-turn", ref_audio="unused.wav")
def test_config_cannot_close_its_script_element():
page = TestClient(build_app(model="</script><script>alert(1)</script>")).get("/").text
assert "</script><script>alert(1)" not in page
assert "\\u003c/script>" in page
def test_proxy_reports_unreachable_backend(monkeypatch):
from examples.online_serving.realtime_web import server as shared_server
class RefusedConnection:
async def __aenter__(self):
raise ConnectionRefusedError(111, "Connection refused")
async def __aexit__(self, *args):
pass
monkeypatch.setattr(shared_server.websockets, "connect", lambda *args, **kwargs: RefusedConnection())
client = TestClient(build_app(ws_backend="ws://127.0.0.1:8091"))
with client.websocket_connect("/v1/realtime?duplex=0") as ws:
event = ws.receive_json()
assert event["type"] == "error"
assert event["code"] == "backend_unavailable"
assert "ws://127.0.0.1:8091" in event["error"]
assert "--port" in event["error"]
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import re
import shutil
import subprocess
import textwrap
@@ -9,7 +8,7 @@ from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2] / "examples" / "online_serving" / "minicpmo" / "realtime_web"
ROOT = Path(__file__).resolve().parents[2] / "examples" / "online_serving" / "realtime_web"
APP_ROOT = ROOT / "app"
STATIC_ROOT = APP_ROOT / "static"
@@ -30,73 +29,37 @@ def test_page_exposes_focused_call_conversation_and_log_surfaces():
assert "Server VAD" not in html
def test_client_uses_proxy_relative_realtime_url_and_model_policy_session():
def test_shared_shell_delegates_wire_contract_to_profiles():
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
assert "profile.url(config, window.location.href)" in source
assert "profile.initialMessages(config, instructions)" in source
assert "profile.append(" in source
assert "profile.mapEvent(event)" in source
assert "native_duplex" not in source
assert "server_vad" not in source
assert "new URL(config.realtimePath, window.location.href)" in source
assert "url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'" in source
assert "url.searchParams.set('autostart', '0')" in source
def test_native_profile_preserves_session_and_capture_policy():
source = (STATIC_ROOT / "profiles/minicpm_native.js").read_text(encoding="utf-8")
assert "native_duplex" not in source
assert "auto_response: true" in source
assert "input_audio_buffer.append" in source
assert "input_audio_buffer.commit" not in source
assert "playback.ack" in source
assert "event.event || event" in source
assert "response.output_audio.delta" in source
assert "response.output_audio_transcript.delta" in source
assert "conversation.item.input_audio_transcription" in source
assert "force_barge_in" not in source
assert "server_vad" not in source
assert "type: 'response.create'" not in source
assert 'type: "response.create"' not in source
def test_web_server_requires_ref_audio_for_audio_output_session():
source = (ROOT / "server.py").read_text(encoding="utf-8")
ref_audio_arg = re.search(
r"parser\.add_argument\(\s*\"--ref-audio\",(?P<body>.*?)\n\s*\)",
source,
re.DOTALL,
)
assert ref_audio_arg is not None
assert "required=True" in ref_audio_arg.group("body")
assert "Optional reference voice" not in ref_audio_arg.group("body")
def test_client_sends_ref_audio_in_realtime_session_contract():
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
assert "if (config.refAudio) session.ref_audio = config.refAudio;" in source
assert "extraBody.ref_audio" not in source
assert "halfDuplex: false" in source
assert "commitMessages: () => []" in source
assert "video_frames" in source
assert "playback.ack" in source
def test_client_has_transactional_cleanup_and_visible_event_logging():
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
assert "async function stopSession" in source
assert "type: 'session.close'" in source
assert "async function cleanupSession" in source
assert "waitForSessionClosed" in source
assert "SESSION_CLOSE_TIMEOUT_MS" in source
assert "case 'session.closed':" in source
assert "track.stop()" in source
assert "clearInterval(sendTimer)" in source
assert "appendEventLog(event)" in source
assert "stopSession({ terminal: false })" in source
def test_client_keeps_microphone_upload_active_during_assistant_playback():
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
upload_gate = re.search(r"function microphoneUploadEnabled\(\) \{(?P<body>.*?)\n \}", source, re.DOTALL)
begin_assistant = re.search(r"function beginAssistant\(responseId\) \{(?P<body>.*?)\n \}", source, re.DOTALL)
assert upload_gate is not None
assert "return running && !muted;" in upload_gate.group("body")
assert "assistantActive" not in upload_gate.group("body")
assert begin_assistant is not None
assert "pendingCapture = []" not in begin_assistant.group("body")
def test_audio_worklets_define_capture_and_playback_processors():
capture = (STATIC_ROOT / "pcm_worklet.js").read_text(encoding="utf-8")
playback = (STATIC_ROOT / "playback_worklet.js").read_text(encoding="utf-8")
@@ -138,6 +101,7 @@ def test_playback_worklet_waits_before_playing_and_rebuffers_after_underrun():
node = shutil.which("node")
if node is None:
pytest.skip("node is required for the AudioWorklet regression test")
assert node is not None
script = textwrap.dedent(
"""
@@ -216,6 +180,7 @@ def test_playback_worklet_fades_terminal_drain_to_zero():
node = shutil.which("node")
if node is None:
pytest.skip("node is required for the AudioWorklet regression test")
assert node is not None
script = textwrap.dedent(
"""
@@ -0,0 +1,412 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const root = path.resolve(__dirname, '../../examples/online_serving/realtime_web/app/static');
const context = vm.createContext({ URL });
for (const name of ['common', 'minicpm_native', 'qwen3_turn']) {
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), context);
}
const profiles = context.OmniRealtimeProfiles;
const plain = (value) => JSON.parse(JSON.stringify(value));
const config = { model: 'Qwen/Qwen3-Omni-30B-A3B-Instruct', realtimePath: 'v1/realtime' };
test('MiniCPM keeps native flags, reference voice, camera, acknowledgements and no commits', () => {
const native = profiles['minicpm-native']();
const url = new URL(native.url({ ...config, model: 'MiniCPM' }, 'https://localhost/ui/'));
assert.equal(url.protocol, 'wss:');
assert.equal(url.searchParams.get('duplex'), '1');
assert.equal(url.searchParams.has('native_duplex'), false);
const [update] = native.initialMessages({ refAudio: 'data:audio/wav;base64,AA==' }, 'Prompt');
assert.equal(update.session.ref_audio, 'data:audio/wav;base64,AA==');
assert.equal(update.session.extra_body.native_duplex, undefined);
assert.equal(update.session.instructions, 'Prompt');
assert.deepEqual(plain(native.commitMessages()), []);
assert.equal(native.append('PCM', 'JPEG').video_frames[0], 'JPEG');
assert.equal(native.ack('r1', 200).type, 'playback.ack');
assert.equal(native.halfDuplex, false);
});
test('STT uses the shipped commit sequence and audio payload, including each new turn', () => {
const stt = profiles['qwen3-turn'](config);
for (let turn = 0; turn < 2; turn++) {
assert.deepEqual(plain(stt.initialMessages(config, 'ignored')), [
{ type: 'session.update', model: config.model },
{ type: 'input_audio_buffer.commit', final: false },
]);
assert.deepEqual(plain(stt.commitMessages()), [{ type: 'input_audio_buffer.commit', final: true }]);
}
assert.equal(stt.reconnectEachTurn, true);
assert.equal(stt.mapEvent({ type: 'response.output_audio.delta', audio: 'PCM' }).event.delta, 'PCM');
assert.equal(stt.mapEvent({ type: 'response.output_audio.done' }).kind, 'done');
assert.equal(stt.mapEvent({ type: 'transcription.delta', delta: 'hello' }).role, 'assistant');
});
test('Qwen enables camera and playback ACK only for the duplex VAD profile', () => {
for (const adapter of ['stt', 'vad']) {
const p = profiles['qwen3-turn']({ ...config, adapter });
const url = new URL(p.url({ ...config, realtimePath: 'wss://backend/v1/realtime?native_duplex=1&minicpmo45_native_duplex=1' }, 'http://localhost/'));
assert.equal(url.protocol, 'wss:');
assert.equal(url.searchParams.get('duplex'), adapter === 'vad' ? '1' : '0');
assert.equal(url.searchParams.has('native_duplex'), false);
assert.equal(url.searchParams.has('minicpmo45_native_duplex'), false);
assert.equal(p.append('PCM', 'JPEG').video_frames, undefined);
assert.equal(plain(p.imageMessages('JPEG')).length, adapter === 'vad' ? 1 : 0);
assert.equal(p.ack('r', 100)?.type || null, adapter === 'vad' ? 'playback.ack' : null);
assert.equal(p.camera, adapter === 'vad');
assert.equal(p.mapEvent({ type: 'response.listen' }).kind, 'ignore');
}
});
test('VAD uses nested format and interruptible endpoint detection', () => {
const vad = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
const [update] = vad.initialMessages(config, 'help');
assert.deepEqual(plain(update.session.audio.input), {
format: { type: 'audio/pcm', rate: 24000 },
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
silence_duration_ms: 500, create_response: true, interrupt_response: true },
});
assert.equal(update.session.extra_body, undefined);
assert.deepEqual(plain(vad.commitMessages()), []);
assert.equal(vad.mapEvent({ type: 'response.audio.done' }).kind, 'drain');
assert.equal(vad.mapEvent({ type: 'response.done' }).kind, 'done');
assert.equal(vad.mapEvent({ type: 'response.output_text.delta', delta: 'hello' }).text, 'hello');
assert.equal(vad.mapEvent({ type: 'error', error: { code: 'input_backpressure' } }).kind, 'backpressure');
assert.equal(vad.mapEvent({ type: 'input_audio_buffer.cleared' }).kind, 'backpressure');
assert.match(vad.mapEvent({ type: 'error', code: 'unsupported', error: 'Unavailable' }).message, /session_mode: duplex/);
assert.match(vad.mapEvent({ type: 'error', code: 'server_vad_initialization_failed' }).message, /Silero/);
});
function shell(profileName, adapter = 'stt', options = {}) {
const elements = new Map();
class Element {
constructor() { this.style = {}; this.listeners = {}; this.children = []; this.value = ''; this.textContent = ''; this.classList = { add() {}, remove() {}, toggle() {} }; }
addEventListener(name, callback) { this.listeners[name] = callback; }
append(...children) { this.children.push(...children); }
appendChild(child) { this.children.push(child); }
replaceChildren() { this.children = []; }
remove() {}
}
const document = {
getElementById(id) { if (!elements.has(id)) elements.set(id, new Element()); return elements.get(id); },
createElement() { return new Element(); },
};
const timers = new Map();
let timerId = 0;
const sockets = [];
const nodes = [];
class Socket {
static OPEN = 1;
constructor() {
this.readyState = 1; this.sent = []; sockets.push(this);
queueMicrotask(() => {
this.onopen?.();
if (!options.silent) this.onmessage?.({ data: JSON.stringify(options.error || { type: adapter === 'stt' && profileName === 'qwen3-turn' ? 'session.created' : 'session.updated' }) });
});
}
send(value) { this.sent.push(JSON.parse(value)); }
close() { this.readyState = 3; this.onclose?.({ code: 1000 }); }
}
class AudioContext {
constructor(options) { this.sampleRate = options.sampleRate; this.audioWorklet = { addModule: async () => {} }; this.destination = {}; }
createMediaStreamSource() { return { connect() {} }; }
createGain() { return { gain: {}, connect() {} }; }
async resume() {}
async close() {}
}
class AudioWorkletNode {
constructor(_context, name) { this.name = name; this.sent = []; nodes.push(this); this.port = { postMessage: (message) => this.sent.push(message) }; }
connect(target) { return target; }
}
const ctx = vm.createContext({
URL, document, AudioContext, AudioWorkletNode, WebSocket: Socket,
navigator: { mediaDevices: { getUserMedia: async () => ({ getTracks: () => [{ stop() {} }] }) } },
location: { href: 'https://localhost/' },
OMNI_REALTIME_CONFIG: { ...config, profile: profileName, adapter },
addEventListener() {},
setTimeout(fn, ms) { const id = ++timerId; timers.set(id, { fn, ms }); return id; },
clearTimeout(id) { timers.delete(id); },
setInterval() { return ++timerId; }, clearInterval() {},
btoa: (value) => Buffer.from(value, 'binary').toString('base64'),
atob: (value) => Buffer.from(value, 'base64').toString('binary'),
});
ctx.window = ctx;
for (const name of ['common', 'minicpm_native', 'qwen3_turn']) {
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), ctx);
}
// Expose closure controls only in the test VM; production has no test API.
const source = fs.readFileSync(path.join(root, 'app.js'), 'utf8').replace(/\}\)\(\);\s*$/, `
globalThis.testUI = { startSession, stopSession, handleEvent, playbackDrained,
microphoneUploadEnabled, flushCapture,
capture() { pendingCapture.push(new Int16Array([100, 200])); },
state() { return { running, connectionReady, assistantActive }; }
};
})();`);
vm.runInContext(source, ctx);
return { ui: ctx.testUI, elements, sockets, nodes,
timeout() { for (const timer of timers.values()) if (timer.ms === 15000) timer.fn(); },
async echo() {
for (const [id, timer] of timers) if (timer.ms === 300) { timers.delete(id); await timer.fn(); }
},
async turnWatchdog() {
for (const [id, timer] of timers) if (timer.ms === 120000) { timers.delete(id); await timer.fn(); }
},
};
}
test('shared shell STT sends final commit and starts a fresh second turn only after playback', async () => {
const app = shell('qwen3-turn');
await app.ui.startSession();
assert.equal(app.ui.microphoneUploadEnabled(), true);
app.ui.capture(); app.ui.flushCapture();
app.elements.get('sendTurnButton').listeners.click();
assert.equal(app.sockets[0].sent.at(-1).final, true);
assert.equal(app.ui.microphoneUploadEnabled(), false);
await app.ui.handleEvent({ type: 'response.output_audio.delta', audio: 'AAAAAA==' });
await app.ui.handleEvent({ type: 'response.output_audio.done' });
await app.echo();
assert.equal(app.sockets.length, 1, 'wait for actual speaker drain');
app.ui.playbackDrained({ responseId: 'turn-1', playedMs: 1 });
await app.echo();
assert.equal(app.sockets.length, 2);
assert.equal(app.sockets[1].sent[1].final, false);
assert.equal(app.ui.microphoneUploadEnabled(), true);
assert.equal(app.elements.get('cameraButton').hidden, true);
assert.equal(app.sockets.flatMap(s => s.sent).some(e => e.type === 'playback.ack'), false);
await app.ui.stopSession({ terminal: false });
});
test('VAD keeps uploading while generating and waits for response.done after audio drain', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'error', code: 'input_backpressure' });
assert.equal(app.ui.microphoneUploadEnabled(), true);
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
await app.ui.handleEvent({ type: 'response.audio.delta', delta: 'AAAAAA==', response_id: 'r1' });
await app.ui.handleEvent({ type: 'response.audio.done', response_id: 'r1' });
app.ui.playbackDrained({ responseId: 'r1', playedMs: 1 });
await app.echo();
assert.equal(app.ui.microphoneUploadEnabled(), true, 'continuous capture remains enabled');
assert.equal(app.ui.state().assistantActive, true, 'audio drain must wait for response.done');
assert.equal(app.sockets[0].sent.at(-1).type, 'playback.ack');
await app.ui.handleEvent({ type: 'response.done', response: { id: 'r1' } });
await app.echo();
assert.equal(app.ui.microphoneUploadEnabled(), true);
assert.equal(app.sockets.length, 1, 'VAD keeps its session');
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.commit'), false);
await app.ui.stopSession({ terminal: false });
});
test('a rejected operation is reported without hanging up the call', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'error', code: 'invalid_image', error: 'input_image requires a JPEG or PNG base64 data URL' });
assert.equal(app.ui.state().running, true, 'one refused image must not end the session');
assert.equal(app.ui.state().connectionReady, true);
assert.match(app.elements.get('runtimeDetail').textContent, /JPEG or PNG/);
await app.ui.stopSession({ terminal: false });
});
test('a session-level error still tears the call down', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'error', code: 'unknown_session', error: 'Unknown or closed duplex session' });
assert.equal(app.ui.state().running, false);
assert.equal(app.elements.get('connectionState').textContent, 'Error');
});
test('an interrupted response disarms the turn watchdog it left armed', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
await app.ui.handleEvent({ type: 'error', code: 'input_backpressure' });
await app.ui.handleEvent({ type: 'response.done', response: { id: 'r1', status: 'cancelled' } });
await app.turnWatchdog();
assert.equal(app.ui.state().running, true, 'barge-in must not time the session out two minutes later');
await app.ui.stopSession({ terminal: false });
});
test('MiniCPM retains microphone upload while speaking and acknowledges speaker drain', async () => {
const app = shell('minicpm-native');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'response.speak', response_id: 'native-r1' });
assert.equal(app.ui.microphoneUploadEnabled(), true);
await app.ui.handleEvent({ type: 'response.output_audio.delta', delta: 'AAAAAA==', response_id: 'native-r1' });
app.ui.playbackDrained({ responseId: 'native-r1', playedMs: 100 });
assert.equal(app.sockets[0].sent.at(-1).type, 'playback.ack');
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.commit'), false);
await app.ui.stopSession({ terminal: false });
await app.echo();
assert.equal(app.ui.state().running, false);
});
test('VAD handshake errors remain visible after resource cleanup', async () => {
const app = shell('qwen3-turn', 'vad', { error: { type: 'error', code: 'unsupported', error: 'Realtime API is not available' } });
await app.ui.startSession();
assert.equal(app.ui.state().running, false);
assert.equal(app.elements.get('connectionState').textContent, 'Error');
assert.match(app.elements.get('runtimeDetail').textContent, /session_mode: duplex/);
assert.equal(app.sockets[0].readyState, 3);
});
test('missing session acknowledgement fails instead of leaving Start disabled forever', async () => {
const app = shell('qwen3-turn', 'vad', { silent: true });
const starting = app.ui.startSession();
for (let i = 0; i < 20; i++) await Promise.resolve();
app.timeout();
await starting;
assert.equal(app.ui.state().running, false);
assert.equal(app.elements.get('callButton').disabled, false);
assert.match(app.elements.get('runtimeDetail').textContent, /handshake timed out/);
});
test('Qwen does not duplicate text and audio-transcript channels', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'response.audio_transcript.delta', delta: '' });
await app.ui.handleEvent({ type: 'response.output_text.delta', delta: 'Hello' });
await app.ui.handleEvent({ type: 'response.audio_transcript.delta', delta: 'Hello' });
await app.ui.handleEvent({ type: 'response.output_text.done', text: 'Hello' });
await app.ui.handleEvent({ type: 'response.done' });
const turns = app.elements.get('conversation').children;
assert.equal(turns.length, 1);
assert.equal(turns[0].children[1].textContent, 'Hello');
await app.ui.stopSession({ terminal: false });
});
const flushTasks = async () => { for (let i = 0; i < 40; i++) await Promise.resolve(); };
test('closing socket accepts only session.closed and completes without the fallback timer', async () => {
const app = shell('minicpm-native');
await app.ui.startSession();
const old = app.sockets[0];
let stopped = false;
const stopping = app.ui.stopSession().then(() => { stopped = true; });
assert.equal(old.sent.at(-1).type, 'session.close');
old.onmessage({ data: JSON.stringify({ type: 'session.updated' }) });
await flushTasks();
assert.equal(stopped, false);
old.onmessage({ data: JSON.stringify({ type: 'session.closed' }) });
await flushTasks();
assert.equal(stopped, true, 'must finish without firing the timeout');
await stopping;
await app.ui.startSession();
old.onmessage({ data: JSON.stringify({ type: 'session.closed' }) });
old.onmessage({ data: '{broken' });
await flushTasks();
assert.equal(app.ui.state().running, true, 'stale socket cannot affect a new session');
await app.ui.stopSession({ terminal: false });
});
test('malformed JSON before readiness rejects the handshake and cleans up', async () => {
const app = shell('qwen3-turn', 'vad', { silent: true });
const starting = app.ui.startSession();
await flushTasks();
app.sockets[0].onmessage({ data: '{broken' });
await starting;
assert.equal(app.ui.state().running, false);
assert.equal(app.sockets[0].readyState, 3);
assert.equal(app.elements.get('connectionState').textContent, 'Error');
assert.match(app.elements.get('runtimeDetail').textContent, /Invalid server event/);
});
test('malformed JSON after readiness fails the session and releases resources', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
app.sockets[0].onmessage({ data: '{broken' });
await flushTasks();
assert.equal(app.ui.state().running, false);
assert.equal(app.ui.microphoneUploadEnabled(), false);
assert.equal(app.sockets[0].readyState, 3);
assert.equal(app.elements.get('connectionState').textContent, 'Error');
assert.match(app.elements.get('runtimeDetail').textContent, /Invalid server event/);
});
test('Qwen VAD interruption clears playback while continuing microphone upload', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
await app.ui.handleEvent({ type: 'response.audio.delta', delta: 'AAAAAA==', response_id: 'r1' });
await app.ui.handleEvent({ type: 'output_audio_buffer.cleared', response_id: 'r1' });
assert.equal(app.ui.state().assistantActive, false);
assert.equal(app.ui.microphoneUploadEnabled(), true);
assert.ok(app.nodes.some(node => node.sent.some(message => message.type === 'clear')));
await app.ui.stopSession({ terminal: false });
});
test('Qwen runtime errors do not suggest changing a working deployment', () => {
const p = profiles['qwen3-turn']({ adapter: 'vad' });
const message = 'playback.ack arrived after a later user input was committed.';
assert.equal(p.mapEvent({ type: 'error', code: 'playback_ack_too_late', error: message }).message, message);
});
test('interrupted playback reports its cursor before clear resets it', () => {
let Playback;
const messages = [];
const ctx = vm.createContext({
sampleRate: 24000,
AudioWorkletProcessor: class { constructor() { this.port = { postMessage: m => messages.push(m) }; } },
registerProcessor: (_name, cls) => { Playback = cls; },
});
vm.runInContext(fs.readFileSync(path.join(root, 'playback_worklet.js'), 'utf8'), ctx);
const player = new Playback();
player.handleMessage({ type: 'audio', responseId: 'old', pcm: new Int16Array(24000), initialBufferMs: 0 });
player.process([], [[new Float32Array(2400)]]);
player.handleMessage({ type: 'clear' });
const report = messages.find(m => m.type === 'playback-stopped');
assert.equal(report.responseId, 'old');
assert.equal(report.playedMs, 100);
assert.equal(player.playedFrames, 0);
assert.equal(player.bufferedFrames(), 0);
player.handleMessage({ type: 'clear' });
assert.equal(messages.filter(m => m.type === 'playback-stopped').length, 1);
});
test('late interruption cursor acknowledges old response without finishing the new response', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
const player = app.nodes.find(n => n.name === 'fullduplex-pcm-playback');
await app.ui.handleEvent({ type: 'response.created', response: { id: 'old' } });
await app.ui.handleEvent({ type: 'output_audio_buffer.cleared', response_id: 'old' });
await app.ui.handleEvent({ type: 'response.created', response: { id: 'new' } });
player.port.onmessage({ data: { type: 'playback-stopped', responseId: 'old', playedMs: 100 } });
const ack = app.sockets[0].sent.find(m => m.type === 'playback.ack');
assert.equal(ack.response_id, 'old');
assert.equal(ack.played_ms, 100);
assert.equal(app.ui.state().assistantActive, true);
});
test('Qwen camera frames ride the OpenAI image interface, not the audio append', () => {
const p = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
assert.deepEqual(plain(p.append('PCM', 'JPEG')), { type: 'input_audio_buffer.append', audio: 'PCM' });
const [created] = plain(p.imageMessages('JPEG'));
assert.equal(created.type, 'conversation.item.create');
assert.equal(created.item.role, 'user');
assert.deepEqual(created.item.content, [
{ type: 'input_image', image_url: 'data:image/jpeg;base64,JPEG' },
]);
});
test('the camera retires its oldest image instead of exhausting the session budget', () => {
const p = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
const ids = [];
for (let i = 0; i < 8; i++) {
const messages = plain(p.imageMessages('JPEG'));
assert.equal(messages.length, 1, 'nothing to retire while under the budget');
ids.push(messages[0].item.id);
}
// The engine counts what it already stores, so the delete has to lead.
const ninth = plain(p.imageMessages('JPEG'));
assert.deepEqual(ninth[0], { type: 'conversation.item.delete', item_id: ids[0] });
assert.equal(ninth[1].type, 'conversation.item.create');
assert.equal(new Set(ids).size, 8, 'item ids must be distinct to be deletable');
// A new call reopens the budget; stale ids from the old one must not linger.
assert.equal(plain(p.initialMessages(config, 'x')).length > 0, true);
assert.equal(plain(p.imageMessages('JPEG')).length, 1);
});
@@ -0,0 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import shutil
import subprocess
from pathlib import Path
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_realtime_profiles_and_shell_lifecycle():
node = shutil.which("node")
if node is None:
pytest.skip("node is required for browser protocol tests")
assert node is not None
subprocess.run(
[node, "--test", str(Path(__file__).with_suffix(".cjs"))],
check=True,
capture_output=True,
text=True,
timeout=30,
)
@@ -0,0 +1,625 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import asyncio
import base64
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
from tests.engine.duplex.test_session_runner import (
SESSION_ID,
Harness,
RecordingStagePort,
_speech_then_stop,
append_audio,
tts_output,
)
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
from vllm_omni.engine.duplex.commands import AckPlayback, CancelResponse, Commit, UpdateSession
from vllm_omni.engine.duplex.config import DuplexSessionConfig
from vllm_omni.engine.duplex.messages import OpenDuplexSessionMessage
from vllm_omni.engine.duplex.plugin import DuplexRuntimeConfigError
from vllm_omni.engine.duplex.session.manager import DuplexSessionManager
from vllm_omni.model_executor.models.qwen3_omni.duplex.input import QwenPcmBuffer
from vllm_omni.model_executor.models.qwen3_omni.duplex.plugin import MAX_PROMPT_IMAGES, Qwen3OmniDuplexPlugin
from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_buffer_commit_and_rollback_preserve_all_samples():
buffer = QwenPcmBuffer()
raw = np.ones(101, dtype="<f4").tobytes()
payload = {"audio": base64.b64encode(raw).decode(), "format": "pcm_f32le", "sample_rate_hz": 16000}
assert buffer.prepare_append(payload, operation_id="a", chunk_period_ms=1000, allow_emit=True) is None
reservation = buffer.prepare_commit(operation_id="c", chunk_period_ms=1000)
assert base64.b64decode(reservation.payload["audio"]) == raw
reservation.rollback()
assert buffer.pending_byte_count == len(raw)
final = buffer.prepare_commit(operation_id="d", chunk_period_ms=1000)
final.commit()
assert not buffer.has_pending() and not buffer.has_reserved()
def test_buffer_refuses_video_frames_and_points_at_the_openai_interface():
buffer = QwenPcmBuffer()
payload = {
"audio": base64.b64encode(np.ones(16, dtype="<f4").tobytes()).decode(),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"video_frames": ["a-frame"],
}
with pytest.raises(ValueError, match="input_image"):
buffer.prepare_append(payload, operation_id="first", chunk_period_ms=200, allow_emit=True)
# The refusal must not swallow the audio it was sent with.
assert not buffer.has_pending()
del payload["video_frames"]
buffer.prepare_append(payload, operation_id="second", chunk_period_ms=200, allow_emit=True)
committed = buffer.prepare_commit(operation_id="second-commit", chunk_period_ms=200)
assert "video_frames" not in committed.payload
committed.commit()
async def open_qwen():
plugin = Qwen3OmniDuplexPlugin(lambda audio, *args: "AAAA")
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kw: repr(messages))
port = RecordingStagePort(stage_count=3)
output: asyncio.Queue[Any] = asyncio.Queue()
results: asyncio.Queue[Any] = asyncio.Queue()
manager = DuplexSessionManager(
plugin=plugin,
stage_port=port,
output_sink=output,
result_sink=results,
runtime_config=DuplexSessionRuntimeConfig(),
model_config=None,
)
config = DuplexSessionConfig(model="qwen", modalities=["text", "audio"], overlap_policy="barge_in_on_speech")
await manager.handle(OpenDuplexSessionMessage(control_id="open", session_id=SESSION_ID, session_config=config))
result = await results.get()
assert result.ok, result
harness = Harness(manager, port, output, results, manager.runners[SESSION_ID])
await harness.settle()
return harness
@pytest.mark.asyncio
async def test_two_committed_turns_stream_text_and_audio_and_release_requests():
h = await open_qwen()
try:
ids = []
for i in range(2):
await h.run(append_audio())
assert len(h.port.submissions) == i
await h.run(Commit(final=True, create_response=True))
assert len(h.port.submissions) == i + 1, [e.to_realtime() for e in h.events]
request_id = h.port.submissions[-1].context.request_id
ids.append(request_id)
assert h.session.active_request_id == request_id
assert len(h.port.submissions[-1].prompt["multi_modal_data"]["audio"]) == i + 1
text = OmniRequestOutput.from_stage_output(
tts_output(request_id, samples=0, text="hello", finished=True),
request_id=request_id,
stage_id=None,
)
assert not h.deliver(text, stage_id=0), "Thinker output must still forward to Talker"
await h.settle()
assert text.finished, "projection must not mutate the output forwarded to Talker"
assert h.session.active_response_id is not None, "Thinker finished must not end the response"
response_id = h.session.active_response_id
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
assert h.session.active_response_id is None
assert h.port.cleanups[-1][0] == [request_id]
assert len(set(ids)) == 2
assert h.session.capabilities.as_dict()["implementation_level"] == "turn_based_duplex"
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_cancel_drops_old_output_and_accepts_next_turn():
h = await open_qwen()
try:
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
old = h.port.submissions[-1].context
await h.run(CancelResponse())
await h.deliver_and_settle(tts_output(old.request_id, finished=True), stage_id=2, epoch=old.fence.epoch)
assert h.session.active_response_id is None
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
assert len(h.port.submissions) == 2
assert h.port.submissions[-1].context.request_id != old.request_id
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_vad_commits_and_speech_interrupts_active_qwen_response():
h = await open_qwen()
try:
h.runner.control._detector = _speech_then_stop()
await h.run(append_audio())
await h.run(append_audio(is_speech=False, value=0.0))
assert len(h.port.submissions) == 1
old = h.port.submissions[-1].context
h.runner.control._detector = _speech_then_stop()
await h.run(append_audio())
assert h.session.epoch > old.fence.epoch
await h.run(append_audio(is_speech=False, value=0.0))
assert len(h.port.submissions) == 2
assert h.port.submissions[-1].context.request_id != old.request_id
finally:
await h.manager.shutdown()
def test_audio_projector_handles_delta_tensor_lists_and_sample_rate():
import torch
encoded: list[tuple[Any, object]] = []
def _encode(audio: object, rate: int, *_args: object) -> str:
encoded.append((audio, rate))
return "encoded"
plugin = Qwen3OmniDuplexPlugin(_encode)
output = SimpleNamespace(
request_id="r",
stage_id=2,
finished=True,
outputs=[],
multimodal_output={"audio": [torch.ones(8), torch.ones(16)], "sr": [torch.tensor(24000)]},
)
events = list(plugin.data_plane.project({"data_plane_outputs": [output]}, context={"modalities": ["audio"]}))
assert len(encoded) == 1 and encoded[0][0].shape == (24,)
assert events[0]["audio_duration_ms"] == 1
assert events[0]["end_of_turn"] is True
def test_audio_history_is_bounded_and_does_not_cross_sessions():
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
state, other = plugin.create_session_state(), plugin.create_session_state()
history = []
for i in range(6):
history.append({"role": "user", "content": [{"type": "audio_url"}]})
config = plugin.prepare_prompt_config({"conversation": history}, state=state, payload={"audio": str(i)})
history.append({"role": "assistant", "content": f"reply-{i}"})
assert len(state.audio_history) == 4
assert [m["audio_payload"]["audio"] for m in config["qwen_messages"] if "audio_payload" in m] == [
"2",
"3",
"4",
"5",
]
assert other.audio_history == []
@pytest.mark.asyncio
async def test_late_playback_ack_keeps_answer_before_new_user_input():
h = await open_qwen()
try:
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
request_id = h.port.submissions[-1].context.request_id
response_id = h.session.active_response_id
await h.deliver_and_settle(tts_output(request_id, samples=0, text="first answer", finished=True), stage_id=0)
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
# Playback has not drained yet. A new utterance commits first.
assert [m["role"] for m in h.session.history] == ["user"]
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
current_response = h.session.active_response_id
events = await h.run(AckPlayback(response_id=response_id, played_ms=1000))
assert not any(e.to_realtime().get("type") == "error" for e in events)
assert [m["role"] for m in h.session.history] == ["user", "assistant", "user"]
assert h.session.history[1]["content"] == "first answer"
assert h.session.active_response_id == current_response
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming_vad", [False, True])
async def test_finished_qwen_response_does_not_repeat_during_silence(streaming_vad):
h = await open_qwen()
try:
if streaming_vad:
from vllm_omni.engine.duplex.turn_detection import ServerTurnDetector, TurnDetectionConfig
detector = ServerTurnDetector(TurnDetectionConfig())
# Exercise framing, endpointing and reset across the entire response.
# A deterministic scorer avoids requiring an ONNX artifact in CPU tests.
detector.vad._frame_scorer = lambda frame: float(np.max(np.abs(frame)) > 0.01)
h.runner.control._detector = detector
else:
h.runner.control._detector = _speech_then_stop()
await h.run(append_audio())
await h.run(append_audio(is_speech=False, value=0.0))
if not streaming_vad:
h.runner.control._detector = None
assert len(h.port.submissions) == 1
request_id = h.port.submissions[0].context.request_id
response_id = h.session.active_response_id
await h.deliver_and_settle(tts_output(request_id, samples=0, text="story", finished=True), stage_id=0)
for _ in range(3):
await h.run(append_audio(is_speech=False, value=0.0))
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
for _ in range(3):
await h.run(append_audio(is_speech=False, value=0.0))
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
await h.settle()
assert len(h.port.submissions) == 1
assert not h.runner.model_state.audio_buffer.has_pending()
assert h.runner.model_state.committed_audio_payload is None
finally:
await h.manager.shutdown()
def test_current_audio_is_not_replaced_by_history_ending_with_assistant():
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
state = plugin.create_session_state()
first = {"audio": base64.b64encode(np.full(160, 0.1, dtype="<f4").tobytes()).decode()}
current = {"audio": base64.b64encode(np.full(320, 0.2, dtype="<f4").tobytes()).decode()}
history = [{"role": "user", "content": [{"type": "audio_url"}]}]
plugin.prepare_prompt_config({"conversation": history}, state=state, payload=first)
history.append({"role": "assistant", "content": "Previous answer"})
config = plugin.prepare_prompt_config({"conversation": history}, state=state, payload=current)
plan = plugin.plan_append(
request_id="new",
fence=None,
session_config=config,
runtime_config={},
seq=1,
turn_seq=1,
payload=current,
final=True,
sampling_params=None,
)
audios = plan.prompt["multi_modal_data"]["audio"]
assert len(audios) == 2
np.testing.assert_allclose(audios[-1][0], np.full(320, 0.2, dtype="<f4"))
assert config["qwen_messages"][-1]["audio_payload"] is current
@pytest.mark.asyncio
async def test_partial_playback_does_not_commit_unaligned_thinker_text():
h = await open_qwen()
try:
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
request_id = h.port.submissions[-1].context.request_id
response_id = h.session.active_response_id
await h.deliver_and_settle(tts_output(request_id, samples=0, text="entire answer", finished=True), stage_id=0)
await h.deliver_and_settle(tts_output(request_id, samples=24000, finished=False), stage_id=2)
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
assert not any(m["role"] == "assistant" for m in h.session.history)
await h.deliver_and_settle(tts_output(request_id, samples=216000, finished=True), stage_id=2)
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
assert not any(m["role"] == "assistant" for m in h.session.history)
await h.run(AckPlayback(response_id=response_id, played_ms=10000))
assert h.session.history[-1]["content"] == "entire answer"
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_prompt_preparation_runs_off_loop_and_cancellation_prevents_submission():
import threading
h = await open_qwen()
entered, release = threading.Event(), threading.Event()
thread_ids = []
original = h.manager.plugin.plan_append
def slow_plan(**kwargs):
thread_ids.append(threading.get_ident())
entered.set()
assert release.wait(5)
return original(**kwargs)
h.manager.plugin.plan_append = slow_plan
try:
await h.run(append_audio())
commit = asyncio.create_task(h.run(Commit(final=True, create_response=True)))
for _ in range(100):
if entered.is_set():
break
await asyncio.sleep(0.01)
assert entered.is_set()
assert thread_ids != [threading.get_ident()]
await h.run(CancelResponse())
release.set()
await commit
await h.settle()
assert not h.port.submissions
finally:
release.set()
await h.manager.shutdown()
def camera_frame():
import io
from PIL import Image
buf = io.BytesIO()
Image.new("RGB", (16, 16), "red").save(buf, format="JPEG")
return base64.b64encode(buf.getvalue()).decode()
def pcm16_base64(samples=16000):
"""Loud PCM16, so the projection reads it as speech rather than silence."""
pcm = (np.sin(np.arange(samples) * 0.05) * 20000).astype("<i2")
return base64.b64encode(pcm.tobytes()).decode()
def image_item(item_id="camera"):
return {
"id": item_id,
"type": "message",
"role": "user",
"content": [
{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()},
{"type": "input_text", "text": "What color is this?"},
],
}
@pytest.mark.asyncio
async def test_image_item_is_context_until_response_create_and_delete_removes_it():
from vllm_omni.engine.duplex.commands import CreateResponse, DeleteItem
h = await open_qwen()
try:
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": image_item()}))
assert not h.port.submissions
await h.run(CreateResponse())
assert len(h.port.submissions) == 1
prompt = h.port.submissions[-1].prompt
assert len(prompt["multi_modal_data"]["image"]) == 1
assert "audio" not in prompt["multi_modal_data"]
assert "What color is this?" in prompt["prompt"]
await h.run(CancelResponse())
await h.run(DeleteItem(item_id="camera"))
await h.run(append_audio())
await h.run(Commit(final=True, create_response=False))
assert len(h.port.submissions) == 1
await h.run(CreateResponse())
assert "image" not in h.port.submissions[-1].prompt["multi_modal_data"]
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_image_context_reaches_audio_turn_and_survives_commit():
from vllm_omni.engine.duplex.commands import CreateItem
h = await open_qwen()
try:
await h.run(CreateItem(item=image_item()))
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
mm = h.port.submissions[-1].prompt["multi_modal_data"]
assert len(mm["image"]) == 1 and len(mm["audio"]) == 1
assert any(
isinstance(m["content"], list) and any(p.get("type") == "image_url" for p in m["content"])
for m in h.session.history
)
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_image_only_item_reaches_the_prompt_the_way_the_camera_sends_it():
"""The camera sends each frame as its own item, with no text part.
A user item used to be forwarded to the session only when it carried text
or audio, so an image-only item was acknowledged to the client and then
dropped. Nothing reported it and every prompt that followed was blind.
"""
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
h = await open_qwen()
try:
item = {
"id": "camera_1",
"type": "message",
"role": "user",
"content": [{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()}],
}
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": item}))
assert [
part
for message in h.session.history
if isinstance(message.get("content"), list)
for part in message["content"]
if part.get("type") == "image_url"
], "an image-only item never reached the session history"
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
mm = h.port.submissions[-1].prompt["multi_modal_data"]
assert len(mm["image"]) == 1 and len(mm["audio"]) == 1
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_one_item_carrying_both_speech_and_a_picture_keeps_both():
"""Audio and images leave the same item by different routes.
The audio becomes buffered appends that a commit seals under this item's
id; registering the spoken message there would overwrite an image stored
under the same id, so the picture has to become its own item.
"""
from vllm_omni.engine.duplex.commands import CreateResponse
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
h = await open_qwen()
try:
item = {
"id": "mixed_1",
"type": "message",
"role": "user",
"content": [
{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()},
{"type": "input_audio", "audio": pcm16_base64(), "format": "pcm16", "sample_rate_hz": 16000},
],
}
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": item}))
await h.run(CreateResponse())
mm = h.port.submissions[-1].prompt["multi_modal_data"]
assert len(mm["image"]) == 1, "the picture was dropped on its way out of a mixed item"
assert len(mm["audio"]) == 1
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_image_limit_rejects_atomically_and_delete_reclaims_capacity():
from vllm_omni.engine.duplex.commands import CreateItem, DeleteItem
h = await open_qwen()
try:
for i in range(8):
await h.run(CreateItem(item=image_item(f"camera_{i}")))
events = await h.run(CreateItem(item=image_item("overflow")))
assert any(e.to_realtime()["type"] == "error" for e in events)
assert len(h.session.history) == 8
await h.run(DeleteItem(item_id="camera_0"))
await h.run(CreateItem(item=image_item("replacement")))
assert len(h.session.history) == 8
assert not h.port.submissions
finally:
await h.manager.shutdown()
def test_prompt_holds_stored_images_to_the_stage_limit():
"""The prompt is the last guard before ``limit_mm_per_prompt``.
A dropped image has to take its ``{"type": "image"}`` placeholder with
it, because the processor pairs placeholders with ``mm["image"]``
positionally.
"""
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
url = "data:image/jpeg;base64," + camera_frame()
history: list[dict[str, Any]] = [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": url}}, {"type": "text", "text": f"i{i}"}],
}
for i in range(MAX_PROMPT_IMAGES + 2)
]
audio = base64.b64encode(np.full(160, 0.1, dtype="<f4").tobytes()).decode()
history.append({"role": "user", "audio_payload": {"audio": audio}})
plan = plugin.plan_append(
request_id="r",
fence=None,
session_config={"qwen_messages": history},
runtime_config={},
seq=1,
turn_seq=1,
payload={"audio": audio},
final=True,
sampling_params=None,
)
prompt = plan.prompt
assert len(prompt["multi_modal_data"]["image"]) == MAX_PROMPT_IMAGES
assert prompt["prompt"].count("{'type': 'image'}") == MAX_PROMPT_IMAGES
# Only the image part is retired; the message keeps what it said.
assert "'i0'" in prompt["prompt"]
def test_text_only_turn_without_prepared_messages_is_refused_not_crashed():
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
with pytest.raises(DuplexRuntimeConfigError):
plugin.plan_append(
request_id="r",
fence=None,
# An empty prepared list is not a missing one: it must not fall
# back to a payload that has no audio to read.
session_config={"qwen_messages": []},
runtime_config={},
seq=1,
turn_seq=1,
payload={"is_speech": True},
final=True,
sampling_params=None,
)
@pytest.mark.asyncio
async def test_commit_all_on_done_keeps_unaligned_text_the_client_never_acks():
h = await open_qwen()
try:
# The runner starts every session ack-gated; this client opts out.
await h.run(UpdateSession(patch={"playback_commit_policy": "commit_all_on_done"}))
assert h.session.config.playback_commit_policy == "commit_all_on_done"
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
request_id = h.port.submissions[-1].context.request_id
await h.deliver_and_settle(tts_output(request_id, samples=0, text="entire answer", finished=True), stage_id=0)
await h.deliver_and_settle(tts_output(request_id, samples=24000, finished=True), stage_id=2)
# Qwen text is unaligned to its audio, but waiting for an ACK this
# client will never send would drop the turn from history entirely.
assert h.session.active_response_id is None
assert h.session.history[-1]["content"] == "entire answer"
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_the_bound_request_id_is_the_one_the_stage_received():
h = await open_qwen()
try:
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
submitted = h.port.submissions[-1].context.request_id
# Qwen has no resumable core request, so every turn gets its own stage
# id. The runner names that id before the append runs and the model
# channel names it again when it submits; they have to agree.
assert submitted.endswith(".r.stage0-turn0"), submitted
assert h.session.active_request_id == submitted
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_visual_capture_records_submitted_pixels_and_request(tmp_path, monkeypatch):
import json
from PIL import Image
from vllm_omni.engine.duplex.commands import CreateItem
monkeypatch.setenv("VLLM_OMNI_QWEN_VISUAL_DEBUG_DIR", str(tmp_path))
h = await open_qwen()
try:
await h.run(CreateItem(item=image_item()))
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
submission = h.port.submissions[-1]
captures = list(tmp_path.glob("*/input.json"))
assert len(captures) == 1
metadata = json.loads(captures[0].read_text())
assert metadata["request_id"] == submission.context.request_id
assert metadata["audio_count"] == 1
assert metadata["prompt"] == submission.prompt["prompt"]
assert len(metadata["images"]) == 1
with Image.open(captures[0].parent / metadata["images"][0]) as saved:
actual = submission.prompt["multi_modal_data"]["image"][0]
assert saved.size == actual.size
assert saved.tobytes() == actual.tobytes()
finally:
await h.manager.shutdown()