Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
eb1abbadf8
commit
882bf8bc95
53 files changed
+3438
-633
No files matched your search
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
EnvironmentVariableCategory.PUBLIC_OMNI: 28,
|
||||
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 64,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 65,
|
||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
|
||||
EnvironmentVariableCategory.INTERNAL: 2,
|
||||
}
|
||||
@@ -181,7 +181,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
ModelEnvironmentVariableDisposition.PROMOTE: 38,
|
||||
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
|
||||
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
|
||||
ModelEnvironmentVariableDisposition.INTERNALIZE: 15,
|
||||
ModelEnvironmentVariableDisposition.INTERNALIZE: 16,
|
||||
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
|
||||
}
|
||||
|
||||
|
||||
@@ -567,10 +567,10 @@ def test_overlap_policy_defaults_and_invalid_values_to_listen_only():
|
||||
assert DuplexSessionConfig.from_event({"session": {"overlap_policy": "bogus"}}).overlap_policy == "listen_only"
|
||||
|
||||
|
||||
def test_capabilities_as_dict_reports_model_native_duplex_constants():
|
||||
def test_capabilities_as_dict_reports_declared_implementation():
|
||||
caps = DuplexCapabilities().as_dict()
|
||||
|
||||
assert caps["implementation_level"] == "model_native_duplex"
|
||||
assert caps["implementation_level"] == "turn_based_duplex"
|
||||
assert caps["input_modes"] == ["append_audio_chunk"]
|
||||
assert caps["supports_kv_lease"] is False
|
||||
assert caps["supports_core_kv_lease"] is False
|
||||
|
||||
@@ -196,7 +196,7 @@ class FakePlugin(DuplexModelPlugin):
|
||||
|
||||
def capabilities(self, *, max_sessions: int) -> DuplexCapabilities:
|
||||
del max_sessions
|
||||
return DuplexCapabilities(supports_input_append=True)
|
||||
return DuplexCapabilities(supports_input_append=True, supports_core_resumable_request=True)
|
||||
|
||||
def validate_client_extra_body(self, extra_body: object) -> None:
|
||||
pass
|
||||
@@ -408,7 +408,9 @@ async def test_open_answers_with_capabilities_and_emits_session_created() -> Non
|
||||
assert result.control_id == "open-sid-open"
|
||||
assert result.session_id == "sid-open"
|
||||
assert result.lease_generation == 0
|
||||
assert result.capabilities == DuplexCapabilities(supports_input_append=True)
|
||||
assert result.capabilities == DuplexCapabilities(
|
||||
supports_input_append=True, supports_core_resumable_request=True
|
||||
)
|
||||
assert result.public_session is not None
|
||||
assert result.public_session["id"] == "sid-open"
|
||||
assert result.public_session["voice"] == "test"
|
||||
|
||||
@@ -107,8 +107,8 @@ def test_endpoint_decisions_match_the_serving_side_policy(
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_a_loud_frame_does_not_restart_the_silence_timer() -> None:
|
||||
"""Silero v6.2 hysteresis: once silence is running, a loud frame only delays it.
|
||||
def test_hysteresis_band_does_not_restart_the_silence_timer() -> None:
|
||||
"""Scores between the negative and activation thresholds preserve the timer.
|
||||
|
||||
The frame at 0.45 sits above the negative threshold (0.35) but below the
|
||||
activation threshold. It must not cancel the pending endpoint, or a speaker
|
||||
@@ -164,9 +164,14 @@ def _pcm16_b64(samples: np.ndarray) -> str:
|
||||
|
||||
def test_pcm16_input_is_accepted_alongside_float32() -> None:
|
||||
config = SileroVADConfig(threshold=0.5)
|
||||
tone = np.zeros(FRAME * 2, dtype=np.float32)
|
||||
tone: np.ndarray = np.zeros(FRAME * 2, dtype=np.float32)
|
||||
seen: list[int] = []
|
||||
vad = SileroStreamingVAD(config, frame_scorer=lambda frame: seen.append(frame.size) or 0.0)
|
||||
|
||||
def _score_frame(frame: np.ndarray) -> float:
|
||||
seen.append(frame.size)
|
||||
return 0.0
|
||||
|
||||
vad = SileroStreamingVAD(config, frame_scorer=_score_frame)
|
||||
vad.process_base64(_pcm16_b64(tone), fmt="pcm16", sample_rate_hz=SAMPLE_RATE_HZ)
|
||||
assert seen == [FRAME, FRAME]
|
||||
|
||||
@@ -272,7 +277,7 @@ def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tm
|
||||
intra_op_num_threads = 0
|
||||
|
||||
class FakeInferenceSession:
|
||||
def __init__(self, path: str, *, providers: list[str], sess_options: object) -> None:
|
||||
def __init__(self, path: str, *, providers: list[str], sess_options: FakeSessionOptions) -> None:
|
||||
self.providers = providers
|
||||
self.sess_options = sess_options
|
||||
self.calls: list[dict[str, np.ndarray]] = []
|
||||
@@ -313,7 +318,7 @@ def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tm
|
||||
assert session.calls[0]["sr"].item() == SAMPLE_RATE_HZ
|
||||
|
||||
state = backend.new_state()
|
||||
first_frame = np.arange(backend.frame_samples, dtype=np.float32)
|
||||
first_frame: np.ndarray = np.arange(backend.frame_samples, dtype=np.float32)
|
||||
probability, state = backend.infer(first_frame, state)
|
||||
first_call = session.calls[1]
|
||||
|
||||
@@ -377,3 +382,17 @@ def test_the_documented_server_vad_fields_are_all_accepted() -> None:
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("speech_probability", [0.5, 0.9])
|
||||
def test_resumed_speech_cancels_pending_silence(speech_probability):
|
||||
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=96, min_speech_duration_ms=32)
|
||||
# 32 ms pause, resumed speech, then a full 96 ms pause: one utterance.
|
||||
scores = [0.9, 0.1, speech_probability, speech_probability, 0.1, 0.1, 0.1]
|
||||
assert _drive(config, scores) == [("start", 0), ("stop", 224)]
|
||||
|
||||
|
||||
def test_short_pauses_do_not_split_continuous_speech_into_repeated_turns():
|
||||
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=500, min_speech_duration_ms=32)
|
||||
scores = [0.9] + [0.1, 0.9, 0.9, 0.9] * 20
|
||||
assert _drive(config, scores) == [("start", 0)]
|
||||
@@ -515,3 +515,48 @@ async def test_a_dead_replica_closes_the_sessions_it_was_serving() -> None:
|
||||
messages = [output_q.get_nowait() for _ in range(output_q.qsize())]
|
||||
types = [getattr(getattr(m, "event", None), "type", type(m).__name__) for m in messages]
|
||||
assert types[-1] in {"session.expired", "session.closed"}, f"the client needs a terminal event, got {types}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_turn_plugin_processes_multimodal_prompt_before_stage_submission(monkeypatch):
|
||||
from vllm_omni.engine.orchestrator import build_engine_core_request_from_tokens
|
||||
from vllm_omni.model_executor.models.qwen3_omni.duplex.plugin import Qwen3OmniDuplexPlugin
|
||||
|
||||
orchestrator, clients, rpc_q, _ = _build()
|
||||
plugin = Qwen3OmniDuplexPlugin(_encode_audio)
|
||||
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: "audio prompt")
|
||||
orchestrator.plugin = orchestrator.session_manager.plugin = plugin
|
||||
seen = []
|
||||
|
||||
def process_inputs(**kwargs):
|
||||
seen.append(kwargs)
|
||||
return build_engine_core_request_from_tokens(
|
||||
request_id=kwargs["request_id"],
|
||||
prompt={"prompt_token_ids": [1, 2]},
|
||||
params=kwargs["params"],
|
||||
model_config=orchestrator.stage_pools[0].stage_vllm_config.model_config,
|
||||
resumable=kwargs["resumable"],
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
orchestrator, "_get_stage_input_processor", lambda stage_id: SimpleNamespace(process_inputs=process_inputs)
|
||||
)
|
||||
await orchestrator._dispatch_message(
|
||||
OpenDuplexSessionMessage(
|
||||
control_id="open-qwen",
|
||||
session_id=SESSION_ID,
|
||||
session_config=DuplexSessionConfig(model="qwen", modalities=["text", "audio"]),
|
||||
)
|
||||
)
|
||||
assert (await rpc_q.get()).ok
|
||||
try:
|
||||
await _submit(orchestrator, _append_audio())
|
||||
assert not clients[0].add_request_calls
|
||||
await _submit(orchestrator, commands.Commit(final=True, create_response=True))
|
||||
assert len(seen) == 1
|
||||
assert seen[0]["prompt"]["multi_modal_data"]["audio"][0][0].shape == (16000,)
|
||||
submitted = clients[0].add_request_calls[0][0]
|
||||
assert submitted.resumable is False
|
||||
assert not orchestrator.request_states[submitted.request_id].streaming.enabled
|
||||
finally:
|
||||
await _close(orchestrator, rpc_q)
|
||||
@@ -47,6 +47,7 @@ from argparse import Namespace
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, HTTPException
|
||||
@@ -564,7 +565,7 @@ async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch
|
||||
Fails if HTTP requests lose ``request_timestamp``, or WebSocket scopes
|
||||
start getting stamped / mutated incorrectly.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_build_openai_app(args, supported_tasks):
|
||||
return FastAPI()
|
||||
@@ -592,8 +593,8 @@ async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch
|
||||
await api_server.omni_run_server_worker("127.0.0.1:8000", _FakeSocket(), _minimal_args())
|
||||
middleware = captured["served_app"]
|
||||
|
||||
http_scope = {"type": "http", "state": {}}
|
||||
ws_scope = {"type": "websocket"}
|
||||
http_scope: dict[str, Any] = {"type": "http", "state": {}}
|
||||
ws_scope: dict[str, Any] = {"type": "websocket"}
|
||||
|
||||
async def _receive():
|
||||
return {"type": "http.disconnect"}
|
||||
@@ -920,11 +921,10 @@ async def test_pure_diffusion_app_state_key_snapshot(monkeypatch) -> None:
|
||||
engine = _FakeEngineClient(stage_configs=[stage])
|
||||
|
||||
def _for_diffusion_factory(label: str):
|
||||
@classmethod
|
||||
def _factory(cls, *args, **kwargs):
|
||||
return _marker(label)
|
||||
|
||||
return _factory
|
||||
return classmethod(_factory)
|
||||
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingChat, "for_diffusion", _for_diffusion_factory("chat"))
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingChatBatch, "for_diffusion", _for_diffusion_factory("chat_batch"))
|
||||
@@ -965,13 +965,11 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
|
||||
speech_kwargs = {}
|
||||
|
||||
def _for_diffusion_factory(label: str):
|
||||
@classmethod
|
||||
def _factory(cls, *args, **kwargs):
|
||||
return _marker(label)
|
||||
|
||||
return _factory
|
||||
return classmethod(_factory)
|
||||
|
||||
@classmethod
|
||||
def _speech_factory(cls, *args, **kwargs):
|
||||
speech_kwargs.update(kwargs)
|
||||
return _marker("speech")
|
||||
@@ -985,7 +983,7 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
|
||||
)
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingVideo, "for_diffusion", _for_diffusion_factory("video"))
|
||||
monkeypatch.setattr(api_server.OmniStreamingVideoOutputHandler, "__init__", lambda self, *a, **k: None)
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", _speech_factory)
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", classmethod(_speech_factory))
|
||||
monkeypatch.setattr(
|
||||
api_server.ServingRealtimeRobotOpenPI,
|
||||
"create_policy_server",
|
||||
|
||||
@@ -490,3 +490,31 @@ async def test_a_model_that_does_not_declare_chat_completions_does_not_get_the_r
|
||||
|
||||
assert state.openai_serving_chat is None
|
||||
assert isinstance(state.openai_serving_duplex, OmniDuplexSessionHandler)
|
||||
|
||||
|
||||
def test_qwen_plugin_preserves_custom_turn_deployment_default(monkeypatch, tmp_path):
|
||||
from vllm_omni.model_executor.models.qwen3_omni.pipeline import QWEN3_OMNI_PIPELINE
|
||||
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **kwargs: QWEN3_OMNI_PIPELINE)
|
||||
deploy = tmp_path / "qwen.yaml"
|
||||
deploy.write_text("stages: []\n")
|
||||
assert not api_server._should_serve_duplex("qwen", {"deploy_config": str(deploy)})
|
||||
deploy.write_text("session_mode: duplex\nstages: []\n")
|
||||
assert api_server._should_serve_duplex("qwen", {"deploy_config": str(deploy)})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["1", "true", "on"])
|
||||
def test_turn_deployment_rejects_explicit_duplex_without_falling_back_to_stt(flag: str) -> None:
|
||||
app = _duplex_app(None)
|
||||
# Even an available STT service must not silently accept a duplex request.
|
||||
app.state.openai_serving_realtime = object()
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect(f"/v1/realtime?duplex={flag}") as websocket:
|
||||
assert websocket.receive_json() == {
|
||||
"type": "error",
|
||||
"code": "unsupported",
|
||||
"error": "VAD realtime is not enabled",
|
||||
}
|
||||
with pytest.raises(WebSocketDisconnect) as exc:
|
||||
websocket.receive_text()
|
||||
assert exc.value.code == 1008
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import sys
|
||||
@@ -70,7 +73,12 @@ def test_build_app_injects_public_realtime_url():
|
||||
def test_build_app_versions_client_bundle_and_disables_index_cache():
|
||||
client = TestClient(build_app())
|
||||
expected_version_hash = hashlib.sha256()
|
||||
for asset_name in ("app.js", "pcm_worklet.js", "playback_worklet.js"):
|
||||
for asset_name in (
|
||||
"app.js",
|
||||
"pcm_worklet.js",
|
||||
"playback_worklet.js",
|
||||
*[str(p.relative_to(STATIC_DIR)) for p in sorted((STATIC_DIR / "profiles").glob("*.js"))],
|
||||
):
|
||||
expected_version_hash.update((STATIC_DIR / asset_name).read_bytes())
|
||||
expected_version = expected_version_hash.hexdigest()[:12]
|
||||
|
||||
@@ -97,3 +105,41 @@ def test_build_app_exposes_realtime_websocket_and_static_assets():
|
||||
assert client.get("/static/app.js").status_code == 200
|
||||
assert client.get("/static/pcm_worklet.js").status_code == 200
|
||||
assert client.get("/static/playback_worklet.js").status_code == 200
|
||||
|
||||
|
||||
def test_qwen_config_does_not_inject_reference_voice():
|
||||
client = TestClient(build_app(profile="qwen3-turn", adapter="vad", model="Qwen/Qwen3-Omni-30B-A3B-Instruct"))
|
||||
page = client.get("/").text
|
||||
assert '"profile": "qwen3-turn"' in page
|
||||
assert '"adapter": "vad"' in page
|
||||
assert '"refAudio": null' in page
|
||||
for name in ("common", "minicpm_native", "qwen3_turn"):
|
||||
assert client.get(f"/static/profiles/{name}.js").status_code == 200
|
||||
with pytest.raises(ValueError, match="ref-audio"):
|
||||
build_app(profile="qwen3-turn", ref_audio="unused.wav")
|
||||
|
||||
|
||||
def test_config_cannot_close_its_script_element():
|
||||
page = TestClient(build_app(model="</script><script>alert(1)</script>")).get("/").text
|
||||
assert "</script><script>alert(1)" not in page
|
||||
assert "\\u003c/script>" in page
|
||||
|
||||
|
||||
def test_proxy_reports_unreachable_backend(monkeypatch):
|
||||
from examples.online_serving.realtime_web import server as shared_server
|
||||
|
||||
class RefusedConnection:
|
||||
async def __aenter__(self):
|
||||
raise ConnectionRefusedError(111, "Connection refused")
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(shared_server.websockets, "connect", lambda *args, **kwargs: RefusedConnection())
|
||||
client = TestClient(build_app(ws_backend="ws://127.0.0.1:8091"))
|
||||
with client.websocket_connect("/v1/realtime?duplex=0") as ws:
|
||||
event = ws.receive_json()
|
||||
assert event["type"] == "error"
|
||||
assert event["code"] == "backend_unavailable"
|
||||
assert "ws://127.0.0.1:8091" in event["error"]
|
||||
assert "--port" in event["error"]
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import textwrap
|
||||
@@ -9,7 +8,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2] / "examples" / "online_serving" / "minicpmo" / "realtime_web"
|
||||
ROOT = Path(__file__).resolve().parents[2] / "examples" / "online_serving" / "realtime_web"
|
||||
APP_ROOT = ROOT / "app"
|
||||
STATIC_ROOT = APP_ROOT / "static"
|
||||
|
||||
@@ -30,73 +29,37 @@ def test_page_exposes_focused_call_conversation_and_log_surfaces():
|
||||
assert "Server VAD" not in html
|
||||
|
||||
|
||||
def test_client_uses_proxy_relative_realtime_url_and_model_policy_session():
|
||||
def test_shared_shell_delegates_wire_contract_to_profiles():
|
||||
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
|
||||
assert "profile.url(config, window.location.href)" in source
|
||||
assert "profile.initialMessages(config, instructions)" in source
|
||||
assert "profile.append(" in source
|
||||
assert "profile.mapEvent(event)" in source
|
||||
assert "native_duplex" not in source
|
||||
assert "server_vad" not in source
|
||||
|
||||
assert "new URL(config.realtimePath, window.location.href)" in source
|
||||
assert "url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'" in source
|
||||
assert "url.searchParams.set('autostart', '0')" in source
|
||||
|
||||
def test_native_profile_preserves_session_and_capture_policy():
|
||||
source = (STATIC_ROOT / "profiles/minicpm_native.js").read_text(encoding="utf-8")
|
||||
assert "native_duplex" not in source
|
||||
assert "auto_response: true" in source
|
||||
assert "input_audio_buffer.append" in source
|
||||
assert "input_audio_buffer.commit" not in source
|
||||
assert "playback.ack" in source
|
||||
assert "event.event || event" in source
|
||||
assert "response.output_audio.delta" in source
|
||||
assert "response.output_audio_transcript.delta" in source
|
||||
assert "conversation.item.input_audio_transcription" in source
|
||||
assert "force_barge_in" not in source
|
||||
assert "server_vad" not in source
|
||||
assert "type: 'response.create'" not in source
|
||||
assert 'type: "response.create"' not in source
|
||||
|
||||
|
||||
def test_web_server_requires_ref_audio_for_audio_output_session():
|
||||
source = (ROOT / "server.py").read_text(encoding="utf-8")
|
||||
|
||||
ref_audio_arg = re.search(
|
||||
r"parser\.add_argument\(\s*\"--ref-audio\",(?P<body>.*?)\n\s*\)",
|
||||
source,
|
||||
re.DOTALL,
|
||||
)
|
||||
assert ref_audio_arg is not None
|
||||
assert "required=True" in ref_audio_arg.group("body")
|
||||
assert "Optional reference voice" not in ref_audio_arg.group("body")
|
||||
|
||||
|
||||
def test_client_sends_ref_audio_in_realtime_session_contract():
|
||||
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
|
||||
|
||||
assert "if (config.refAudio) session.ref_audio = config.refAudio;" in source
|
||||
assert "extraBody.ref_audio" not in source
|
||||
assert "halfDuplex: false" in source
|
||||
assert "commitMessages: () => []" in source
|
||||
assert "video_frames" in source
|
||||
assert "playback.ack" in source
|
||||
|
||||
|
||||
def test_client_has_transactional_cleanup_and_visible_event_logging():
|
||||
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
|
||||
|
||||
assert "async function stopSession" in source
|
||||
assert "type: 'session.close'" in source
|
||||
assert "async function cleanupSession" in source
|
||||
assert "waitForSessionClosed" in source
|
||||
assert "SESSION_CLOSE_TIMEOUT_MS" in source
|
||||
assert "case 'session.closed':" in source
|
||||
assert "track.stop()" in source
|
||||
assert "clearInterval(sendTimer)" in source
|
||||
assert "appendEventLog(event)" in source
|
||||
assert "stopSession({ terminal: false })" in source
|
||||
|
||||
|
||||
def test_client_keeps_microphone_upload_active_during_assistant_playback():
|
||||
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
|
||||
upload_gate = re.search(r"function microphoneUploadEnabled\(\) \{(?P<body>.*?)\n \}", source, re.DOTALL)
|
||||
begin_assistant = re.search(r"function beginAssistant\(responseId\) \{(?P<body>.*?)\n \}", source, re.DOTALL)
|
||||
|
||||
assert upload_gate is not None
|
||||
assert "return running && !muted;" in upload_gate.group("body")
|
||||
assert "assistantActive" not in upload_gate.group("body")
|
||||
assert begin_assistant is not None
|
||||
assert "pendingCapture = []" not in begin_assistant.group("body")
|
||||
|
||||
|
||||
def test_audio_worklets_define_capture_and_playback_processors():
|
||||
capture = (STATIC_ROOT / "pcm_worklet.js").read_text(encoding="utf-8")
|
||||
playback = (STATIC_ROOT / "playback_worklet.js").read_text(encoding="utf-8")
|
||||
@@ -138,6 +101,7 @@ def test_playback_worklet_waits_before_playing_and_rebuffers_after_underrun():
|
||||
node = shutil.which("node")
|
||||
if node is None:
|
||||
pytest.skip("node is required for the AudioWorklet regression test")
|
||||
assert node is not None
|
||||
|
||||
script = textwrap.dedent(
|
||||
"""
|
||||
@@ -216,6 +180,7 @@ def test_playback_worklet_fades_terminal_drain_to_zero():
|
||||
node = shutil.which("node")
|
||||
if node is None:
|
||||
pytest.skip("node is required for the AudioWorklet regression test")
|
||||
assert node is not None
|
||||
|
||||
script = textwrap.dedent(
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const root = path.resolve(__dirname, '../../examples/online_serving/realtime_web/app/static');
|
||||
const context = vm.createContext({ URL });
|
||||
for (const name of ['common', 'minicpm_native', 'qwen3_turn']) {
|
||||
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), context);
|
||||
}
|
||||
const profiles = context.OmniRealtimeProfiles;
|
||||
const plain = (value) => JSON.parse(JSON.stringify(value));
|
||||
const config = { model: 'Qwen/Qwen3-Omni-30B-A3B-Instruct', realtimePath: 'v1/realtime' };
|
||||
|
||||
test('MiniCPM keeps native flags, reference voice, camera, acknowledgements and no commits', () => {
|
||||
const native = profiles['minicpm-native']();
|
||||
const url = new URL(native.url({ ...config, model: 'MiniCPM' }, 'https://localhost/ui/'));
|
||||
assert.equal(url.protocol, 'wss:');
|
||||
assert.equal(url.searchParams.get('duplex'), '1');
|
||||
assert.equal(url.searchParams.has('native_duplex'), false);
|
||||
const [update] = native.initialMessages({ refAudio: 'data:audio/wav;base64,AA==' }, 'Prompt');
|
||||
assert.equal(update.session.ref_audio, 'data:audio/wav;base64,AA==');
|
||||
assert.equal(update.session.extra_body.native_duplex, undefined);
|
||||
assert.equal(update.session.instructions, 'Prompt');
|
||||
assert.deepEqual(plain(native.commitMessages()), []);
|
||||
assert.equal(native.append('PCM', 'JPEG').video_frames[0], 'JPEG');
|
||||
assert.equal(native.ack('r1', 200).type, 'playback.ack');
|
||||
assert.equal(native.halfDuplex, false);
|
||||
});
|
||||
|
||||
test('STT uses the shipped commit sequence and audio payload, including each new turn', () => {
|
||||
const stt = profiles['qwen3-turn'](config);
|
||||
for (let turn = 0; turn < 2; turn++) {
|
||||
assert.deepEqual(plain(stt.initialMessages(config, 'ignored')), [
|
||||
{ type: 'session.update', model: config.model },
|
||||
{ type: 'input_audio_buffer.commit', final: false },
|
||||
]);
|
||||
assert.deepEqual(plain(stt.commitMessages()), [{ type: 'input_audio_buffer.commit', final: true }]);
|
||||
}
|
||||
assert.equal(stt.reconnectEachTurn, true);
|
||||
assert.equal(stt.mapEvent({ type: 'response.output_audio.delta', audio: 'PCM' }).event.delta, 'PCM');
|
||||
assert.equal(stt.mapEvent({ type: 'response.output_audio.done' }).kind, 'done');
|
||||
assert.equal(stt.mapEvent({ type: 'transcription.delta', delta: 'hello' }).role, 'assistant');
|
||||
});
|
||||
|
||||
test('Qwen enables camera and playback ACK only for the duplex VAD profile', () => {
|
||||
for (const adapter of ['stt', 'vad']) {
|
||||
const p = profiles['qwen3-turn']({ ...config, adapter });
|
||||
const url = new URL(p.url({ ...config, realtimePath: 'wss://backend/v1/realtime?native_duplex=1&minicpmo45_native_duplex=1' }, 'http://localhost/'));
|
||||
assert.equal(url.protocol, 'wss:');
|
||||
assert.equal(url.searchParams.get('duplex'), adapter === 'vad' ? '1' : '0');
|
||||
assert.equal(url.searchParams.has('native_duplex'), false);
|
||||
assert.equal(url.searchParams.has('minicpmo45_native_duplex'), false);
|
||||
assert.equal(p.append('PCM', 'JPEG').video_frames, undefined);
|
||||
assert.equal(plain(p.imageMessages('JPEG')).length, adapter === 'vad' ? 1 : 0);
|
||||
assert.equal(p.ack('r', 100)?.type || null, adapter === 'vad' ? 'playback.ack' : null);
|
||||
assert.equal(p.camera, adapter === 'vad');
|
||||
assert.equal(p.mapEvent({ type: 'response.listen' }).kind, 'ignore');
|
||||
}
|
||||
});
|
||||
|
||||
test('VAD uses nested format and interruptible endpoint detection', () => {
|
||||
const vad = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
|
||||
const [update] = vad.initialMessages(config, 'help');
|
||||
assert.deepEqual(plain(update.session.audio.input), {
|
||||
format: { type: 'audio/pcm', rate: 24000 },
|
||||
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
|
||||
silence_duration_ms: 500, create_response: true, interrupt_response: true },
|
||||
});
|
||||
assert.equal(update.session.extra_body, undefined);
|
||||
assert.deepEqual(plain(vad.commitMessages()), []);
|
||||
assert.equal(vad.mapEvent({ type: 'response.audio.done' }).kind, 'drain');
|
||||
assert.equal(vad.mapEvent({ type: 'response.done' }).kind, 'done');
|
||||
assert.equal(vad.mapEvent({ type: 'response.output_text.delta', delta: 'hello' }).text, 'hello');
|
||||
assert.equal(vad.mapEvent({ type: 'error', error: { code: 'input_backpressure' } }).kind, 'backpressure');
|
||||
assert.equal(vad.mapEvent({ type: 'input_audio_buffer.cleared' }).kind, 'backpressure');
|
||||
assert.match(vad.mapEvent({ type: 'error', code: 'unsupported', error: 'Unavailable' }).message, /session_mode: duplex/);
|
||||
assert.match(vad.mapEvent({ type: 'error', code: 'server_vad_initialization_failed' }).message, /Silero/);
|
||||
});
|
||||
|
||||
function shell(profileName, adapter = 'stt', options = {}) {
|
||||
const elements = new Map();
|
||||
class Element {
|
||||
constructor() { this.style = {}; this.listeners = {}; this.children = []; this.value = ''; this.textContent = ''; this.classList = { add() {}, remove() {}, toggle() {} }; }
|
||||
addEventListener(name, callback) { this.listeners[name] = callback; }
|
||||
append(...children) { this.children.push(...children); }
|
||||
appendChild(child) { this.children.push(child); }
|
||||
replaceChildren() { this.children = []; }
|
||||
remove() {}
|
||||
}
|
||||
const document = {
|
||||
getElementById(id) { if (!elements.has(id)) elements.set(id, new Element()); return elements.get(id); },
|
||||
createElement() { return new Element(); },
|
||||
};
|
||||
const timers = new Map();
|
||||
let timerId = 0;
|
||||
const sockets = [];
|
||||
const nodes = [];
|
||||
class Socket {
|
||||
static OPEN = 1;
|
||||
constructor() {
|
||||
this.readyState = 1; this.sent = []; sockets.push(this);
|
||||
queueMicrotask(() => {
|
||||
this.onopen?.();
|
||||
if (!options.silent) this.onmessage?.({ data: JSON.stringify(options.error || { type: adapter === 'stt' && profileName === 'qwen3-turn' ? 'session.created' : 'session.updated' }) });
|
||||
});
|
||||
}
|
||||
send(value) { this.sent.push(JSON.parse(value)); }
|
||||
close() { this.readyState = 3; this.onclose?.({ code: 1000 }); }
|
||||
}
|
||||
class AudioContext {
|
||||
constructor(options) { this.sampleRate = options.sampleRate; this.audioWorklet = { addModule: async () => {} }; this.destination = {}; }
|
||||
createMediaStreamSource() { return { connect() {} }; }
|
||||
createGain() { return { gain: {}, connect() {} }; }
|
||||
async resume() {}
|
||||
async close() {}
|
||||
}
|
||||
class AudioWorkletNode {
|
||||
constructor(_context, name) { this.name = name; this.sent = []; nodes.push(this); this.port = { postMessage: (message) => this.sent.push(message) }; }
|
||||
connect(target) { return target; }
|
||||
}
|
||||
const ctx = vm.createContext({
|
||||
URL, document, AudioContext, AudioWorkletNode, WebSocket: Socket,
|
||||
navigator: { mediaDevices: { getUserMedia: async () => ({ getTracks: () => [{ stop() {} }] }) } },
|
||||
location: { href: 'https://localhost/' },
|
||||
OMNI_REALTIME_CONFIG: { ...config, profile: profileName, adapter },
|
||||
addEventListener() {},
|
||||
setTimeout(fn, ms) { const id = ++timerId; timers.set(id, { fn, ms }); return id; },
|
||||
clearTimeout(id) { timers.delete(id); },
|
||||
setInterval() { return ++timerId; }, clearInterval() {},
|
||||
btoa: (value) => Buffer.from(value, 'binary').toString('base64'),
|
||||
atob: (value) => Buffer.from(value, 'base64').toString('binary'),
|
||||
});
|
||||
ctx.window = ctx;
|
||||
for (const name of ['common', 'minicpm_native', 'qwen3_turn']) {
|
||||
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), ctx);
|
||||
}
|
||||
// Expose closure controls only in the test VM; production has no test API.
|
||||
const source = fs.readFileSync(path.join(root, 'app.js'), 'utf8').replace(/\}\)\(\);\s*$/, `
|
||||
globalThis.testUI = { startSession, stopSession, handleEvent, playbackDrained,
|
||||
microphoneUploadEnabled, flushCapture,
|
||||
capture() { pendingCapture.push(new Int16Array([100, 200])); },
|
||||
state() { return { running, connectionReady, assistantActive }; }
|
||||
};
|
||||
})();`);
|
||||
vm.runInContext(source, ctx);
|
||||
return { ui: ctx.testUI, elements, sockets, nodes,
|
||||
timeout() { for (const timer of timers.values()) if (timer.ms === 15000) timer.fn(); },
|
||||
async echo() {
|
||||
for (const [id, timer] of timers) if (timer.ms === 300) { timers.delete(id); await timer.fn(); }
|
||||
},
|
||||
async turnWatchdog() {
|
||||
for (const [id, timer] of timers) if (timer.ms === 120000) { timers.delete(id); await timer.fn(); }
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('shared shell STT sends final commit and starts a fresh second turn only after playback', async () => {
|
||||
const app = shell('qwen3-turn');
|
||||
await app.ui.startSession();
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
||||
app.ui.capture(); app.ui.flushCapture();
|
||||
app.elements.get('sendTurnButton').listeners.click();
|
||||
assert.equal(app.sockets[0].sent.at(-1).final, true);
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), false);
|
||||
await app.ui.handleEvent({ type: 'response.output_audio.delta', audio: 'AAAAAA==' });
|
||||
await app.ui.handleEvent({ type: 'response.output_audio.done' });
|
||||
await app.echo();
|
||||
assert.equal(app.sockets.length, 1, 'wait for actual speaker drain');
|
||||
app.ui.playbackDrained({ responseId: 'turn-1', playedMs: 1 });
|
||||
await app.echo();
|
||||
assert.equal(app.sockets.length, 2);
|
||||
assert.equal(app.sockets[1].sent[1].final, false);
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
||||
assert.equal(app.elements.get('cameraButton').hidden, true);
|
||||
assert.equal(app.sockets.flatMap(s => s.sent).some(e => e.type === 'playback.ack'), false);
|
||||
await app.ui.stopSession({ terminal: false });
|
||||
});
|
||||
|
||||
test('VAD keeps uploading while generating and waits for response.done after audio drain', async () => {
|
||||
const app = shell('qwen3-turn', 'vad');
|
||||
await app.ui.startSession();
|
||||
await app.ui.handleEvent({ type: 'error', code: 'input_backpressure' });
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
||||
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
|
||||
await app.ui.handleEvent({ type: 'response.audio.delta', delta: 'AAAAAA==', response_id: 'r1' });
|
||||
await app.ui.handleEvent({ type: 'response.audio.done', response_id: 'r1' });
|
||||
app.ui.playbackDrained({ responseId: 'r1', playedMs: 1 });
|
||||
await app.echo();
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), true, 'continuous capture remains enabled');
|
||||
assert.equal(app.ui.state().assistantActive, true, 'audio drain must wait for response.done');
|
||||
assert.equal(app.sockets[0].sent.at(-1).type, 'playback.ack');
|
||||
await app.ui.handleEvent({ type: 'response.done', response: { id: 'r1' } });
|
||||
await app.echo();
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
||||
assert.equal(app.sockets.length, 1, 'VAD keeps its session');
|
||||
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.commit'), false);
|
||||
await app.ui.stopSession({ terminal: false });
|
||||
});
|
||||
|
||||
test('a rejected operation is reported without hanging up the call', async () => {
|
||||
const app = shell('qwen3-turn', 'vad');
|
||||
await app.ui.startSession();
|
||||
await app.ui.handleEvent({ type: 'error', code: 'invalid_image', error: 'input_image requires a JPEG or PNG base64 data URL' });
|
||||
assert.equal(app.ui.state().running, true, 'one refused image must not end the session');
|
||||
assert.equal(app.ui.state().connectionReady, true);
|
||||
assert.match(app.elements.get('runtimeDetail').textContent, /JPEG or PNG/);
|
||||
await app.ui.stopSession({ terminal: false });
|
||||
});
|
||||
|
||||
test('a session-level error still tears the call down', async () => {
|
||||
const app = shell('qwen3-turn', 'vad');
|
||||
await app.ui.startSession();
|
||||
await app.ui.handleEvent({ type: 'error', code: 'unknown_session', error: 'Unknown or closed duplex session' });
|
||||
assert.equal(app.ui.state().running, false);
|
||||
assert.equal(app.elements.get('connectionState').textContent, 'Error');
|
||||
});
|
||||
|
||||
test('an interrupted response disarms the turn watchdog it left armed', async () => {
|
||||
const app = shell('qwen3-turn', 'vad');
|
||||
await app.ui.startSession();
|
||||
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
|
||||
await app.ui.handleEvent({ type: 'error', code: 'input_backpressure' });
|
||||
await app.ui.handleEvent({ type: 'response.done', response: { id: 'r1', status: 'cancelled' } });
|
||||
await app.turnWatchdog();
|
||||
assert.equal(app.ui.state().running, true, 'barge-in must not time the session out two minutes later');
|
||||
await app.ui.stopSession({ terminal: false });
|
||||
});
|
||||
|
||||
test('MiniCPM retains microphone upload while speaking and acknowledges speaker drain', async () => {
|
||||
const app = shell('minicpm-native');
|
||||
await app.ui.startSession();
|
||||
await app.ui.handleEvent({ type: 'response.speak', response_id: 'native-r1' });
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
||||
await app.ui.handleEvent({ type: 'response.output_audio.delta', delta: 'AAAAAA==', response_id: 'native-r1' });
|
||||
app.ui.playbackDrained({ responseId: 'native-r1', playedMs: 100 });
|
||||
assert.equal(app.sockets[0].sent.at(-1).type, 'playback.ack');
|
||||
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.commit'), false);
|
||||
await app.ui.stopSession({ terminal: false });
|
||||
await app.echo();
|
||||
assert.equal(app.ui.state().running, false);
|
||||
});
|
||||
|
||||
test('VAD handshake errors remain visible after resource cleanup', async () => {
|
||||
const app = shell('qwen3-turn', 'vad', { error: { type: 'error', code: 'unsupported', error: 'Realtime API is not available' } });
|
||||
await app.ui.startSession();
|
||||
assert.equal(app.ui.state().running, false);
|
||||
assert.equal(app.elements.get('connectionState').textContent, 'Error');
|
||||
assert.match(app.elements.get('runtimeDetail').textContent, /session_mode: duplex/);
|
||||
assert.equal(app.sockets[0].readyState, 3);
|
||||
});
|
||||
|
||||
test('missing session acknowledgement fails instead of leaving Start disabled forever', async () => {
|
||||
const app = shell('qwen3-turn', 'vad', { silent: true });
|
||||
const starting = app.ui.startSession();
|
||||
for (let i = 0; i < 20; i++) await Promise.resolve();
|
||||
app.timeout();
|
||||
await starting;
|
||||
assert.equal(app.ui.state().running, false);
|
||||
assert.equal(app.elements.get('callButton').disabled, false);
|
||||
assert.match(app.elements.get('runtimeDetail').textContent, /handshake timed out/);
|
||||
});
|
||||
|
||||
test('Qwen does not duplicate text and audio-transcript channels', async () => {
|
||||
const app = shell('qwen3-turn', 'vad');
|
||||
await app.ui.startSession();
|
||||
await app.ui.handleEvent({ type: 'response.audio_transcript.delta', delta: '' });
|
||||
await app.ui.handleEvent({ type: 'response.output_text.delta', delta: 'Hello' });
|
||||
await app.ui.handleEvent({ type: 'response.audio_transcript.delta', delta: 'Hello' });
|
||||
await app.ui.handleEvent({ type: 'response.output_text.done', text: 'Hello' });
|
||||
await app.ui.handleEvent({ type: 'response.done' });
|
||||
const turns = app.elements.get('conversation').children;
|
||||
assert.equal(turns.length, 1);
|
||||
assert.equal(turns[0].children[1].textContent, 'Hello');
|
||||
await app.ui.stopSession({ terminal: false });
|
||||
});
|
||||
|
||||
const flushTasks = async () => { for (let i = 0; i < 40; i++) await Promise.resolve(); };
|
||||
|
||||
test('closing socket accepts only session.closed and completes without the fallback timer', async () => {
|
||||
const app = shell('minicpm-native');
|
||||
await app.ui.startSession();
|
||||
const old = app.sockets[0];
|
||||
let stopped = false;
|
||||
const stopping = app.ui.stopSession().then(() => { stopped = true; });
|
||||
assert.equal(old.sent.at(-1).type, 'session.close');
|
||||
old.onmessage({ data: JSON.stringify({ type: 'session.updated' }) });
|
||||
await flushTasks();
|
||||
assert.equal(stopped, false);
|
||||
old.onmessage({ data: JSON.stringify({ type: 'session.closed' }) });
|
||||
await flushTasks();
|
||||
assert.equal(stopped, true, 'must finish without firing the timeout');
|
||||
await stopping;
|
||||
await app.ui.startSession();
|
||||
old.onmessage({ data: JSON.stringify({ type: 'session.closed' }) });
|
||||
old.onmessage({ data: '{broken' });
|
||||
await flushTasks();
|
||||
assert.equal(app.ui.state().running, true, 'stale socket cannot affect a new session');
|
||||
await app.ui.stopSession({ terminal: false });
|
||||
});
|
||||
|
||||
test('malformed JSON before readiness rejects the handshake and cleans up', async () => {
|
||||
const app = shell('qwen3-turn', 'vad', { silent: true });
|
||||
const starting = app.ui.startSession();
|
||||
await flushTasks();
|
||||
app.sockets[0].onmessage({ data: '{broken' });
|
||||
await starting;
|
||||
assert.equal(app.ui.state().running, false);
|
||||
assert.equal(app.sockets[0].readyState, 3);
|
||||
assert.equal(app.elements.get('connectionState').textContent, 'Error');
|
||||
assert.match(app.elements.get('runtimeDetail').textContent, /Invalid server event/);
|
||||
});
|
||||
|
||||
test('malformed JSON after readiness fails the session and releases resources', async () => {
|
||||
const app = shell('qwen3-turn', 'vad');
|
||||
await app.ui.startSession();
|
||||
app.sockets[0].onmessage({ data: '{broken' });
|
||||
await flushTasks();
|
||||
assert.equal(app.ui.state().running, false);
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), false);
|
||||
assert.equal(app.sockets[0].readyState, 3);
|
||||
assert.equal(app.elements.get('connectionState').textContent, 'Error');
|
||||
assert.match(app.elements.get('runtimeDetail').textContent, /Invalid server event/);
|
||||
});
|
||||
|
||||
|
||||
test('Qwen VAD interruption clears playback while continuing microphone upload', async () => {
|
||||
const app = shell('qwen3-turn', 'vad');
|
||||
await app.ui.startSession();
|
||||
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
|
||||
await app.ui.handleEvent({ type: 'response.audio.delta', delta: 'AAAAAA==', response_id: 'r1' });
|
||||
await app.ui.handleEvent({ type: 'output_audio_buffer.cleared', response_id: 'r1' });
|
||||
assert.equal(app.ui.state().assistantActive, false);
|
||||
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
||||
assert.ok(app.nodes.some(node => node.sent.some(message => message.type === 'clear')));
|
||||
await app.ui.stopSession({ terminal: false });
|
||||
});
|
||||
|
||||
|
||||
test('Qwen runtime errors do not suggest changing a working deployment', () => {
|
||||
const p = profiles['qwen3-turn']({ adapter: 'vad' });
|
||||
const message = 'playback.ack arrived after a later user input was committed.';
|
||||
assert.equal(p.mapEvent({ type: 'error', code: 'playback_ack_too_late', error: message }).message, message);
|
||||
});
|
||||
|
||||
test('interrupted playback reports its cursor before clear resets it', () => {
|
||||
let Playback;
|
||||
const messages = [];
|
||||
const ctx = vm.createContext({
|
||||
sampleRate: 24000,
|
||||
AudioWorkletProcessor: class { constructor() { this.port = { postMessage: m => messages.push(m) }; } },
|
||||
registerProcessor: (_name, cls) => { Playback = cls; },
|
||||
});
|
||||
vm.runInContext(fs.readFileSync(path.join(root, 'playback_worklet.js'), 'utf8'), ctx);
|
||||
const player = new Playback();
|
||||
player.handleMessage({ type: 'audio', responseId: 'old', pcm: new Int16Array(24000), initialBufferMs: 0 });
|
||||
player.process([], [[new Float32Array(2400)]]);
|
||||
player.handleMessage({ type: 'clear' });
|
||||
const report = messages.find(m => m.type === 'playback-stopped');
|
||||
assert.equal(report.responseId, 'old');
|
||||
assert.equal(report.playedMs, 100);
|
||||
assert.equal(player.playedFrames, 0);
|
||||
assert.equal(player.bufferedFrames(), 0);
|
||||
player.handleMessage({ type: 'clear' });
|
||||
assert.equal(messages.filter(m => m.type === 'playback-stopped').length, 1);
|
||||
});
|
||||
|
||||
test('late interruption cursor acknowledges old response without finishing the new response', async () => {
|
||||
const app = shell('qwen3-turn', 'vad');
|
||||
await app.ui.startSession();
|
||||
const player = app.nodes.find(n => n.name === 'fullduplex-pcm-playback');
|
||||
await app.ui.handleEvent({ type: 'response.created', response: { id: 'old' } });
|
||||
await app.ui.handleEvent({ type: 'output_audio_buffer.cleared', response_id: 'old' });
|
||||
await app.ui.handleEvent({ type: 'response.created', response: { id: 'new' } });
|
||||
player.port.onmessage({ data: { type: 'playback-stopped', responseId: 'old', playedMs: 100 } });
|
||||
const ack = app.sockets[0].sent.find(m => m.type === 'playback.ack');
|
||||
assert.equal(ack.response_id, 'old');
|
||||
assert.equal(ack.played_ms, 100);
|
||||
assert.equal(app.ui.state().assistantActive, true);
|
||||
});
|
||||
|
||||
|
||||
test('Qwen camera frames ride the OpenAI image interface, not the audio append', () => {
|
||||
const p = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
|
||||
assert.deepEqual(plain(p.append('PCM', 'JPEG')), { type: 'input_audio_buffer.append', audio: 'PCM' });
|
||||
|
||||
const [created] = plain(p.imageMessages('JPEG'));
|
||||
assert.equal(created.type, 'conversation.item.create');
|
||||
assert.equal(created.item.role, 'user');
|
||||
assert.deepEqual(created.item.content, [
|
||||
{ type: 'input_image', image_url: 'data:image/jpeg;base64,JPEG' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('the camera retires its oldest image instead of exhausting the session budget', () => {
|
||||
const p = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
|
||||
const ids = [];
|
||||
for (let i = 0; i < 8; i++) {
|
||||
const messages = plain(p.imageMessages('JPEG'));
|
||||
assert.equal(messages.length, 1, 'nothing to retire while under the budget');
|
||||
ids.push(messages[0].item.id);
|
||||
}
|
||||
// The engine counts what it already stores, so the delete has to lead.
|
||||
const ninth = plain(p.imageMessages('JPEG'));
|
||||
assert.deepEqual(ninth[0], { type: 'conversation.item.delete', item_id: ids[0] });
|
||||
assert.equal(ninth[1].type, 'conversation.item.create');
|
||||
assert.equal(new Set(ids).size, 8, 'item ids must be distinct to be deletable');
|
||||
|
||||
// A new call reopens the budget; stale ids from the old one must not linger.
|
||||
assert.equal(plain(p.initialMessages(config, 'x')).length > 0, true);
|
||||
assert.equal(plain(p.imageMessages('JPEG')).length, 1);
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_realtime_profiles_and_shell_lifecycle():
|
||||
node = shutil.which("node")
|
||||
if node is None:
|
||||
pytest.skip("node is required for browser protocol tests")
|
||||
assert node is not None
|
||||
subprocess.run(
|
||||
[node, "--test", str(Path(__file__).with_suffix(".cjs"))],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
@@ -0,0 +1,625 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
import asyncio
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.engine.duplex.test_session_runner import (
|
||||
SESSION_ID,
|
||||
Harness,
|
||||
RecordingStagePort,
|
||||
_speech_then_stop,
|
||||
append_audio,
|
||||
tts_output,
|
||||
)
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.engine.duplex.commands import AckPlayback, CancelResponse, Commit, UpdateSession
|
||||
from vllm_omni.engine.duplex.config import DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.messages import OpenDuplexSessionMessage
|
||||
from vllm_omni.engine.duplex.plugin import DuplexRuntimeConfigError
|
||||
from vllm_omni.engine.duplex.session.manager import DuplexSessionManager
|
||||
from vllm_omni.model_executor.models.qwen3_omni.duplex.input import QwenPcmBuffer
|
||||
from vllm_omni.model_executor.models.qwen3_omni.duplex.plugin import MAX_PROMPT_IMAGES, Qwen3OmniDuplexPlugin
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_buffer_commit_and_rollback_preserve_all_samples():
|
||||
buffer = QwenPcmBuffer()
|
||||
raw = np.ones(101, dtype="<f4").tobytes()
|
||||
payload = {"audio": base64.b64encode(raw).decode(), "format": "pcm_f32le", "sample_rate_hz": 16000}
|
||||
assert buffer.prepare_append(payload, operation_id="a", chunk_period_ms=1000, allow_emit=True) is None
|
||||
reservation = buffer.prepare_commit(operation_id="c", chunk_period_ms=1000)
|
||||
assert base64.b64decode(reservation.payload["audio"]) == raw
|
||||
reservation.rollback()
|
||||
assert buffer.pending_byte_count == len(raw)
|
||||
final = buffer.prepare_commit(operation_id="d", chunk_period_ms=1000)
|
||||
final.commit()
|
||||
assert not buffer.has_pending() and not buffer.has_reserved()
|
||||
|
||||
|
||||
def test_buffer_refuses_video_frames_and_points_at_the_openai_interface():
|
||||
buffer = QwenPcmBuffer()
|
||||
payload = {
|
||||
"audio": base64.b64encode(np.ones(16, dtype="<f4").tobytes()).decode(),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
"video_frames": ["a-frame"],
|
||||
}
|
||||
with pytest.raises(ValueError, match="input_image"):
|
||||
buffer.prepare_append(payload, operation_id="first", chunk_period_ms=200, allow_emit=True)
|
||||
# The refusal must not swallow the audio it was sent with.
|
||||
assert not buffer.has_pending()
|
||||
|
||||
del payload["video_frames"]
|
||||
buffer.prepare_append(payload, operation_id="second", chunk_period_ms=200, allow_emit=True)
|
||||
committed = buffer.prepare_commit(operation_id="second-commit", chunk_period_ms=200)
|
||||
assert "video_frames" not in committed.payload
|
||||
committed.commit()
|
||||
|
||||
|
||||
async def open_qwen():
|
||||
plugin = Qwen3OmniDuplexPlugin(lambda audio, *args: "AAAA")
|
||||
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kw: repr(messages))
|
||||
port = RecordingStagePort(stage_count=3)
|
||||
output: asyncio.Queue[Any] = asyncio.Queue()
|
||||
results: asyncio.Queue[Any] = asyncio.Queue()
|
||||
manager = DuplexSessionManager(
|
||||
plugin=plugin,
|
||||
stage_port=port,
|
||||
output_sink=output,
|
||||
result_sink=results,
|
||||
runtime_config=DuplexSessionRuntimeConfig(),
|
||||
model_config=None,
|
||||
)
|
||||
config = DuplexSessionConfig(model="qwen", modalities=["text", "audio"], overlap_policy="barge_in_on_speech")
|
||||
await manager.handle(OpenDuplexSessionMessage(control_id="open", session_id=SESSION_ID, session_config=config))
|
||||
result = await results.get()
|
||||
assert result.ok, result
|
||||
harness = Harness(manager, port, output, results, manager.runners[SESSION_ID])
|
||||
await harness.settle()
|
||||
return harness
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_committed_turns_stream_text_and_audio_and_release_requests():
|
||||
h = await open_qwen()
|
||||
try:
|
||||
ids = []
|
||||
for i in range(2):
|
||||
await h.run(append_audio())
|
||||
assert len(h.port.submissions) == i
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
assert len(h.port.submissions) == i + 1, [e.to_realtime() for e in h.events]
|
||||
request_id = h.port.submissions[-1].context.request_id
|
||||
ids.append(request_id)
|
||||
assert h.session.active_request_id == request_id
|
||||
assert len(h.port.submissions[-1].prompt["multi_modal_data"]["audio"]) == i + 1
|
||||
text = OmniRequestOutput.from_stage_output(
|
||||
tts_output(request_id, samples=0, text="hello", finished=True),
|
||||
request_id=request_id,
|
||||
stage_id=None,
|
||||
)
|
||||
assert not h.deliver(text, stage_id=0), "Thinker output must still forward to Talker"
|
||||
await h.settle()
|
||||
assert text.finished, "projection must not mutate the output forwarded to Talker"
|
||||
assert h.session.active_response_id is not None, "Thinker finished must not end the response"
|
||||
response_id = h.session.active_response_id
|
||||
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
|
||||
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
|
||||
assert h.session.active_response_id is None
|
||||
assert h.port.cleanups[-1][0] == [request_id]
|
||||
assert len(set(ids)) == 2
|
||||
assert h.session.capabilities.as_dict()["implementation_level"] == "turn_based_duplex"
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_drops_old_output_and_accepts_next_turn():
|
||||
h = await open_qwen()
|
||||
try:
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
old = h.port.submissions[-1].context
|
||||
await h.run(CancelResponse())
|
||||
await h.deliver_and_settle(tts_output(old.request_id, finished=True), stage_id=2, epoch=old.fence.epoch)
|
||||
assert h.session.active_response_id is None
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
assert len(h.port.submissions) == 2
|
||||
assert h.port.submissions[-1].context.request_id != old.request_id
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vad_commits_and_speech_interrupts_active_qwen_response():
|
||||
h = await open_qwen()
|
||||
try:
|
||||
h.runner.control._detector = _speech_then_stop()
|
||||
await h.run(append_audio())
|
||||
await h.run(append_audio(is_speech=False, value=0.0))
|
||||
assert len(h.port.submissions) == 1
|
||||
old = h.port.submissions[-1].context
|
||||
h.runner.control._detector = _speech_then_stop()
|
||||
await h.run(append_audio())
|
||||
assert h.session.epoch > old.fence.epoch
|
||||
await h.run(append_audio(is_speech=False, value=0.0))
|
||||
assert len(h.port.submissions) == 2
|
||||
assert h.port.submissions[-1].context.request_id != old.request_id
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
def test_audio_projector_handles_delta_tensor_lists_and_sample_rate():
|
||||
import torch
|
||||
|
||||
encoded: list[tuple[Any, object]] = []
|
||||
|
||||
def _encode(audio: object, rate: int, *_args: object) -> str:
|
||||
encoded.append((audio, rate))
|
||||
return "encoded"
|
||||
|
||||
plugin = Qwen3OmniDuplexPlugin(_encode)
|
||||
output = SimpleNamespace(
|
||||
request_id="r",
|
||||
stage_id=2,
|
||||
finished=True,
|
||||
outputs=[],
|
||||
multimodal_output={"audio": [torch.ones(8), torch.ones(16)], "sr": [torch.tensor(24000)]},
|
||||
)
|
||||
events = list(plugin.data_plane.project({"data_plane_outputs": [output]}, context={"modalities": ["audio"]}))
|
||||
assert len(encoded) == 1 and encoded[0][0].shape == (24,)
|
||||
assert events[0]["audio_duration_ms"] == 1
|
||||
assert events[0]["end_of_turn"] is True
|
||||
|
||||
|
||||
def test_audio_history_is_bounded_and_does_not_cross_sessions():
|
||||
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
|
||||
state, other = plugin.create_session_state(), plugin.create_session_state()
|
||||
history = []
|
||||
for i in range(6):
|
||||
history.append({"role": "user", "content": [{"type": "audio_url"}]})
|
||||
config = plugin.prepare_prompt_config({"conversation": history}, state=state, payload={"audio": str(i)})
|
||||
history.append({"role": "assistant", "content": f"reply-{i}"})
|
||||
assert len(state.audio_history) == 4
|
||||
assert [m["audio_payload"]["audio"] for m in config["qwen_messages"] if "audio_payload" in m] == [
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
]
|
||||
assert other.audio_history == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_late_playback_ack_keeps_answer_before_new_user_input():
|
||||
h = await open_qwen()
|
||||
try:
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
request_id = h.port.submissions[-1].context.request_id
|
||||
response_id = h.session.active_response_id
|
||||
await h.deliver_and_settle(tts_output(request_id, samples=0, text="first answer", finished=True), stage_id=0)
|
||||
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
|
||||
# Playback has not drained yet. A new utterance commits first.
|
||||
assert [m["role"] for m in h.session.history] == ["user"]
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
current_response = h.session.active_response_id
|
||||
events = await h.run(AckPlayback(response_id=response_id, played_ms=1000))
|
||||
assert not any(e.to_realtime().get("type") == "error" for e in events)
|
||||
assert [m["role"] for m in h.session.history] == ["user", "assistant", "user"]
|
||||
assert h.session.history[1]["content"] == "first answer"
|
||||
assert h.session.active_response_id == current_response
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming_vad", [False, True])
|
||||
async def test_finished_qwen_response_does_not_repeat_during_silence(streaming_vad):
|
||||
h = await open_qwen()
|
||||
try:
|
||||
if streaming_vad:
|
||||
from vllm_omni.engine.duplex.turn_detection import ServerTurnDetector, TurnDetectionConfig
|
||||
|
||||
detector = ServerTurnDetector(TurnDetectionConfig())
|
||||
# Exercise framing, endpointing and reset across the entire response.
|
||||
# A deterministic scorer avoids requiring an ONNX artifact in CPU tests.
|
||||
detector.vad._frame_scorer = lambda frame: float(np.max(np.abs(frame)) > 0.01)
|
||||
h.runner.control._detector = detector
|
||||
else:
|
||||
h.runner.control._detector = _speech_then_stop()
|
||||
await h.run(append_audio())
|
||||
await h.run(append_audio(is_speech=False, value=0.0))
|
||||
if not streaming_vad:
|
||||
h.runner.control._detector = None
|
||||
assert len(h.port.submissions) == 1
|
||||
request_id = h.port.submissions[0].context.request_id
|
||||
response_id = h.session.active_response_id
|
||||
await h.deliver_and_settle(tts_output(request_id, samples=0, text="story", finished=True), stage_id=0)
|
||||
for _ in range(3):
|
||||
await h.run(append_audio(is_speech=False, value=0.0))
|
||||
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
|
||||
for _ in range(3):
|
||||
await h.run(append_audio(is_speech=False, value=0.0))
|
||||
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
|
||||
await h.settle()
|
||||
assert len(h.port.submissions) == 1
|
||||
assert not h.runner.model_state.audio_buffer.has_pending()
|
||||
assert h.runner.model_state.committed_audio_payload is None
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
def test_current_audio_is_not_replaced_by_history_ending_with_assistant():
|
||||
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
|
||||
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
|
||||
state = plugin.create_session_state()
|
||||
first = {"audio": base64.b64encode(np.full(160, 0.1, dtype="<f4").tobytes()).decode()}
|
||||
current = {"audio": base64.b64encode(np.full(320, 0.2, dtype="<f4").tobytes()).decode()}
|
||||
history = [{"role": "user", "content": [{"type": "audio_url"}]}]
|
||||
plugin.prepare_prompt_config({"conversation": history}, state=state, payload=first)
|
||||
history.append({"role": "assistant", "content": "Previous answer"})
|
||||
config = plugin.prepare_prompt_config({"conversation": history}, state=state, payload=current)
|
||||
plan = plugin.plan_append(
|
||||
request_id="new",
|
||||
fence=None,
|
||||
session_config=config,
|
||||
runtime_config={},
|
||||
seq=1,
|
||||
turn_seq=1,
|
||||
payload=current,
|
||||
final=True,
|
||||
sampling_params=None,
|
||||
)
|
||||
audios = plan.prompt["multi_modal_data"]["audio"]
|
||||
assert len(audios) == 2
|
||||
np.testing.assert_allclose(audios[-1][0], np.full(320, 0.2, dtype="<f4"))
|
||||
assert config["qwen_messages"][-1]["audio_payload"] is current
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_playback_does_not_commit_unaligned_thinker_text():
|
||||
h = await open_qwen()
|
||||
try:
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
request_id = h.port.submissions[-1].context.request_id
|
||||
response_id = h.session.active_response_id
|
||||
await h.deliver_and_settle(tts_output(request_id, samples=0, text="entire answer", finished=True), stage_id=0)
|
||||
await h.deliver_and_settle(tts_output(request_id, samples=24000, finished=False), stage_id=2)
|
||||
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
|
||||
assert not any(m["role"] == "assistant" for m in h.session.history)
|
||||
await h.deliver_and_settle(tts_output(request_id, samples=216000, finished=True), stage_id=2)
|
||||
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
|
||||
assert not any(m["role"] == "assistant" for m in h.session.history)
|
||||
await h.run(AckPlayback(response_id=response_id, played_ms=10000))
|
||||
assert h.session.history[-1]["content"] == "entire answer"
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_preparation_runs_off_loop_and_cancellation_prevents_submission():
|
||||
import threading
|
||||
|
||||
h = await open_qwen()
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
thread_ids = []
|
||||
original = h.manager.plugin.plan_append
|
||||
|
||||
def slow_plan(**kwargs):
|
||||
thread_ids.append(threading.get_ident())
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original(**kwargs)
|
||||
|
||||
h.manager.plugin.plan_append = slow_plan
|
||||
try:
|
||||
await h.run(append_audio())
|
||||
commit = asyncio.create_task(h.run(Commit(final=True, create_response=True)))
|
||||
for _ in range(100):
|
||||
if entered.is_set():
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert entered.is_set()
|
||||
assert thread_ids != [threading.get_ident()]
|
||||
await h.run(CancelResponse())
|
||||
release.set()
|
||||
await commit
|
||||
await h.settle()
|
||||
assert not h.port.submissions
|
||||
finally:
|
||||
release.set()
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
def camera_frame():
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (16, 16), "red").save(buf, format="JPEG")
|
||||
return base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def pcm16_base64(samples=16000):
|
||||
"""Loud PCM16, so the projection reads it as speech rather than silence."""
|
||||
pcm = (np.sin(np.arange(samples) * 0.05) * 20000).astype("<i2")
|
||||
return base64.b64encode(pcm.tobytes()).decode()
|
||||
|
||||
|
||||
def image_item(item_id="camera"):
|
||||
return {
|
||||
"id": item_id,
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()},
|
||||
{"type": "input_text", "text": "What color is this?"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_item_is_context_until_response_create_and_delete_removes_it():
|
||||
from vllm_omni.engine.duplex.commands import CreateResponse, DeleteItem
|
||||
|
||||
h = await open_qwen()
|
||||
try:
|
||||
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
|
||||
|
||||
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": image_item()}))
|
||||
assert not h.port.submissions
|
||||
await h.run(CreateResponse())
|
||||
assert len(h.port.submissions) == 1
|
||||
prompt = h.port.submissions[-1].prompt
|
||||
assert len(prompt["multi_modal_data"]["image"]) == 1
|
||||
assert "audio" not in prompt["multi_modal_data"]
|
||||
assert "What color is this?" in prompt["prompt"]
|
||||
await h.run(CancelResponse())
|
||||
await h.run(DeleteItem(item_id="camera"))
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=False))
|
||||
assert len(h.port.submissions) == 1
|
||||
await h.run(CreateResponse())
|
||||
assert "image" not in h.port.submissions[-1].prompt["multi_modal_data"]
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_context_reaches_audio_turn_and_survives_commit():
|
||||
from vllm_omni.engine.duplex.commands import CreateItem
|
||||
|
||||
h = await open_qwen()
|
||||
try:
|
||||
await h.run(CreateItem(item=image_item()))
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
mm = h.port.submissions[-1].prompt["multi_modal_data"]
|
||||
assert len(mm["image"]) == 1 and len(mm["audio"]) == 1
|
||||
assert any(
|
||||
isinstance(m["content"], list) and any(p.get("type") == "image_url" for p in m["content"])
|
||||
for m in h.session.history
|
||||
)
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_only_item_reaches_the_prompt_the_way_the_camera_sends_it():
|
||||
"""The camera sends each frame as its own item, with no text part.
|
||||
|
||||
A user item used to be forwarded to the session only when it carried text
|
||||
or audio, so an image-only item was acknowledged to the client and then
|
||||
dropped. Nothing reported it and every prompt that followed was blind.
|
||||
"""
|
||||
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
|
||||
|
||||
h = await open_qwen()
|
||||
try:
|
||||
item = {
|
||||
"id": "camera_1",
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()}],
|
||||
}
|
||||
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": item}))
|
||||
assert [
|
||||
part
|
||||
for message in h.session.history
|
||||
if isinstance(message.get("content"), list)
|
||||
for part in message["content"]
|
||||
if part.get("type") == "image_url"
|
||||
], "an image-only item never reached the session history"
|
||||
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
mm = h.port.submissions[-1].prompt["multi_modal_data"]
|
||||
assert len(mm["image"]) == 1 and len(mm["audio"]) == 1
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_item_carrying_both_speech_and_a_picture_keeps_both():
|
||||
"""Audio and images leave the same item by different routes.
|
||||
|
||||
The audio becomes buffered appends that a commit seals under this item's
|
||||
id; registering the spoken message there would overwrite an image stored
|
||||
under the same id, so the picture has to become its own item.
|
||||
"""
|
||||
from vllm_omni.engine.duplex.commands import CreateResponse
|
||||
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
|
||||
|
||||
h = await open_qwen()
|
||||
try:
|
||||
item = {
|
||||
"id": "mixed_1",
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()},
|
||||
{"type": "input_audio", "audio": pcm16_base64(), "format": "pcm16", "sample_rate_hz": 16000},
|
||||
],
|
||||
}
|
||||
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": item}))
|
||||
await h.run(CreateResponse())
|
||||
mm = h.port.submissions[-1].prompt["multi_modal_data"]
|
||||
assert len(mm["image"]) == 1, "the picture was dropped on its way out of a mixed item"
|
||||
assert len(mm["audio"]) == 1
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_limit_rejects_atomically_and_delete_reclaims_capacity():
|
||||
from vllm_omni.engine.duplex.commands import CreateItem, DeleteItem
|
||||
|
||||
h = await open_qwen()
|
||||
try:
|
||||
for i in range(8):
|
||||
await h.run(CreateItem(item=image_item(f"camera_{i}")))
|
||||
events = await h.run(CreateItem(item=image_item("overflow")))
|
||||
assert any(e.to_realtime()["type"] == "error" for e in events)
|
||||
assert len(h.session.history) == 8
|
||||
await h.run(DeleteItem(item_id="camera_0"))
|
||||
await h.run(CreateItem(item=image_item("replacement")))
|
||||
assert len(h.session.history) == 8
|
||||
assert not h.port.submissions
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
def test_prompt_holds_stored_images_to_the_stage_limit():
|
||||
"""The prompt is the last guard before ``limit_mm_per_prompt``.
|
||||
|
||||
A dropped image has to take its ``{"type": "image"}`` placeholder with
|
||||
it, because the processor pairs placeholders with ``mm["image"]``
|
||||
positionally.
|
||||
"""
|
||||
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
|
||||
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
|
||||
url = "data:image/jpeg;base64," + camera_frame()
|
||||
history: list[dict[str, Any]] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": url}}, {"type": "text", "text": f"i{i}"}],
|
||||
}
|
||||
for i in range(MAX_PROMPT_IMAGES + 2)
|
||||
]
|
||||
audio = base64.b64encode(np.full(160, 0.1, dtype="<f4").tobytes()).decode()
|
||||
history.append({"role": "user", "audio_payload": {"audio": audio}})
|
||||
|
||||
plan = plugin.plan_append(
|
||||
request_id="r",
|
||||
fence=None,
|
||||
session_config={"qwen_messages": history},
|
||||
runtime_config={},
|
||||
seq=1,
|
||||
turn_seq=1,
|
||||
payload={"audio": audio},
|
||||
final=True,
|
||||
sampling_params=None,
|
||||
)
|
||||
|
||||
prompt = plan.prompt
|
||||
assert len(prompt["multi_modal_data"]["image"]) == MAX_PROMPT_IMAGES
|
||||
assert prompt["prompt"].count("{'type': 'image'}") == MAX_PROMPT_IMAGES
|
||||
# Only the image part is retired; the message keeps what it said.
|
||||
assert "'i0'" in prompt["prompt"]
|
||||
|
||||
|
||||
def test_text_only_turn_without_prepared_messages_is_refused_not_crashed():
|
||||
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
|
||||
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
|
||||
with pytest.raises(DuplexRuntimeConfigError):
|
||||
plugin.plan_append(
|
||||
request_id="r",
|
||||
fence=None,
|
||||
# An empty prepared list is not a missing one: it must not fall
|
||||
# back to a payload that has no audio to read.
|
||||
session_config={"qwen_messages": []},
|
||||
runtime_config={},
|
||||
seq=1,
|
||||
turn_seq=1,
|
||||
payload={"is_speech": True},
|
||||
final=True,
|
||||
sampling_params=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_commit_all_on_done_keeps_unaligned_text_the_client_never_acks():
|
||||
h = await open_qwen()
|
||||
try:
|
||||
# The runner starts every session ack-gated; this client opts out.
|
||||
await h.run(UpdateSession(patch={"playback_commit_policy": "commit_all_on_done"}))
|
||||
assert h.session.config.playback_commit_policy == "commit_all_on_done"
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
request_id = h.port.submissions[-1].context.request_id
|
||||
await h.deliver_and_settle(tts_output(request_id, samples=0, text="entire answer", finished=True), stage_id=0)
|
||||
await h.deliver_and_settle(tts_output(request_id, samples=24000, finished=True), stage_id=2)
|
||||
# Qwen text is unaligned to its audio, but waiting for an ACK this
|
||||
# client will never send would drop the turn from history entirely.
|
||||
assert h.session.active_response_id is None
|
||||
assert h.session.history[-1]["content"] == "entire answer"
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_bound_request_id_is_the_one_the_stage_received():
|
||||
h = await open_qwen()
|
||||
try:
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
submitted = h.port.submissions[-1].context.request_id
|
||||
# Qwen has no resumable core request, so every turn gets its own stage
|
||||
# id. The runner names that id before the append runs and the model
|
||||
# channel names it again when it submits; they have to agree.
|
||||
assert submitted.endswith(".r.stage0-turn0"), submitted
|
||||
assert h.session.active_request_id == submitted
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_visual_capture_records_submitted_pixels_and_request(tmp_path, monkeypatch):
|
||||
import json
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.engine.duplex.commands import CreateItem
|
||||
|
||||
monkeypatch.setenv("VLLM_OMNI_QWEN_VISUAL_DEBUG_DIR", str(tmp_path))
|
||||
h = await open_qwen()
|
||||
try:
|
||||
await h.run(CreateItem(item=image_item()))
|
||||
await h.run(append_audio())
|
||||
await h.run(Commit(final=True, create_response=True))
|
||||
submission = h.port.submissions[-1]
|
||||
captures = list(tmp_path.glob("*/input.json"))
|
||||
assert len(captures) == 1
|
||||
metadata = json.loads(captures[0].read_text())
|
||||
assert metadata["request_id"] == submission.context.request_id
|
||||
assert metadata["audio_count"] == 1
|
||||
assert metadata["prompt"] == submission.prompt["prompt"]
|
||||
assert len(metadata["images"]) == 1
|
||||
with Image.open(captures[0].parent / metadata["images"][0]) as saved:
|
||||
actual = submission.prompt["multi_modal_data"]["image"][0]
|
||||
assert saved.size == actual.size
|
||||
assert saved.tobytes() == actual.tobytes()
|
||||
finally:
|
||||
await h.manager.shutdown()
|
||||
Reference in new issue
Block a user