anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Frontend] Add a shared realtime web UI for MiniCPM-o and Qwen3-Omni (#7222) (#7585)

Browse Source 
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-09-17 23:32:31 +08:00
1 parent eb1abbadf8
commit 882bf8bc95
53 files changed
+3438 -633
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/serving/realtime_duplex_api.md
+4 -1
View File
@@ -768,7 +768,10 @@ events with `server_event_seq > 40` are replayed in order)
```
with camera frames (omni video; rides the append that closes a 1 s model
unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix)
unit; each entry is a **bare** base64 JPEG/PNG — no `data:` URL prefix).
Only a model whose Stage 0 interleaves a frame track at unit boundaries takes
this field; a turn model has no such boundary and rejects it, expecting
`conversation.item.create` with `input_image` content instead.
```json
{
examples/online_serving/minicpmo/README.md
+9
View File
@@ -316,3 +316,12 @@ them unless `--allow-invalid-clock` is explicit.
[`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/)
- Recipe:
[`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md)
### Shared realtime UI implementation
The `python -m examples.online_serving.minicpmo.realtime_web` command remains
available with the same MiniCPM defaults and required `--ref-audio`. Its assets
now live in [the shared realtime UI](../realtime_web/README.md), with a dedicated
`minicpm-native` profile preserving native duplex input, playback ACKs and camera
frames. Qwen3 uses a separate profile in the same shell, supporting manual STT
turns or Server VAD turns with speech interruption.
examples/online_serving/minicpmo/realtime_web/server.py
+14 -176
View File
@@ -1,185 +1,23 @@
"""Static host and same-origin Realtime proxy for the full-duplex demo."""
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
"""Backward-compatible MiniCPM entry point for the shared realtime UI."""
import argparse
import asyncio
import base64
import contextlib
import hashlib
import json
import logging
from pathlib import Path
from urllib.parse import urlencode
from examples.online_serving.realtime_web.server import (
APP_DIR,
STATIC_DIR,
_join_ws_url,
build_app,
)
from examples.online_serving.realtime_web.server import (
main as _main,
)
import uvicorn
import websockets
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse, Response
from fastapi.staticfiles import StaticFiles
logger = logging.getLogger(__name__)
APP_DIR = Path(__file__).parent / "app"
STATIC_DIR = APP_DIR / "static"
def _join_ws_url(base: str, path: str, query: str) -> str:
return base.rstrip("/") + path + (("?" + query) if query else "")
async def _pump_client_to_backend(client: WebSocket, backend) -> None:
try:
while True:
message = await client.receive()
if message["type"] == "websocket.disconnect":
await backend.close()
return
if message.get("text") is not None:
await backend.send(message["text"])
elif message.get("bytes") is not None:
await backend.send(message["bytes"])
except (WebSocketDisconnect, websockets.ConnectionClosed):
with contextlib.suppress(Exception):
await backend.close()
async def _pump_backend_to_client(client: WebSocket, backend) -> None:
try:
async for message in backend:
if isinstance(message, bytes):
await client.send_bytes(message)
else:
await client.send_text(message)
except (WebSocketDisconnect, websockets.ConnectionClosed):
return
except RuntimeError as exc:
if "websocket.send" in str(exc) and "websocket.close" in str(exc):
return
raise
def _expected_proxy_close(exc: BaseException) -> bool:
if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
return True
return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
def build_app(
*,
ws_backend: str = "ws://127.0.0.1:8099",
model: str = "openbmb/MiniCPM-o-4_5",
public_realtime_url: str | None = None,
ref_audio: str | None = None,
) -> FastAPI:
app = FastAPI(title="Experimental Full-Duplex Web Demo")
index_path = APP_DIR / "index.html"
app_version_hash = hashlib.sha256()
for asset_path in (
STATIC_DIR / "app.js",
STATIC_DIR / "pcm_worklet.js",
STATIC_DIR / "playback_worklet.js",
):
app_version_hash.update(asset_path.read_bytes())
app_version = app_version_hash.hexdigest()[:12]
ref_audio_uri: str | None = None
if ref_audio:
ref_path = Path(ref_audio)
if not ref_path.is_file():
raise SystemExit(f"--ref-audio not found: {ref_audio}")
encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
ref_audio_uri = f"data:audio/wav;base64,{encoded}"
@app.get("/", response_class=HTMLResponse)
def index() -> HTMLResponse:
config = json.dumps(
{
"model": model,
"realtimePath": public_realtime_url or "v1/realtime",
"refAudio": ref_audio_uri,
"appVersion": app_version,
},
ensure_ascii=True,
)
html = (
index_path.read_text(encoding="utf-8")
.replace("__FULL_DUPLEX_CONFIG__", config)
.replace("__FULL_DUPLEX_APP_VERSION__", app_version)
)
return HTMLResponse(html, headers={"Cache-Control": "no-store"})
@app.get("/healthz")
def healthz() -> Response:
return Response(content="ok", media_type="text/plain")
@app.websocket("/v1/realtime")
async def realtime_proxy(websocket: WebSocket) -> None:
await websocket.accept()
query = urlencode(websocket.query_params.multi_items())
backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
logger.info("Proxying Realtime WebSocket to %s", backend_url)
try:
async with websockets.connect(
backend_url,
max_size=64 * 1024 * 1024,
) as backend:
tasks = {
asyncio.create_task(_pump_client_to_backend(websocket, backend)),
asyncio.create_task(_pump_backend_to_client(websocket, backend)),
}
done, pending = await asyncio.wait(
tasks,
return_when=asyncio.FIRST_COMPLETED,
)
for task in pending:
task.cancel()
await asyncio.gather(*pending, return_exceptions=True)
for result in await asyncio.gather(*done, return_exceptions=True):
if isinstance(result, BaseException) and not _expected_proxy_close(result):
raise result
except (WebSocketDisconnect, websockets.ConnectionClosed):
return
except Exception:
logger.exception("Realtime WebSocket proxy failed")
with contextlib.suppress(Exception):
await websocket.close(code=1011)
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
return app
__all__ = ["APP_DIR", "STATIC_DIR", "_join_ws_url", "build_app", "main"]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=7862)
parser.add_argument("--ws-backend", default="ws://127.0.0.1:8099")
parser.add_argument(
"--public-realtime-url",
help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
)
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
parser.add_argument(
"--ref-audio",
required=True,
help=(
"Reference voice wav for TTS voice cloning, e.g. the "
"official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
),
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
uvicorn.run(
build_app(
ws_backend=args.ws_backend,
model=args.model,
public_realtime_url=args.public_realtime_url,
ref_audio=args.ref_audio,
),
host=args.host,
port=args.port,
log_level="info",
)
_main(default_profile="minicpm-native")
if __name__ == "__main__":
examples/online_serving/qwen3_omni/README.md
+57 -14
View File
@@ -320,24 +320,17 @@ python openai_realtime_client.py \
Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
```yaml
base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
duplex_session:
server_vad_model_path: /models/silero_vad.onnx
```
Start from the bundled Qwen3-Omni deployment YAML, add the fields above at the top level, and serve that complete configuration:
```bash
cp vllm_omni/deploy/qwen3_omni_moe.yaml /path/to/qwen3_omni_server_vad.yaml
# Edit /path/to/qwen3_omni_server_vad.yaml and add duplex_session.
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
--omni \
--port 8091 \
--deploy-config /path/to/qwen3_omni_server_vad.yaml
```
Keep Qwen's default `session_mode: turn`; `duplex_session` enables the Realtime handler without changing scheduler
semantics. Bare `/v1/realtime` selects that handler. The client uses `?duplex=0` only for the existing non-Server-VAD
wire flow; `?duplex=1` remains a supported compatibility alias.
For the engine-owned plugin, use an overlay based on
`vllm_omni/deploy/qwen3_omni_duplex.yaml`, with `duplex_session.server_vad_model_path`
pointing to your local artifact. See the [complete duplex setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
The default `qwen3_omni_moe.yaml` remains a turn deployment. Duplex mode serves
`/v1/realtime?duplex=1` and `/v1/chat/completions`; the legacy STT handler requires
a turn deployment.
The Python client supports the following command-line arguments:
@@ -584,3 +577,53 @@ The gradio script supports the following arguments:
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
- `--port`: Port for Gradio server (default: 7861)
- `--share`: Share the Gradio demo publicly (creates a public link)
## Browser voice call (shared realtime UI)
Run these commands from the repository root with the vLLM-Omni environment activated.
### Without VAD: manually submit each turn
Start the backend:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
```
In another terminal, check backend readiness and start the UI:
```bash
curl --fail http://127.0.0.1:8091/health
python -m examples.online_serving.qwen3_omni.realtime_web \
--backend ws://127.0.0.1:8091 --stt --port 7863
```
Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
`--stt` is the default. Each turn uses a separate connection without shared model
history; the conversation shown in the browser is a local log.
### With VAD: automatically submit after silence
Start the backend with a Server VAD deployment overlay and a compatible Silero
ONNX artifact, following the
[complete VAD setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
After its health check succeeds, start the UI:
```bash
python -m examples.online_serving.qwen3_omni.realtime_web \
--backend ws://127.0.0.1:8091 --vad --port 7863
```
Speak and pause for 500 ms to submit automatically. `--vad` configures the client;
it does not load a VAD model or change the backend deployment by itself.
VAD mode keeps uploading audio during replies and supports speech interruption.
The engine-owned Qwen duplex plugin maintains conversation history and tracks
played replies through playback acknowledgements. Each committed utterance starts
a new generation request; the model does not use native streaming-input KV decoding.
STT mode pauses microphone upload while the reply is generated and played.
In VAD mode, click **Camera** to upload sampled frames with your spoken question;
see [camera setup and limits](../realtime_web/README.md#qwen-vad-camera-input).
STT remains audio-only. To switch back to manual turns, use the default turn
deployment, restart the UI with `--stt`, refresh the page, and reconnect. Stop the existing UI before reusing port 7863.
See the [shared UI guide](../realtime_web/README.md) for HTTPS access and testing.
examples/online_serving/qwen3_omni/openai_realtime_client.py
+2 -1
View File
@@ -130,7 +130,7 @@ async def run_client(
"prefix_padding_ms": 300,
"silence_duration_ms": 500,
"create_response": True,
"interrupt_response": False,
"interrupt_response": True,
},
}
},
@@ -233,6 +233,7 @@ async def run_client(
continue
if event_type == "response.done" and server_vad:
await ws.send(json.dumps({"type": "session.close"}))
break
if event_type == "error":
examples/online_serving/qwen3_omni/realtime_web/__init__.py
+4
View File
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Qwen3-Omni turn-based browser call UI."""
examples/online_serving/qwen3_omni/realtime_web/__main__.py
+7
View File
@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from examples.online_serving.realtime_web.server import main
if __name__ == "__main__":
main(default_profile="qwen3-turn")
examples/online_serving/realtime_web/README.md
+221
View File
@@ -0,0 +1,221 @@
# Shared Realtime Web UI
One browser shell provides microphone capture, speaker playback, a level meter,
conversation text, and an event log. Model profiles own the wire protocol.
This implements [RFC #7222](https://github.com/vllm-project/vllm-omni/issues/7222).
| Profile | Turn control | Camera | Playback acknowledgement |
| --- | --- | --- | --- |
| `minicpm-native` | Model-controlled listen/speak, continuous audio input | Frames accompany audio | Yes |
| `qwen3-turn --stt` (default) | User presses **Send turn** | No | No |
| `qwen3-turn --vad` | Server detects trailing silence; speech interrupts replies | Sampled frames with each spoken turn | Yes |
Qwen3 VAD uses an engine-owned duplex plugin: microphone upload continues
while replies stream, and speech can interrupt generation and playback. The
checkpoint still generates committed turns; this is not native streaming-input
KV decoding. STT remains explicit-turn and audio-only.
## MiniCPM: existing command stays valid
Start your MiniCPM backend using the [existing deployment instructions](../minicpmo/README.md), then:
```bash
python -m examples.online_serving.minicpmo.realtime_web \
--ws-backend ws://127.0.0.1:8099 \
--ref-audio /path/to/ref_minicpm_signature.wav
```
The compatibility wrapper serves the shared assets with `minicpm-native`.
The current native query, `extra_body`, reference voice, continuous microphone
upload, camera frames and playback acknowledgements are retained.
## Qwen3: explicit-turn STT adapter
Start the backend, then the UI in a second terminal:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
python -m examples.online_serving.qwen3_omni.realtime_web \
--backend ws://127.0.0.1:8091 --stt --port 7863
```
Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
Wait for the answer to finish playing before speaking again. The UI opens a
fresh STT connection for each turn; the displayed conversation is a local log,
and **previous turns are not provided as model history**. The STT endpoint
accepts only model selection, so system-prompt controls are hidden in this mode.
The adapter follows the shipped `qwen3_omni/openai_realtime_client.py` and
`vllm_omni/entrypoints/openai/realtime_connection.py`:
1. Explicit `duplex=0` selects the legacy STT handler on a turn deployment.
It does not enable STT on a duplex deployment.
2. Send `{type: "session.update", model: ...}` and `commit(final=false)`.
3. Stream mono PCM16 at 16 kHz; **Send turn** flushes buffered audio, then sends `commit(final=true)`.
4. `response.output_audio.delta.audio` contains PCM; `response.output_audio.done`
terminates this adapter, including a response with no audio.
5. The local implementation emits **model output** on `transcription.*`, so the
UI displays it as assistant text. This differs from the early RFC's suggested
user-transcript mapping. The VAD adapter also supports `response.output_text.*`
and audio-transcript events without duplicating the answer.
## Qwen3 with Server VAD (automatic turns)
Run commands from the repository root with the vLLM-Omni environment activated.
Prepare the pinned Silero v6.2 ONNX artifact as described in the
[Qwen3 Server VAD instructions](../qwen3_omni/README.md#realtime-websocket-client-openai_realtime_clientpy).
The backend requires ONNX Runtime and a compatible local artifact; setting
`--vad` on the UI alone does not enable VAD on the backend.
Create an overlay named `qwen3_vad.yaml`, replacing both paths with absolute paths
on the backend host:
```yaml
base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
duplex_session:
server_vad_model_path: /path/to/silero_vad.onnx
```
The base configuration enables duplex mode and sets the per-prompt audio and image limits.
Start the backend:
```bash
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
--deploy-config qwen3_vad.yaml
```
In a second terminal, wait until the health check succeeds, then start the UI:
```bash
curl --fail http://127.0.0.1:8091/health
python -m examples.online_serving.qwen3_omni.realtime_web \
--backend ws://127.0.0.1:8091 --vad --port 7863
```
Open `http://localhost:7863` and start a session. Speak and pause: the server
commits the turn automatically after 500 ms of silence. No **Send turn** action
is needed. Defaults are a speech threshold of 0.5 and 300 ms of prefix padding.
Speak again to interrupt an answer. The engine owns input buffering, VAD,
request cancellation, playback acknowledgements, and conversation history.
Each committed utterance starts a normal Thinker → Talker → Code2Wav request;
it does not append input to a live Qwen KV cache. The capability response reports
`implementation_level: turn_based_duplex` and `supports_core_resumable_request: false`.
The plugin retains up to four audio inputs (including the current turn), pruning
older audio at an 8 MiB base64 budget; one current utterance is bounded by the
engine's pending-input limit. The UI reports playback progress both on normal
drain and on interruption. Qwen does not provide text/audio alignment, so assistant
text enters history only after the complete audio response has been generated and
acknowledged as played. A partially played answer contributes no guessed text
prefix. Image conversation items remain available until deleted. Model context limits still apply.
The bundled duplex deploy allows four audio inputs and eight images per prompt.
Use headphones to avoid speaker audio triggering VAD interruption. Reference
voices and tool calls are not supported by this plugin.
### Switching between VAD and manual turns
| Mode | Backend | UI flag | Submit a turn |
| --- | --- | --- | --- |
| Without VAD | Default Qwen turn deployment | `--stt` (default) | Press **Send turn** |
| With VAD | Server VAD configuration and Silero artifact | `--vad` | Pause after speaking |
Stop the existing UI process before starting another on the same port, then
refresh the browser and reconnect. To use STT, start a backend with the default
turn deployment and restart the UI with `--stt`. The duplex backend exposes
Realtime duplex and ChatCompletion; it does not expose the legacy STT handler.
To enable VAD on a default backend, first restart that backend with the overlay.
Make `--backend` match the actual backend port; the UI and backend use separate
ports. `Connection refused` means the target backend is unavailable: check its
startup log and `/health` before connecting.
VAD connects with `duplex=1` and sends nested `session.audio.input` configuration
with `create_response: true` and `interrupt_response: true`. It sends playback
ACKs when speaker audio drains. `response.audio.done` / `response.output_audio.done`
only drain playback; `response.done` terminates a turn. Backpressure displays a
notice without pausing the microphone. Failed handshakes time out after 15 seconds;
Silero initialization failures display a specific error.
After updating the backend Python code, restart the backend, refresh the UI, and
start a new session so the plugin and browser assets use the updated code. A
successful `/health` check confirms readiness, not which deployment mode is active.
## Qwen VAD camera input
After starting a session with `--vad`, click **Camera** and allow access. The UI
sends one JPEG per second, resized to a longest side of 448 pixels, independently
of microphone uploads (including while muted). Each frame is a user conversation
item using the [OpenAI Realtime image input](https://developers.openai.com/api/docs/guides/realtime-conversations#image-inputs) shape:
```json
{"type":"conversation.item.create","item":{"id":"camera_1","type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/jpeg;base64,..."}]}}
```
Images become conversation context immediately; they do not start a response.
Speak and let VAD commit your audio, or send `response.create` after adding an
image/text item. A manual `input_audio_buffer.commit` only commits audio; follow
it with `response.create` to generate. Images persist across audio commits,
interruptions, and audio-buffer clears. Remove them with `conversation.item.delete`.
The browser retains eight camera items and deletes the oldest before adding a
ninth. Camera-off stops capture but leaves those items in the conversation.
This deployment accepts JPEG/PNG Base64 data URLs with a total image-context
budget of eight images and 4 MiB; external image URLs are not supported. Clients
must delete old image items when reaching the limit. There is no precise
video/audio timestamp alignment or native streaming-video KV update.
### Protocol compatibility boundary
Qwen VAD uses `session.update`, 24 kHz PCM16 `input_audio_buffer.append`,
`conversation.item.create` with `input_image`/`input_text`, and `response.create`.
The engine resamples input to Qwen's 16 kHz processing rate. Qwen rejects the
custom `video_frames` field on an append: it answers whole turns, so it has no
audio unit for a frame track to align to. MiniCPM keeps that field for its
native camera protocol, where Stage 0 interleaves frames at unit boundaries.
This is a supported subset, not full OpenAI Realtime API compatibility.
`duplex=1`, `overlap_policy`, `playback.ack`, and `session.close` remain vLLM-Omni
extensions. The legacy `--stt` mode is unchanged and is not the standard Realtime
conversation API. This demo does not add WebRTC, semantic VAD, or tool calling.
## Shared host options
The shared entry point can select either profile explicitly:
```bash
python -m examples.online_serving.realtime_web --profile qwen3-turn \
--backend ws://127.0.0.1:8091 --stt --port 7863
```
- `--backend` / `--ws-backend`: backend WebSocket origin (not the full `/v1/realtime` URL).
- `--model`: override the wrapper's model name.
- `--public-realtime-url`: optional browser-visible WebSocket URL; otherwise the
static host proxies `/v1/realtime` on the same origin.
- `--host`, `--port`: UI bind address and port (default port 7862).
- `--ref-audio`: required for MiniCPM; rejected for Qwen3.
Microphone and camera access require `localhost` or HTTPS. For a remote backend,
use an SSH tunnel to the UI host or an HTTPS reverse proxy with WebSocket support.
An explicit `wss://` public URL retains its scheme.
## Validation
```bash
node --test tests/examples/test_realtime_web_profiles.cjs
pytest tests/examples/test_minicpmo_realtime_web_server.py \
tests/examples/test_minicpmo_realtime_web_static.py \
tests/examples/test_realtime_web_profiles.py
pytest -m cpu tests/model_executor/models/qwen3_omni/test_duplex_plugin.py \
tests/engine/duplex/test_session_runner.py \
tests/engine/test_duplex_orchestrator.py
```
These CPU tests cover profile messages, two STT turns, VAD terminal/drain ordering,
backpressure, native capture during playback, server config injection, and shared
worklets. Backend regressions cover current audio inclusion in history-based
prompts, delayed playback ACK ordering, and no extra generation during silence
after a completed VAD turn. Hardware acceptance still requires a MiniCPM call
with camera/barge-in and two Qwen turns against each enabled backend, including
speech interruption in VAD mode; simulated tests do not
establish model quality or hardware latency.
examples/online_serving/realtime_web/__init__.py
+4
View File
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared browser audio UI and model-specific realtime profiles."""
examples/online_serving/realtime_web/__main__.py
+7
View File
@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from .server import main
if __name__ == "__main__":
main()
examples/online_serving/minicpmo/realtime_web/app/index.html → examples/online_serving/realtime_web/app/index.html
+12 -11
View File
@@ -4,37 +4,35 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="color-scheme" content="light">
<title>MiniCPM-o 4.5 Voice</title>
<title>Omni Realtime Voice</title>
<link rel="stylesheet" href="static/styles.css">
</head>
<body>
<header class="app-header">
<div>
<p class="eyebrow">Experimental full-duplex runtime</p>
<h1>MiniCPM-o 4.5 Voice</h1>
<p id="pageEyebrow" class="eyebrow"></p>
<h1 id="pageTitle">Omni Realtime Voice</h1>
</div>
<span id="connectionState" class="status status-offline">Offline</span>
</header>
<main>
<p id="profileDescription" role="note"></p>
<section class="call-bar" aria-label="Session controls">
<div class="call-actions">
<button id="callButton" class="primary" type="button">Start session</button>
<button id="sendTurnButton" class="secondary" type="button" hidden disabled>Send turn</button>
<button id="muteButton" class="secondary" type="button" disabled>Mute</button>
<button id="cameraButton" class="secondary" type="button" disabled>Camera</button>
<video id="cameraPreview" muted playsinline style="display:none;width:96px;height:72px;border-radius:8px;object-fit:cover;background:#000"></video>
</div>
<div style="display:flex;gap:8px;align-items:flex-start;margin-top:8px;width:100%">
<div id="promptControls" style="display:flex;gap:8px;align-items:flex-start;margin-top:8px;width:100%">
<label style="display:inline-flex;flex-direction:column;gap:4px;font-size:13px;opacity:.85;white-space:nowrap">
Prompt preset
<select id="promptPreset" style="font-size:13px;padding:4px 6px;border-radius:6px">
<option value="omni" selected>Omni (official omni preset)</option>
<option value="chinese_call">中文通话 (official audio preset)</option>
<option value="english_call">English Call (official audio preset)</option>
<option value="custom">Custom</option>
</select>
</label>
<textarea id="systemPrompt" rows="2" placeholder="System prompt" style="flex:1;font-size:13px;padding:6px 8px;border-radius:6px;resize:vertical">Streaming Omni Conversation.</textarea>
<textarea id="systemPrompt" rows="2" placeholder="System prompt" style="flex:1;font-size:13px;padding:6px 8px;border-radius:6px;resize:vertical"></textarea>
</div>
<div class="meter" aria-label="Microphone level">
<span>Mic</span>
@@ -53,7 +51,7 @@
<p class="eyebrow">Live transcript</p>
<h2 id="conversationTitle">Conversation</h2>
</div>
<span class="policy-label">Model-controlled listen / speak</span>
<span id="policyLabel" class="policy-label"></span>
</div>
<div id="conversation" class="conversation" aria-live="polite">
<p id="emptyConversation" class="empty-state">Start a session and speak naturally.</p>
@@ -73,7 +71,10 @@
</details>
</main>
<script>window.FULL_DUPLEX_CONFIG = __FULL_DUPLEX_CONFIG__;</script>
<script>window.OMNI_REALTIME_CONFIG = __FULL_DUPLEX_CONFIG__; window.FULL_DUPLEX_CONFIG = window.OMNI_REALTIME_CONFIG;</script>
<script src="static/profiles/common.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
<script src="static/profiles/minicpm_native.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
<script src="static/profiles/qwen3_turn.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
<script src="static/app.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
</body>
</html>
examples/online_serving/minicpmo/realtime_web/app/static/app.js → examples/online_serving/realtime_web/app/static/app.js
+296 -140
View File
@@ -1,8 +1,10 @@
(() => {
'use strict';
const config = window.FULL_DUPLEX_CONFIG || {};
const config = window.OMNI_REALTIME_CONFIG || window.FULL_DUPLEX_CONFIG || {};
const profile = window.OmniRealtimeProfiles[config.profile || 'minicpm-native'](config);
const callButton = document.getElementById('callButton');
const sendTurnButton = document.getElementById('sendTurnButton');
const muteButton = document.getElementById('muteButton');
const cameraButton = document.getElementById('cameraButton');
const cameraPreview = document.getElementById('cameraPreview');
@@ -27,20 +29,26 @@
const INITIAL_PLAYBACK_BUFFER_MS = 400;
const SESSION_CLOSE_TIMEOUT_MS = 1000;
// Default prompts mirroring the official MiniCPM-o-Demo presets
// (assets/presets/{omni,audio_duplex}/*.yaml).
const PROMPT_PRESETS = {
omni: 'Streaming Omni Conversation.',
chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
+ '请用高自然度的方式和用户聊天。你处于双工模式，可以一边听、一边说。'
+ '你是由面壁智能开发的人工智能助手：面壁小钢炮。',
english_call: 'Replicate the tone and style from the input audio. Your task is to be '
+ 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
+ 'seriously and in a high quality. Please chat with the user in a high naturalness '
+ 'style. You are in duplex mode, where you can listen and speak at the same time.',
};
const PROMPT_PRESETS = profile.presets;
document.title = profile.title;
document.getElementById('pageTitle').textContent = profile.title;
document.getElementById('pageEyebrow').textContent = profile.eyebrow;
document.getElementById('profileDescription').textContent = profile.description;
document.getElementById('policyLabel').textContent = profile.policy;
cameraButton.hidden = !profile.camera;
sendTurnButton.hidden = !profile.clientCommit;
promptPreset.replaceChildren();
for (const name of [...Object.keys(PROMPT_PRESETS), 'custom']) {
const option = document.createElement('option');
option.value = name;
option.textContent = name.replaceAll('_', ' ');
promptPreset.appendChild(option);
}
systemPromptInput.value = Object.values(PROMPT_PRESETS)[0];
document.getElementById('promptControls').hidden = !profile.instructions;
let socket = null;
let closingSocket = null;
let mediaStream = null;
let captureContext = null;
let captureNode = null;
@@ -65,6 +73,63 @@
let liveUserTurn = null;
let liveAssistantTurn = null;
let sessionCloseResolver = null;
let responseComplete = false;
let playbackComplete = true;
let turnSubmitted = false;
let echoTimer = null;
let sessionGeneration = 0;
let audioChain = Promise.resolve();
let assistantTextChannel = null;
let connectionReady = false;
let turnCounter = 0;
let stopping = null;
let turnTimeout = null;
function markBusy() {
assistantActive = true;
if (profile.halfDuplex) pendingCapture = [];
}
function armTurnTimeout() {
clearTimeout(turnTimeout);
turnTimeout = window.setTimeout(() => {
failSession('Timed out waiting for the model response. Check the backend event log and restart the session.');
}, 120000);
}
async function failSession(message) {
appendLog(message, true);
await stopSession({ terminal: false });
setConnection('Error', 'error');
runtimeDetail.textContent = message;
}
function finishResponseIfReady() {
if (!responseComplete || !playbackComplete || echoTimer !== null) return;
const generation = sessionGeneration;
echoTimer = window.setTimeout(async () => {
echoTimer = null;
if (!running || generation !== sessionGeneration) return;
if (profile.reconnectEachTurn) {
connectionReady = false;
const previous = socket;
socket = null;
if (previous) { previous.onclose = null; previous.onmessage = null; previous.close(); }
try { await openSocket(); }
catch (error) { await failSession(error.message); return; }
if (!running || generation !== sessionGeneration) return;
}
assistantActive = false;
responseComplete = false;
turnSubmitted = false;
currentResponseId = null;
responseHasAudio = false;
assistantTextChannel = null;
if (profile.halfDuplex) pendingCapture = [];
sendTurnButton.disabled = !profile.clientCommit;
setModel(profile.waiting);
}, ECHO_GUARD_MS);
}
function staticAssetUrl(path) {
const version = String(config.appVersion || '').trim();
@@ -82,12 +147,7 @@
}
function realtimeUrl() {
const url = new URL(config.realtimePath, window.location.href);
url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
url.searchParams.set('duplex', '1');
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
url.searchParams.set('autostart', '0');
return url.toString();
return profile.url(config, window.location.href);
}
function setConnection(label, kind) {
@@ -250,7 +310,7 @@
}
function microphoneUploadEnabled() {
return running && !muted;
return running && connectionReady && !muted && (!profile.halfDuplex || !assistantActive);
}
function flushCapture() {
@@ -267,19 +327,9 @@
offset += chunk.length;
}
pendingCapture = [];
const pcm = resampleInt16(merged, captureRate, INPUT_RATE);
const appendEvent = {
type: 'input_audio_buffer.append',
audio: int16ToBase64(pcm),
format: 'pcm16',
sample_rate_hz: INPUT_RATE,
};
// Omni duplex: ~1 fps camera frame rides the audio append (official
// MiniCPM-o-Demo contract: one base64 JPEG per ~1 s chunk).
if (cameraPendingFrame) {
appendEvent.video_frames = [cameraPendingFrame];
cameraPendingFrame = null;
}
const pcm = resampleInt16(merged, captureRate, profile.inputSampleRate || INPUT_RATE);
const appendEvent = profile.append(int16ToBase64(pcm), cameraPendingFrame);
cameraPendingFrame = null;
socket.send(JSON.stringify(appendEvent));
}
@@ -287,13 +337,17 @@
currentResponseId = responseId || currentResponseId;
responseHasAudio = false;
assistantActive = true;
setModel('Speaking');
responseComplete = false;
playbackComplete = true;
if (profile.halfDuplex) pendingCapture = [];
setModel(profile.halfDuplex ? 'Thinking' : 'Speaking');
}
function feedPlayback(decoded, responseId) {
if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return;
const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate);
responseHasAudio = true;
playbackComplete = false;
assistantActive = true;
setPlayback('Buffering');
playbackNode.port.postMessage({
@@ -305,6 +359,7 @@
}
function requestPlaybackDrain(responseId) {
if (!responseHasAudio) { playbackComplete = true; finishResponseIfReady(); return; }
if (!playbackNode) return;
playbackNode.port.postMessage({ type: 'drain', responseId: responseId || currentResponseId });
}
@@ -314,90 +369,118 @@
if (!responseId && playedMs > 0) appendLog('playback ack skipped: missing response id', true);
return;
}
socket.send(JSON.stringify({
type: 'playback.ack',
response_id: responseId,
item_id: `item_${responseId}`,
played_ms: playedMs,
committed_ms: playedMs,
}));
const ack = profile.ack(responseId, playedMs);
if (ack) socket.send(JSON.stringify(ack));
}
function playbackDrained(message) {
const responseId = message.responseId || currentResponseId;
sendPlaybackAck(responseId, Number(message.playedMs) || 0);
if (currentResponseId && responseId !== currentResponseId) return;
if (profile.playbackAck) sendPlaybackAck(responseId, Number(message.playedMs) || 0);
setPlayback('Idle');
if (message.underrunMs > 0) {
appendLog(`playback underrun ${message.underrunMs} ms`);
}
window.setTimeout(() => {
assistantActive = false;
currentResponseId = null;
responseHasAudio = false;
if (running) setModel('Listening');
}, ECHO_GUARD_MS);
playbackComplete = true;
if (message.underrunMs > 0) appendLog(`playback underrun ${message.underrunMs} ms`);
if (!profile.halfDuplex && !profile.waitForResponseDone) responseComplete = true;
finishResponseIfReady();
}
function handleEvent(event) {
async function handleAudioEvent(action) {
markBusy();
currentResponseId = action.responseId || currentResponseId || `turn-${turnCounter}`;
setModel('Speaking');
const generation = sessionGeneration;
const decoded = await decodeAudioDelta(action.event);
if (generation === sessionGeneration) feedPlayback(decoded, currentResponseId);
}
function handleTranscriptEvent(action) {
if (action.role === 'assistant') {
// Qwen may emit both text and audio-transcript representations.
// Display one channel per response instead of duplicating the answer.
if (profile.deduplicateTranscript && assistantTextChannel && action.channel !== assistantTextChannel) return;
if (action.text) assistantTextChannel = action.channel || assistantTextChannel;
}
if (action.kind === 'text') addTranscript(action.role, action.text);
else if (profile.deduplicateTranscript && action.role === 'assistant') {
if (action.text) {
const turn = ensureTurn('assistant');
turn.value = action.text;
turn.text.textContent = action.text;
}
} else finishTranscript(action.role, action.text);
}
async function handleEvent(event) {
appendEventLog(event);
const responseId = responseIdOf(event);
switch (event.type) {
case 'session.created':
case 'session.updated':
const action = profile.mapEvent(event);
const responseId = action.responseId;
switch (action.kind) {
case 'connected':
setConnection('Connected', 'online');
setModel('Listening');
if (!assistantActive) setModel(profile.waiting);
break;
case 'response.listen':
case 'interrupt':
sessionGeneration += 1;
// A cancelled response never reaches 'done', so nothing else would
// disarm the turn watchdog it left armed.
clearTimeout(turnTimeout);
if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
responseComplete = true;
playbackComplete = true;
responseHasAudio = false;
assistantActive = false;
setModel('Listening');
currentResponseId = null;
assistantTextChannel = null;
finishTranscript('assistant');
setPlayback('Idle');
setModel(profile.waiting);
break;
case 'response.created':
case 'response.speak':
case 'listen':
assistantActive = false;
setModel(profile.waiting);
break;
case 'begin':
if (profile.halfDuplex) armTurnTimeout();
if (echoTimer !== null) { clearTimeout(echoTimer); echoTimer = null; }
beginAssistant(responseId);
break;
case 'response.output_audio.delta':
currentResponseId = responseId || currentResponseId;
assistantActive = true;
setModel('Speaking');
decodeAudioDelta(event)
.then((decoded) => feedPlayback(decoded, responseId))
.catch((error) => appendLog(`audio decode failed: ${error.message || error}`, true));
case 'audio':
await handleAudioEvent(action);
break;
case 'response.output_audio.done':
case 'drain':
requestPlaybackDrain(responseId);
break;
case 'response.output_audio_transcript.delta':
addTranscript('assistant', event.delta || '');
case 'text': case 'text-final':
handleTranscriptEvent(action);
break;
case 'response.output_audio_transcript.done':
finishTranscript('assistant', event.transcript || '');
break;
case 'conversation.item.input_audio_transcription.delta':
addTranscript('user', event.delta || '');
break;
case 'conversation.item.input_audio_transcription.completed':
finishTranscript('user', event.transcript || '');
break;
case 'response.done':
case 'done':
clearTimeout(turnTimeout);
responseComplete = true;
finishTranscript('assistant');
if (!responseHasAudio) requestPlaybackDrain(responseId);
requestPlaybackDrain(responseId);
finishResponseIfReady();
break;
case 'playback.acknowledged':
{
const acknowledgement = event.event || event;
runtimeDetail.textContent = `Playback committed ${acknowledgement.committed_ms || 0} ms`;
}
case 'backpressure':
armTurnTimeout();
markBusy();
setModel('Thinking / Speaking');
runtimeDetail.textContent = action.message;
break;
case 'session.closed':
case 'ack':
runtimeDetail.textContent = `Playback committed ${action.committedMs} ms`;
break;
case 'closed':
if (sessionCloseResolver) sessionCloseResolver();
sessionCloseResolver = null;
break;
case 'error':
setConnection('Error', 'error');
runtimeDetail.textContent = String(event.error || event.code || 'Server error');
break;
default:
if (action.fatal) { await failSession(action.message); break; }
// A rejected frame or a refused update costs one operation, not the
// call: report it and keep listening.
appendLog(action.message, true);
runtimeDetail.textContent = action.message;
break;
default: break;
}
}
@@ -406,8 +489,15 @@
playbackRate = playbackContext.sampleRate;
await playbackContext.audioWorklet.addModule(staticAssetUrl('static/playback_worklet.js'));
playbackNode = new AudioWorkletNode(playbackContext, 'fullduplex-pcm-playback');
const currentPlaybackNode = playbackNode;
playbackNode.port.onmessage = (message) => {
if (playbackNode !== currentPlaybackNode) return;
if (message.data.type === 'playback-started') setPlayback('Playing');
else if (message.data.type === 'playback-stopped') {
// The old response may report after the next one has started. ACK its
// own cursor without changing the current response's UI state.
if (profile.playbackAck) sendPlaybackAck(message.data.responseId, Number(message.data.playedMs) || 0);
}
else if (message.data.type === 'playback-drained') playbackDrained(message.data);
else if (message.data.type === 'playback-underrun') {
runtimeDetail.textContent = `Playback underrun ${message.data.underrunMs || 0} ms`;
@@ -451,48 +541,69 @@
function openSocket() {
return new Promise((resolve, reject) => {
const url = realtimeUrl();
socket = new WebSocket(url);
const current = new WebSocket(url);
socket = current;
connectionReady = false;
turnCounter += 1;
let settled = false;
socket.onopen = () => {
const timer = window.setTimeout(() => {
if (settled) return;
settled = true;
const extraBody = {
auto_response: true,
};
const session = {
modalities: ['audio', 'text'],
voice: 'default',
extra_body: extraBody,
};
// Reference voice for TTS cloning, provided by the server via
// --ref-audio (mirrors the official demo's default ref audio).
if (config.refAudio) session.ref_audio = config.refAudio;
const instructions = systemPromptInput ? systemPromptInput.value.trim() : '';
if (instructions) session.instructions = instructions;
socket.send(JSON.stringify({ type: 'session.update', session }));
runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
current.onclose = null;
current.close();
reject(new Error(`Session handshake timed out. ${profile.connectionHint}`));
}, 15000);
const rejectOnce = (message) => {
clearTimeout(timer);
if (!settled) { settled = true; reject(new Error(message)); }
};
current.onopen = () => {
const instructions = systemPromptInput.value.trim();
for (const event of profile.initialMessages(config, instructions)) current.send(JSON.stringify(event));
appendLog(`websocket open ${url}`);
resolve();
};
socket.onmessage = (message) => {
if (typeof message.data !== 'string') return;
try {
handleEvent(JSON.parse(message.data));
} catch (error) {
appendLog(`invalid server event: ${error.message || error}`, true);
current.onmessage = (message) => {
if (typeof message.data !== 'string' || (socket !== current && closingSocket !== current)) return;
let event;
try { event = JSON.parse(message.data); }
catch (error) {
if (socket !== current) return;
const detail = `Invalid server event: ${error.message}`;
if (!settled) rejectOnce(detail);
else void failSession(detail);
return;
}
};
socket.onerror = () => {
if (!settled) {
// Shutdown has detached the active socket. Only its close acknowledgement
// may pass, directly: the event queue can itself be awaiting cleanup.
if (closingSocket === current) {
if (event?.type === 'session.closed' && sessionCloseResolver) sessionCloseResolver();
return;
}
const action = profile.mapEvent(event);
if (action.kind === 'error' && !settled) {
rejectOnce(action.message);
return;
}
if (event.type === profile.readyEvent && !settled) {
clearTimeout(timer);
settled = true;
reject(new Error(`WebSocket connection failed: ${url}`));
connectionReady = true;
runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
resolve();
}
// Serialize decoding with terminal events: a drain must never overtake
// an asynchronously decoded WAV chunk.
audioChain = audioChain.then(() => {
if (socket === current || action.kind === 'closed') return handleEvent(event);
}).catch((error) => failSession(`Server event failed: ${error.message}`));
};
socket.onclose = (event) => {
current.onerror = () => rejectOnce(`WebSocket connection failed. ${profile.connectionHint}`);
current.onclose = (event) => {
clearTimeout(timer);
connectionReady = false;
appendLog(`websocket closed code=${event.code}`);
if (running) {
setConnection('Disconnected', 'error');
stopSession();
}
if (!settled) rejectOnce(`Connection closed before session ready. ${profile.connectionHint}`);
else if (running && socket === current) failSession(`Backend disconnected (${event.code}). ${profile.connectionHint}`);
};
});
}
@@ -515,46 +626,59 @@
setConnection('Connecting', 'connecting');
runtimeDetail.textContent = 'Requesting microphone access';
try {
sessionGeneration += 1;
audioChain = Promise.resolve();
await openPlayback();
await openCapture();
await openSocket();
running = true;
muted = false;
assistantActive = false;
sendTimer = window.setInterval(flushCapture, SEND_INTERVAL_MS);
sendTimer = window.setInterval(flushCapture, profile.sendIntervalMs || SEND_INTERVAL_MS);
startClock();
callButton.textContent = 'End session';
callButton.classList.add('is-active');
muteButton.disabled = false;
cameraButton.disabled = false;
cameraButton.disabled = !profile.camera;
sendTurnButton.disabled = !profile.clientCommit;
setConnection('Connected', 'online');
setModel('Listening');
setModel(profile.waiting);
appendLog('session started');
} catch (error) {
appendLog(`start failed: ${error.message || error}`, true);
await stopSession({ terminal: false });
setConnection('Error', 'error');
runtimeDetail.textContent = String(error.message || error);
await stopSession();
} finally {
callButton.disabled = false;
}
}
async function startCamera() {
if (cameraStream) return;
if (!profile.camera || cameraStream) return;
cameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
cameraPreview.srcObject = cameraStream;
cameraPreview.style.display = '';
await cameraPreview.play().catch(() => {});
// Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk,
// no client-side resize (the server normalizes at scale_resolution=448).
cameraTimer = window.setInterval(() => {
const captureCameraFrame = () => {
if (!cameraStream || cameraPreview.videoWidth === 0) return;
cameraCanvas.width = cameraPreview.videoWidth;
cameraCanvas.height = cameraPreview.videoHeight;
cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0);
cameraPendingFrame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
}, 1000);
const scale = profile.cameraMaxDimension
? Math.min(1, profile.cameraMaxDimension / Math.max(cameraPreview.videoWidth, cameraPreview.videoHeight)) : 1;
cameraCanvas.width = Math.max(1, Math.round(cameraPreview.videoWidth * scale));
cameraCanvas.height = Math.max(1, Math.round(cameraPreview.videoHeight * scale));
cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0, cameraCanvas.width, cameraCanvas.height);
const frame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
if (profile.imageMessages) {
if (connectionReady && socket?.readyState === WebSocket.OPEN) {
for (const event of profile.imageMessages(frame)) socket.send(JSON.stringify(event));
}
} else cameraPendingFrame = frame;
};
// Do not make the first spoken turn race a one-second timer.
captureCameraFrame();
cameraTimer = window.setInterval(captureCameraFrame, 1000);
cameraButton.textContent = 'Camera off';
cameraButton.classList.add('is-active');
appendLog('camera on (1 fps omni frames)');
@@ -590,15 +714,32 @@
const finish = () => {
if (done) return;
done = true;
clearTimeout(timer);
if (sessionCloseResolver === finish) sessionCloseResolver = null;
resolve();
};
sessionCloseResolver = finish;
window.setTimeout(finish, timeoutMs);
const timer = window.setTimeout(finish, timeoutMs);
});
}
async function stopSession({ terminal = true } = {}) {
function stopSession(options = {}) {
if (stopping) return stopping;
stopping = cleanupSession(options).finally(() => { stopping = null; });
return stopping;
}
async function cleanupSession({ terminal = true } = {}) {
sessionGeneration += 1;
connectionReady = false;
clearTimeout(echoTimer);
clearTimeout(turnTimeout);
echoTimer = null;
responseComplete = false;
playbackComplete = true;
turnSubmitted = false;
assistantTextChannel = null;
sendTurnButton.disabled = true;
running = false;
assistantActive = false;
pendingCapture = [];
@@ -607,15 +748,16 @@
sendTimer = null;
clockTimer = null;
if (socket) {
const closingSocket = socket;
closingSocket = socket;
socket = null;
closingSocket.onclose = null;
if (terminal && closingSocket.readyState === WebSocket.OPEN) {
if (terminal && profile.closeSession && closingSocket.readyState === WebSocket.OPEN) {
const closed = waitForSessionClosed(closingSocket, SESSION_CLOSE_TIMEOUT_MS);
closingSocket.send(JSON.stringify({ type: 'session.close' }));
await closed;
}
closingSocket.close(1000, 'client stop');
closingSocket = null;
}
if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
if (mediaStream) {
@@ -632,6 +774,8 @@
playbackNode = null;
currentResponseId = null;
responseHasAudio = false;
finishTranscript('user');
finishTranscript('assistant');
meterFill.style.width = '0%';
sessionTimer.textContent = '00:00';
callButton.textContent = 'Start session';
@@ -659,6 +803,18 @@
if (running) stopSession();
else startSession();
});
sendTurnButton.addEventListener('click', () => {
if (!running || !connectionReady || !profile.clientCommit || turnSubmitted) return;
flushCapture();
for (const event of profile.commitMessages()) socket.send(JSON.stringify(event));
turnSubmitted = true;
responseComplete = false;
markBusy();
armTurnTimeout();
sendTurnButton.disabled = true;
setModel('Thinking');
appendLog('turn submitted');
});
muteButton.addEventListener('click', toggleMute);
clearLogButton.addEventListener('click', () => {
eventLog.textContent = '';
examples/online_serving/minicpmo/realtime_web/app/static/pcm_worklet.js → examples/online_serving/realtime_web/app/static/pcm_worklet.js
View File
File renamed without changes.
examples/online_serving/minicpmo/realtime_web/app/static/playback_worklet.js → examples/online_serving/realtime_web/app/static/playback_worklet.js
+7
View File
@@ -39,6 +39,13 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
}
this.notifyIfDrained();
} else if (message.type === 'clear') {
if (this.activeResponseId) {
this.port.postMessage({
type: 'playback-stopped',
responseId: this.activeResponseId,
playedMs: Math.round((this.playedFrames * 1000) / sampleRate),
});
}
this.queue = [];
this.offset = 0;
this.playedFrames = 0;
examples/online_serving/realtime_web/app/static/profiles/common.js
+56
View File
@@ -0,0 +1,56 @@
/* Shared wire helpers. Profiles translate server events into shell actions. */
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles = root.OmniRealtimeProfiles || {};
profiles.url = (config, location) => {
const url = new URL(config.realtimePath || 'v1/realtime', location);
if (url.protocol === 'http:') url.protocol = 'ws:';
if (url.protocol === 'https:') url.protocol = 'wss:';
return url;
};
/* Errors that leave the session unusable, so the shell has to tear it down.
Everything else costs one operation --- a rejected image, a refused
session.update, a late playback ack --- and the call carries on.
'runtime_append_failed' is deliberately absent: the engine reports it and
keeps the session, unlike 'runtime_append_task_failed' which kills it. */
profiles.fatalErrorCodes = new Set([
'unknown_session',
'session_closed',
'resource_exhausted',
'unsupported',
'unsupported_turn_detection',
'server_vad_unavailable',
'runtime_append_task_failed',
'internal_error',
]);
profiles.event = (event) => {
const responseId = event.response_id || event.response?.id || null;
const action = (kind, extra = {}) => ({ kind, responseId, ...extra });
switch (event.type) {
case 'session.created': case 'session.updated': return action('connected');
case 'response.created': return action('begin');
case 'response.audio.delta': case 'response.output_audio.delta':
return action('audio', { event: { ...event, delta: event.delta || event.audio || event.response?.audio } });
case 'response.audio.done': case 'response.output_audio.done': return action('drain');
case 'response.audio_transcript.delta': case 'response.output_audio_transcript.delta':
return action('text', { role: 'assistant', channel: 'audio', text: event.delta || '' });
case 'response.audio_transcript.done': case 'response.output_audio_transcript.done':
return action('text-final', { role: 'assistant', channel: 'audio', text: event.transcript || '' });
case 'conversation.item.input_audio_transcription.delta':
return action('text', { role: 'user', text: event.delta || '' });
case 'conversation.item.input_audio_transcription.completed':
return action('text-final', { role: 'user', text: event.transcript || '' });
case 'response.done': return action('done');
case 'session.closed': return action('closed');
case 'error': {
const code = event.code || event.error?.code;
return action('error', {
code,
fatal: profiles.fatalErrorCodes.has(code),
message: typeof event.error === 'string' ? event.error : event.error?.message || event.message || code || 'Server error',
});
}
default: return action('ignore');
}
};
})(globalThis);
examples/online_serving/realtime_web/app/static/profiles/minicpm_native.js
+55
View File
@@ -0,0 +1,55 @@
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles;
profiles['minicpm-native'] = () => ({
title: 'MiniCPM-o 4.5 Voice', eyebrow: 'Experimental full-duplex runtime',
policy: 'Model-controlled listen / speak',
description: 'Speak naturally. The model can listen and speak at the same time. Camera frames are sent with your audio.',
waiting: 'Listening', camera: true, playbackAck: true, clientCommit: false,
halfDuplex: false, closeSession: true, reconnectEachTurn: false,
readyEvent: 'session.updated', instructions: true, sendIntervalMs: 200,
presets: {
omni: 'Streaming Omni Conversation.',
chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
+ '请用高自然度的方式和用户聊天。你处于双工模式，可以一边听、一边说。'
+ '你是由面壁智能开发的人工智能助手：面壁小钢炮。',
english_call: 'Replicate the tone and style from the input audio. Your task is to be '
+ 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
+ 'seriously and in a high quality. Please chat with the user in a high naturalness '
+ 'style. You are in duplex mode, where you can listen and speak at the same time.',
},
url(config, location) {
const url = profiles.url(config, location);
url.searchParams.set('duplex', '1');
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
url.searchParams.set('autostart', '0');
return url.toString();
},
initialMessages(config, instructions) {
const session = { modalities: ['audio', 'text'], voice: 'default',
extra_body: { auto_response: true } };
if (config.refAudio) session.ref_audio = config.refAudio;
if (instructions) session.instructions = instructions;
return [{ type: 'session.update', session }];
},
append(audio, frame) {
const event = { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
if (frame) event.video_frames = [frame];
return event;
},
commitMessages: () => [],
ack(responseId, playedMs) {
return { type: 'playback.ack', response_id: responseId, item_id: `item_${responseId}`,
played_ms: playedMs, committed_ms: playedMs };
},
mapEvent(event) {
if (event.type === 'response.listen') return { kind: 'listen' };
if (event.type === 'response.speak') return { kind: 'begin', responseId: event.response_id || event.response?.id };
if (event.type === 'playback.acknowledged') {
return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
}
return profiles.event(event);
},
connectionHint: 'Check the MiniCPM duplex deployment and reference voice configuration.',
});
})(globalThis);
examples/online_serving/realtime_web/app/static/profiles/qwen3_turn.js
+100
View File
@@ -0,0 +1,100 @@
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles;
profiles['qwen3-turn'] = (config) => {
const vad = config.adapter === 'vad';
let imageSeq = 0;
let imageItems = [];
// The engine refuses a ninth image (or 4 MiB of them) for the whole
// session, so the camera track retires its own oldest frame instead.
const MAX_IMAGE_ITEMS = 8;
const hint = 'This Qwen deploy needs session_mode: duplex and Server VAD support; use --stt or enable Server VAD.';
return {
title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based realtime call',
policy: vad ? 'Server VAD • interruptible responses' : 'Send turn • one response at a time',
description: vad
? 'Pause to send your turn; speak again to interrupt. Microphone upload continues during playback. Requires the Qwen duplex plugin and Silero. Optional camera frames accompany each spoken turn.'
: 'Speak, then press Send turn. Microphone upload pauses while the model answers. Each turn uses a fresh connection; conversation history is not carried between turns. No barge-in or camera input.',
waiting: 'Waiting for you', camera: vad, cameraMaxDimension: 448, playbackAck: vad, clientCommit: !vad,
waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad, reconnectEachTurn: !vad,
inputSampleRate: vad ? 24000 : 16000,
readyEvent: vad ? 'session.updated' : 'session.created', instructions: vad, sendIntervalMs: 200,
presets: { assistant: 'You are a helpful assistant. Answer clearly and concisely.' },
url(config, location) {
const url = profiles.url(config, location);
for (const key of ['native_duplex', 'minicpmo45_native_duplex', 'autostart']) url.searchParams.delete(key);
// Explicit opt-out matches the local file client, including servers
// that auto-select the duplex endpoint when the query is omitted.
url.searchParams.set('duplex', vad ? '1' : '0');
url.searchParams.set('model', config.model || 'Qwen/Qwen3-Omni-30B-A3B-Instruct');
return url.toString();
},
initialMessages(config, instructions) {
imageItems = [];
if (!vad) return [
{ type: 'session.update', model: config.model },
{ type: 'input_audio_buffer.commit', final: false },
];
const session = { model: config.model, overlap_policy: 'barge_in_on_speech', audio: { input: {
format: { type: 'audio/pcm', rate: 24000 },
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
silence_duration_ms: 500, create_response: true, interrupt_response: true },
} } };
if (instructions) session.instructions = instructions;
return [{ type: 'session.update', session }];
},
append(audio) {
return vad ? { type: 'input_audio_buffer.append', audio }
: { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
},
// Qwen is a turn model: a frame is a picture added to the conversation,
// not a track interleaved into the audio. That is exactly the OpenAI
// Realtime image interface, so the camera sends conversation items
// rather than an extension field on the append.
imageMessages(frame) {
if (!vad) return [];
const messages = [];
if (imageItems.length >= MAX_IMAGE_ITEMS) {
// Retire before creating: the engine counts what is already stored.
messages.push({ type: 'conversation.item.delete', item_id: imageItems.shift() });
}
const id = `camera_${++imageSeq}`;
imageItems.push(id);
messages.push({ type: 'conversation.item.create', item: {
id, type: 'message', role: 'user',
content: [{ type: 'input_image', image_url: `data:image/jpeg;base64,${frame}` }],
} });
return messages;
},
commitMessages: () => vad ? [] : [{ type: 'input_audio_buffer.commit', final: true }],
ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
mapEvent(event) {
if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
// On the shipped STT route transcription.* is model-generated text,
// not a separate ASR transcript of the user. See realtime_connection.py.
return { kind: 'text', role: 'assistant', channel: 'text', text: event.delta || '' };
}
if (event.type === 'response.output_text.done' || event.type === 'transcription.done') {
return { kind: 'text-final', role: 'assistant', channel: 'text', text: event.text || '' };
}
if (vad && (event.type === 'output_audio_buffer.cleared' ||
(event.type === 'response.done' && event.response?.status === 'cancelled'))) {
return { kind: 'interrupt' };
}
if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };
if (!vad && event.type === 'response.output_audio.done') return { kind: 'done' };
const action = profiles.event(event);
if (action.kind === 'error') {
if (action.code === 'input_backpressure') return { ...action, kind: 'backpressure', message: 'Input buffer is full; please repeat dropped speech after the response.' };
if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact or restart this UI with --stt.';
else if (vad && action.code === 'unsupported') action.message += ` ${hint}`;
}
return action;
},
connectionHint: vad ? hint : 'Check the Qwen /v1/realtime STT endpoint and model name.',
};
};
})(globalThis);
examples/online_serving/minicpmo/realtime_web/app/static/styles.css → examples/online_serving/realtime_web/app/static/styles.css
+3
View File
@@ -125,3 +125,6 @@ button:disabled { opacity: .46; cursor: default; }
@media (prefers-reduced-motion: reduce) {
*, *::before, *::after { animation-duration: .01ms !important; animation-iteration-count: 1 !important; }
}
/* Profile-specific controls may have inline flex styles. */
[hidden] { display: none !important; }
examples/online_serving/realtime_web/server.py
+229
View File
@@ -0,0 +1,229 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Static host and same-origin proxy for the shared realtime browser UI."""
from __future__ import annotations
import argparse
import asyncio
import base64
import contextlib
import hashlib
import json
import logging
from pathlib import Path
from urllib.parse import urlencode
import uvicorn
import websockets
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse, Response
from fastapi.staticfiles import StaticFiles
logger = logging.getLogger(__name__)
APP_DIR = Path(__file__).parent / "app"
STATIC_DIR = APP_DIR / "static"
def _join_ws_url(base: str, path: str, query: str) -> str:
return base.rstrip("/") + path + (("?" + query) if query else "")
async def _pump_client_to_backend(client: WebSocket, backend) -> None:
try:
while True:
message = await client.receive()
if message["type"] == "websocket.disconnect":
await backend.close()
return
if message.get("text") is not None:
await backend.send(message["text"])
elif message.get("bytes") is not None:
await backend.send(message["bytes"])
except (WebSocketDisconnect, websockets.ConnectionClosed):
with contextlib.suppress(Exception):
await backend.close()
async def _pump_backend_to_client(client: WebSocket, backend) -> None:
try:
async for message in backend:
if isinstance(message, bytes):
await client.send_bytes(message)
else:
await client.send_text(message)
except (WebSocketDisconnect, websockets.ConnectionClosed):
return
except RuntimeError as exc:
if "websocket.send" in str(exc) and "websocket.close" in str(exc):
return
raise
def _expected_proxy_close(exc: BaseException) -> bool:
if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
return True
return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
def build_app(
*,
ws_backend: str | None = None,
model: str | None = None,
public_realtime_url: str | None = None,
ref_audio: str | None = None,
profile: str = "minicpm-native",
adapter: str = "stt",
) -> FastAPI:
if profile not in {"minicpm-native", "qwen3-turn"}:
raise ValueError(f"Unknown profile: {profile}")
if adapter not in {"stt", "vad"}:
raise ValueError(f"Unknown adapter: {adapter}")
if profile == "qwen3-turn" and ref_audio:
raise ValueError("Qwen3 profile does not accept --ref-audio")
native = profile == "minicpm-native"
ws_backend = ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
model = model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
app = FastAPI(title="Omni Realtime Web UI")
index_path = APP_DIR / "index.html"
app_version_hash = hashlib.sha256()
for asset_path in (
STATIC_DIR / "app.js",
STATIC_DIR / "pcm_worklet.js",
STATIC_DIR / "playback_worklet.js",
*sorted((STATIC_DIR / "profiles").glob("*.js")),
):
app_version_hash.update(asset_path.read_bytes())
app_version = app_version_hash.hexdigest()[:12]
ref_audio_uri: str | None = None
if ref_audio:
ref_path = Path(ref_audio)
if not ref_path.is_file():
raise SystemExit(f"--ref-audio not found: {ref_audio}")
encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
ref_audio_uri = f"data:audio/wav;base64,{encoded}"
@app.get("/", response_class=HTMLResponse)
def index() -> HTMLResponse:
config = json.dumps(
{
"model": model,
"profile": profile,
"adapter": adapter,
"realtimePath": public_realtime_url or "v1/realtime",
"refAudio": ref_audio_uri,
"appVersion": app_version,
},
ensure_ascii=True,
).replace("<", "\\u003c")
html = (
index_path.read_text(encoding="utf-8")
.replace("__FULL_DUPLEX_CONFIG__", config)
.replace("__FULL_DUPLEX_APP_VERSION__", app_version)
)
return HTMLResponse(html, headers={"Cache-Control": "no-store"})
@app.get("/healthz")
def healthz() -> Response:
return Response(content="ok", media_type="text/plain")
@app.websocket("/v1/realtime")
async def realtime_proxy(websocket: WebSocket) -> None:
await websocket.accept()
query = urlencode(websocket.query_params.multi_items())
backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
logger.info("Proxying Realtime WebSocket to %s", backend_url)
try:
async with websockets.connect(
backend_url,
max_size=64 * 1024 * 1024,
) as backend:
tasks = {
asyncio.create_task(_pump_client_to_backend(websocket, backend)),
asyncio.create_task(_pump_backend_to_client(websocket, backend)),
}
done, pending = await asyncio.wait(
tasks,
return_when=asyncio.FIRST_COMPLETED,
)
for task in pending:
task.cancel()
await asyncio.gather(*pending, return_exceptions=True)
for result in await asyncio.gather(*done, return_exceptions=True):
if isinstance(result, BaseException) and not _expected_proxy_close(result):
raise result
except (WebSocketDisconnect, websockets.ConnectionClosed):
return
except (OSError, TimeoutError) as exc:
message = (
f"Cannot connect to realtime backend {ws_backend}. "
"Check that vllm serve is running and --backend matches its --port. "
"For containers or remote hosts, use the backend address reachable from the UI server."
)
logger.warning("%s (%s)", message, exc)
with contextlib.suppress(Exception):
await websocket.send_json({"type": "error", "code": "backend_unavailable", "error": message})
await websocket.close(code=1011)
except Exception:
logger.exception("Realtime WebSocket proxy failed")
with contextlib.suppress(Exception):
await websocket.close(code=1011)
finally:
with contextlib.suppress(Exception):
await websocket.close()
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
return app
def main(default_profile: str = "minicpm-native") -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=7862)
parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn"), default=default_profile)
parser.add_argument("--ws-backend", "--backend", dest="ws_backend")
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--stt", dest="adapter", action="store_const", const="stt")
mode.add_argument("--vad", dest="adapter", action="store_const", const="vad")
parser.set_defaults(adapter="stt")
parser.add_argument(
"--public-realtime-url",
help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
)
parser.add_argument("--model")
parser.add_argument(
"--ref-audio",
help=(
"Reference voice wav for TTS voice cloning, e.g. the "
"official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
),
)
args = parser.parse_args()
native = args.profile == "minicpm-native"
if native and not args.ref_audio:
parser.error("--ref-audio is required for the minicpm-native profile")
if not native and args.ref_audio:
parser.error("--ref-audio is only supported by minicpm-native")
args.model = args.model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
args.ws_backend = args.ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
logging.basicConfig(level=logging.INFO)
uvicorn.run(
build_app(
ws_backend=args.ws_backend,
model=args.model,
public_realtime_url=args.public_realtime_url,
ref_audio=args.ref_audio,
profile=args.profile,
adapter=args.adapter,
),
host=args.host,
port=args.port,
log_level="info",
)
if __name__ == "__main__":
main()
tests/config/test_environment_variables.py
+2 -2
View File
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
EnvironmentVariableCategory.PUBLIC_OMNI: 28,
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
EnvironmentVariableCategory.MODEL_SPECIFIC: 64,
EnvironmentVariableCategory.MODEL_SPECIFIC: 65,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
EnvironmentVariableCategory.INTERNAL: 2,
}
@@ -181,7 +181,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
ModelEnvironmentVariableDisposition.PROMOTE: 38,
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
ModelEnvironmentVariableDisposition.INTERNALIZE: 15,
ModelEnvironmentVariableDisposition.INTERNALIZE: 16,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
}
tests/engine/duplex/test_engine_session.py
+2 -2
View File
@@ -567,10 +567,10 @@ def test_overlap_policy_defaults_and_invalid_values_to_listen_only():
assert DuplexSessionConfig.from_event({"session": {"overlap_policy": "bogus"}}).overlap_policy == "listen_only"
def test_capabilities_as_dict_reports_model_native_duplex_constants():
def test_capabilities_as_dict_reports_declared_implementation():
caps = DuplexCapabilities().as_dict()
assert caps["implementation_level"] == "model_native_duplex"
assert caps["implementation_level"] == "turn_based_duplex"
assert caps["input_modes"] == ["append_audio_chunk"]
assert caps["supports_kv_lease"] is False
assert caps["supports_core_kv_lease"] is False
tests/engine/duplex/test_session_manager.py
+4 -2
View File
@@ -196,7 +196,7 @@ class FakePlugin(DuplexModelPlugin):
def capabilities(self, *, max_sessions: int) -> DuplexCapabilities:
del max_sessions
return DuplexCapabilities(supports_input_append=True)
return DuplexCapabilities(supports_input_append=True, supports_core_resumable_request=True)
def validate_client_extra_body(self, extra_body: object) -> None:
pass
@@ -408,7 +408,9 @@ async def test_open_answers_with_capabilities_and_emits_session_created() -> Non
assert result.control_id == "open-sid-open"
assert result.session_id == "sid-open"
assert result.lease_generation == 0
assert result.capabilities == DuplexCapabilities(supports_input_append=True)
assert result.capabilities == DuplexCapabilities(
supports_input_append=True, supports_core_resumable_request=True
)
assert result.public_session is not None
assert result.public_session["id"] == "sid-open"
assert result.public_session["voice"] == "test"
tests/engine/duplex/test_vad_backend.py
+25 -6
View File
@@ -107,8 +107,8 @@ def test_endpoint_decisions_match_the_serving_side_policy(
# --------------------------------------------------------------------------- #
def test_a_loud_frame_does_not_restart_the_silence_timer() -> None:
"""Silero v6.2 hysteresis: once silence is running, a loud frame only delays it.
def test_hysteresis_band_does_not_restart_the_silence_timer() -> None:
"""Scores between the negative and activation thresholds preserve the timer.
The frame at 0.45 sits above the negative threshold (0.35) but below the
activation threshold. It must not cancel the pending endpoint, or a speaker
@@ -164,9 +164,14 @@ def _pcm16_b64(samples: np.ndarray) -> str:
def test_pcm16_input_is_accepted_alongside_float32() -> None:
config = SileroVADConfig(threshold=0.5)
tone = np.zeros(FRAME * 2, dtype=np.float32)
tone: np.ndarray = np.zeros(FRAME * 2, dtype=np.float32)
seen: list[int] = []
vad = SileroStreamingVAD(config, frame_scorer=lambda frame: seen.append(frame.size) or 0.0)
def _score_frame(frame: np.ndarray) -> float:
seen.append(frame.size)
return 0.0
vad = SileroStreamingVAD(config, frame_scorer=_score_frame)
vad.process_base64(_pcm16_b64(tone), fmt="pcm16", sample_rate_hz=SAMPLE_RATE_HZ)
assert seen == [FRAME, FRAME]
@@ -272,7 +277,7 @@ def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tm
intra_op_num_threads = 0
class FakeInferenceSession:
def __init__(self, path: str, *, providers: list[str], sess_options: object) -> None:
def __init__(self, path: str, *, providers: list[str], sess_options: FakeSessionOptions) -> None:
self.providers = providers
self.sess_options = sess_options
self.calls: list[dict[str, np.ndarray]] = []
@@ -313,7 +318,7 @@ def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tm
assert session.calls[0]["sr"].item() == SAMPLE_RATE_HZ
state = backend.new_state()
first_frame = np.arange(backend.frame_samples, dtype=np.float32)
first_frame: np.ndarray = np.arange(backend.frame_samples, dtype=np.float32)
probability, state = backend.infer(first_frame, state)
first_call = session.calls[1]
@@ -377,3 +382,17 @@ def test_the_documented_server_vad_fields_are_all_accepted() -> None:
)
is None
)
@pytest.mark.parametrize("speech_probability", [0.5, 0.9])
def test_resumed_speech_cancels_pending_silence(speech_probability):
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=96, min_speech_duration_ms=32)
# 32 ms pause, resumed speech, then a full 96 ms pause: one utterance.
scores = [0.9, 0.1, speech_probability, speech_probability, 0.1, 0.1, 0.1]
assert _drive(config, scores) == [("start", 0), ("stop", 224)]
def test_short_pauses_do_not_split_continuous_speech_into_repeated_turns():
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=500, min_speech_duration_ms=32)
scores = [0.9] + [0.1, 0.9, 0.9, 0.9] * 20
assert _drive(config, scores) == [("start", 0)]
tests/engine/test_duplex_orchestrator.py
+45
View File
@@ -515,3 +515,48 @@ async def test_a_dead_replica_closes_the_sessions_it_was_serving() -> None:
messages = [output_q.get_nowait() for _ in range(output_q.qsize())]
types = [getattr(getattr(m, "event", None), "type", type(m).__name__) for m in messages]
assert types[-1] in {"session.expired", "session.closed"}, f"the client needs a terminal event, got {types}"
@pytest.mark.asyncio
async def test_turn_plugin_processes_multimodal_prompt_before_stage_submission(monkeypatch):
from vllm_omni.engine.orchestrator import build_engine_core_request_from_tokens
from vllm_omni.model_executor.models.qwen3_omni.duplex.plugin import Qwen3OmniDuplexPlugin
orchestrator, clients, rpc_q, _ = _build()
plugin = Qwen3OmniDuplexPlugin(_encode_audio)
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: "audio prompt")
orchestrator.plugin = orchestrator.session_manager.plugin = plugin
seen = []
def process_inputs(**kwargs):
seen.append(kwargs)
return build_engine_core_request_from_tokens(
request_id=kwargs["request_id"],
prompt={"prompt_token_ids": [1, 2]},
params=kwargs["params"],
model_config=orchestrator.stage_pools[0].stage_vllm_config.model_config,
resumable=kwargs["resumable"],
)
monkeypatch.setattr(
orchestrator, "_get_stage_input_processor", lambda stage_id: SimpleNamespace(process_inputs=process_inputs)
)
await orchestrator._dispatch_message(
OpenDuplexSessionMessage(
control_id="open-qwen",
session_id=SESSION_ID,
session_config=DuplexSessionConfig(model="qwen", modalities=["text", "audio"]),
)
)
assert (await rpc_q.get()).ok
try:
await _submit(orchestrator, _append_audio())
assert not clients[0].add_request_calls
await _submit(orchestrator, commands.Commit(final=True, create_response=True))
assert len(seen) == 1
assert seen[0]["prompt"]["multi_modal_data"]["audio"][0][0].shape == (16000,)
submitted = clients[0].add_request_calls[0][0]
assert submitted.resumable is False
assert not orchestrator.request_states[submitted.request_id].streaming.enabled
finally:
await _close(orchestrator, rpc_q)
tests/entrypoints/openai_api/test_api_server_guards.py
+7 -9
View File
@@ -47,6 +47,7 @@ from argparse import Namespace
from contextlib import asynccontextmanager
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
import pytest
from fastapi import FastAPI, HTTPException
@@ -564,7 +565,7 @@ async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch
Fails if HTTP requests lose ``request_timestamp``, or WebSocket scopes
start getting stamped / mutated incorrectly.
"""
captured: dict[str, object] = {}
captured: dict[str, Any] = {}
def fake_build_openai_app(args, supported_tasks):
return FastAPI()
@@ -592,8 +593,8 @@ async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch
await api_server.omni_run_server_worker("127.0.0.1:8000", _FakeSocket(), _minimal_args())
middleware = captured["served_app"]
http_scope = {"type": "http", "state": {}}
ws_scope = {"type": "websocket"}
http_scope: dict[str, Any] = {"type": "http", "state": {}}
ws_scope: dict[str, Any] = {"type": "websocket"}
async def _receive():
return {"type": "http.disconnect"}
@@ -920,11 +921,10 @@ async def test_pure_diffusion_app_state_key_snapshot(monkeypatch) -> None:
engine = _FakeEngineClient(stage_configs=[stage])
def _for_diffusion_factory(label: str):
@classmethod
def _factory(cls, *args, **kwargs):
return _marker(label)
return _factory
return classmethod(_factory)
monkeypatch.setattr(api_server.OmniOpenAIServingChat, "for_diffusion", _for_diffusion_factory("chat"))
monkeypatch.setattr(api_server.OmniOpenAIServingChatBatch, "for_diffusion", _for_diffusion_factory("chat_batch"))
@@ -965,13 +965,11 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
speech_kwargs = {}
def _for_diffusion_factory(label: str):
@classmethod
def _factory(cls, *args, **kwargs):
return _marker(label)
return _factory
return classmethod(_factory)
@classmethod
def _speech_factory(cls, *args, **kwargs):
speech_kwargs.update(kwargs)
return _marker("speech")
@@ -985,7 +983,7 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
)
monkeypatch.setattr(api_server.OmniOpenAIServingVideo, "for_diffusion", _for_diffusion_factory("video"))
monkeypatch.setattr(api_server.OmniStreamingVideoOutputHandler, "__init__", lambda self, *a, **k: None)
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", _speech_factory)
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", classmethod(_speech_factory))
monkeypatch.setattr(
api_server.ServingRealtimeRobotOpenPI,
"create_policy_server",
tests/entrypoints/openai_api/test_duplex_api_server.py
+28
View File
@@ -490,3 +490,31 @@ async def test_a_model_that_does_not_declare_chat_completions_does_not_get_the_r
assert state.openai_serving_chat is None
assert isinstance(state.openai_serving_duplex, OmniDuplexSessionHandler)
def test_qwen_plugin_preserves_custom_turn_deployment_default(monkeypatch, tmp_path):
from vllm_omni.model_executor.models.qwen3_omni.pipeline import QWEN3_OMNI_PIPELINE
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **kwargs: QWEN3_OMNI_PIPELINE)
deploy = tmp_path / "qwen.yaml"
deploy.write_text("stages: []\n")
assert not api_server._should_serve_duplex("qwen", {"deploy_config": str(deploy)})
deploy.write_text("session_mode: duplex\nstages: []\n")
assert api_server._should_serve_duplex("qwen", {"deploy_config": str(deploy)})
@pytest.mark.parametrize("flag", ["1", "true", "on"])
def test_turn_deployment_rejects_explicit_duplex_without_falling_back_to_stt(flag: str) -> None:
app = _duplex_app(None)
# Even an available STT service must not silently accept a duplex request.
app.state.openai_serving_realtime = object()
with TestClient(app) as client:
with client.websocket_connect(f"/v1/realtime?duplex={flag}") as websocket:
assert websocket.receive_json() == {
"type": "error",
"code": "unsupported",
"error": "VAD realtime is not enabled",
}
with pytest.raises(WebSocketDisconnect) as exc:
websocket.receive_text()
assert exc.value.code == 1008
tests/examples/test_minicpmo_realtime_web_server.py
+47 -1
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import hashlib
import importlib.util
import sys
@@ -70,7 +73,12 @@ def test_build_app_injects_public_realtime_url():
def test_build_app_versions_client_bundle_and_disables_index_cache():
client = TestClient(build_app())
expected_version_hash = hashlib.sha256()
for asset_name in ("app.js", "pcm_worklet.js", "playback_worklet.js"):
for asset_name in (
"app.js",
"pcm_worklet.js",
"playback_worklet.js",
*[str(p.relative_to(STATIC_DIR)) for p in sorted((STATIC_DIR / "profiles").glob("*.js"))],
):
expected_version_hash.update((STATIC_DIR / asset_name).read_bytes())
expected_version = expected_version_hash.hexdigest()[:12]
@@ -97,3 +105,41 @@ def test_build_app_exposes_realtime_websocket_and_static_assets():
assert client.get("/static/app.js").status_code == 200
assert client.get("/static/pcm_worklet.js").status_code == 200
assert client.get("/static/playback_worklet.js").status_code == 200
def test_qwen_config_does_not_inject_reference_voice():
client = TestClient(build_app(profile="qwen3-turn", adapter="vad", model="Qwen/Qwen3-Omni-30B-A3B-Instruct"))
page = client.get("/").text
assert '"profile": "qwen3-turn"' in page
assert '"adapter": "vad"' in page
assert '"refAudio": null' in page
for name in ("common", "minicpm_native", "qwen3_turn"):
assert client.get(f"/static/profiles/{name}.js").status_code == 200
with pytest.raises(ValueError, match="ref-audio"):
build_app(profile="qwen3-turn", ref_audio="unused.wav")
def test_config_cannot_close_its_script_element():
page = TestClient(build_app(model="</script><script>alert(1)</script>")).get("/").text
assert "</script><script>alert(1)" not in page
assert "\\u003c/script>" in page
def test_proxy_reports_unreachable_backend(monkeypatch):
from examples.online_serving.realtime_web import server as shared_server
class RefusedConnection:
async def __aenter__(self):
raise ConnectionRefusedError(111, "Connection refused")
async def __aexit__(self, *args):
pass
monkeypatch.setattr(shared_server.websockets, "connect", lambda *args, **kwargs: RefusedConnection())
client = TestClient(build_app(ws_backend="ws://127.0.0.1:8091"))
with client.websocket_connect("/v1/realtime?duplex=0") as ws:
event = ws.receive_json()
assert event["type"] == "error"
assert event["code"] == "backend_unavailable"
assert "ws://127.0.0.1:8091" in event["error"]
assert "--port" in event["error"]
tests/examples/test_minicpmo_realtime_web_static.py
+18 -53
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import re
import shutil
import subprocess
import textwrap
@@ -9,7 +8,7 @@ from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2] / "examples" / "online_serving" / "minicpmo" / "realtime_web"
ROOT = Path(__file__).resolve().parents[2] / "examples" / "online_serving" / "realtime_web"
APP_ROOT = ROOT / "app"
STATIC_ROOT = APP_ROOT / "static"
@@ -30,73 +29,37 @@ def test_page_exposes_focused_call_conversation_and_log_surfaces():
assert "Server VAD" not in html
def test_client_uses_proxy_relative_realtime_url_and_model_policy_session():
def test_shared_shell_delegates_wire_contract_to_profiles():
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
assert "profile.url(config, window.location.href)" in source
assert "profile.initialMessages(config, instructions)" in source
assert "profile.append(" in source
assert "profile.mapEvent(event)" in source
assert "native_duplex" not in source
assert "server_vad" not in source
assert "new URL(config.realtimePath, window.location.href)" in source
assert "url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'" in source
assert "url.searchParams.set('autostart', '0')" in source
def test_native_profile_preserves_session_and_capture_policy():
source = (STATIC_ROOT / "profiles/minicpm_native.js").read_text(encoding="utf-8")
assert "native_duplex" not in source
assert "auto_response: true" in source
assert "input_audio_buffer.append" in source
assert "input_audio_buffer.commit" not in source
assert "playback.ack" in source
assert "event.event || event" in source
assert "response.output_audio.delta" in source
assert "response.output_audio_transcript.delta" in source
assert "conversation.item.input_audio_transcription" in source
assert "force_barge_in" not in source
assert "server_vad" not in source
assert "type: 'response.create'" not in source
assert 'type: "response.create"' not in source
def test_web_server_requires_ref_audio_for_audio_output_session():
source = (ROOT / "server.py").read_text(encoding="utf-8")
ref_audio_arg = re.search(
r"parser\.add_argument\(\s*\"--ref-audio\",(?P<body>.*?)\n\s*\)",
source,
re.DOTALL,
)
assert ref_audio_arg is not None
assert "required=True" in ref_audio_arg.group("body")
assert "Optional reference voice" not in ref_audio_arg.group("body")
def test_client_sends_ref_audio_in_realtime_session_contract():
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
assert "if (config.refAudio) session.ref_audio = config.refAudio;" in source
assert "extraBody.ref_audio" not in source
assert "halfDuplex: false" in source
assert "commitMessages: () => []" in source
assert "video_frames" in source
assert "playback.ack" in source
def test_client_has_transactional_cleanup_and_visible_event_logging():
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
assert "async function stopSession" in source
assert "type: 'session.close'" in source
assert "async function cleanupSession" in source
assert "waitForSessionClosed" in source
assert "SESSION_CLOSE_TIMEOUT_MS" in source
assert "case 'session.closed':" in source
assert "track.stop()" in source
assert "clearInterval(sendTimer)" in source
assert "appendEventLog(event)" in source
assert "stopSession({ terminal: false })" in source
def test_client_keeps_microphone_upload_active_during_assistant_playback():
source = (STATIC_ROOT / "app.js").read_text(encoding="utf-8")
upload_gate = re.search(r"function microphoneUploadEnabled\(\) \{(?P<body>.*?)\n \}", source, re.DOTALL)
begin_assistant = re.search(r"function beginAssistant\(responseId\) \{(?P<body>.*?)\n \}", source, re.DOTALL)
assert upload_gate is not None
assert "return running && !muted;" in upload_gate.group("body")
assert "assistantActive" not in upload_gate.group("body")
assert begin_assistant is not None
assert "pendingCapture = []" not in begin_assistant.group("body")
def test_audio_worklets_define_capture_and_playback_processors():
capture = (STATIC_ROOT / "pcm_worklet.js").read_text(encoding="utf-8")
playback = (STATIC_ROOT / "playback_worklet.js").read_text(encoding="utf-8")
@@ -138,6 +101,7 @@ def test_playback_worklet_waits_before_playing_and_rebuffers_after_underrun():
node = shutil.which("node")
if node is None:
pytest.skip("node is required for the AudioWorklet regression test")
assert node is not None
script = textwrap.dedent(
"""
@@ -216,6 +180,7 @@ def test_playback_worklet_fades_terminal_drain_to_zero():
node = shutil.which("node")
if node is None:
pytest.skip("node is required for the AudioWorklet regression test")
assert node is not None
script = textwrap.dedent(
"""
tests/examples/test_realtime_web_profiles.cjs
+412
View File
@@ -0,0 +1,412 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const root = path.resolve(__dirname, '../../examples/online_serving/realtime_web/app/static');
const context = vm.createContext({ URL });
for (const name of ['common', 'minicpm_native', 'qwen3_turn']) {
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), context);
}
const profiles = context.OmniRealtimeProfiles;
const plain = (value) => JSON.parse(JSON.stringify(value));
const config = { model: 'Qwen/Qwen3-Omni-30B-A3B-Instruct', realtimePath: 'v1/realtime' };
test('MiniCPM keeps native flags, reference voice, camera, acknowledgements and no commits', () => {
const native = profiles['minicpm-native']();
const url = new URL(native.url({ ...config, model: 'MiniCPM' }, 'https://localhost/ui/'));
assert.equal(url.protocol, 'wss:');
assert.equal(url.searchParams.get('duplex'), '1');
assert.equal(url.searchParams.has('native_duplex'), false);
const [update] = native.initialMessages({ refAudio: 'data:audio/wav;base64,AA==' }, 'Prompt');
assert.equal(update.session.ref_audio, 'data:audio/wav;base64,AA==');
assert.equal(update.session.extra_body.native_duplex, undefined);
assert.equal(update.session.instructions, 'Prompt');
assert.deepEqual(plain(native.commitMessages()), []);
assert.equal(native.append('PCM', 'JPEG').video_frames[0], 'JPEG');
assert.equal(native.ack('r1', 200).type, 'playback.ack');
assert.equal(native.halfDuplex, false);
});
test('STT uses the shipped commit sequence and audio payload, including each new turn', () => {
const stt = profiles['qwen3-turn'](config);
for (let turn = 0; turn < 2; turn++) {
assert.deepEqual(plain(stt.initialMessages(config, 'ignored')), [
{ type: 'session.update', model: config.model },
{ type: 'input_audio_buffer.commit', final: false },
]);
assert.deepEqual(plain(stt.commitMessages()), [{ type: 'input_audio_buffer.commit', final: true }]);
}
assert.equal(stt.reconnectEachTurn, true);
assert.equal(stt.mapEvent({ type: 'response.output_audio.delta', audio: 'PCM' }).event.delta, 'PCM');
assert.equal(stt.mapEvent({ type: 'response.output_audio.done' }).kind, 'done');
assert.equal(stt.mapEvent({ type: 'transcription.delta', delta: 'hello' }).role, 'assistant');
});
test('Qwen enables camera and playback ACK only for the duplex VAD profile', () => {
for (const adapter of ['stt', 'vad']) {
const p = profiles['qwen3-turn']({ ...config, adapter });
const url = new URL(p.url({ ...config, realtimePath: 'wss://backend/v1/realtime?native_duplex=1&minicpmo45_native_duplex=1' }, 'http://localhost/'));
assert.equal(url.protocol, 'wss:');
assert.equal(url.searchParams.get('duplex'), adapter === 'vad' ? '1' : '0');
assert.equal(url.searchParams.has('native_duplex'), false);
assert.equal(url.searchParams.has('minicpmo45_native_duplex'), false);
assert.equal(p.append('PCM', 'JPEG').video_frames, undefined);
assert.equal(plain(p.imageMessages('JPEG')).length, adapter === 'vad' ? 1 : 0);
assert.equal(p.ack('r', 100)?.type || null, adapter === 'vad' ? 'playback.ack' : null);
assert.equal(p.camera, adapter === 'vad');
assert.equal(p.mapEvent({ type: 'response.listen' }).kind, 'ignore');
}
});
test('VAD uses nested format and interruptible endpoint detection', () => {
const vad = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
const [update] = vad.initialMessages(config, 'help');
assert.deepEqual(plain(update.session.audio.input), {
format: { type: 'audio/pcm', rate: 24000 },
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
silence_duration_ms: 500, create_response: true, interrupt_response: true },
});
assert.equal(update.session.extra_body, undefined);
assert.deepEqual(plain(vad.commitMessages()), []);
assert.equal(vad.mapEvent({ type: 'response.audio.done' }).kind, 'drain');
assert.equal(vad.mapEvent({ type: 'response.done' }).kind, 'done');
assert.equal(vad.mapEvent({ type: 'response.output_text.delta', delta: 'hello' }).text, 'hello');
assert.equal(vad.mapEvent({ type: 'error', error: { code: 'input_backpressure' } }).kind, 'backpressure');
assert.equal(vad.mapEvent({ type: 'input_audio_buffer.cleared' }).kind, 'backpressure');
assert.match(vad.mapEvent({ type: 'error', code: 'unsupported', error: 'Unavailable' }).message, /session_mode: duplex/);
assert.match(vad.mapEvent({ type: 'error', code: 'server_vad_initialization_failed' }).message, /Silero/);
});
function shell(profileName, adapter = 'stt', options = {}) {
const elements = new Map();
class Element {
constructor() { this.style = {}; this.listeners = {}; this.children = []; this.value = ''; this.textContent = ''; this.classList = { add() {}, remove() {}, toggle() {} }; }
addEventListener(name, callback) { this.listeners[name] = callback; }
append(...children) { this.children.push(...children); }
appendChild(child) { this.children.push(child); }
replaceChildren() { this.children = []; }
remove() {}
}
const document = {
getElementById(id) { if (!elements.has(id)) elements.set(id, new Element()); return elements.get(id); },
createElement() { return new Element(); },
};
const timers = new Map();
let timerId = 0;
const sockets = [];
const nodes = [];
class Socket {
static OPEN = 1;
constructor() {
this.readyState = 1; this.sent = []; sockets.push(this);
queueMicrotask(() => {
this.onopen?.();
if (!options.silent) this.onmessage?.({ data: JSON.stringify(options.error || { type: adapter === 'stt' && profileName === 'qwen3-turn' ? 'session.created' : 'session.updated' }) });
});
}
send(value) { this.sent.push(JSON.parse(value)); }
close() { this.readyState = 3; this.onclose?.({ code: 1000 }); }
}
class AudioContext {
constructor(options) { this.sampleRate = options.sampleRate; this.audioWorklet = { addModule: async () => {} }; this.destination = {}; }
createMediaStreamSource() { return { connect() {} }; }
createGain() { return { gain: {}, connect() {} }; }
async resume() {}
async close() {}
}
class AudioWorkletNode {
constructor(_context, name) { this.name = name; this.sent = []; nodes.push(this); this.port = { postMessage: (message) => this.sent.push(message) }; }
connect(target) { return target; }
}
const ctx = vm.createContext({
URL, document, AudioContext, AudioWorkletNode, WebSocket: Socket,
navigator: { mediaDevices: { getUserMedia: async () => ({ getTracks: () => [{ stop() {} }] }) } },
location: { href: 'https://localhost/' },
OMNI_REALTIME_CONFIG: { ...config, profile: profileName, adapter },
addEventListener() {},
setTimeout(fn, ms) { const id = ++timerId; timers.set(id, { fn, ms }); return id; },
clearTimeout(id) { timers.delete(id); },
setInterval() { return ++timerId; }, clearInterval() {},
btoa: (value) => Buffer.from(value, 'binary').toString('base64'),
atob: (value) => Buffer.from(value, 'base64').toString('binary'),
});
ctx.window = ctx;
for (const name of ['common', 'minicpm_native', 'qwen3_turn']) {
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), ctx);
}
// Expose closure controls only in the test VM; production has no test API.
const source = fs.readFileSync(path.join(root, 'app.js'), 'utf8').replace(/\}\)\(\);\s*$/, `
globalThis.testUI = { startSession, stopSession, handleEvent, playbackDrained,
microphoneUploadEnabled, flushCapture,
capture() { pendingCapture.push(new Int16Array([100, 200])); },
state() { return { running, connectionReady, assistantActive }; }
};
})();`);
vm.runInContext(source, ctx);
return { ui: ctx.testUI, elements, sockets, nodes,
timeout() { for (const timer of timers.values()) if (timer.ms === 15000) timer.fn(); },
async echo() {
for (const [id, timer] of timers) if (timer.ms === 300) { timers.delete(id); await timer.fn(); }
},
async turnWatchdog() {
for (const [id, timer] of timers) if (timer.ms === 120000) { timers.delete(id); await timer.fn(); }
},
};
}
test('shared shell STT sends final commit and starts a fresh second turn only after playback', async () => {
const app = shell('qwen3-turn');
await app.ui.startSession();
assert.equal(app.ui.microphoneUploadEnabled(), true);
app.ui.capture(); app.ui.flushCapture();
app.elements.get('sendTurnButton').listeners.click();
assert.equal(app.sockets[0].sent.at(-1).final, true);
assert.equal(app.ui.microphoneUploadEnabled(), false);
await app.ui.handleEvent({ type: 'response.output_audio.delta', audio: 'AAAAAA==' });
await app.ui.handleEvent({ type: 'response.output_audio.done' });
await app.echo();
assert.equal(app.sockets.length, 1, 'wait for actual speaker drain');
app.ui.playbackDrained({ responseId: 'turn-1', playedMs: 1 });
await app.echo();
assert.equal(app.sockets.length, 2);
assert.equal(app.sockets[1].sent[1].final, false);
assert.equal(app.ui.microphoneUploadEnabled(), true);
assert.equal(app.elements.get('cameraButton').hidden, true);
assert.equal(app.sockets.flatMap(s => s.sent).some(e => e.type === 'playback.ack'), false);
await app.ui.stopSession({ terminal: false });
});
test('VAD keeps uploading while generating and waits for response.done after audio drain', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'error', code: 'input_backpressure' });
assert.equal(app.ui.microphoneUploadEnabled(), true);
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
await app.ui.handleEvent({ type: 'response.audio.delta', delta: 'AAAAAA==', response_id: 'r1' });
await app.ui.handleEvent({ type: 'response.audio.done', response_id: 'r1' });
app.ui.playbackDrained({ responseId: 'r1', playedMs: 1 });
await app.echo();
assert.equal(app.ui.microphoneUploadEnabled(), true, 'continuous capture remains enabled');
assert.equal(app.ui.state().assistantActive, true, 'audio drain must wait for response.done');
assert.equal(app.sockets[0].sent.at(-1).type, 'playback.ack');
await app.ui.handleEvent({ type: 'response.done', response: { id: 'r1' } });
await app.echo();
assert.equal(app.ui.microphoneUploadEnabled(), true);
assert.equal(app.sockets.length, 1, 'VAD keeps its session');
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.commit'), false);
await app.ui.stopSession({ terminal: false });
});
test('a rejected operation is reported without hanging up the call', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'error', code: 'invalid_image', error: 'input_image requires a JPEG or PNG base64 data URL' });
assert.equal(app.ui.state().running, true, 'one refused image must not end the session');
assert.equal(app.ui.state().connectionReady, true);
assert.match(app.elements.get('runtimeDetail').textContent, /JPEG or PNG/);
await app.ui.stopSession({ terminal: false });
});
test('a session-level error still tears the call down', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'error', code: 'unknown_session', error: 'Unknown or closed duplex session' });
assert.equal(app.ui.state().running, false);
assert.equal(app.elements.get('connectionState').textContent, 'Error');
});
test('an interrupted response disarms the turn watchdog it left armed', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
await app.ui.handleEvent({ type: 'error', code: 'input_backpressure' });
await app.ui.handleEvent({ type: 'response.done', response: { id: 'r1', status: 'cancelled' } });
await app.turnWatchdog();
assert.equal(app.ui.state().running, true, 'barge-in must not time the session out two minutes later');
await app.ui.stopSession({ terminal: false });
});
test('MiniCPM retains microphone upload while speaking and acknowledges speaker drain', async () => {
const app = shell('minicpm-native');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'response.speak', response_id: 'native-r1' });
assert.equal(app.ui.microphoneUploadEnabled(), true);
await app.ui.handleEvent({ type: 'response.output_audio.delta', delta: 'AAAAAA==', response_id: 'native-r1' });
app.ui.playbackDrained({ responseId: 'native-r1', playedMs: 100 });
assert.equal(app.sockets[0].sent.at(-1).type, 'playback.ack');
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.commit'), false);
await app.ui.stopSession({ terminal: false });
await app.echo();
assert.equal(app.ui.state().running, false);
});
test('VAD handshake errors remain visible after resource cleanup', async () => {
const app = shell('qwen3-turn', 'vad', { error: { type: 'error', code: 'unsupported', error: 'Realtime API is not available' } });
await app.ui.startSession();
assert.equal(app.ui.state().running, false);
assert.equal(app.elements.get('connectionState').textContent, 'Error');
assert.match(app.elements.get('runtimeDetail').textContent, /session_mode: duplex/);
assert.equal(app.sockets[0].readyState, 3);
});
test('missing session acknowledgement fails instead of leaving Start disabled forever', async () => {
const app = shell('qwen3-turn', 'vad', { silent: true });
const starting = app.ui.startSession();
for (let i = 0; i < 20; i++) await Promise.resolve();
app.timeout();
await starting;
assert.equal(app.ui.state().running, false);
assert.equal(app.elements.get('callButton').disabled, false);
assert.match(app.elements.get('runtimeDetail').textContent, /handshake timed out/);
});
test('Qwen does not duplicate text and audio-transcript channels', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'response.audio_transcript.delta', delta: '' });
await app.ui.handleEvent({ type: 'response.output_text.delta', delta: 'Hello' });
await app.ui.handleEvent({ type: 'response.audio_transcript.delta', delta: 'Hello' });
await app.ui.handleEvent({ type: 'response.output_text.done', text: 'Hello' });
await app.ui.handleEvent({ type: 'response.done' });
const turns = app.elements.get('conversation').children;
assert.equal(turns.length, 1);
assert.equal(turns[0].children[1].textContent, 'Hello');
await app.ui.stopSession({ terminal: false });
});
const flushTasks = async () => { for (let i = 0; i < 40; i++) await Promise.resolve(); };
test('closing socket accepts only session.closed and completes without the fallback timer', async () => {
const app = shell('minicpm-native');
await app.ui.startSession();
const old = app.sockets[0];
let stopped = false;
const stopping = app.ui.stopSession().then(() => { stopped = true; });
assert.equal(old.sent.at(-1).type, 'session.close');
old.onmessage({ data: JSON.stringify({ type: 'session.updated' }) });
await flushTasks();
assert.equal(stopped, false);
old.onmessage({ data: JSON.stringify({ type: 'session.closed' }) });
await flushTasks();
assert.equal(stopped, true, 'must finish without firing the timeout');
await stopping;
await app.ui.startSession();
old.onmessage({ data: JSON.stringify({ type: 'session.closed' }) });
old.onmessage({ data: '{broken' });
await flushTasks();
assert.equal(app.ui.state().running, true, 'stale socket cannot affect a new session');
await app.ui.stopSession({ terminal: false });
});
test('malformed JSON before readiness rejects the handshake and cleans up', async () => {
const app = shell('qwen3-turn', 'vad', { silent: true });
const starting = app.ui.startSession();
await flushTasks();
app.sockets[0].onmessage({ data: '{broken' });
await starting;
assert.equal(app.ui.state().running, false);
assert.equal(app.sockets[0].readyState, 3);
assert.equal(app.elements.get('connectionState').textContent, 'Error');
assert.match(app.elements.get('runtimeDetail').textContent, /Invalid server event/);
});
test('malformed JSON after readiness fails the session and releases resources', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
app.sockets[0].onmessage({ data: '{broken' });
await flushTasks();
assert.equal(app.ui.state().running, false);
assert.equal(app.ui.microphoneUploadEnabled(), false);
assert.equal(app.sockets[0].readyState, 3);
assert.equal(app.elements.get('connectionState').textContent, 'Error');
assert.match(app.elements.get('runtimeDetail').textContent, /Invalid server event/);
});
test('Qwen VAD interruption clears playback while continuing microphone upload', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
await app.ui.handleEvent({ type: 'response.audio.delta', delta: 'AAAAAA==', response_id: 'r1' });
await app.ui.handleEvent({ type: 'output_audio_buffer.cleared', response_id: 'r1' });
assert.equal(app.ui.state().assistantActive, false);
assert.equal(app.ui.microphoneUploadEnabled(), true);
assert.ok(app.nodes.some(node => node.sent.some(message => message.type === 'clear')));
await app.ui.stopSession({ terminal: false });
});
test('Qwen runtime errors do not suggest changing a working deployment', () => {
const p = profiles['qwen3-turn']({ adapter: 'vad' });
const message = 'playback.ack arrived after a later user input was committed.';
assert.equal(p.mapEvent({ type: 'error', code: 'playback_ack_too_late', error: message }).message, message);
});
test('interrupted playback reports its cursor before clear resets it', () => {
let Playback;
const messages = [];
const ctx = vm.createContext({
sampleRate: 24000,
AudioWorkletProcessor: class { constructor() { this.port = { postMessage: m => messages.push(m) }; } },
registerProcessor: (_name, cls) => { Playback = cls; },
});
vm.runInContext(fs.readFileSync(path.join(root, 'playback_worklet.js'), 'utf8'), ctx);
const player = new Playback();
player.handleMessage({ type: 'audio', responseId: 'old', pcm: new Int16Array(24000), initialBufferMs: 0 });
player.process([], [[new Float32Array(2400)]]);
player.handleMessage({ type: 'clear' });
const report = messages.find(m => m.type === 'playback-stopped');
assert.equal(report.responseId, 'old');
assert.equal(report.playedMs, 100);
assert.equal(player.playedFrames, 0);
assert.equal(player.bufferedFrames(), 0);
player.handleMessage({ type: 'clear' });
assert.equal(messages.filter(m => m.type === 'playback-stopped').length, 1);
});
test('late interruption cursor acknowledges old response without finishing the new response', async () => {
const app = shell('qwen3-turn', 'vad');
await app.ui.startSession();
const player = app.nodes.find(n => n.name === 'fullduplex-pcm-playback');
await app.ui.handleEvent({ type: 'response.created', response: { id: 'old' } });
await app.ui.handleEvent({ type: 'output_audio_buffer.cleared', response_id: 'old' });
await app.ui.handleEvent({ type: 'response.created', response: { id: 'new' } });
player.port.onmessage({ data: { type: 'playback-stopped', responseId: 'old', playedMs: 100 } });
const ack = app.sockets[0].sent.find(m => m.type === 'playback.ack');
assert.equal(ack.response_id, 'old');
assert.equal(ack.played_ms, 100);
assert.equal(app.ui.state().assistantActive, true);
});
test('Qwen camera frames ride the OpenAI image interface, not the audio append', () => {
const p = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
assert.deepEqual(plain(p.append('PCM', 'JPEG')), { type: 'input_audio_buffer.append', audio: 'PCM' });
const [created] = plain(p.imageMessages('JPEG'));
assert.equal(created.type, 'conversation.item.create');
assert.equal(created.item.role, 'user');
assert.deepEqual(created.item.content, [
{ type: 'input_image', image_url: 'data:image/jpeg;base64,JPEG' },
]);
});
test('the camera retires its oldest image instead of exhausting the session budget', () => {
const p = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
const ids = [];
for (let i = 0; i < 8; i++) {
const messages = plain(p.imageMessages('JPEG'));
assert.equal(messages.length, 1, 'nothing to retire while under the budget');
ids.push(messages[0].item.id);
}
// The engine counts what it already stores, so the delete has to lead.
const ninth = plain(p.imageMessages('JPEG'));
assert.deepEqual(ninth[0], { type: 'conversation.item.delete', item_id: ids[0] });
assert.equal(ninth[1].type, 'conversation.item.create');
assert.equal(new Set(ids).size, 8, 'item ids must be distinct to be deletable');
// A new call reopens the budget; stale ids from the old one must not linger.
assert.equal(plain(p.initialMessages(config, 'x')).length > 0, true);
assert.equal(plain(p.imageMessages('JPEG')).length, 1);
});
tests/examples/test_realtime_web_profiles.py
+24
View File
@@ -0,0 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import shutil
import subprocess
from pathlib import Path
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_realtime_profiles_and_shell_lifecycle():
node = shutil.which("node")
if node is None:
pytest.skip("node is required for browser protocol tests")
assert node is not None
subprocess.run(
[node, "--test", str(Path(__file__).with_suffix(".cjs"))],
check=True,
capture_output=True,
text=True,
timeout=30,
)
tests/model_executor/models/qwen3_omni/test_duplex_plugin.py
+625
View File
@@ -0,0 +1,625 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import asyncio
import base64
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
from tests.engine.duplex.test_session_runner import (
SESSION_ID,
Harness,
RecordingStagePort,
_speech_then_stop,
append_audio,
tts_output,
)
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
from vllm_omni.engine.duplex.commands import AckPlayback, CancelResponse, Commit, UpdateSession
from vllm_omni.engine.duplex.config import DuplexSessionConfig
from vllm_omni.engine.duplex.messages import OpenDuplexSessionMessage
from vllm_omni.engine.duplex.plugin import DuplexRuntimeConfigError
from vllm_omni.engine.duplex.session.manager import DuplexSessionManager
from vllm_omni.model_executor.models.qwen3_omni.duplex.input import QwenPcmBuffer
from vllm_omni.model_executor.models.qwen3_omni.duplex.plugin import MAX_PROMPT_IMAGES, Qwen3OmniDuplexPlugin
from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_buffer_commit_and_rollback_preserve_all_samples():
buffer = QwenPcmBuffer()
raw = np.ones(101, dtype="<f4").tobytes()
payload = {"audio": base64.b64encode(raw).decode(), "format": "pcm_f32le", "sample_rate_hz": 16000}
assert buffer.prepare_append(payload, operation_id="a", chunk_period_ms=1000, allow_emit=True) is None
reservation = buffer.prepare_commit(operation_id="c", chunk_period_ms=1000)
assert base64.b64decode(reservation.payload["audio"]) == raw
reservation.rollback()
assert buffer.pending_byte_count == len(raw)
final = buffer.prepare_commit(operation_id="d", chunk_period_ms=1000)
final.commit()
assert not buffer.has_pending() and not buffer.has_reserved()
def test_buffer_refuses_video_frames_and_points_at_the_openai_interface():
buffer = QwenPcmBuffer()
payload = {
"audio": base64.b64encode(np.ones(16, dtype="<f4").tobytes()).decode(),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"video_frames": ["a-frame"],
}
with pytest.raises(ValueError, match="input_image"):
buffer.prepare_append(payload, operation_id="first", chunk_period_ms=200, allow_emit=True)
# The refusal must not swallow the audio it was sent with.
assert not buffer.has_pending()
del payload["video_frames"]
buffer.prepare_append(payload, operation_id="second", chunk_period_ms=200, allow_emit=True)
committed = buffer.prepare_commit(operation_id="second-commit", chunk_period_ms=200)
assert "video_frames" not in committed.payload
committed.commit()
async def open_qwen():
plugin = Qwen3OmniDuplexPlugin(lambda audio, *args: "AAAA")
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kw: repr(messages))
port = RecordingStagePort(stage_count=3)
output: asyncio.Queue[Any] = asyncio.Queue()
results: asyncio.Queue[Any] = asyncio.Queue()
manager = DuplexSessionManager(
plugin=plugin,
stage_port=port,
output_sink=output,
result_sink=results,
runtime_config=DuplexSessionRuntimeConfig(),
model_config=None,
)
config = DuplexSessionConfig(model="qwen", modalities=["text", "audio"], overlap_policy="barge_in_on_speech")
await manager.handle(OpenDuplexSessionMessage(control_id="open", session_id=SESSION_ID, session_config=config))
result = await results.get()
assert result.ok, result
harness = Harness(manager, port, output, results, manager.runners[SESSION_ID])
await harness.settle()
return harness
@pytest.mark.asyncio
async def test_two_committed_turns_stream_text_and_audio_and_release_requests():
h = await open_qwen()
try:
ids = []
for i in range(2):
await h.run(append_audio())
assert len(h.port.submissions) == i
await h.run(Commit(final=True, create_response=True))
assert len(h.port.submissions) == i + 1, [e.to_realtime() for e in h.events]
request_id = h.port.submissions[-1].context.request_id
ids.append(request_id)
assert h.session.active_request_id == request_id
assert len(h.port.submissions[-1].prompt["multi_modal_data"]["audio"]) == i + 1
text = OmniRequestOutput.from_stage_output(
tts_output(request_id, samples=0, text="hello", finished=True),
request_id=request_id,
stage_id=None,
)
assert not h.deliver(text, stage_id=0), "Thinker output must still forward to Talker"
await h.settle()
assert text.finished, "projection must not mutate the output forwarded to Talker"
assert h.session.active_response_id is not None, "Thinker finished must not end the response"
response_id = h.session.active_response_id
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
assert h.session.active_response_id is None
assert h.port.cleanups[-1][0] == [request_id]
assert len(set(ids)) == 2
assert h.session.capabilities.as_dict()["implementation_level"] == "turn_based_duplex"
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_cancel_drops_old_output_and_accepts_next_turn():
h = await open_qwen()
try:
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
old = h.port.submissions[-1].context
await h.run(CancelResponse())
await h.deliver_and_settle(tts_output(old.request_id, finished=True), stage_id=2, epoch=old.fence.epoch)
assert h.session.active_response_id is None
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
assert len(h.port.submissions) == 2
assert h.port.submissions[-1].context.request_id != old.request_id
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_vad_commits_and_speech_interrupts_active_qwen_response():
h = await open_qwen()
try:
h.runner.control._detector = _speech_then_stop()
await h.run(append_audio())
await h.run(append_audio(is_speech=False, value=0.0))
assert len(h.port.submissions) == 1
old = h.port.submissions[-1].context
h.runner.control._detector = _speech_then_stop()
await h.run(append_audio())
assert h.session.epoch > old.fence.epoch
await h.run(append_audio(is_speech=False, value=0.0))
assert len(h.port.submissions) == 2
assert h.port.submissions[-1].context.request_id != old.request_id
finally:
await h.manager.shutdown()
def test_audio_projector_handles_delta_tensor_lists_and_sample_rate():
import torch
encoded: list[tuple[Any, object]] = []
def _encode(audio: object, rate: int, *_args: object) -> str:
encoded.append((audio, rate))
return "encoded"
plugin = Qwen3OmniDuplexPlugin(_encode)
output = SimpleNamespace(
request_id="r",
stage_id=2,
finished=True,
outputs=[],
multimodal_output={"audio": [torch.ones(8), torch.ones(16)], "sr": [torch.tensor(24000)]},
)
events = list(plugin.data_plane.project({"data_plane_outputs": [output]}, context={"modalities": ["audio"]}))
assert len(encoded) == 1 and encoded[0][0].shape == (24,)
assert events[0]["audio_duration_ms"] == 1
assert events[0]["end_of_turn"] is True
def test_audio_history_is_bounded_and_does_not_cross_sessions():
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
state, other = plugin.create_session_state(), plugin.create_session_state()
history = []
for i in range(6):
history.append({"role": "user", "content": [{"type": "audio_url"}]})
config = plugin.prepare_prompt_config({"conversation": history}, state=state, payload={"audio": str(i)})
history.append({"role": "assistant", "content": f"reply-{i}"})
assert len(state.audio_history) == 4
assert [m["audio_payload"]["audio"] for m in config["qwen_messages"] if "audio_payload" in m] == [
"2",
"3",
"4",
"5",
]
assert other.audio_history == []
@pytest.mark.asyncio
async def test_late_playback_ack_keeps_answer_before_new_user_input():
h = await open_qwen()
try:
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
request_id = h.port.submissions[-1].context.request_id
response_id = h.session.active_response_id
await h.deliver_and_settle(tts_output(request_id, samples=0, text="first answer", finished=True), stage_id=0)
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
# Playback has not drained yet. A new utterance commits first.
assert [m["role"] for m in h.session.history] == ["user"]
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
current_response = h.session.active_response_id
events = await h.run(AckPlayback(response_id=response_id, played_ms=1000))
assert not any(e.to_realtime().get("type") == "error" for e in events)
assert [m["role"] for m in h.session.history] == ["user", "assistant", "user"]
assert h.session.history[1]["content"] == "first answer"
assert h.session.active_response_id == current_response
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
@pytest.mark.parametrize("streaming_vad", [False, True])
async def test_finished_qwen_response_does_not_repeat_during_silence(streaming_vad):
h = await open_qwen()
try:
if streaming_vad:
from vllm_omni.engine.duplex.turn_detection import ServerTurnDetector, TurnDetectionConfig
detector = ServerTurnDetector(TurnDetectionConfig())
# Exercise framing, endpointing and reset across the entire response.
# A deterministic scorer avoids requiring an ONNX artifact in CPU tests.
detector.vad._frame_scorer = lambda frame: float(np.max(np.abs(frame)) > 0.01)
h.runner.control._detector = detector
else:
h.runner.control._detector = _speech_then_stop()
await h.run(append_audio())
await h.run(append_audio(is_speech=False, value=0.0))
if not streaming_vad:
h.runner.control._detector = None
assert len(h.port.submissions) == 1
request_id = h.port.submissions[0].context.request_id
response_id = h.session.active_response_id
await h.deliver_and_settle(tts_output(request_id, samples=0, text="story", finished=True), stage_id=0)
for _ in range(3):
await h.run(append_audio(is_speech=False, value=0.0))
await h.deliver_and_settle(tts_output(request_id, finished=True), stage_id=2)
for _ in range(3):
await h.run(append_audio(is_speech=False, value=0.0))
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
await h.settle()
assert len(h.port.submissions) == 1
assert not h.runner.model_state.audio_buffer.has_pending()
assert h.runner.model_state.committed_audio_payload is None
finally:
await h.manager.shutdown()
def test_current_audio_is_not_replaced_by_history_ending_with_assistant():
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
state = plugin.create_session_state()
first = {"audio": base64.b64encode(np.full(160, 0.1, dtype="<f4").tobytes()).decode()}
current = {"audio": base64.b64encode(np.full(320, 0.2, dtype="<f4").tobytes()).decode()}
history = [{"role": "user", "content": [{"type": "audio_url"}]}]
plugin.prepare_prompt_config({"conversation": history}, state=state, payload=first)
history.append({"role": "assistant", "content": "Previous answer"})
config = plugin.prepare_prompt_config({"conversation": history}, state=state, payload=current)
plan = plugin.plan_append(
request_id="new",
fence=None,
session_config=config,
runtime_config={},
seq=1,
turn_seq=1,
payload=current,
final=True,
sampling_params=None,
)
audios = plan.prompt["multi_modal_data"]["audio"]
assert len(audios) == 2
np.testing.assert_allclose(audios[-1][0], np.full(320, 0.2, dtype="<f4"))
assert config["qwen_messages"][-1]["audio_payload"] is current
@pytest.mark.asyncio
async def test_partial_playback_does_not_commit_unaligned_thinker_text():
h = await open_qwen()
try:
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
request_id = h.port.submissions[-1].context.request_id
response_id = h.session.active_response_id
await h.deliver_and_settle(tts_output(request_id, samples=0, text="entire answer", finished=True), stage_id=0)
await h.deliver_and_settle(tts_output(request_id, samples=24000, finished=False), stage_id=2)
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
assert not any(m["role"] == "assistant" for m in h.session.history)
await h.deliver_and_settle(tts_output(request_id, samples=216000, finished=True), stage_id=2)
await h.run(AckPlayback(response_id=response_id, played_ms=1000))
assert not any(m["role"] == "assistant" for m in h.session.history)
await h.run(AckPlayback(response_id=response_id, played_ms=10000))
assert h.session.history[-1]["content"] == "entire answer"
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_prompt_preparation_runs_off_loop_and_cancellation_prevents_submission():
import threading
h = await open_qwen()
entered, release = threading.Event(), threading.Event()
thread_ids = []
original = h.manager.plugin.plan_append
def slow_plan(**kwargs):
thread_ids.append(threading.get_ident())
entered.set()
assert release.wait(5)
return original(**kwargs)
h.manager.plugin.plan_append = slow_plan
try:
await h.run(append_audio())
commit = asyncio.create_task(h.run(Commit(final=True, create_response=True)))
for _ in range(100):
if entered.is_set():
break
await asyncio.sleep(0.01)
assert entered.is_set()
assert thread_ids != [threading.get_ident()]
await h.run(CancelResponse())
release.set()
await commit
await h.settle()
assert not h.port.submissions
finally:
release.set()
await h.manager.shutdown()
def camera_frame():
import io
from PIL import Image
buf = io.BytesIO()
Image.new("RGB", (16, 16), "red").save(buf, format="JPEG")
return base64.b64encode(buf.getvalue()).decode()
def pcm16_base64(samples=16000):
"""Loud PCM16, so the projection reads it as speech rather than silence."""
pcm = (np.sin(np.arange(samples) * 0.05) * 20000).astype("<i2")
return base64.b64encode(pcm.tobytes()).decode()
def image_item(item_id="camera"):
return {
"id": item_id,
"type": "message",
"role": "user",
"content": [
{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()},
{"type": "input_text", "text": "What color is this?"},
],
}
@pytest.mark.asyncio
async def test_image_item_is_context_until_response_create_and_delete_removes_it():
from vllm_omni.engine.duplex.commands import CreateResponse, DeleteItem
h = await open_qwen()
try:
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": image_item()}))
assert not h.port.submissions
await h.run(CreateResponse())
assert len(h.port.submissions) == 1
prompt = h.port.submissions[-1].prompt
assert len(prompt["multi_modal_data"]["image"]) == 1
assert "audio" not in prompt["multi_modal_data"]
assert "What color is this?" in prompt["prompt"]
await h.run(CancelResponse())
await h.run(DeleteItem(item_id="camera"))
await h.run(append_audio())
await h.run(Commit(final=True, create_response=False))
assert len(h.port.submissions) == 1
await h.run(CreateResponse())
assert "image" not in h.port.submissions[-1].prompt["multi_modal_data"]
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_image_context_reaches_audio_turn_and_survives_commit():
from vllm_omni.engine.duplex.commands import CreateItem
h = await open_qwen()
try:
await h.run(CreateItem(item=image_item()))
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
mm = h.port.submissions[-1].prompt["multi_modal_data"]
assert len(mm["image"]) == 1 and len(mm["audio"]) == 1
assert any(
isinstance(m["content"], list) and any(p.get("type") == "image_url" for p in m["content"])
for m in h.session.history
)
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_image_only_item_reaches_the_prompt_the_way_the_camera_sends_it():
"""The camera sends each frame as its own item, with no text part.
A user item used to be forwarded to the session only when it carried text
or audio, so an image-only item was acknowledged to the client and then
dropped. Nothing reported it and every prompt that followed was blind.
"""
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
h = await open_qwen()
try:
item = {
"id": "camera_1",
"type": "message",
"role": "user",
"content": [{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()}],
}
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": item}))
assert [
part
for message in h.session.history
if isinstance(message.get("content"), list)
for part in message["content"]
if part.get("type") == "image_url"
], "an image-only item never reached the session history"
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
mm = h.port.submissions[-1].prompt["multi_modal_data"]
assert len(mm["image"]) == 1 and len(mm["audio"]) == 1
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_one_item_carrying_both_speech_and_a_picture_keeps_both():
"""Audio and images leave the same item by different routes.
The audio becomes buffered appends that a commit seals under this item's
id; registering the spoken message there would overwrite an image stored
under the same id, so the picture has to become its own item.
"""
from vllm_omni.engine.duplex.commands import CreateResponse
from vllm_omni.engine.duplex.realtime_commands import translate_realtime_command
h = await open_qwen()
try:
item = {
"id": "mixed_1",
"type": "message",
"role": "user",
"content": [
{"type": "input_image", "image_url": "data:image/jpeg;base64," + camera_frame()},
{"type": "input_audio", "audio": pcm16_base64(), "format": "pcm16", "sample_rate_hz": 16000},
],
}
await h.run(translate_realtime_command({"type": "conversation.item.create", "item": item}))
await h.run(CreateResponse())
mm = h.port.submissions[-1].prompt["multi_modal_data"]
assert len(mm["image"]) == 1, "the picture was dropped on its way out of a mixed item"
assert len(mm["audio"]) == 1
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_image_limit_rejects_atomically_and_delete_reclaims_capacity():
from vllm_omni.engine.duplex.commands import CreateItem, DeleteItem
h = await open_qwen()
try:
for i in range(8):
await h.run(CreateItem(item=image_item(f"camera_{i}")))
events = await h.run(CreateItem(item=image_item("overflow")))
assert any(e.to_realtime()["type"] == "error" for e in events)
assert len(h.session.history) == 8
await h.run(DeleteItem(item_id="camera_0"))
await h.run(CreateItem(item=image_item("replacement")))
assert len(h.session.history) == 8
assert not h.port.submissions
finally:
await h.manager.shutdown()
def test_prompt_holds_stored_images_to_the_stage_limit():
"""The prompt is the last guard before ``limit_mm_per_prompt``.
A dropped image has to take its ``{"type": "image"}`` placeholder with
it, because the processor pairs placeholders with ``mm["image"]``
positionally.
"""
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
url = "data:image/jpeg;base64," + camera_frame()
history: list[dict[str, Any]] = [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": url}}, {"type": "text", "text": f"i{i}"}],
}
for i in range(MAX_PROMPT_IMAGES + 2)
]
audio = base64.b64encode(np.full(160, 0.1, dtype="<f4").tobytes()).decode()
history.append({"role": "user", "audio_payload": {"audio": audio}})
plan = plugin.plan_append(
request_id="r",
fence=None,
session_config={"qwen_messages": history},
runtime_config={},
seq=1,
turn_seq=1,
payload={"audio": audio},
final=True,
sampling_params=None,
)
prompt = plan.prompt
assert len(prompt["multi_modal_data"]["image"]) == MAX_PROMPT_IMAGES
assert prompt["prompt"].count("{'type': 'image'}") == MAX_PROMPT_IMAGES
# Only the image part is retired; the message keeps what it said.
assert "'i0'" in prompt["prompt"]
def test_text_only_turn_without_prepared_messages_is_refused_not_crashed():
plugin = Qwen3OmniDuplexPlugin(lambda *args: None)
plugin.processor = SimpleNamespace(apply_chat_template=lambda messages, **kwargs: repr(messages))
with pytest.raises(DuplexRuntimeConfigError):
plugin.plan_append(
request_id="r",
fence=None,
# An empty prepared list is not a missing one: it must not fall
# back to a payload that has no audio to read.
session_config={"qwen_messages": []},
runtime_config={},
seq=1,
turn_seq=1,
payload={"is_speech": True},
final=True,
sampling_params=None,
)
@pytest.mark.asyncio
async def test_commit_all_on_done_keeps_unaligned_text_the_client_never_acks():
h = await open_qwen()
try:
# The runner starts every session ack-gated; this client opts out.
await h.run(UpdateSession(patch={"playback_commit_policy": "commit_all_on_done"}))
assert h.session.config.playback_commit_policy == "commit_all_on_done"
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
request_id = h.port.submissions[-1].context.request_id
await h.deliver_and_settle(tts_output(request_id, samples=0, text="entire answer", finished=True), stage_id=0)
await h.deliver_and_settle(tts_output(request_id, samples=24000, finished=True), stage_id=2)
# Qwen text is unaligned to its audio, but waiting for an ACK this
# client will never send would drop the turn from history entirely.
assert h.session.active_response_id is None
assert h.session.history[-1]["content"] == "entire answer"
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_the_bound_request_id_is_the_one_the_stage_received():
h = await open_qwen()
try:
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
submitted = h.port.submissions[-1].context.request_id
# Qwen has no resumable core request, so every turn gets its own stage
# id. The runner names that id before the append runs and the model
# channel names it again when it submits; they have to agree.
assert submitted.endswith(".r.stage0-turn0"), submitted
assert h.session.active_request_id == submitted
finally:
await h.manager.shutdown()
@pytest.mark.asyncio
async def test_visual_capture_records_submitted_pixels_and_request(tmp_path, monkeypatch):
import json
from PIL import Image
from vllm_omni.engine.duplex.commands import CreateItem
monkeypatch.setenv("VLLM_OMNI_QWEN_VISUAL_DEBUG_DIR", str(tmp_path))
h = await open_qwen()
try:
await h.run(CreateItem(item=image_item()))
await h.run(append_audio())
await h.run(Commit(final=True, create_response=True))
submission = h.port.submissions[-1]
captures = list(tmp_path.glob("*/input.json"))
assert len(captures) == 1
metadata = json.loads(captures[0].read_text())
assert metadata["request_id"] == submission.context.request_id
assert metadata["audio_count"] == 1
assert metadata["prompt"] == submission.prompt["prompt"]
assert len(metadata["images"]) == 1
with Image.open(captures[0].parent / metadata["images"][0]) as saved:
actual = submission.prompt["multi_modal_data"]["image"][0]
assert saved.size == actual.size
assert saved.tobytes() == actual.tobytes()
finally:
await h.manager.shutdown()
vllm_omni/config/environment_variable_inventory.py
+1
View File
@@ -203,6 +203,7 @@ _MODEL_INTERNALIZE = (
"MOSS_TTS_DEBUG_STOP",
"NEMOTRON_VOICECHAT_DEBUG_FUNCTION_TIMELINE",
"VLLM_OMNI_BREEZE_GOLDEN_DIR",
"VLLM_OMNI_QWEN_VISUAL_DEBUG_DIR",
"VLLM_OMNI_QWEN3_CODE2WAV_BATCH_STATS",
"VLLM_OMNI_QWEN3_CODE2WAV_BATCH_STATS_LOG_EVERY",
"VLLM_OMNI_QWEN3_CODE2WAV_CUDAGRAPH_STATS",
vllm_omni/config/stage_config.py
+2
View File
@@ -312,6 +312,8 @@ class PipelineConfig:
# Dotted path of the model's ``DuplexModelPlugin``. Online serving uses
# DuplexOmni only when the deploy configuration selects session_mode: duplex.
duplex_plugin: str | None = None
# Preserve legacy turn deployments when adding an optional duplex plugin.
default_session_mode: str | None = None
# Legacy duplex wiring of the models that are not ported to the plugin
# framework yet (PersonaPlex, Nemotron VoiceChat). Nothing reads them: a
# pipeline that only declares these is served turn-based. Each field goes
vllm_omni/deploy/qwen3_omni_duplex.yaml
+8
View File
@@ -0,0 +1,8 @@
# Concurrent input / interruptible VAD turns; not native streaming input KV.
base_config: qwen3_omni_moe.yaml
session_mode: duplex
stages:
- stage_id: 0
limit_mm_per_prompt:
audio: 4
image: 8
vllm_omni/engine/duplex/config.py
+42 -23
View File
@@ -111,6 +111,8 @@ class DuplexCapabilities:
#: seeding a text prompt has no representation and the turn never
#: completes. ``text_turn_priming_units`` is how many silence units such a
#: seeded turn must be given to generate on -- it still speaks per unit.
supports_image_input: bool = False
supports_text_only_turn: bool = False
supports_chat_completions: bool = False
text_turn_priming_units: int = 0
requires_model_runner_kv: bool = False
@@ -149,11 +151,13 @@ class DuplexCapabilities:
"supports_session_resume": self.supports_session_resume,
"session_admission_mode": self.session_admission_mode,
"supports_audio_truncate": self.supports_audio_truncate,
"supports_image_input": self.supports_image_input,
"supports_text_only_turn": self.supports_text_only_turn,
"requires_model_runner_kv": self.requires_model_runner_kv,
"requires_native_stage_role": self.requires_native_stage_role,
# Every duplex model is model-native and appends audio chunks (§7 D4);
# kept on the wire as constants for clients that still read them.
"implementation_level": "model_native_duplex",
"implementation_level": (
"model_native_duplex" if self.supports_model_native_turn_policy else "turn_based_duplex"
),
"adapter_patterns": self.adapter_patterns,
"input_modes": ["append_audio_chunk"],
"signal_sources": self.signal_sources,
@@ -169,6 +173,8 @@ class DuplexPlaybackCursor:
sent_ms: int = 0
played_ms: int = 0
committed_ms: int = 0
text_requires_complete_audio: bool = False
audio_complete: bool = False
def acknowledge(self, played_ms: int, committed_ms: int | None = None) -> None:
self.played_ms = max(self.played_ms, max(0, int(played_ms)))
@@ -188,7 +194,11 @@ class DuplexPlaybackCursor:
}
def snapshot(self) -> DuplexPlaybackView:
return DuplexPlaybackView(**self.as_dict())
return DuplexPlaybackView(
**self.as_dict(),
text_requires_complete_audio=self.text_requires_complete_audio,
audio_complete=self.audio_complete,
)
@dataclass(frozen=True, slots=True)
@@ -197,6 +207,8 @@ class DuplexPlaybackView:
sent_ms: int = 0
played_ms: int = 0
committed_ms: int = 0
text_requires_complete_audio: bool = False
audio_complete: bool = False
def as_dict(self) -> dict[str, int]:
return {
@@ -310,16 +322,21 @@ class DuplexSessionConfig:
if isinstance(source.get("extra_body"), dict):
config.extra_body = dict(source["extra_body"])
extra = config.extra_body
if isinstance(extra.get("overlap_policy"), str):
config.overlap_policy = cls._normalize_overlap_policy(extra["overlap_policy"])
if isinstance(extra.get("overlap_short_ack_ms"), int | float):
config.overlap_short_ack_ms = max(0, int(extra["overlap_short_ack_ms"]))
if isinstance(extra.get("overlap_barge_in_ms"), int | float):
config.overlap_barge_in_ms = max(0, int(extra["overlap_barge_in_ms"]))
if isinstance(extra.get("overlap_silence_rms"), int | float):
config.overlap_silence_rms = max(0.0, float(extra["overlap_silence_rms"]))
if isinstance(extra.get("playback_commit_policy"), str):
config.playback_commit_policy = cls._normalize_playback_commit_policy(extra["playback_commit_policy"])
overlap_policy = extra.get("overlap_policy")
if isinstance(overlap_policy, str):
config.overlap_policy = cls._normalize_overlap_policy(overlap_policy)
short_ack_ms = extra.get("overlap_short_ack_ms")
if isinstance(short_ack_ms, int | float):
config.overlap_short_ack_ms = max(0, int(short_ack_ms))
barge_in_ms = extra.get("overlap_barge_in_ms")
if isinstance(barge_in_ms, int | float):
config.overlap_barge_in_ms = max(0, int(barge_in_ms))
silence_rms = extra.get("overlap_silence_rms")
if isinstance(silence_rms, int | float):
config.overlap_silence_rms = max(0.0, float(silence_rms))
playback_commit_policy = extra.get("playback_commit_policy")
if isinstance(playback_commit_policy, str):
config.playback_commit_policy = cls._normalize_playback_commit_policy(playback_commit_policy)
return config
@classmethod
@@ -633,7 +650,6 @@ class ResponseCreateOptions:
from vllm_omni.protocol.duplex import (
REALTIME_OUTPUT_AUDIO_FORMATS,
parse_realtime_audio_format,
realtime_max_output_tokens,
)
payload: dict[str, object] = dict(response_payload)
@@ -665,11 +681,12 @@ class ResponseCreateOptions:
response_format = payload.get("output_audio_format") or payload.get("response_format")
if response_format is None and isinstance(audio_output, dict):
response_format = audio_output.get("format")
response_format, _ = parse_realtime_audio_format(response_format)
if isinstance(response_format, str) and response_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
response_format = duplex_response_format(response_format)
parsed_format, _ = parse_realtime_audio_format(response_format)
output_audio_format: str | None
if isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
output_audio_format = str(duplex_response_format(parsed_format))
else:
response_format = None
output_audio_format = None
temperature = (
float(cast("int | float", payload["temperature"]))
if isinstance(payload.get("temperature"), int | float)
@@ -687,9 +704,9 @@ class ResponseCreateOptions:
else payload.get("max_tokens")
)
if "max_response_output_tokens" in payload or "max_output_tokens" in payload or "max_tokens" in payload:
max_tokens = realtime_max_output_tokens(max_tokens)
token_limit = realtime_max_output_tokens(max_tokens)
else:
max_tokens = None
token_limit = None
modalities = payload.get("modalities") or payload.get("output_modalities")
if isinstance(modalities, list) and all(isinstance(item, str) for item in modalities):
modalities = tuple(modalities)
@@ -717,9 +734,9 @@ class ResponseCreateOptions:
return cls(
instructions=instructions,
voice=voice,
response_format=response_format,
response_format=output_audio_format,
temperature=temperature,
max_tokens=max_tokens,
max_tokens=token_limit,
speed=speed,
modalities=modalities,
extra_body=response_extra,
@@ -761,6 +778,8 @@ def realtime_item_to_history_message(item: object) -> dict[str, object] | None:
part_type = part.get("type")
if part_type in {"input_text", "text", "output_text"} and isinstance(part.get("text"), str):
text_chunks.append(str(part["text"]))
elif part_type == "input_image" and isinstance(part.get("image_url"), str):
audio_chunks.append({"type": "image_url", "image_url": {"url": part["image_url"]}})
elif part_type in {"input_audio", "audio"}:
audio = part.get("audio") or part.get("data")
fmt = part.get("format") if isinstance(part.get("format"), str) else "wav"
vllm_omni/engine/duplex/plugin.py
+21 -4
View File
@@ -179,6 +179,7 @@ class DuplexModelPlugin(ABC):
two halves is impossible by construction.
"""
projects_intermediate_outputs: bool = False
plugin_id: str = ""
private_runtime_config_keys: frozenset[str] = frozenset()
#: Samples per silence unit the runner appends to keep a model turn going.
@@ -199,6 +200,16 @@ class DuplexModelPlugin(ABC):
defaults: tuple[object, ...],
) -> tuple[object, ...]: ...
def prepare_prompt_config(
self, config: dict[str, object], *, state: DuplexModelSessionState, payload: dict[str, object]
) -> dict[str, object]:
"""Add model-owned context before planning an append on the session loop."""
return config
async def prepare_append_plan(self, **kwargs) -> DuplexAppendPlan:
"""Prepare a plan; plugins may offload expensive work on owned snapshots."""
return self.plan_append(**kwargs)
@abstractmethod
def plan_append(
self,
@@ -312,10 +323,16 @@ def payload_turn_id(payload: object) -> int | None:
def coerce_int(value: object) -> int | None:
try:
return int(value) if value is not None else None
except (TypeError, ValueError):
return None
if isinstance(value, bool):
return int(value)
if isinstance(value, int):
return value
if isinstance(value, float | str | bytes | bytearray):
try:
return int(value)
except (TypeError, ValueError, OverflowError):
return None
return None
__all__ = [
vllm_omni/engine/duplex/realtime_events.py
+69 -48
View File
@@ -327,7 +327,7 @@ def _response_done_output_item(
)
if text:
content.append(_response_item_text_content_part(text=text))
item = {
item: dict[str, object] = {
"id": item_id,
"object": "realtime.item",
"type": "message",
@@ -796,7 +796,7 @@ def _project(state: RealtimeProjectionState, event: dict[str, object]) -> list[D
item_id = _response_item_id(state, response_id)
modalities = event.get("modalities")
has_audio_modality = not isinstance(modalities, list) or "audio" in modalities
item = {
item: dict[str, object] = {
"id": item_id,
"object": "realtime.item",
"type": "message",
@@ -900,13 +900,13 @@ def _project(state: RealtimeProjectionState, event: dict[str, object]) -> list[D
item_id = (
event_item_id if isinstance(event_item_id, str) and event_item_id else _pop_pending_commit_item_id(state)
)
item = state.conversation_items.get(item_id)
events = []
if item is None:
committed_item = state.conversation_items.get(item_id)
commit_events: list[DuplexEvent] = []
if committed_item is None:
message = event.get("message")
no_response = event.get("no_response") is True
is_speech = event.get("is_speech")
item = {
created_commit_item: dict[str, object] = {
"id": item_id,
"object": "realtime.item",
"type": "message",
@@ -918,17 +918,18 @@ def _project(state: RealtimeProjectionState, event: dict[str, object]) -> list[D
else _user_item_content_from_duplex_message(message)
),
}
state.conversation_items[item_id] = item
events.extend(_conversation_item_added_events(state, item))
item["status"] = "completed"
events.append(
committed_item = created_commit_item
state.conversation_items[item_id] = committed_item
commit_events.extend(_conversation_item_added_events(state, committed_item))
committed_item["status"] = "completed"
commit_events.append(
InputCommitted(previous_item_id=_previous_item_id(state, item_id), item_id=item_id, details=event)
)
transcription_event = _input_audio_transcription_completed_event(item_id, item)
transcription_event = _input_audio_transcription_completed_event(item_id, committed_item)
if transcription_event is not None:
events.append(transcription_event)
events.append(_conversation_item_done_event(state, item))
return events
commit_events.append(transcription_event)
commit_events.append(_conversation_item_done_event(state, committed_item))
return commit_events
if event_type == "input.cancelled":
return [InputCleared()]
if event_type == "audio.cancelled":
@@ -951,9 +952,9 @@ def _project(state: RealtimeProjectionState, event: dict[str, object]) -> list[D
item_id = _response_item_id(state, response_id)
committed_audio_ms = max(0, int(committed_ms))
state.item_truncation_cursors[item_id] = (0, committed_audio_ms)
item = state.conversation_items.get(item_id)
if item is not None:
truncate_realtime_item_content(item, content_index=0, audio_end_ms=committed_audio_ms)
cancelled_item = state.conversation_items.get(item_id)
if cancelled_item is not None:
truncate_realtime_item_content(cancelled_item, content_index=0, audio_end_ms=committed_audio_ms)
events.extend(_realtime_audio_done_events(state, event, response_id))
events.extend(
_realtime_response_terminal_events(
@@ -968,48 +969,49 @@ def _project(state: RealtimeProjectionState, event: dict[str, object]) -> list[D
state.active_response_id = None
return events
if event_type == "conversation.item.created":
item = event.get("item")
if isinstance(item, dict) and isinstance(item.get("id"), str):
item_id = str(item["id"])
already_known = item_id in state.conversation_items
state.conversation_items[item_id] = item
created_raw = event.get("item")
if isinstance(created_raw, dict) and isinstance(created_raw.get("id"), str):
created_item_id = str(created_raw["id"])
created_item = cast("dict[str, object]", created_raw)
already_known = created_item_id in state.conversation_items
state.conversation_items[created_item_id] = created_item
if already_known:
if item.get("status") == "completed":
return [_conversation_item_done_event(state, item)]
if created_item.get("status") == "completed":
return [_conversation_item_done_event(state, created_item)]
return []
events = _conversation_item_added_events(state, item)
if item.get("status") == "completed":
events.append(_conversation_item_done_event(state, item))
return events
return [ItemCreated(item=item if isinstance(item, Mapping) else {})]
created_events = _conversation_item_added_events(state, created_item)
if created_item.get("status") == "completed":
created_events.append(_conversation_item_done_event(state, created_item))
return created_events
return [ItemCreated(item=created_raw if isinstance(created_raw, Mapping) else {})]
if event_type == "conversation.item.deleted":
item_id = event.get("item_id")
if isinstance(item_id, str):
_remove_conversation_item(state, item_id)
return [ItemDeleted(item_id=_str_or_none(item_id), details=event)]
deleted_item_id = event.get("item_id")
if isinstance(deleted_item_id, str):
_remove_conversation_item(state, deleted_item_id)
return [ItemDeleted(item_id=_str_or_none(deleted_item_id), details=event)]
if event_type == "conversation.item.truncated":
item_id = event.get("item_id")
truncated_item_id = event.get("item_id")
audio_end_ms = event.get("audio_end_ms")
content_index = event.get("content_index", 0)
if isinstance(item_id, str):
item = state.conversation_items.get(item_id)
if item is not None:
if isinstance(truncated_item_id, str):
truncated_item = state.conversation_items.get(truncated_item_id)
if truncated_item is not None:
truncate_realtime_item_content(
item,
truncated_item,
content_index=_int_or(content_index),
audio_end_ms=_int_or(audio_end_ms),
)
return [
ItemTruncated(
item_id=_str_or_none(item_id),
item_id=_str_or_none(truncated_item_id),
content_index=_int_or(content_index),
audio_end_ms=_int_or(audio_end_ms),
details=event,
)
]
if event_type == "conversation.item.retrieved":
item = event.get("item")
return [ItemRetrieved(item=item if isinstance(item, Mapping) else {})]
retrieved_item = event.get("item")
return [ItemRetrieved(item=retrieved_item if isinstance(retrieved_item, Mapping) else {})]
if event_type == "response.output_item.done":
response_id = event.get("response_id")
return _realtime_response_terminal_events(
@@ -1044,7 +1046,7 @@ def _function_call_done_events(state: RealtimeProjectionState, event: dict[str,
arguments = str(arguments)
response_id = f"resp_{uuid4().hex}"
item_id = f"item_{uuid4().hex}"
item = {
item: dict[str, object] = {
"id": item_id,
"object": "realtime.item",
"type": "function_call",
@@ -1431,7 +1433,7 @@ def resolve_create_item(state: RealtimeProjectionState, command: CreateItem) ->
],
)
ack_events = register_user_item(state, item) if item_type == "message" and role == "user" else []
signal_payload = {
signal_payload: dict[str, object] = {
"type": "turn.signal",
"event": "conversation.item.create",
"payload": {"item": item},
@@ -1444,12 +1446,15 @@ def resolve_create_item(state: RealtimeProjectionState, command: CreateItem) ->
if not isinstance(content, list):
return ResolvedControl(payloads=[], events=ack_events)
text_chunks: list[str] = []
image_parts: list[Mapping[str, object]] = []
audio_payloads: list[dict[str, object]] = []
for part in content:
if not isinstance(part, dict):
continue
if part.get("type") in {"input_text", "text"} and isinstance(part.get("text"), str):
text_chunks.append(str(part["text"]))
if part.get("type") == "input_image":
image_parts.append(part)
if part.get("type") in {"input_audio", "audio"}:
audio = part.get("audio") or part.get("data")
if not isinstance(audio, str) or not audio:
@@ -1494,12 +1499,28 @@ def resolve_create_item(state: RealtimeProjectionState, command: CreateItem) ->
}
if transcript:
commit_payload["transcript"] = transcript
return ResolvedControl(payloads=[*audio_payloads, commit_payload], events=ack_events)
if not text_chunks:
payloads: list[dict[str, object]] = []
if image_parts:
# The audio leaves as buffered appends that the commit seals, and
# that commit registers *this* item id against the spoken message.
# Images cannot ride it: the commit would overwrite them. So they
# travel as their own item, stored ahead of the audio describing
# them, under an id derived from the one the client named.
image_item = dict(item)
image_item["id"] = f"{item_id}_image"
image_item["status"] = "completed"
image_item["content"] = list(image_parts)
payloads.append({**signal_payload, "payload": {"item": image_item}})
payloads.extend([*audio_payloads, commit_payload])
return ResolvedControl(payloads=payloads, events=ack_events)
if not text_chunks and not image_parts:
return ResolvedControl(payloads=[], events=ack_events)
text_item = dict(item)
text_item["status"] = "completed"
signal_payload["payload"] = {"item": text_item}
# An image is content in its own right, not a decoration on text. Dropping
# the payload here would still ack the item to the client, so the picture
# would be silently missing from every prompt that followed.
stored_item = dict(item)
stored_item["status"] = "completed"
signal_payload["payload"] = {"item": stored_item}
return ResolvedControl(payloads=[signal_payload], events=ack_events)
vllm_omni/engine/duplex/session/append_task.py
+5 -4
View File
@@ -110,8 +110,7 @@ class AppendAttempt:
try:
predecessor_ok = await predecessor
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
if helpers.task_is_cancelling(asyncio.current_task()):
raise
predecessor_ok = False
except Exception:
@@ -159,8 +158,10 @@ class AppendAttempt:
self.ctx.run.runtime_closed = True
return False
if not emitted_response and session.epoch == self.epoch:
if session.active_request_id == helpers.stage0_request_id(session, self.epoch):
session.clear_request(self.request_id)
# Only if the session still points at this append's request:
# ``clear_request`` compares before it clears, and the id
# carries a turn suffix when the core request is not resumable.
session.clear_request(self.request_id)
if self.final:
self.out.emit_events([session.signal_turn(DuplexTurnEventType.USER_STARTED.value)])
return append_ok
vllm_omni/engine/duplex/session/control.py
+54
View File
@@ -113,6 +113,19 @@ class SessionControl:
except ValueError as exc:
self._out.emit_error("bad_audio", str(exc))
return None
if result.speech_started or result.speech_stopped:
logger.info(
"Duplex VAD session=%s started=%s stopped=%s active=%s probability=%.3f "
"start_ms=%s end_ms=%s commit=%s",
self._ctx.session.session_id,
result.speech_started,
result.speech_stopped,
result.speech_active,
result.speech_probability,
result.audio_start_ms,
result.audio_end_ms,
result.should_commit,
)
apply_turn_detection_result(event, result)
return result
@@ -310,6 +323,47 @@ class SessionControl:
name="duplex-continue",
)
return
item_payload = item if isinstance(item, dict) else None
raw_parts = item_payload.get("content", []) if item_payload is not None else []
parts: list[object] = [part for part in raw_parts] if isinstance(raw_parts, list) else []
images = [p for p in parts if isinstance(p, dict) and p.get("type") == "input_image"]
if images:
if item_payload is None or item_payload.get("role") != "user":
self._out.emit_error("invalid_image", "input_image is supported only in user messages")
return
if not session.capabilities.supports_image_input:
self._out.emit_error("unsupported", "This model does not support input_image conversation items")
return
from vllm_omni.engine.duplex.realtime_commands import validate_realtime_video_frames
urls: list[str] = []
for part in images:
url = part.get("image_url")
if not isinstance(url, str) or not url.startswith(
("data:image/jpeg;base64,", "data:image/png;base64,")
):
self._out.emit_error("invalid_image", "input_image requires a JPEG or PNG base64 data URL")
return
error = validate_realtime_video_frames([url.split(",", 1)[1]], None)
if error:
self._out.emit_error("invalid_image", error)
return
urls.append(url)
existing: list[str] = []
for history_message in session.history:
content = history_message.get("content")
if not isinstance(content, list):
continue
for part in content:
if not isinstance(part, dict) or part.get("type") != "image_url":
continue
image_url = part.get("image_url")
nested_url = image_url.get("url") if isinstance(image_url, dict) else None
if isinstance(nested_url, str):
existing.append(nested_url)
if len(existing) + len(urls) > 8 or sum(map(len, existing + urls)) > 4 * 1024 * 1024:
self._out.emit_error("input_backpressure", "Image context exceeds 8 images or 4 MiB; delete old items")
return
message = realtime_item_to_history_message(item)
item_id = item.get("id") if isinstance(item, dict) else None
if message is not None:
vllm_omni/engine/duplex/session/engine_session.py
+117 -45
View File
@@ -20,7 +20,7 @@ import time
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from uuid import uuid4
from vllm_omni.engine.duplex.config import (
@@ -53,6 +53,42 @@ def _default_lease() -> DuplexLeaseState:
return DuplexLeaseState(config=DuplexLeaseConfig(), generation=0, last_activity=time.monotonic())
def _object_dict(**fields: object) -> dict[str, object]:
return dict(fields)
def _as_int(value: object, default: int = 0) -> int:
if isinstance(value, bool):
return default
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
return default
def _as_optional_int(value: object) -> int | None:
if value is None or isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
return None
def _copy_mapping(value: object) -> dict[str, object] | None:
if not isinstance(value, dict):
return None
return {str(key): item for key, item in value.items()}
def _copy_list(value: object) -> list[object] | None:
if not isinstance(value, list):
return None
return [item for item in value]
@dataclass
class InputBufferState:
commit_seq: int = 0
@@ -624,7 +660,7 @@ class DuplexEngineSession:
input_audio_part["transcript"] = transcript
self._bind_active_response_to_input_commit(self._input.commit_seq + 1)
self._input.commit_seq += 1
message = {"role": "user", "content": [input_audio_part]}
message = _object_dict(role="user", content=[input_audio_part])
if transcript:
message["transcript"] = transcript
self._conversation.messages.append(message)
@@ -694,7 +730,7 @@ class DuplexEngineSession:
self._response.stage_metric_tpot_weighted_ms.clear()
self._response.stage_metric_tpot_weight.clear()
def stash_stage_metrics(self, stage_metrics: Mapping[object, object] | None) -> None:
def stash_stage_metrics(self, stage_metrics: Mapping[Any, Any] | None) -> None:
"""Hold a stage snapshot until a response exists to attribute it to.
A stage that hands its output to the next stage instead of the client
@@ -706,9 +742,10 @@ class DuplexEngineSession:
"""
if not isinstance(stage_metrics, Mapping):
return
snapshot = {
str(stage_id): dict(values) for stage_id, values in stage_metrics.items() if isinstance(values, Mapping)
}
snapshot: dict[str, dict[str, object]] = {}
for stage_id, values in stage_metrics.items():
if isinstance(values, Mapping):
snapshot[str(stage_id)] = {str(key): value for key, value in values.items()}
if not snapshot:
return
if self.active_response_id is not None:
@@ -718,7 +755,7 @@ class DuplexEngineSession:
def accumulate_response_stage_metrics(
self,
stage_metrics: Mapping[object, object] | None,
stage_metrics: Mapping[Any, Any] | None,
) -> dict[str, dict[str, object]]:
if self.active_response_id is None:
return copy.deepcopy(self._response.stage_metrics)
@@ -762,7 +799,9 @@ class DuplexEngineSession:
for name in additive_fields:
value = raw_values.get(name)
if isinstance(value, int | float) and not isinstance(value, bool):
current[name] = current.get(name, 0) + value
previous = current.get(name, 0)
addend = previous if isinstance(previous, int | float) and not isinstance(previous, bool) else 0
current[name] = addend + value
for name in first_positive_fields:
value = raw_values.get(name)
current_value = current.get(name)
@@ -776,7 +815,8 @@ class DuplexEngineSession:
for list_name, mean_name in interval_fields:
values = raw_values.get(list_name)
if isinstance(values, list):
combined = list(current.get(list_name, []))
existing = current.get(list_name, [])
combined = list(existing) if isinstance(existing, list) else []
combined.extend(
value for value in values if isinstance(value, int | float) and not isinstance(value, bool)
)
@@ -805,7 +845,7 @@ class DuplexEngineSession:
def replace_response_stage_metric_snapshots(
self,
stage_metrics: Mapping[object, object] | None,
stage_metrics: Mapping[Any, Any] | None,
) -> dict[str, dict[str, object]]:
"""Merge cumulative chat snapshots by replacing each stage's latest value."""
if self.active_response_id is None or not isinstance(stage_metrics, Mapping):
@@ -839,8 +879,12 @@ class DuplexEngineSession:
*,
text_chars: int | None = None,
audio_text_marks: list[dict[str, object]] | None = None,
text_requires_complete_audio: bool = False,
audio_complete: bool = False,
) -> None:
playback = self._playback.current
playback.text_requires_complete_audio |= text_requires_complete_audio
playback.audio_complete |= audio_complete
if duration_ms is not None:
playback.generated_ms = max(playback.generated_ms, duration_ms)
playback.sent_ms = max(playback.sent_ms, duration_ms)
@@ -925,9 +969,9 @@ class DuplexEngineSession:
response_input_commit_seq = self.input_commit_seq
response_history_is_late = self.input_commit_seq > response_input_commit_seq
assistant_text = "".join(self._response.assistant_text_buffer).strip()
message = None
message: dict[str, object] | None = None
if assistant_text:
self._conversation.last_assistant_full_message = {"role": "assistant", "content": assistant_text}
self._conversation.last_assistant_full_message = _object_dict(role="assistant", content=assistant_text)
self._conversation.last_assistant_audio_text_marks = list(self._response.assistant_audio_text_marks)
if response_id is not None:
self._conversation.assistant_response_snapshots[response_id] = (
@@ -943,7 +987,7 @@ class DuplexEngineSession:
else:
committed_text = ""
if commit_text and committed_text:
message = {"role": "assistant", "content": committed_text}
message = _object_dict(role="assistant", content=committed_text)
item_id = f"item_{response_id}" if response_id is not None else None
if item_id is not None and item_id in self._conversation.history_item_placeholders:
self._store_history_item_message(item_id, message)
@@ -1078,8 +1122,8 @@ class DuplexEngineSession:
self._conversation.pending_truncations_ms.pop(item_id, None)
return True
message = self._conversation.item_ids.get(item_id)
if message is None:
stored_message = self._conversation.item_ids.get(item_id)
if stored_message is None:
pending = self._conversation.pending_item_ids.get(item_id)
if pending is None:
self._conversation.pending_truncations_ms[item_id] = max(0, int(audio_end_ms))
@@ -1113,6 +1157,7 @@ class DuplexEngineSession:
if item_id.startswith("item_"):
self._conversation.assistant_response_snapshots.pop(item_id.removeprefix("item_"), None)
return True
message = stored_message
changed = self._truncate_message_to_audio_ms(
message,
audio_end_ms=audio_end_ms,
@@ -1209,12 +1254,20 @@ class DuplexEngineSession:
sent_ms = max(self.playback.sent_ms, self.playback.generated_ms)
committed_ms = self.playback.committed_ms
policy = playback_commit_policy or self.config.playback_commit_policy
commit_all = policy == DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value
if self.playback.text_requires_complete_audio:
# Unaligned text has no spoken prefix to cut at, so history gets
# the whole answer or none of it. Under commit_all_on_done that
# choice is the whole answer --- which is also the only thing a
# client that never acks playback can be credited with.
if commit_all:
return assistant_text
keep_chars = self._text_chars_for_audio_ms(committed_ms, len(assistant_text))
return assistant_text[:keep_chars].rstrip()
if sent_ms <= 0 or committed_ms >= sent_ms:
return assistant_text
if committed_ms <= 0:
if policy == DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value:
return assistant_text
return ""
return assistant_text if commit_all else ""
keep_chars = self._text_chars_for_audio_ms(committed_ms, len(assistant_text))
if keep_chars <= 0:
return ""
@@ -1233,6 +1286,11 @@ class DuplexEngineSession:
audio_end_ms = max(0, int(audio_end_ms))
marks = marks if marks is not None else self._response.assistant_audio_text_marks
playback = playback or self._playback.current
if playback.text_requires_complete_audio:
# Unaligned streams cannot establish a spoken prefix. A terminal
# boundary plus its playback ACK can establish the whole answer.
total_ms = max(playback.sent_ms, playback.generated_ms)
return text_len if playback.audio_complete and total_ms > 0 and audio_end_ms >= total_ms else 0
if not marks:
sent_ms = max(1, playback.sent_ms, playback.generated_ms)
return int(text_len * max(0.0, min(1.0, audio_end_ms / sent_ms)))
@@ -1307,11 +1365,9 @@ class DuplexEngineSession:
elif event_type == DuplexTurnEventType.ASSISTANT_DONE.value:
self.transition_turn(DuplexTurnState.IDLE)
elif event_type == DuplexTurnEventType.PLAYBACK_ACK.value:
played_ms = int(payload.get("played_ms", 0) or 0)
committed_ms = payload.get("committed_ms")
self.acknowledge_playback(
played_ms,
int(committed_ms) if isinstance(committed_ms, int | float) else None,
_as_int(payload.get("played_ms", 0)),
_as_optional_int(payload.get("committed_ms")),
)
elif event_type == DuplexTurnEventType.BARGE_IN.value:
self.transition_turn(DuplexTurnState.BARGE_IN)
@@ -1346,29 +1402,45 @@ class DuplexEngineSession:
"playback": self.playback.as_dict(),
"capabilities": self.capabilities.as_dict(),
}
if isinstance(self.config.extra_body.get("realtime_tools"), list):
payload["tools"] = self.config.extra_body["realtime_tools"]
if isinstance(self.config.extra_body.get("realtime_tool_choice"), str | dict):
payload["tool_choice"] = self.config.extra_body["realtime_tool_choice"]
if isinstance(self.config.extra_body.get("realtime_metadata"), dict):
payload["metadata"] = dict(self.config.extra_body["realtime_metadata"])
if isinstance(self.config.extra_body.get("realtime_include"), list):
payload["include"] = list(self.config.extra_body["realtime_include"])
if isinstance(self.config.extra_body.get("realtime_prompt"), dict):
payload["prompt"] = dict(self.config.extra_body["realtime_prompt"])
if isinstance(self.config.extra_body.get("realtime_input_audio_transcription"), dict):
payload["input_audio_transcription"] = dict(self.config.extra_body["realtime_input_audio_transcription"])
if isinstance(self.config.extra_body.get("realtime_input_audio_noise_reduction"), dict):
payload["input_audio_noise_reduction"] = dict(
self.config.extra_body["realtime_input_audio_noise_reduction"]
)
if isinstance(self.config.extra_body.get("realtime_audio"), dict):
payload["audio"] = dict(self.config.extra_body["realtime_audio"])
if isinstance(self.config.extra_body.get("realtime_tracing"), str | dict):
payload["tracing"] = self.config.extra_body["realtime_tracing"]
raw_realtime_session = self.config.extra_body.get("realtime_session_payload")
if isinstance(raw_realtime_session, dict):
for key, value in raw_realtime_session.items():
extra_body = self.config.extra_body
realtime_tools = _copy_list(extra_body.get("realtime_tools"))
if realtime_tools is not None:
payload["tools"] = realtime_tools
realtime_tool_choice = extra_body.get("realtime_tool_choice")
if isinstance(realtime_tool_choice, str):
payload["tool_choice"] = realtime_tool_choice
else:
copied_tool_choice = _copy_mapping(realtime_tool_choice)
if copied_tool_choice is not None:
payload["tool_choice"] = copied_tool_choice
realtime_metadata = _copy_mapping(extra_body.get("realtime_metadata"))
if realtime_metadata is not None:
payload["metadata"] = realtime_metadata
realtime_include = _copy_list(extra_body.get("realtime_include"))
if realtime_include is not None:
payload["include"] = realtime_include
realtime_prompt = _copy_mapping(extra_body.get("realtime_prompt"))
if realtime_prompt is not None:
payload["prompt"] = realtime_prompt
realtime_transcription = _copy_mapping(extra_body.get("realtime_input_audio_transcription"))
if realtime_transcription is not None:
payload["input_audio_transcription"] = realtime_transcription
realtime_noise_reduction = _copy_mapping(extra_body.get("realtime_input_audio_noise_reduction"))
if realtime_noise_reduction is not None:
payload["input_audio_noise_reduction"] = realtime_noise_reduction
realtime_audio = _copy_mapping(extra_body.get("realtime_audio"))
if realtime_audio is not None:
payload["audio"] = realtime_audio
realtime_tracing = extra_body.get("realtime_tracing")
if isinstance(realtime_tracing, str):
payload["tracing"] = realtime_tracing
else:
copied_tracing = _copy_mapping(realtime_tracing)
if copied_tracing is not None:
payload["tracing"] = copied_tracing
copied_session = _copy_mapping(extra_body.get("realtime_session_payload"))
if copied_session is not None:
for key, value in copied_session.items():
if key not in payload and key != "extra_body":
payload[key] = value
return payload
vllm_omni/engine/duplex/session/helpers.py
+30 -3
View File
@@ -13,14 +13,16 @@ Functions, not a component, because there is nothing here to own.
from __future__ import annotations
import asyncio
import binascii
from typing import TYPE_CHECKING
import pybase64 as base64
from vllm_omni.engine.duplex.config import DuplexPlaybackCommitPolicy
from vllm_omni.engine.duplex.contracts import DuplexFence, duplex_resource_request_id
from vllm_omni.engine.duplex.contracts import DuplexFence
from vllm_omni.engine.duplex.events import ErrorEvent, OverlapDecision, error_event
from vllm_omni.engine.duplex.plugin import payload_turn_id
if TYPE_CHECKING:
from collections.abc import Mapping
@@ -29,13 +31,38 @@ if TYPE_CHECKING:
from vllm_omni.engine.duplex.session.engine_session import DuplexCommittedInput, DuplexEngineSession
def task_is_cancelling(task: asyncio.Task[object] | None) -> bool:
"""True when *task* has a pending cancellation (Python 3.11+ ``Task.cancelling``)."""
if task is None:
return False
cancelling = getattr(task, "cancelling", None)
return bool(cancelling()) if callable(cancelling) else False
# --------------------------------------------------------------------------- #
# Questions about the session #
# --------------------------------------------------------------------------- #
def stage0_request_id(session: DuplexEngineSession, epoch: int) -> str:
return duplex_resource_request_id(DuplexFence(session.session_id, epoch=epoch), "stage0")
def append_fence(session: DuplexEngineSession, payload: object, *, epoch: int | None = None) -> DuplexFence:
"""The fence one append is submitted under.
A model without a resumable core request gets one stage request per turn,
so the turn this resolves to is part of the request id. The runner names
that id when it queues the append and the model channel names it again
when it submits, and the two must not disagree --- otherwise the session
binds a request id it never submitted.
"""
turn_id = payload_turn_id(payload)
if turn_id is None:
turn_id = session.active_response_turn_id
if turn_id is None:
turn_id = session.turn_id
return DuplexFence(
session.session_id,
epoch=session.epoch if epoch is None else epoch,
turn_id=turn_id,
)
def response_in_progress(session: DuplexEngineSession, tasks: DuplexSessionTasks) -> bool:
vllm_omni/engine/duplex/session/manager.py
+8 -3
View File
@@ -408,8 +408,11 @@ class DuplexSessionManager:
return configured
@staticmethod
def stage_request_id(fence: DuplexFence, *, stage_id: int) -> str:
return duplex_resource_request_id(fence, f"stage{stage_id}")
def stage_request_id(fence: DuplexFence, *, stage_id: int, resumable: bool = True) -> str:
role = f"stage{stage_id}"
if not resumable:
role += f"-turn{fence.turn_id}"
return duplex_resource_request_id(fence, role)
def ensure_stage_request(
self,
@@ -422,7 +425,9 @@ class DuplexSessionManager:
if stage_id >= self.stage_port.stage_count:
return None
effective_fence = fence or session.fence
request_id = self.stage_request_id(effective_fence, stage_id=stage_id)
request_id = self.stage_request_id(
effective_fence, stage_id=stage_id, resumable=session.capabilities.supports_core_resumable_request
)
session.reserve_stage_request(stage_id, request_id, fence=effective_fence)
context = DuplexStageRequestContext(
request_id=request_id,
vllm_omni/engine/duplex/session/model_channel.py
+42 -21
View File
@@ -22,6 +22,7 @@ tracked task, and aborting a stage request in the background.
from __future__ import annotations
import asyncio
import copy
import time
import uuid
from collections.abc import Awaitable, Callable, Mapping
@@ -43,8 +44,8 @@ from vllm_omni.engine.duplex.contracts import (
from vllm_omni.engine.duplex.plugin import (
DuplexRuntimeConfigError,
coerce_int,
payload_turn_id,
)
from vllm_omni.engine.duplex.session import helpers
from vllm_omni.engine.duplex.session.context import DuplexSessionContext, StageOutput
from vllm_omni.engine.duplex.session.emitter import SessionEmitter
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession, DuplexFenceMismatchError
@@ -114,6 +115,13 @@ class ModelChannel:
) -> dict[str, object]:
session = self._ctx.session
response_config = session.response_config
if not session.capabilities.supports_core_resumable_request and self.should_commit_response_to_history(
session, response_id
):
# Committed-turn models can receive the next utterance before the
# speaker drains. Reserve the response's chronological slot now;
# an ACK fills it later without moving it past newer user input.
session.reserve_history_item(f"item_{response_id}")
payload: dict[str, object] = {
"type": "response.created",
"session_id": session.session_id,
@@ -202,22 +210,13 @@ class ModelChannel:
session = self._ctx.session
if self._ctx.stage_port.stage_count == 0:
raise RuntimeError("duplex_data_plane_has_no_stage")
payload_turn = payload_turn_id(payload)
fence = DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=(
payload_turn
if payload_turn is not None
else (
session.active_response_turn_id if session.active_response_turn_id is not None else session.turn_id
)
),
)
fence = helpers.append_fence(session, payload)
lease_operation_id = f"append:{operation_id or uuid.uuid4().hex}"
operation_started = False
stage_id = 0
request_id = self._ctx.manager.stage_request_id(fence, stage_id=stage_id)
request_id = self._ctx.manager.stage_request_id(
fence, stage_id=stage_id, resumable=session.capabilities.supports_core_resumable_request
)
try:
session.begin_lease_operation(fence, lease_operation_id)
operation_started = True
@@ -226,10 +225,17 @@ class ModelChannel:
request_context = self._ctx.manager.ensure_stage_request(session, stage_id=stage_id, fence=fence)
if request_context is None:
raise RuntimeError("duplex_data_plane_has_no_stage")
append_plan = self._ctx.plugin.plan_append(
prompt_payload: dict[str, object] = (
{str(key): value for key, value in payload.items()} if isinstance(payload, Mapping) else {}
)
append_plan = await self._ctx.plugin.prepare_append_plan(
request_id=request_id,
fence=fence,
session_config=dict(request_context.session_config),
session_config=self._ctx.plugin.prepare_prompt_config(
{**request_context.session_config, "conversation": list(session.history)},
state=self._ctx.model_state,
payload=prompt_payload,
),
runtime_config=dict(request_context.runtime_config),
seq=reservation.update.seq,
turn_seq=reservation.update.turn_seq,
@@ -237,6 +243,10 @@ class ModelChannel:
final=final,
sampling_params=request_context.stage_sampling_params,
)
# Preparation may yield while cancellation or session teardown runs.
# Never submit the worker's stale result to the model.
if session.epoch != fence.epoch or session.state != DuplexSessionState.OPEN:
raise DuplexFenceMismatchError(session.fence, fence)
if not isinstance(append_plan, DuplexAppendPlan):
raise TypeError("duplex plugin plan_append() must return DuplexAppendPlan")
submission = DuplexStageSubmission(
@@ -279,7 +289,7 @@ class ModelChannel:
"seq": update.seq,
"turn_id": update.turn_id,
"turn_seq": update.turn_seq,
"resumable": True,
"resumable": session.capabilities.supports_core_resumable_request,
},
}
],
@@ -413,7 +423,7 @@ class ModelChannel:
engine_output.finished = True
engine_output = attach_duplex_output_decision(engine_output, item.decision)
elif isinstance(output, OmniRequestOutput):
engine_output = output
engine_output = copy.copy(output)
if finished:
engine_output.finished = True
else:
@@ -425,6 +435,9 @@ class ModelChannel:
stage_id=item.stage_id,
final_output_type=final_output_type if isinstance(final_output_type, str) else "audio",
)
engine_output.stage_id = item.stage_id
if self._ctx.plugin.projects_intermediate_outputs and item.stage_id < item.context.final_stage_id:
engine_output.finished = False
snapshot = self.stage_metrics_snapshot(item.stage_id, item.metrics, output)
if snapshot is not None:
existing = engine_output.metrics if isinstance(engine_output.metrics, dict) else {}
@@ -461,6 +474,11 @@ class ModelChannel:
await self._close_from_runtime(close_reason)
return
finished = self._data_plane_outputs_finished(drain_result)
if finished and not session.capabilities.supports_core_resumable_request:
if self._ctx.run.stream_request_id == item.request_id:
self._ctx.run.stream_request_id = None
await self._ctx.stage_port.cleanup([item.request_id])
return
if finished and emitted_response and not self._out.auto_responds():
# A finished, emitted response releases the per-request projector
# cursor on its way out and offers the model another
@@ -759,8 +777,9 @@ class ModelChannel:
response_id = session.begin_response(turn_id=model_turn_id)
response_created = True
self._out.emit(self.response_created_payload(response_id, epoch=session.epoch))
stage_metrics = model_result.get("stage_metrics")
response_stage_metrics = session.accumulate_response_stage_metrics(
model_result.get("stage_metrics") if isinstance(model_result.get("stage_metrics"), Mapping) else None
stage_metrics if isinstance(stage_metrics, Mapping) else None
)
if response_created:
speak_payload = {
@@ -805,6 +824,8 @@ class ModelChannel:
mark_duration_ms,
text_chars=mark_text_chars if mark_duration_ms is not None else None,
audio_text_marks=audio_text_marks,
text_requires_complete_audio=model_result.get("text_requires_complete_audio") is True,
audio_complete=model_result.get("audio_complete") is True,
)
payload = {
"type": "response.output_audio.delta",
@@ -904,10 +925,10 @@ class ModelChannel:
*,
audio_offset_ms: int,
text_offset_chars: int,
) -> list[dict[str, int]] | None:
) -> list[dict[str, object]] | None:
if not audio_text_marks:
return None
normalized: list[dict[str, int]] = []
normalized: list[dict[str, object]] = []
for raw_mark in audio_text_marks:
if not isinstance(raw_mark, dict):
continue
vllm_omni/engine/duplex/session/runner.py
+65 -34
View File
@@ -71,7 +71,6 @@ from vllm_omni.engine.duplex.plugin import (
DuplexModelPlugin,
DuplexModelSessionState,
PcmAppendReservation,
payload_turn_id,
)
from vllm_omni.engine.duplex.realtime_events import (
RealtimeProjectionState,
@@ -251,7 +250,8 @@ class DuplexSessionRunner:
if stage_id < context.final_stage_id:
decision = self.model.decide_output(stage_id, output, context)
consume = decision is not None or stage_id >= context.final_stage_id
if not consume:
project_intermediate = self.plugin.projects_intermediate_outputs and stage_id == 0
if not consume and not project_intermediate:
# Stage0 text without a direct decision feeds the TTS stage as before.
# Its metrics still have to reach the client: before sessions moved
# into the engine the orchestrator published them as a standalone
@@ -274,7 +274,7 @@ class DuplexSessionRunner:
decision=decision,
)
)
return True
return consume
def on_stage_failure(self, stage_id: int, exc: BaseException) -> None:
"""A stage rejected this session's request: fail the active response now.
@@ -434,12 +434,11 @@ class DuplexSessionRunner:
worker.cancel()
await asyncio.gather(worker, return_exceptions=True)
def spawn(self, coro: Awaitable[None], *, name: str) -> asyncio.Task[None]:
def spawn(self, coro: Awaitable[None], *, name: str) -> None:
task = asyncio.ensure_future(coro)
task.set_name(name)
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
return task
async def offload(self, fn: Callable[..., _OffloadT], *args: object, **kwargs: object) -> _OffloadT:
loop = self._loop or asyncio.get_running_loop()
@@ -539,14 +538,14 @@ class DuplexSessionRunner:
elif isinstance(command, ClearInput):
self._on_clear_input()
elif isinstance(command, CancelResponse):
resolved = resolve_cancel_response(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_cancel_response(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self._on_cancel(payload)
elif isinstance(command, ClearOutputAudio):
resolved = resolve_clear_output_audio(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_clear_output_audio(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self._on_cancel(payload)
elif isinstance(command, CancelInput | BargeIn):
await self._on_cancel(command.payload())
@@ -564,7 +563,8 @@ class DuplexSessionRunner:
return
payload = command.payload()
if command.event in _CANCEL_EVENTS:
normalized = dict(payload.get("payload") or {})
inner = payload.get("payload")
normalized: dict[str, object] = dict(inner) if isinstance(inner, Mapping) else {}
normalized.update(payload)
normalized["type"] = command.event
await self._on_cancel(normalized)
@@ -577,19 +577,19 @@ class DuplexSessionRunner:
elif isinstance(command, Heartbeat):
self._on_heartbeat(command)
elif isinstance(command, CreateItem):
resolved = resolve_create_item(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_create_item(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self._run_internal_payload(payload)
elif isinstance(command, DeleteItem):
resolved = resolve_delete_item(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_delete_item(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self.control.on_turn_signal(payload)
elif isinstance(command, TruncateItem):
resolved = resolve_truncate_item(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_truncate_item(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self._run_internal_payload(payload)
elif isinstance(command, CloseSession):
await self._on_close_command(command.reason)
@@ -776,7 +776,7 @@ class DuplexSessionRunner:
sr_raw = event.get("sample_rate_hz") or event.get("sample_rate")
sample_rate_hz = sr_raw if isinstance(sr_raw, int | float) else 16000
try:
audio, fmt, sample_rate_hz = await self.offload(
converted: tuple[object, object, int | float | None] = await self.offload(
convert_input_audio_with_rate,
audio,
fmt,
@@ -785,13 +785,25 @@ class DuplexSessionRunner:
except ValueError as exc:
self._emit_error("bad_event", str(exc))
return
audio, fmt, converted_rate = converted
if converted_rate is not None:
sample_rate_hz = converted_rate
if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le"}:
self._emit_error("bad_audio", "input_audio_buffer.append pcm16 audio could not be decoded")
return
event["audio"] = audio
event["format"] = fmt
event["sample_rate_hz"] = sample_rate_hz
client_force_listen = bool(event.get("force_listen", False))
vad_result = await self.control.run_turn_detection(event)
if (
vad_result is not None
and not session.capabilities.supports_core_resumable_request
and not client_force_listen
):
# VAD's force_listen hint controls native model decoding. Committed
# turn models already buffer speech; it must not suppress barge-in.
event.pop("force_listen", None)
projector = self._require_projector()
self._emit_events(note_input_append(projector, event, vad_result=vad_result))
if self.run.closing or session.state != DuplexSessionState.OPEN:
@@ -896,8 +908,11 @@ class DuplexSessionRunner:
return
if pcm_reservation.byte_count == 0:
session.release_input_bytes(raw_audio_bytes)
payload = pcm_reservation.payload
await self._start_append(payload, final=False, pcm_reservation=pcm_reservation)
append_payload = pcm_reservation.payload
if append_payload is None:
self._maybe_schedule_vad_commit(vad_result)
return
await self._start_append(append_payload, final=False, pcm_reservation=pcm_reservation)
self._maybe_schedule_vad_commit(vad_result)
def _maybe_schedule_vad_commit(self, vad_result: TurnDetectionResult | None) -> None:
@@ -968,10 +983,13 @@ class DuplexSessionRunner:
on_append_accepted = _reanchor_chain
append_epoch = session.epoch
append_turn_id = payload_turn_id(payload)
if append_turn_id is None:
append_turn_id = session.turn_id
request_id = helpers.stage0_request_id(self.session, append_epoch)
append_fence = helpers.append_fence(session, payload, epoch=append_epoch)
append_turn_id = append_fence.turn_id
request_id = self.ctx.manager.stage_request_id(
append_fence,
stage_id=0,
resumable=session.capabilities.supports_core_resumable_request,
)
if final or precreate_response:
session.bind_request(request_id)
if precreate_response:
@@ -979,6 +997,14 @@ class DuplexSessionRunner:
if precreate_response and session.active_response_id is None:
response_id = session.begin_response(turn_id=append_turn_id)
self.emit(self.model.response_created_payload(response_id, epoch=append_epoch))
if final and not session.capabilities.supports_core_resumable_request:
logger.info(
"Duplex committed turn session=%s request=%s response=%s audio_bytes=%s",
session.session_id,
request_id,
session.active_response_id,
helpers.audio_payload_size_bytes(payload),
)
attempt = AppendAttempt(
ctx=self.ctx,
out=self.out,
@@ -1034,8 +1060,7 @@ class DuplexSessionRunner:
try:
return await predecessor
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
if helpers.task_is_cancelling(asyncio.current_task()):
raise
return False
except Exception:
@@ -1063,8 +1088,7 @@ class DuplexSessionRunner:
if not await pending_silence:
return False
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
if helpers.task_is_cancelling(asyncio.current_task()):
raise
return False
except Exception:
@@ -1523,6 +1547,10 @@ class DuplexSessionRunner:
retained_committed_payload=committed_payload,
)
return
if session.unanswered_user_items() and session.capabilities.supports_text_only_turn:
session.reset_unanswered_user_items()
await self._start_append({"type": "conversation"}, final=True, precreate_response=True)
return
if session.unanswered_user_items():
# Conversation items are context, not a turn. A model-native model
# decides to speak from the audio it hears, and there is no audio
@@ -1601,13 +1629,16 @@ class DuplexSessionRunner:
if deferred_payload is None:
commit_reservation.commit()
return False
retained_payload: dict[str, object]
if model_state.committed_audio_payload is not None:
deferred_payload = overlap_policy.merge_audio_payloads(
retained_payload = overlap_policy.merge_audio_payloads(
model_state.committed_audio_payload,
deferred_payload,
)
else:
retained_payload = deferred_payload
model_state.retain_committed_audio(
deferred_payload,
retained_payload,
operation_id=commit_reservation.operation_id,
reserved_bytes=commit_reservation.byte_count,
)
vllm_omni/engine/duplex/vad.py
+8 -2
View File
@@ -428,11 +428,17 @@ class SileroStreamingVAD:
if self._speech_active:
contained_speech = True
if probability >= self.config.threshold:
# Confirmed speech cancels a pending endpoint. Otherwise a
# brief pause can age through resumed speech and split the
# utterance at the next low-probability frame.
self._silence_samples = 0
continue
below_negative_threshold = probability < negative_threshold
if not below_negative_threshold and self._silence_samples == 0:
continue
# A silence candidate is running: louder frames keep its clock
# moving but cannot themselves close the turn.
# Scores inside the hysteresis band keep the pending timer,
# but cannot themselves close the turn.
self._silence_samples += self._WINDOW_SAMPLES
if not below_negative_threshold or self._silence_samples < min_silence_samples:
continue
vllm_omni/engine/duplex_orchestrator.py
+45 -8
View File
@@ -12,6 +12,8 @@ and applies the session-owned policy.
from __future__ import annotations
import asyncio
import threading
import time as _time
from collections.abc import Coroutine
from dataclasses import dataclass, field
@@ -20,6 +22,7 @@ from typing import TYPE_CHECKING, Any
from vllm.logger import init_logger
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
from vllm_omni.engine import OmniEngineCoreRequest
from vllm_omni.engine.duplex.contracts import (
DuplexFence,
DuplexOutputContext,
@@ -77,6 +80,7 @@ class DuplexOrchestrator(Orchestrator, DuplexStagePort):
**kwargs: Any,
) -> None:
super().__init__(*args, **kwargs)
self._prompt_processing_lock = threading.Lock()
self.plugin = plugin
self.duplex_session_config = duplex_session_config
self.session_manager = DuplexSessionManager(
@@ -288,7 +292,9 @@ class DuplexOrchestrator(Orchestrator, DuplexStagePort):
fence=context.fence,
config_generation=context.config_generation,
)
request_state.streaming.enabled = True
request_state.streaming.enabled = self.plugin.capabilities(
max_sessions=self.duplex_session_config.max_sessions
).supports_core_resumable_request
self.request_states[context.request_id] = request_state
elif isinstance(request_state, DuplexOrchestratorRequestState):
if request_state.config_generation != context.config_generation:
@@ -301,18 +307,49 @@ class DuplexOrchestrator(Orchestrator, DuplexStagePort):
self._sync_bridge_state(request_state, context)
self.session_manager.register_request(context.request_id, context.session_id)
def _upgrade_processed_stage_request(self, request, raw_prompt):
request = super()._upgrade_processed_stage_request(request, raw_prompt)
if not self.plugin.capabilities(
max_sessions=self.duplex_session_config.max_sessions
).supports_core_resumable_request and not isinstance(request, OmniEngineCoreRequest):
request = OmniEngineCoreRequest.from_request(request)
return request
def _process_turn_prompt(self, *args, **kwargs):
# Input processors own mutable caches. Hold a thread lock even if the
# awaiting session is cancelled while its preprocessing is still running.
with self._prompt_processing_lock:
return self._build_next_stage_request(*args, **kwargs)
async def submit(self, submission: DuplexStageSubmission) -> DuplexStageSubmissionResult:
context = submission.context
request_state = self.request_states.get(context.request_id)
if not isinstance(request_state, DuplexOrchestratorRequestState):
raise RuntimeError(f"duplex request was not preregistered: {context.request_id}")
request = build_engine_core_request_from_tokens(
request_id=context.request_id,
prompt=dict(submission.prompt),
params=context.stage_sampling_params,
model_config=self.stage_pools[context.stage_id].stage_vllm_config.model_config,
resumable=True,
)
if self.plugin.capabilities(
max_sessions=self.duplex_session_config.max_sessions
).supports_core_resumable_request:
request = build_engine_core_request_from_tokens(
request_id=context.request_id,
prompt=dict(submission.prompt),
params=context.stage_sampling_params,
model_config=self.stage_pools[context.stage_id].stage_vllm_config.model_config,
resumable=True,
)
else:
# Use the ordinary multimodal input processor for turn-model plugins.
# Its CPU preprocessing runs off the session/orchestrator event loop.
request = await asyncio.to_thread(
self._process_turn_prompt,
context.request_id,
context.stage_id,
dict(submission.prompt),
context.stage_sampling_params,
resumable=False,
)
if self.request_states.get(context.request_id) is not request_state:
raise RuntimeError("duplex request cancelled during input preprocessing")
request_state.prompt = dict(submission.prompt)
request.external_req_id = request.request_id
pool = self.stage_pools[context.stage_id]
if submission.already_submitted:
vllm_omni/entrypoints/duplex/warmup.py
+4 -1
View File
@@ -46,6 +46,7 @@ async def _warmup_duplex_realtime(app, args, warmup_frames: int) -> None:
# One silence unit as the engine-side plugin defines it (DuplexOmniEngine.plugin).
plugin = getattr(getattr(app.state.engine_client, "engine", None), "plugin", None)
frame_samples = int(getattr(plugin, "silence_continuation_samples", 16000))
native_append = plugin.capabilities(max_sessions=1).supports_core_resumable_request if plugin else True
silence = base64.b64encode(bytes(frame_samples * 4)).decode("ascii")
from vllm_omni.clients.duplex import build_realtime_url
@@ -81,7 +82,7 @@ async def _warmup_duplex_realtime(app, args, warmup_frames: int) -> None:
"output_audio_format": "pcm16",
"idle_timeout_s": 60,
"turn_detection": None,
"extra_body": {"auto_response": True},
"extra_body": {"auto_response": native_append},
},
}
)
@@ -108,6 +109,8 @@ async def _warmup_duplex_realtime(app, args, warmup_frames: int) -> None:
await ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": silence}))
sent += 1
await asyncio.sleep(0.08)
if not native_append:
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
# Wait for the pipeline's first audio output so every stage ran.
saw_audio = await _recv_until(lambda e: e.get("type") == "response.output_audio.delta", 30)
# Close the session EXPLICITLY: a bare websocket close parks the
vllm_omni/entrypoints/openai/api_server.py
+34 -18
View File
@@ -521,7 +521,9 @@ def _should_serve_duplex(model: str, kwargs: dict[str, Any]) -> bool:
deploy_path = _DEPLOY_DIR / deploy_path
# Resolve base_config too, so the API and stage workers use the same mode.
session_mode = resolve_deploy_yaml(deploy_path).get("session_mode")
session_mode = resolve_deploy_yaml(deploy_path).get(
"session_mode", getattr(pipeline_config, "default_session_mode", None)
)
if session_mode not in ("turn", "duplex"):
raise ValueError("A duplex-capable model requires session_mode: turn or duplex in its deploy config")
return session_mode == "duplex"
@@ -1742,6 +1744,12 @@ async def realtime_websocket(websocket: WebSocket):
await duplex_handler.handle_realtime_session(websocket)
return
if isinstance(duplex_query, str) and duplex_query.lower() in {"1", "true", "on"}:
await websocket.accept()
await websocket.send_json({"type": "error", "code": "unsupported", "error": "VAD realtime is not enabled"})
await websocket.close(code=1008)
return
serving = getattr(websocket.app.state, "openai_serving_realtime", None)
if serving is None:
await websocket.accept()
@@ -2042,6 +2050,8 @@ async def generate_images(
)
try:
width: int | None = None
height: int | None = None
# Unify request construction for any multi-stage pipeline to avoid
# divergence between /v1/images and /v1/chat/completions.
if len(stage_configs) > 1:
@@ -2059,7 +2069,6 @@ async def generate_images(
"num_outputs_per_prompt": request.n,
}
if request.size is not None:
parse_size(request.size)
width, height = parse_size(request.size)
app_state_args = getattr(raw_request.app.state, "args", None)
_check_max_generated_image_size(app_state_args, width, height)
@@ -2113,7 +2122,7 @@ async def generate_images(
)
# Build params - pass through user values directly
prompt: OmniTextPrompt = {"prompt": request.prompt, "modalities": ["image"]}
prompt = OmniTextPrompt(prompt=request.prompt, modalities=["image"])
if request.negative_prompt is not None:
prompt["negative_prompt"] = request.negative_prompt
gen_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=request.n)
@@ -2129,12 +2138,11 @@ async def generate_images(
if extra_args:
gen_params.extra_args = extra_args
# Parse per-request LoRA (compatible with chat's extra_body.lora shape).
lora_request, lora_scale = _parse_lora_request(request.lora)
lora_request, lora_scale = _parse_lora_request(request.lora) if request.lora is not None else (None, None)
_update_if_not_none(gen_params, "lora_request", lora_request)
_update_if_not_none(gen_params, "lora_scale", lora_scale)
# Parse and add size if provided
width, height = None, None
if request.size:
width, height = parse_size(request.size)
size_str = f"{width}x{height}"
@@ -2300,9 +2308,9 @@ async def edit_images(
try:
# 2. Build prompt & images params
cot_output = None
prompt: OmniTextPrompt = {"prompt": prompt, "modalities": ["image"]}
omni_prompt = OmniTextPrompt(prompt=prompt, modalities=["image"])
if negative_prompt is not None:
prompt["negative_prompt"] = negative_prompt
omni_prompt["negative_prompt"] = negative_prompt
input_images_list = []
images = image or image_array
urls = url or url_array
@@ -2333,17 +2341,17 @@ async def edit_images(
# Hunyuan-aware behavior. RGBA/P uploads otherwise diverge from offline.
normalize_edit_images_rgb = bot_task is not None or sys_type is not None
pil_images = await _load_input_images(input_images_list, normalize_rgb=normalize_edit_images_rgb)
prompt["multi_modal_data"] = {}
prompt["multi_modal_data"]["image"] = pil_images
omni_prompt["multi_modal_data"] = {}
omni_prompt["multi_modal_data"]["image"] = pil_images
if mask_image is not None:
# Mask role is different (alpha channel matters); never normalize.
loaded = await _load_input_images([mask_image], normalize_rgb=False)
prompt["multi_modal_data"]["mask_image"] = loaded[0]
omni_prompt["multi_modal_data"]["mask_image"] = loaded[0]
if reference_image is not None:
loaded = await _load_input_images([reference_image], normalize_rgb=normalize_edit_images_rgb)
prompt["multi_modal_data"]["reference_image"] = loaded[0]
omni_prompt["multi_modal_data"]["reference_image"] = loaded[0]
# 3 Build sample params
gen_params = OmniDiffusionSamplingParams()
@@ -2409,13 +2417,13 @@ async def edit_images(
# Keep AR stage target grid in sync with requested output size.
# GLM-Image consumes target_h/target_w via mm_processor_kwargs.
if width is not None and height is not None:
prompt["mm_processor_kwargs"] = {
omni_prompt["mm_processor_kwargs"] = {
"target_h": height,
"target_w": width,
}
# Backward-compatible fallback for processors reading top-level fields.
prompt["height"] = height
prompt["width"] = width
omni_prompt["height"] = height
omni_prompt["width"] = width
_update_if_not_none(gen_params, "width", width)
_update_if_not_none(gen_params, "height", height)
@@ -2513,7 +2521,7 @@ async def edit_images(
if return_stage_metrics is not None:
extra_body["return_stage_metrics"] = return_stage_metrics
prompt_text = prompt.get("prompt", "")
prompt_text = omni_prompt.get("prompt", "")
generation_result = await chat_handler.generate_diffusion_images(
prompt=prompt_text,
extra_body=extra_body,
@@ -2544,7 +2552,7 @@ async def edit_images(
engine_client=engine_client,
gen_params=gen_params,
stage_configs=stage_configs,
prompt=prompt,
prompt=omni_prompt,
request_id=request_id,
)
images = _extract_images_from_result(result)
@@ -2552,6 +2560,8 @@ async def edit_images(
peak_memory_mb = getattr(result, "peak_memory_mb", None)
response_metrics = getattr(result, "metrics", None) if return_stage_metrics else None
if images is None:
images = []
logger.debug(f"Successfully generated {len(images)} image(s)")
# Encode images to base64
@@ -2973,7 +2983,10 @@ async def omni_sleep(request: OmniSleepRequest, raw_request: Request):
acks = await engine_client.sleep(stage_ids=request.stage_ids, level=request.level)
for sid in request.stage_ids:
sleeping_set.add(sid)
return {"status": "SUCCESS", "acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) else a for a in acks]}
return {
"status": "SUCCESS",
"acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) and not isinstance(a, type) else a for a in acks],
}
@router.post("/v1/omni/wakeup")
@@ -2989,7 +3002,10 @@ async def omni_wakeup(request: OmniWakeupRequest, raw_request: Request):
for sid in request.stage_ids:
if sid in sleeping_set:
sleeping_set.remove(sid)
return {"status": "SUCCESS", "acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) else a for a in acks]}
return {
"status": "SUCCESS",
"acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) and not isinstance(a, type) else a for a in acks],
}
if __name__ == "__main__":
vllm_omni/model_executor/models/qwen3_omni/duplex/__init__.py
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
vllm_omni/model_executor/models/qwen3_omni/duplex/input.py
+103
View File
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Transactional PCM buffering for committed Qwen speech turns."""
from __future__ import annotations
from dataclasses import dataclass, field
import pybase64 as base64
from vllm_omni.engine.duplex.plugin import PcmAppendBuffer, PcmAppendReservation
@dataclass
class Reservation(PcmAppendReservation):
owner: QwenPcmBuffer
operation_id: str
payload: dict[str, object] | None
raw: bytes
_active: bool = True
@property
def active(self):
return self._active
@property
def byte_count(self):
return len(self.raw)
def commit(self):
if self._active:
self._active = False
self.owner.reservations.remove(self)
def rollback(self):
if self._active:
self.owner.buffer[:0] = self.raw
self.commit()
@dataclass
class QwenPcmBuffer(PcmAppendBuffer):
buffer: bytearray = field(default_factory=bytearray)
reservations: list[Reservation] = field(default_factory=list)
@property
def pending_byte_count(self):
return len(self.buffer)
def clear(self):
for reservation in self.reservations:
reservation._active = False
self.reservations.clear()
self.buffer.clear()
def clear_force_listen(self):
pass
def has_pending(self):
return bool(self.buffer)
def has_reserved(self):
return bool(self.reservations)
def prepare_append(self, payload, *, operation_id, chunk_period_ms, allow_emit):
if payload.get("format") != "pcm_f32le" or payload.get("sample_rate_hz") != 16000:
raise ValueError("Qwen duplex requires normalized 16 kHz pcm_f32le")
if payload.get("video_frames"):
# Qwen answers whole turns, so it has no audio unit for a frame
# track to interleave against. Images arrive the OpenAI way, as
# input_image conversation items, and stay until deleted.
raise ValueError(
"Qwen duplex does not take video_frames on an audio append; "
"send images as conversation.item.create items with input_image content"
)
raw = base64.b64decode(payload.get("audio", ""), validate=True)
if len(raw) % 4:
raise ValueError("Invalid float32 PCM length")
self.buffer.extend(raw)
# Qwen inference starts on commit, never on an individual audio chunk.
return None
def prepare_commit(self, *, operation_id, chunk_period_ms):
raw = bytes(self.buffer)
payload = (
{
"audio": base64.b64encode(raw).decode(),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"is_speech": True,
}
if raw
else None
)
self.buffer.clear()
reservation = Reservation(self, operation_id, payload, raw)
self.reservations.append(reservation)
return reservation
def flush(self, *, chunk_period_ms):
reservation = self.prepare_commit(operation_id="flush", chunk_period_ms=chunk_period_ms)
reservation.commit()
return reservation.payload
vllm_omni/model_executor/models/qwen3_omni/duplex/plugin.py
+390
View File
@@ -0,0 +1,390 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Engine-owned VAD/commit turns with concurrent input and interruptible output.
Qwen3-Omni is a turn model: this plugin does not advertise native streaming KV
or model-driven turn decisions. Each committed utterance is a fresh request.
"""
from __future__ import annotations
import asyncio
import copy
import hashlib
import io
import json
import os
from collections.abc import Mapping
from pathlib import Path
import numpy as np
import pybase64 as base64
from PIL import Image
from vllm.logger import init_logger
from vllm.sampling_params import RequestOutputKind
from vllm_omni.engine.duplex.config import DuplexCapabilities
from vllm_omni.engine.duplex.contracts import DuplexAppendPlan, duplex_resource_request_belongs_to_session
from vllm_omni.engine.duplex.plugin import DuplexDataPlane, DuplexModelPlugin, DuplexRuntimeConfigError
from vllm_omni.model_executor.models.qwen3_omni.duplex.session import QwenDuplexSessionState
logger = init_logger(__name__)
#: Mirrors ``limit_mm_per_prompt.image`` in ``deploy/qwen3_omni_duplex.yaml``.
#: The session already refuses a ninth stored image, but that budget lives in
#: the engine and this one in the deploy config; the prompt is the only place
#: that can keep them from drifting apart into a rejected submission.
MAX_PROMPT_IMAGES = 8
class QwenDataPlane(DuplexDataPlane):
def __init__(self, encode_audio):
self.encode_audio = encode_audio
self.terminal: set[str] = set()
def begin_request(self, request_id):
self.terminal.discard(request_id)
def is_terminal(self, request_id):
return request_id in self.terminal
def mark_terminal(self, request_id):
self.terminal.add(request_id)
def close_stream(self, request_id):
pass
def close_session(self, session_id, *, active_request_id=None):
self.terminal = {
rid for rid in self.terminal if not duplex_resource_request_belongs_to_session(rid, session_id)
}
def project(self, result, *, context=None):
if not isinstance(result, dict):
return
context = context or {}
for output in result.get("data_plane_outputs", []):
rid = output.request_id
if self.is_terminal(rid):
continue
completion = next(iter(getattr(output, "outputs", ()) or ()), None)
stage_id = getattr(output, "stage_id", 0)
text = getattr(completion, "text", "") if stage_id == 0 else ""
mm = getattr(output, "multimodal_output", None)
if not isinstance(mm, Mapping) or "audio" not in mm:
mm = getattr(completion, "multimodal_output", {}) or {}
audio = mm.get("audio")
encoded = None
duration_ms = 0
rate = mm.get("sr", mm.get("sample_rate", mm.get("sample_rate_hz", 24000)))
if isinstance(rate, (list, tuple)):
rate = next((value for value in rate if value is not None), 24000)
rate = int(rate)
if rate <= 0:
raise ValueError("Invalid output audio sample rate")
if audio is not None and "audio" in context.get("modalities", ()):
chunks = audio if isinstance(audio, (list, tuple)) else [audio]
arrays = [
np.asarray(
chunk.detach().float().cpu().numpy() if hasattr(chunk, "detach") else chunk, dtype=np.float32
).reshape(-1)
for chunk in chunks
]
audio = np.concatenate(arrays) if arrays else np.empty(0, dtype=np.float32)
duration_ms = audio.size * 1000 / rate
if audio.size:
encoded = self.encode_audio(
audio, rate, context.get("response_format", "pcm16"), context.get("speed")
)
yield {
"supported": True,
"data_plane_request_id": rid,
"text": text or "",
"audio_data": encoded or "",
"audio_format": context.get("response_format", "pcm16"),
"audio_duration_ms": duration_ms,
# Thinker text and codec chunks are independent streams. Only
# the final audio boundary can conservatively attest all text.
"audio_text_mark": False,
"text_requires_complete_audio": "audio" in context.get("modalities", ()),
"audio_complete": bool(output.finished) and stage_id == 2,
"sample_rate_hz": rate,
"end_of_turn": bool(output.finished),
"model_turn_id": context.get("active_response_turn_id")
if context.get("active_response_turn_id") is not None
else context.get("turn_id", 0),
"runtime_impl": "qwen3_commit",
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
}
class Qwen3OmniDuplexPlugin(DuplexModelPlugin):
plugin_id = "qwen3-omni"
projects_intermediate_outputs = True
def __init__(self, encode_audio):
self.data_plane = QwenDataPlane(encode_audio)
self.processor = None
self._processor_lock = asyncio.Lock()
def capabilities(self, *, max_sessions):
return DuplexCapabilities(
supports_model_native_turn_policy=False,
supports_input_append=True,
supports_barge_in=True,
supports_replace_latest_chunk=False,
supports_reencode_context=True,
supports_turn_commit_only=True,
supports_core_resumable_request=False,
supports_independent_io_streams=True,
supports_realtime_endpoint=True,
supports_multi_session=max_sessions > 1,
supports_multi_session_same_replica=max_sessions > 1,
supports_session_lease=True,
supports_session_resume=True,
session_admission_mode="engine_managed",
supports_audio_truncate=True,
supports_chat_completions=True,
supports_image_input=True,
supports_text_only_turn=True,
adapter_patterns=["turn_commit"],
signal_sources=["client_event", "server_policy"],
stage_handoff_transport="scheduler_data_plane",
target_barge_in_latency_ms=None,
)
def create_session_state(self):
return QwenDuplexSessionState()
def validate_client_extra_body(self, extra_body):
if not isinstance(extra_body, Mapping):
return
if extra_body.get("auto_response") or extra_body.get("full_duplex"):
raise DuplexRuntimeConfigError(
"Qwen requires server VAD or explicit commits; native auto_response is unsupported"
)
if extra_body.get("realtime_tools"):
raise DuplexRuntimeConfigError("Qwen duplex tool calls are not implemented")
async def prepare_runtime_config(self, config, *, model_config):
self.validate_client_extra_body(config.extra_body)
if config.ref_audio:
raise DuplexRuntimeConfigError("Qwen duplex does not support reference voice audio")
if model_config is None and self.processor is None:
raise DuplexRuntimeConfigError("Qwen duplex requires the stage-0 model configuration")
async with self._processor_lock:
if self.processor is None:
from transformers import AutoProcessor
self.processor = await asyncio.to_thread(
AutoProcessor.from_pretrained,
model_config.model,
trust_remote_code=model_config.trust_remote_code,
revision=getattr(model_config, "revision", None),
)
return self.runtime_config_for_update(config, {})
def runtime_config_for_update(self, config, current):
self.validate_client_extra_body(config.extra_body)
if config.ref_audio:
raise DuplexRuntimeConfigError("Qwen duplex does not support reference voice audio")
return {
"instructions": config.instructions,
"initial_user_text": config.initial_user_text,
"temperature": config.temperature,
"max_tokens": config.max_tokens,
}
def configure_sampling_params(self, *, runtime_config, defaults):
params = copy.deepcopy(defaults)
for value in params:
if value is not None:
value.output_kind = RequestOutputKind.DELTA
if params and params[0] is not None:
if runtime_config.get("temperature") is not None:
params[0].temperature = float(runtime_config["temperature"])
if isinstance(runtime_config.get("max_tokens"), int):
params[0].max_tokens = runtime_config["max_tokens"]
return params
def prepare_prompt_config(self, config, *, state, payload):
history = config.get("conversation", [])
# The history accessor copies message dicts but preserves content identity.
# Keep payloads privately, never put PCM into public conversation events.
has_audio = bool(payload.get("audio"))
last_content = history[-1].get("content") if history else None
current_is_in_history = (
has_audio
and isinstance(last_content, list)
and any(part.get("type") == "audio_url" for part in last_content)
)
if current_is_in_history:
content = history[-1].get("content")
state.audio_history = [(key, value) for key, value in state.audio_history if key is not content]
state.audio_history.append((content, payload))
# Reserve one of the four audio slots for the actual submitted input,
# including when deferred input has no trailing user history entry.
history_limit = 4 if current_is_in_history or not has_audio else 3
retained = state.audio_history[-history_limit:]
while len(retained) > 1 and sum(len(value.get("audio", "")) for _, value in retained) > 8 * 1024 * 1024:
retained.pop(0)
state.audio_history = retained
messages = []
def has_image(message):
content = message.get("content")
return isinstance(content, list) and any(p.get("type") == "image_url" for p in content)
selected = [m for m in history[:-16] if has_image(m)] + history[-16:]
for message in selected:
content = message.get("content")
audio = next((value for key, value in retained if key is content), None)
if audio is not None:
messages.append({"role": "user", "audio_payload": audio})
elif isinstance(content, list) and any(part.get("type") == "image_url" for part in content):
messages.append(message)
elif isinstance(content, str) and content:
messages.append(message)
elif message.get("role") == "user":
# Do not keep an assistant answer after dropping its user audio.
messages = [m for m in messages if has_image(m)]
while messages and messages[0].get("role") == "assistant":
messages.pop(0)
if has_audio and not current_is_in_history:
# History is context, not a replacement for the current request.
# Re-encoding only history here makes Qwen answer an old question.
messages.append({"role": "user", "audio_payload": payload})
return {**config, "qwen_messages": messages}
@staticmethod
def _trim_prompt_images(messages, images):
"""Drop the oldest images, with their placeholders, down to the prompt limit.
The processor pairs ``{"type": "image"}`` parts with ``mm["image"]``
positionally, and the placeholders read across *messages* in the same
order as *images*, so a dropped image has to take its placeholder with
it. Oldest first keeps what the client shared most recently.
"""
excess = len(images) - MAX_PROMPT_IMAGES
if excess <= 0:
return messages, images
logger.warning(
"Qwen duplex prompt carries %d images; dropping the %d oldest to stay within %d",
len(images),
excess,
MAX_PROMPT_IMAGES,
)
remaining = excess
for message in messages:
if remaining <= 0:
break
content = message.get("content")
if not isinstance(content, list):
continue
kept = []
for part in content:
if remaining > 0 and isinstance(part, dict) and part.get("type") == "image":
remaining -= 1
continue
kept.append(part)
message["content"] = kept
return [message for message in messages if message.get("content") != []], images[excess:]
@staticmethod
def _audio_content(payload):
raw = base64.b64decode(payload["audio"], validate=True)
audio = np.frombuffer(raw, dtype="<f4").copy()
if not audio.size or not np.isfinite(audio).all():
raise DuplexRuntimeConfigError("Audio must contain finite PCM samples")
return [{"type": "audio"}], audio
async def prepare_append_plan(self, **kwargs):
# History mutation has already happened on the session loop. The worker
# owns this snapshot and never reads or mutates live session state.
snapshot = copy.deepcopy(kwargs)
return await asyncio.to_thread(self.plan_append, **snapshot)
def plan_append(
self, *, request_id, fence, session_config, runtime_config, seq, turn_seq, payload, final, sampling_params
):
if not final or not isinstance(payload, dict):
raise DuplexRuntimeConfigError("Qwen generation requires a committed audio turn")
messages = []
if runtime_config.get("instructions"):
messages.append({"role": "system", "content": runtime_config["instructions"]})
history = session_config.get("qwen_messages")
if history is None and payload.get("audio"):
# Nothing prepared the conversation, so the committed payload is
# the whole turn. An empty prepared list is not that: it means the
# preparation ran and found nothing to say.
history = [{"role": "user", "audio_payload": payload}]
if not history:
raise DuplexRuntimeConfigError("Qwen generation requires committed audio or conversation history")
audios = []
images = []
for index, message in enumerate(history):
if "audio_payload" not in message:
content = message.get("content")
if isinstance(content, list):
parts = []
for part in content:
if part.get("type") == "image_url":
encoded = part["image_url"]["url"].split(",", 1)[1]
with Image.open(io.BytesIO(base64.b64decode(encoded, validate=True))) as image:
image.thumbnail((448, 448))
images.append(image.convert("RGB").copy())
parts.append({"type": "image"})
elif part.get("type") == "text":
parts.append(part)
messages.append({"role": message["role"], "content": parts})
else:
messages.append(message)
continue
content, audio = self._audio_content(message["audio_payload"])
audios.append((audio, 16000))
if runtime_config.get("initial_user_text") and index == len(history) - 1:
content.append({"type": "text", "text": runtime_config["initial_user_text"]})
messages.append({"role": "user", "content": content})
messages, images = self._trim_prompt_images(messages, images)
prompt = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
mm = {"audio": audios} if audios else {}
if images:
mm["image"] = images
logger.info(
"Qwen duplex visual input request=%s images=%d sizes=%s audios=%d",
request_id,
len(images),
[image.size for image in images],
len(audios),
)
# Opt-in capture of the exact decoded images submitted to the processor.
# Request IDs are hashed because they need not be safe path components.
debug_dir = os.environ.get("VLLM_OMNI_QWEN_VISUAL_DEBUG_DIR")
if debug_dir:
try:
target = Path(debug_dir) / hashlib.sha256(request_id.encode()).hexdigest()[:24]
target.mkdir(parents=True, exist_ok=True)
filenames = []
for index, image in enumerate(images):
filename = f"image_{index:02d}.png"
image.save(target / filename)
filenames.append(filename)
(target / "input.json").write_text(
json.dumps(
{"request_id": request_id, "images": filenames, "audio_count": len(audios), "prompt": prompt},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
logger.info("Qwen duplex visual input capture request=%s path=%s", request_id, target)
except (OSError, ValueError):
logger.exception("Failed to capture Qwen visual input request=%s", request_id)
return DuplexAppendPlan(prompt={"prompt": prompt, "multi_modal_data": mm})
def decide_output(self, **kwargs):
return None
def data_plane_context(self, **kwargs):
return kwargs
vllm_omni/model_executor/models/qwen3_omni/duplex/session.py
+59
View File
@@ -0,0 +1,59 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import asyncio
from dataclasses import dataclass, field
from vllm_omni.engine.duplex.plugin import DuplexModelSessionState
from vllm_omni.model_executor.models.qwen3_omni.duplex.input import (
QwenPcmBuffer,
)
@dataclass(slots=True)
class QwenDuplexSessionState(DuplexModelSessionState):
"""Mutable model-owned state of one Qwen duplex session (owned by the session runner)."""
audio_buffer: QwenPcmBuffer = field(default_factory=QwenPcmBuffer)
# Keep content objects alongside payloads so identity survives history copies.
audio_history: list[tuple[object, dict]] = field(default_factory=list)
input_since_commit: bool = False
speech_since_commit: bool = False
context_locked: bool = False
committed_audio_payload: dict[str, object] | None = None
committed_audio_operation_id: str | None = None
committed_audio_reserved_bytes: int = 0
deferred_response_create: bool = False
deferred_precreate_response: bool = False
continuation_owner_id: str | None = None
continuation_units: int = 0
pending_silence_task: asyncio.Task[bool] | None = None
pending_silence_owner_id: str | None = None
def retain_committed_audio(
self,
payload: dict[str, object],
*,
operation_id: str | None,
reserved_bytes: int = 0,
) -> None:
self.committed_audio_payload = payload
self.committed_audio_operation_id = operation_id
self.committed_audio_reserved_bytes += max(0, int(reserved_bytes))
def clear_committed_audio(self) -> int:
reserved_bytes = self.committed_audio_reserved_bytes
self.committed_audio_payload = None
self.committed_audio_operation_id = None
self.committed_audio_reserved_bytes = 0
self.deferred_response_create = False
self.deferred_precreate_response = False
return reserved_bytes
def clear_continuation(self) -> None:
self.continuation_owner_id = None
self.continuation_units = 0
self.pending_silence_task = None
self.pending_silence_owner_id = None
vllm_omni/model_executor/models/qwen3_omni/pipeline.py
+3 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Qwen3-Omni-MoE pipeline topology (frozen).
Stage 0: Thinker — multimodal understanding + text generation
@@ -21,6 +21,8 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_omni"
QWEN3_OMNI_PIPELINE = PipelineConfig(
model_type="qwen3_omni_moe",
default_session_mode="turn",
duplex_plugin="vllm_omni.model_executor.models.qwen3_omni.duplex.plugin.Qwen3OmniDuplexPlugin",
default_deploy_config_name="qwen3_omni_moe.yaml",
model_arch="Qwen3OmniMoeForConditionalGeneration",
endpoint_restrictions=(