[Frontend] OpenAI-compliant fullduplex /v1/realtime with Qwen3-Omni support (#7285)
Signed-off-by: vraiti <vraiti@redhat.com>
This commit is contained in:
vraiti authored and GitHub committed
1 parent
7266fc6138
commit
d31dbc5b0a
25 files changed
+2854
-455
No files matched your search
@@ -0,0 +1,6 @@
|
||||
agent-starter-react/
|
||||
.git/
|
||||
.claude/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.log
|
||||
@@ -0,0 +1,25 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
FROM python:3.11-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --no-install-recommends -y ca-certificates ffmpeg libsndfile1 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip install \
|
||||
"livekit-agents[openai,turn-detector,silero]>=1.6.10"
|
||||
|
||||
RUN useradd --create-home --uid 10001 agent
|
||||
|
||||
WORKDIR /app
|
||||
COPY --chown=agent:agent agent.py ./agent.py
|
||||
|
||||
USER agent
|
||||
|
||||
ENTRYPOINT ["python", "agent.py", "start"]
|
||||
@@ -0,0 +1,36 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
FROM node:24-bookworm-slim AS dependencies
|
||||
|
||||
ARG FRONTEND_REF=main
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --no-install-recommends -y ca-certificates git \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& npm install --global pnpm@9.15.9
|
||||
|
||||
WORKDIR /app
|
||||
RUN git clone --depth 1 --branch "${FRONTEND_REF}" \
|
||||
https://github.com/livekit-examples/agent-starter-react.git . \
|
||||
&& pnpm install --frozen-lockfile \
|
||||
&& rm -rf .git
|
||||
|
||||
FROM node:24-bookworm-slim
|
||||
|
||||
ENV NODE_ENV=development \
|
||||
NEXT_TELEMETRY_DISABLED=1
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --no-install-recommends -y ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& npm install --global pnpm@9.15.9
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=dependencies --chown=node:node /app /app
|
||||
|
||||
USER node
|
||||
|
||||
EXPOSE 3000
|
||||
CMD ["pnpm", "dev", "--hostname", "0.0.0.0"]
|
||||
@@ -0,0 +1,28 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
FROM golang:1.26-alpine AS builder
|
||||
|
||||
ARG LIVEKIT_REF=v1.10.1
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apk add --no-cache git ca-certificates
|
||||
|
||||
WORKDIR /src
|
||||
RUN git clone --depth 1 --branch "${LIVEKIT_REF}" \
|
||||
https://github.com/livekit/livekit.git . \
|
||||
&& go mod download \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH="${TARGETARCH}" \
|
||||
go build -trimpath -ldflags="-s -w" -o /out/livekit-server ./cmd/server
|
||||
|
||||
FROM alpine:3.22
|
||||
|
||||
RUN apk add --no-cache ca-certificates tzdata \
|
||||
&& addgroup -S livekit \
|
||||
&& adduser -S -G livekit livekit
|
||||
|
||||
COPY --from=builder /out/livekit-server /usr/local/bin/livekit-server
|
||||
|
||||
USER livekit
|
||||
ENTRYPOINT ["/usr/local/bin/livekit-server"]
|
||||
@@ -0,0 +1,32 @@
|
||||
# Qwen3-Omni LiveKit demo
|
||||
|
||||
This directory contains a Docker Compose version of the local LiveKit demo.
|
||||
It builds the LiveKit server, the Next.js frontend, and the Python agent as
|
||||
separate images. The vLLM-Omni server runs outside Compose and is configured
|
||||
with `VLLM_OMNI_HOST`.
|
||||
|
||||
Start vLLM-Omni with port `8000` bound to an address reachable from Docker,
|
||||
for example with `--host 0.0.0.0`, then run:
|
||||
|
||||
```bash
|
||||
cd examples/online_serving/qwen3_omni/fullduplex-livekit-frontend
|
||||
VLLM_OMNI_HOST=host.docker.internal docker compose up --build
|
||||
```
|
||||
|
||||
Open [http://localhost:3000](http://localhost:3000). The LiveKit server is
|
||||
available at `ws://localhost:7880`; its development credentials are `devkey`
|
||||
and `secret`.
|
||||
|
||||
For a machine other than the local host, set both the vLLM address and the
|
||||
address LiveKit advertises to WebRTC clients:
|
||||
|
||||
```bash
|
||||
VLLM_OMNI_HOST=192.168.1.20 \
|
||||
LIVEKIT_NODE_IP=192.168.1.10 \
|
||||
LIVEKIT_PUBLIC_URL=ws://192.168.1.10:7880 \
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
Use `docker compose up --build --abort-on-container-exit` when the Compose
|
||||
process should stop the remaining services after one component exits, matching
|
||||
the behavior of `run-livekit-stack.sh`.
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = [
|
||||
# "livekit-agents[openai,turn-detector,silero]>=1.6.10",
|
||||
# ]
|
||||
# ///
|
||||
import logging
|
||||
import os
|
||||
|
||||
from livekit.agents import (
|
||||
Agent,
|
||||
AgentServer,
|
||||
AgentSession,
|
||||
JobContext,
|
||||
RunContext,
|
||||
TurnHandlingOptions,
|
||||
cli,
|
||||
function_tool,
|
||||
inference,
|
||||
)
|
||||
from livekit.plugins.openai.realtime import RealtimeModel
|
||||
|
||||
logger = logging.getLogger("voice-assistant")
|
||||
|
||||
VLLM_BASE_URL = f"http://{os.environ['VLLM_OMNI_HOST']}:8000/v1"
|
||||
LIVEKIT_URL = os.environ.get("LIVEKIT_URL", "ws://localhost:7880")
|
||||
|
||||
server = AgentServer(
|
||||
ws_url=LIVEKIT_URL,
|
||||
api_key=os.environ.get("LIVEKIT_API_KEY", "devkey"),
|
||||
api_secret=os.environ.get("LIVEKIT_API_SECRET", "secret"),
|
||||
)
|
||||
|
||||
|
||||
class VoiceAssistant(Agent):
|
||||
@function_tool()
|
||||
async def lookup_weather(
|
||||
self,
|
||||
context: RunContext,
|
||||
location: str,
|
||||
) -> dict[str, str | int]:
|
||||
"""Return deterministic sample data for the demo tool."""
|
||||
return {
|
||||
"location": location,
|
||||
"weather": "clear skies",
|
||||
"temperature_f": 72,
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
instructions="You are a helpful voice assistant. Respond naturally and concisely.",
|
||||
)
|
||||
|
||||
|
||||
@server.rtc_session()
|
||||
async def entrypoint(ctx: JobContext):
|
||||
# Qwen3-Omni only supports turn_detection=null (manual mode) -- it does
|
||||
# not perform voice activity detection itself, so turn-taking must be
|
||||
# decided client-side. semantic_vad (MiniCPM-o only) is rejected by
|
||||
# vLLM-Omni's OpenAI-compatible realtime endpoint for Qwen3-Omni.
|
||||
# Omni serves one model, so an empty model lets the server select it.
|
||||
model = RealtimeModel(
|
||||
base_url=VLLM_BASE_URL,
|
||||
model="",
|
||||
api_key="unused",
|
||||
)
|
||||
|
||||
# turn_handling's client-side TurnDetector means the server isn't asked
|
||||
# to decide turns, so turn_detection is left NOT_GIVEN on RealtimeModel.
|
||||
session = AgentSession(
|
||||
llm=model,
|
||||
turn_handling=TurnHandlingOptions(turn_detection=inference.TurnDetector()),
|
||||
)
|
||||
|
||||
await session.start(
|
||||
agent=VoiceAssistant(),
|
||||
room=ctx.room,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Voice assistant started, connected to vLLM-Omni at %s",
|
||||
VLLM_BASE_URL,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli.run_app(server)
|
||||
@@ -0,0 +1,55 @@
|
||||
name: vllm-omni-livekit
|
||||
|
||||
services:
|
||||
livekit:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.livekit
|
||||
args:
|
||||
LIVEKIT_REF: ${LIVEKIT_REF:-v1.10.1}
|
||||
command:
|
||||
- --config
|
||||
- /etc/livekit.yaml
|
||||
- --dev
|
||||
- --node-ip
|
||||
- ${LIVEKIT_NODE_IP:-127.0.0.1}
|
||||
volumes:
|
||||
- ./livekit.yaml:/etc/livekit.yaml:ro
|
||||
ports:
|
||||
- "${LIVEKIT_HTTP_PORT:-7880}:7880"
|
||||
- "${LIVEKIT_TCP_PORT:-7881}:7881/tcp"
|
||||
- "${LIVEKIT_UDP_PORT:-7882}:7882/udp"
|
||||
restart: unless-stopped
|
||||
|
||||
frontend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.frontend
|
||||
args:
|
||||
FRONTEND_REF: ${FRONTEND_REF:-main}
|
||||
environment:
|
||||
NODE_ENV: development
|
||||
LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:-devkey}
|
||||
LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:-secret}
|
||||
LIVEKIT_URL: ${LIVEKIT_PUBLIC_URL:-ws://localhost:7880}
|
||||
AGENT_NAME: ${AGENT_NAME:-}
|
||||
ports:
|
||||
- "${FRONTEND_PORT:-3000}:3000"
|
||||
depends_on:
|
||||
- livekit
|
||||
restart: unless-stopped
|
||||
|
||||
agent:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.agent
|
||||
environment:
|
||||
VLLM_OMNI_HOST: ${VLLM_OMNI_HOST:-host.docker.internal}
|
||||
LIVEKIT_URL: ws://livekit:7880
|
||||
LIVEKIT_API_KEY: ${LIVEKIT_API_KEY:-devkey}
|
||||
LIVEKIT_API_SECRET: ${LIVEKIT_API_SECRET:-secret}
|
||||
extra_hosts:
|
||||
- host.docker.internal:host-gateway
|
||||
depends_on:
|
||||
- livekit
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,10 @@
|
||||
# Local development configuration for the Compose stack.
|
||||
port: 7880
|
||||
bind_addresses:
|
||||
- 0.0.0.0
|
||||
rtc:
|
||||
tcp_port: 7881
|
||||
udp_port: 7882
|
||||
use_external_ip: false
|
||||
keys:
|
||||
devkey: secret
|
||||
@@ -1,398 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Realtime client for vLLM-Omni /v1/realtime (audio + text events).
|
||||
|
||||
This client:
|
||||
1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz),
|
||||
2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events,
|
||||
3) Receives response.output_audio.*, response.output_text.*, and transcript events,
|
||||
4) Saves synthesized audio to an output WAV file and optional text file.
|
||||
|
||||
By default each ``response.output_audio.delta`` is treated as an **incremental PCM**
|
||||
chunk and all chunks are concatenated into the final ``--output-wav``.
|
||||
|
||||
Optional debugging: pass ``--delta-dump-dir DIR`` to write every
|
||||
``response.output_audio.delta`` payload as ``delta_000001.wav``, ``delta_000002.wav``, …
|
||||
|
||||
Usage:
|
||||
python openai_realtime_client.py \
|
||||
--url ws://localhost:8091/v1/realtime \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--input-wav input_16k_mono.wav \
|
||||
--output-wav realtime_output.wav \
|
||||
--server-vad \
|
||||
--delta-dump-dir ./rt_delta_wavs
|
||||
|
||||
Dependencies:
|
||||
pip install websockets
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
print("Please install websockets: pip install websockets")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def _read_wav_pcm16(path: Path) -> bytes:
|
||||
with wave.open(str(path), "rb") as wf:
|
||||
nchannels = wf.getnchannels()
|
||||
sampwidth = wf.getsampwidth()
|
||||
framerate = wf.getframerate()
|
||||
comptype = wf.getcomptype()
|
||||
nframes = wf.getnframes()
|
||||
|
||||
if nchannels != 1:
|
||||
raise ValueError(f"Input WAV must be mono (got {nchannels} channels).")
|
||||
if sampwidth != 2:
|
||||
raise ValueError(f"Input WAV must be 16-bit PCM (got sample width={sampwidth}).")
|
||||
if framerate != 16000:
|
||||
raise ValueError(f"Input WAV must be 16kHz (got {framerate} Hz).")
|
||||
if comptype != "NONE":
|
||||
raise ValueError(f"Input WAV must be uncompressed PCM (got comptype={comptype}).")
|
||||
if nframes <= 0:
|
||||
raise ValueError("Input WAV has no audio frames.")
|
||||
|
||||
return wf.readframes(nframes)
|
||||
|
||||
|
||||
def _write_wav_pcm16(path: Path, pcm16_bytes: bytes, sample_rate_hz: int) -> None:
|
||||
with wave.open(str(path), "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(sample_rate_hz)
|
||||
wf.writeframes(pcm16_bytes)
|
||||
|
||||
|
||||
def _with_realtime_route(url: str, *, server_vad: bool) -> str:
|
||||
parts = urlsplit(url)
|
||||
query = [(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key != "duplex"]
|
||||
if not server_vad:
|
||||
query.append(("duplex", "0"))
|
||||
return urlunsplit(parts._replace(query=urlencode(query)))
|
||||
|
||||
|
||||
async def run_client(
|
||||
url: str,
|
||||
model: str,
|
||||
input_wav: Path,
|
||||
output_wav: Path,
|
||||
output_text: Path | None,
|
||||
chunk_ms: int,
|
||||
send_delay_ms: int,
|
||||
delta_dump_dir: Path | None,
|
||||
server_vad: bool,
|
||||
request_idx: int = 1,
|
||||
total_requests: int = 1,
|
||||
) -> None:
|
||||
log_prefix = f"[req {request_idx:02d}/{total_requests:02d}] " if total_requests > 1 else ""
|
||||
pcm16 = _read_wav_pcm16(input_wav)
|
||||
bytes_per_ms = 16000 * 2 // 1000 # mono PCM16 at 16kHz
|
||||
chunk_bytes = max(bytes_per_ms * chunk_ms, 2)
|
||||
|
||||
incremental_pcm_parts: list[bytes] = []
|
||||
output_sample_rate = 24000
|
||||
delta_index = 0
|
||||
text_chunks: list[str] = []
|
||||
final_text: str = ""
|
||||
final_audio_transcript: str = ""
|
||||
|
||||
if delta_dump_dir is not None:
|
||||
delta_dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Keep the CLI mode authoritative even if the input URL already contains
|
||||
# an explicit duplex query parameter.
|
||||
url = _with_realtime_route(url, server_vad=server_vad)
|
||||
async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws:
|
||||
# 1) Validate model.
|
||||
if server_vad:
|
||||
session_update = {
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"model": model,
|
||||
"audio": {
|
||||
"input": {
|
||||
"format": {"type": "audio/pcm", "rate": 16_000},
|
||||
"turn_detection": {
|
||||
"type": "server_vad",
|
||||
"threshold": 0.5,
|
||||
"prefix_padding_ms": 300,
|
||||
"silence_duration_ms": 500,
|
||||
"create_response": True,
|
||||
"interrupt_response": True,
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
else:
|
||||
# Preserve the existing non-duplex wire format.
|
||||
session_update = {"type": "session.update", "model": model}
|
||||
await ws.send(json.dumps(session_update))
|
||||
|
||||
# 2) Preserve the legacy explicit-commit flow unless Server VAD owns endpointing.
|
||||
if not server_vad:
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
|
||||
|
||||
# 3) Stream audio chunks.
|
||||
for i in range(0, len(pcm16), chunk_bytes):
|
||||
chunk = pcm16[i : i + chunk_bytes]
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(chunk).decode("utf-8"),
|
||||
}
|
||||
)
|
||||
)
|
||||
if send_delay_ms > 0:
|
||||
await asyncio.sleep(send_delay_ms / 1000.0)
|
||||
|
||||
# 4) Finalize the turn explicitly, or append silence for Server VAD endpointing.
|
||||
if server_vad:
|
||||
trailing_silence = bytes(16_000 * 2)
|
||||
for i in range(0, len(trailing_silence), chunk_bytes):
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(trailing_silence[i : i + chunk_bytes]).decode("utf-8"),
|
||||
}
|
||||
)
|
||||
)
|
||||
else:
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||
|
||||
# 5) Receive server events until audio done.
|
||||
while True:
|
||||
message = await ws.recv()
|
||||
if isinstance(message, bytes):
|
||||
# We only expect JSON text frames.
|
||||
continue
|
||||
|
||||
event = json.loads(message)
|
||||
event_type = event.get("type")
|
||||
|
||||
if event_type == "session.created":
|
||||
continue
|
||||
|
||||
if event_type == "response.output_audio.delta":
|
||||
sr = event.get("sample_rate_hz")
|
||||
if isinstance(sr, int) and sr > 0:
|
||||
output_sample_rate = sr
|
||||
audio_b64 = event.get("delta" if server_vad else "audio", "")
|
||||
if audio_b64:
|
||||
pcm_delta = base64.b64decode(audio_b64)
|
||||
incremental_pcm_parts.append(pcm_delta)
|
||||
if delta_dump_dir is not None and pcm_delta:
|
||||
delta_index += 1
|
||||
dump_path = delta_dump_dir / f"delta_{delta_index:06d}.wav"
|
||||
_write_wav_pcm16(dump_path, pcm_delta, output_sample_rate)
|
||||
print(
|
||||
f"{log_prefix}delta dump #{delta_index}: {dump_path} "
|
||||
f"(pcm bytes={len(pcm_delta)}, sr={output_sample_rate})"
|
||||
)
|
||||
continue
|
||||
|
||||
if event_type in {"response.output_text.delta", "transcription.delta"}:
|
||||
delta = event.get("delta", "")
|
||||
if delta:
|
||||
text_chunks.append(delta)
|
||||
print(delta, end="", flush=True)
|
||||
continue
|
||||
|
||||
if event_type in {"response.output_text.done", "transcription.done"}:
|
||||
final_text = event.get("text", "") or "".join(text_chunks)
|
||||
usage = event.get("usage")
|
||||
final_text_with_tag = f"Final transcription: {final_text}"
|
||||
if text_chunks:
|
||||
print()
|
||||
print(f"{log_prefix}{final_text_with_tag}")
|
||||
if usage:
|
||||
print(f"{log_prefix}text usage: {usage}")
|
||||
continue
|
||||
|
||||
if event_type == "response.output_audio_transcript.done":
|
||||
final_audio_transcript = event.get("transcript", "")
|
||||
continue
|
||||
|
||||
if event_type == "response.output_audio.done":
|
||||
if not server_vad:
|
||||
break
|
||||
continue
|
||||
|
||||
if event_type == "response.done" and server_vad:
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
break
|
||||
|
||||
if event_type == "error":
|
||||
raise RuntimeError(f"Server error: {event}")
|
||||
|
||||
all_pcm16 = b"".join(incremental_pcm_parts)
|
||||
if not all_pcm16:
|
||||
raise RuntimeError("No audio received from server.")
|
||||
|
||||
output_wav.parent.mkdir(parents=True, exist_ok=True)
|
||||
_write_wav_pcm16(output_wav, all_pcm16, output_sample_rate)
|
||||
print(f"{log_prefix}Saved realtime audio to: {output_wav} (incremental chunks joined)")
|
||||
|
||||
if output_text is not None:
|
||||
text_to_save = final_text or "".join(text_chunks) or final_audio_transcript
|
||||
output_text.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_text.write_text(text_to_save, encoding="utf-8")
|
||||
print(f"{log_prefix}Saved realtime text to: {output_text}")
|
||||
|
||||
|
||||
def _indexed_output_path(path: Path | None, index: int, total: int) -> Path | None:
|
||||
if path is None or total <= 1:
|
||||
return path
|
||||
return path.with_name(f"{path.stem}_{index:02d}{path.suffix}")
|
||||
|
||||
|
||||
async def run_clients_concurrent(
|
||||
*,
|
||||
url: str,
|
||||
model: str,
|
||||
input_wav: Path,
|
||||
output_wav: Path,
|
||||
output_text: Path | None,
|
||||
chunk_ms: int,
|
||||
send_delay_ms: int,
|
||||
delta_dump_dir: Path | None,
|
||||
server_vad: bool,
|
||||
num_requests: int,
|
||||
concurrency: int,
|
||||
) -> None:
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
|
||||
async def _run_one(index: int) -> tuple[int, bool, str | None]:
|
||||
per_output_wav = _indexed_output_path(output_wav, index, num_requests)
|
||||
per_output_text = _indexed_output_path(output_text, index, num_requests)
|
||||
per_delta_dir = None
|
||||
if delta_dump_dir is not None:
|
||||
per_delta_dir = delta_dump_dir / f"req_{index:02d}"
|
||||
async with sem:
|
||||
try:
|
||||
await run_client(
|
||||
url=url,
|
||||
model=model,
|
||||
input_wav=input_wav,
|
||||
output_wav=per_output_wav,
|
||||
output_text=per_output_text,
|
||||
chunk_ms=chunk_ms,
|
||||
send_delay_ms=send_delay_ms,
|
||||
delta_dump_dir=per_delta_dir,
|
||||
server_vad=server_vad,
|
||||
request_idx=index,
|
||||
total_requests=num_requests,
|
||||
)
|
||||
return index, True, None
|
||||
except Exception as exc:
|
||||
return index, False, str(exc)
|
||||
|
||||
tasks = [asyncio.create_task(_run_one(i), name=f"rt-client-{i}") for i in range(1, num_requests + 1)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
failed = [(idx, err) for idx, ok, err in results if not ok]
|
||||
succeeded = num_requests - len(failed)
|
||||
print(f"[summary] succeeded={succeeded}, failed={len(failed)}, total={num_requests}")
|
||||
if failed:
|
||||
for idx, err in failed:
|
||||
print(f"[summary] req {idx:02d} failed: {err}")
|
||||
raise RuntimeError(f"{len(failed)} concurrent request(s) failed")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Realtime audio/text client for vLLM-Omni")
|
||||
parser.add_argument("--url", default="ws://localhost:8091/v1/realtime", help="WebSocket URL")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
help="Model name for session.update",
|
||||
)
|
||||
parser.add_argument("--input-wav", required=True, type=Path, help="Input WAV (mono, PCM16, 16kHz)")
|
||||
parser.add_argument("--output-wav", default=Path("realtime_output.wav"), type=Path, help="Output WAV path")
|
||||
parser.add_argument(
|
||||
"--output-text",
|
||||
default=None,
|
||||
type=Path,
|
||||
help="Optional output text path for final transcription",
|
||||
)
|
||||
parser.add_argument("--chunk-ms", type=int, default=200, help="Input chunk size in milliseconds")
|
||||
parser.add_argument(
|
||||
"--send-delay-ms",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Delay between chunk sends; set >0 to simulate realtime upload",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--delta-dump-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="If set, each response.output_audio.delta is saved as delta_NNNNNN.wav under this directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server-vad",
|
||||
action="store_true",
|
||||
help="Let the server detect speech endpoints instead of sending input_audio_buffer.commit",
|
||||
)
|
||||
parser.add_argument("--num-requests", type=int, default=1, help="Total number of requests to send")
|
||||
parser.add_argument(
|
||||
"--concurrency",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Maximum number of concurrent websocket requests",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.num_requests <= 0:
|
||||
raise ValueError("--num-requests must be >= 1")
|
||||
if args.concurrency <= 0:
|
||||
raise ValueError("--concurrency must be >= 1")
|
||||
concurrency = min(args.concurrency, args.num_requests)
|
||||
|
||||
if args.num_requests == 1:
|
||||
asyncio.run(
|
||||
run_client(
|
||||
url=args.url,
|
||||
model=args.model,
|
||||
input_wav=args.input_wav,
|
||||
output_wav=args.output_wav,
|
||||
output_text=args.output_text,
|
||||
chunk_ms=args.chunk_ms,
|
||||
send_delay_ms=args.send_delay_ms,
|
||||
delta_dump_dir=args.delta_dump_dir,
|
||||
server_vad=args.server_vad,
|
||||
)
|
||||
)
|
||||
else:
|
||||
asyncio.run(
|
||||
run_clients_concurrent(
|
||||
url=args.url,
|
||||
model=args.model,
|
||||
input_wav=args.input_wav,
|
||||
output_wav=args.output_wav,
|
||||
output_text=args.output_text,
|
||||
chunk_ms=args.chunk_ms,
|
||||
send_delay_ms=args.send_delay_ms,
|
||||
delta_dump_dir=args.delta_dump_dir,
|
||||
server_vad=args.server_vad,
|
||||
num_requests=args.num_requests,
|
||||
concurrency=concurrency,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,4 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Qwen3-Omni turn-based browser call UI."""
|
||||
@@ -1,7 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from examples.online_serving.realtime_web.server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(default_profile="qwen3-turn")
|
||||
Reference in new issue
Block a user