[Feature] JoyAI-VL-Interaction streaming interaction serving layer (#4575)

Signed-off-by: lishunyang12 <125541396+lishunyang12@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Signed-off-by: lishunyang <lishunyang12@163.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-06-21 07:42:51 +00:00
1 parent 950e66a44c
commit e241005a71
45 files changed
+3699 -1

No files matched your search

@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import base64
import io
import json
import struct
import wave
import aiohttp
from aiohttp import web
_HEADER = struct.Struct(">iii")
_SAMPLE_RATE = 16000
_DEFAULT_PROMPT = "Transcribe the audio."
_ASR_TEXT_TAG = "<asr_text>"
def _clean_transcript(raw: str) -> str:
"""Qwen3-ASR emits ``language <Lang><asr_text><transcript>``; keep the transcript."""
text = raw or ""
marker = text.rfind(_ASR_TEXT_TAG)
if marker != -1:
text = text[marker + len(_ASR_TEXT_TAG) :]
return text.replace("</asr_text>", "").strip()
def _pcm_to_wav_b64(pcm: bytes, sample_rate: int) -> str:
buf = io.BytesIO()
with wave.open(buf, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(sample_rate)
wav.writeframes(pcm)
return base64.b64encode(buf.getvalue()).decode()
def _result(text: str, *, final: bool) -> dict:
return {
"asr_response": {
"event_type": "IS_FINAL" if final else "IS_PARTIAL",
"recognition_result": {"hypothesis": [{"text": text, "confidence": None}]},
},
"mid": "",
"code": 0,
"msg": "ok",
}
async def _transcribe(cfg: dict, pcm: bytes) -> str:
if not pcm:
return ""
wav_b64 = _pcm_to_wav_b64(pcm, cfg["sample_rate"])
body = {
"model": cfg["model"],
"messages": [
{
"role": "user",
"content": [
{"type": "input_audio", "input_audio": {"data": wav_b64, "format": "wav"}},
{"type": "text", "text": cfg["prompt"]},
],
}
],
"temperature": 0.0,
"max_tokens": cfg["max_tokens"],
}
url = cfg["backend_url"].rstrip("/") + "/v1/chat/completions"
async with aiohttp.ClientSession() as session:
async with session.post(url, json=body) as resp:
payload = await resp.json()
choices = payload.get("choices") or []
if not choices:
return ""
return _clean_transcript(choices[0].get("message", {}).get("content") or "")
async def _handle(request: web.Request) -> web.WebSocketResponse:
ws = web.WebSocketResponse(max_msg_size=0)
await ws.prepare(request)
cfg = request.app["cfg"]
pcm = bytearray()
try:
async for msg in ws:
if msg.type == aiohttp.WSMsgType.BINARY and len(msg.data) >= _HEADER.size:
seqid, _, _ = _HEADER.unpack(msg.data[: _HEADER.size])
pcm.extend(msg.data[_HEADER.size :])
if seqid < 0:
text = await _transcribe(cfg, bytes(pcm))
await ws.send_str(json.dumps(_result(text, final=True), ensure_ascii=False))
pcm.clear()
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR):
break
finally:
if not ws.closed:
await ws.close()
return ws
def create_app(backend_url: str, model: str, sample_rate: int, prompt: str, max_tokens: int) -> web.Application:
app = web.Application()
app["cfg"] = {
"backend_url": backend_url,
"model": model,
"sample_rate": sample_rate,
"prompt": prompt,
"max_tokens": max_tokens,
}
app.router.add_get("/v1/asr", _handle)
return app
def main() -> None:
parser = argparse.ArgumentParser(description="JoyVL webui <-> Qwen3-ASR (chat input_audio) bridge")
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=8093)
parser.add_argument("--backend-url", default="http://127.0.0.1:8094")
parser.add_argument("--model", default="Qwen/Qwen3-ASR-1.7B")
parser.add_argument("--sample-rate", type=int, default=_SAMPLE_RATE)
parser.add_argument("--prompt", default=_DEFAULT_PROMPT)
parser.add_argument("--max-tokens", type=int, default=256)
args = parser.parse_args()
app = create_app(args.backend_url, args.model, args.sample_rate, args.prompt, args.max_tokens)
web.run_app(app, host=args.host, port=args.port)
if __name__ == "__main__":
main()
@@ -0,0 +1,117 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import aiohttp
from aiohttp import web
try:
import cn2an # optional: Arabic-digit -> Chinese number normalization
except ImportError:
cn2an = None
logger = logging.getLogger("tts_bridge")
def _normalize(text: str) -> str:
"""Spell out Arabic digits in Chinese so short counting outputs synthesize
intelligibly: bare "1次" garbles on Qwen3-TTS (and others), while "一次"
is clean. ``cn2an.transform`` rewrites digits in place ("第3次"->"第三次")
and is pure-Python (~microseconds). No-op if cn2an is unavailable."""
if not cn2an or not text:
return text
try:
return cn2an.transform(text, "an2cn")
except Exception as err: # never let TN break synthesis
logger.debug("cn2an normalize skipped: %s", err)
return text
async def _pump_backend_to_front(back: aiohttp.ClientWebSocketResponse, front: web.WebSocketResponse) -> None:
async for msg in back:
if msg.type == aiohttp.WSMsgType.BINARY:
await front.send_bytes(msg.data)
elif msg.type == aiohttp.WSMsgType.TEXT:
event = json.loads(msg.data).get("type")
if event == "session.done":
await front.send_str(json.dumps({"type": "response.done"}))
return
if event == "error":
await front.send_str(json.dumps({"type": "error", "error": "tts backend error"}))
return
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR):
return
async def _handle(request: web.Request) -> web.WebSocketResponse:
front = web.WebSocketResponse(heartbeat=20, max_msg_size=0)
await front.prepare(request)
cfg = request.app["cfg"]
session = aiohttp.ClientSession()
back: aiohttp.ClientWebSocketResponse | None = None
pump: asyncio.Task | None = None
parts: list[str] = []
try:
async for msg in front:
if msg.type != web.WSMsgType.TEXT:
continue
data = json.loads(msg.data)
if "config" in data and back is None:
voice = (data["config"] or {}).get("voice") or cfg["voice"]
back = await session.ws_connect(cfg["backend_url"], max_msg_size=0)
await back.send_str(
json.dumps({"type": "session.config", "response_format": cfg["response_format"], "voice": voice})
)
pump = asyncio.create_task(_pump_backend_to_front(back, front))
elif data.get("type") == "input_text.append" and back is not None:
parts.append(data.get("text", ""))
elif data.get("type") == "input_text.commit" and back is not None:
# Normalize the full utterance once (digits may span chunks), then send.
await back.send_str(json.dumps({"type": "input.text", "text": _normalize("".join(parts))}))
parts = []
await back.send_str(json.dumps({"type": "input.done"}))
if pump is not None:
await pump
break
except Exception as err:
logger.warning("tts bridge error: %s", err)
finally:
if pump is not None and not pump.done():
pump.cancel()
if back is not None and not back.closed:
await back.close()
await session.close()
if not front.closed:
await front.close()
return front
def create_app(backend_url: str, voice: str, response_format: str) -> web.Application:
app = web.Application()
app["cfg"] = {"backend_url": backend_url, "voice": voice, "response_format": response_format}
app.router.add_get("/v1/tts", _handle)
return app
def main() -> None:
parser = argparse.ArgumentParser(description="JoyVL webui <-> Qwen3-TTS websocket bridge")
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=8092)
parser.add_argument("--backend-url", default="ws://127.0.0.1:8091/v1/audio/speech/stream")
parser.add_argument("--voice", default="vivian")
parser.add_argument("--response-format", default="pcm")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
web.run_app(create_app(args.backend_url, args.voice, args.response_format), host=args.host, port=args.port)
if __name__ == "__main__":
main()
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
from stream_client import stream
ICON = {"response": "SPEAK ", "silence": "silence ", "delegate": "DELEGATE"}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("video")
parser.add_argument("--server", default="http://127.0.0.1:8070")
parser.add_argument("--query", default=None)
parser.add_argument("--fps", type=float, default=1.0)
parser.add_argument("--session-id", default="cli")
args = parser.parse_args()
for tick in stream(args.video, args.server, args.session_id, args.query, args.fps):
if tick.action == "silence":
continue
line = f"[{tick.t:6.1f}s] {ICON.get(tick.action)} {tick.text}"
if tick.delegation:
line += f" (-> {tick.delegation.get('question', '')})"
print(line, flush=True)
if __name__ == "__main__":
main()
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import base64
import time
from collections.abc import Iterator
from dataclasses import dataclass
import cv2
import requests
@dataclass
class Tick:
index: int
t: float
action: str
text: str
skipped: bool
latency_ms: float
delegation: dict | None
def iter_frames(video_path: str, fps: float) -> Iterator[tuple[float, bytes]]:
cap = cv2.VideoCapture(video_path)
src_fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
step = max(1, round(src_fps / fps))
idx = 0
while True:
ok, frame = cap.read()
if not ok:
break
if idx % step == 0:
ok, buf = cv2.imencode(".jpg", frame)
if ok:
yield idx / src_fps, buf.tobytes()
idx += 1
cap.release()
def stream(
video_path: str,
server: str,
session_id: str,
query: str | None = None,
fps: float = 1.0,
realtime: bool = False,
) -> Iterator[Tick]:
"""Send each sampled frame as one chat completion; yield the parsed action.
``query`` is attached to the first tick to arm the session; inject later
queries by posting another frame turn with a new text part.
"""
url = server.rstrip("/") + "/v1/chat/completions"
requests.post(server.rstrip("/") + "/reset", json={"session_id": session_id}, timeout=10)
for i, (t, jpeg) in enumerate(iter_frames(video_path, fps)):
data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
content = [{"type": "image_url", "image_url": {"url": data_url}}]
if i == 0 and query:
content.insert(0, {"type": "text", "text": query})
resp = requests.post(
url,
json={"session_id": session_id, "messages": [{"role": "user", "content": content}]},
timeout=120,
).json()
info = resp.get("interaction", {})
yield Tick(
index=info.get("frame_index", i + 1),
t=t,
action=info.get("action", "?"),
text=resp["choices"][0]["message"]["content"],
skipped=info.get("inference_skipped", False),
latency_ms=info.get("latency_ms", 0.0),
delegation=info.get("delegation"),
)
if realtime:
time.sleep(1.0 / fps)
@@ -0,0 +1,46 @@
# RTSP local streaming
Simulate an RTSP camera from a local video file so the WebUI (or any RTSP client) can
test the RTSP input path without a physical IP camera. RTSP is a **webui-side input** —
the browser pulls the stream and feeds frames to the orchestrator over the normal API;
no serving-layer code is involved.
Needs [`ffmpeg`](https://ffmpeg.org/) and [MediaMTX](https://github.com/bluenviron/mediamtx/releases)
(download the binary + `mediamtx.yml` and keep them next to these scripts, or point
`MEDIAMTX_BIN` / `MEDIAMTX_CONFIG` at them).
## 1. Start the RTSP server
```bash
bash ./mediamtx.sh # MediaMTX, listens on :8554
```
## 2. Push a local video as an RTSP stream
```bash
bash ./rtsp.sh ./videos/example.mp4 rtsp://127.0.0.1:8554/fire1
```
`rtsp.sh` takes `[video-path] [rtsp-url]` (or set `VIDEO_PATH` / `RTSP_URL`). To stream a
whole directory of clips, one stream per file:
```bash
bash ./rtsp_all.sh ./videos rtsp://127.0.0.1:8554
```
## 3. Use it in the WebUI
Enter the URL in the WebUI RTSP box:
```
rtsp://127.0.0.1:8554/fire1
```
If the WebUI runs on another machine, replace `127.0.0.1` with the MediaMTX host's IP.
## Notes
- `rtsp.sh` transcodes audio to AAC; replace `-c:a aac -b:a 128k -ar 44100` with `-an`
if the source clip has no audio track and your `ffmpeg` errors on it.
- If a client cannot connect, check that port `8554` is reachable and not blocked by a
firewall, container network, or security group.
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
# Start a local MediaMTX RTSP server (listens on :8554 by default).
# Download the binary + mediamtx.yml from https://github.com/bluenviron/mediamtx/releases
# and either keep them next to this script or point the vars below at them.
MEDIAMTX_BIN="${MEDIAMTX_BIN:-./mediamtx}"
MEDIAMTX_CONFIG="${MEDIAMTX_CONFIG:-./mediamtx.yml}"
exec "$MEDIAMTX_BIN" "$MEDIAMTX_CONFIG"
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -euo pipefail
# Loop a local video file and push it to an RTSP URL, simulating an RTSP camera.
# bash ./rtsp.sh [video-path] [rtsp-output-url]
# Defaults stream ./videos/example.mp4 to rtsp://127.0.0.1:8554/fire1
VIDEO_PATH="${1:-${VIDEO_PATH:-./videos/example.mp4}}"
RTSP_URL="${2:-${RTSP_URL:-rtsp://127.0.0.1:8554/fire1}}"
if [[ ! -f "$VIDEO_PATH" ]]; then
echo "Video file not found: $VIDEO_PATH" >&2
exit 1
fi
echo "Input video: $VIDEO_PATH"
echo "RTSP output: $RTSP_URL"
# Drop "-c:a aac -b:a 128k -ar 44100" for "-an" if the source has no audio track.
exec ffmpeg \
-hide_banner \
-loglevel info \
-re \
-stream_loop -1 \
-i "$VIDEO_PATH" \
-vf "scale='min(1280,iw)':-2" \
-c:v libx264 \
-preset veryfast \
-tune zerolatency \
-b:v 2500k \
-maxrate 2500k \
-bufsize 5000k \
-c:a aac \
-b:a 128k \
-ar 44100 \
-f rtsp \
-rtsp_transport udp \
"$RTSP_URL"
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
set -euo pipefail
# Push every .mp4 in a directory as its own RTSP stream (one ffmpeg per file),
# named rtsp://<base>/<filename-without-extension>.
# bash ./rtsp_all.sh [video-dir] [rtsp-base-url]
VIDEO_DIR="${1:-./videos}"
RTSP_BASE="${2:-rtsp://127.0.0.1:8554}"
if [[ ! -d "$VIDEO_DIR" ]]; then
echo "Video directory not found: $VIDEO_DIR" >&2
exit 1
fi
shopt -s nullglob
VIDEO_FILES=("$VIDEO_DIR"/*.mp4)
shopt -u nullglob
if [[ ${#VIDEO_FILES[@]} -eq 0 ]]; then
echo "No video files found in: $VIDEO_DIR" >&2
exit 1
fi
echo "Found ${#VIDEO_FILES[@]} video(s) in $VIDEO_DIR"
PIDS=()
cleanup() {
echo "Stopping all streams..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
wait 2>/dev/null
}
trap cleanup EXIT INT TERM
for video in "${VIDEO_FILES[@]}"; do
stream_name="$(basename "${video%.*}")"
rtsp_url="${RTSP_BASE}/${stream_name}"
echo "Starting stream: $rtsp_url <- $(basename "$video")"
ffmpeg \
-hide_banner \
-loglevel error \
-re \
-stream_loop -1 \
-i "$video" \
-vf "format=yuv420p,scale='min(1280,iw)':'min(720,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2" \
-c:v libx264 \
-preset veryfast \
-tune zerolatency \
-b:v 4000k \
-maxrate 4500k \
-bufsize 9000k \
-c:a aac \
-b:a 128k \
-ar 44100 \
-f rtsp \
-rtsp_transport tcp \
"$rtsp_url" &
PIDS+=($!)
done
echo "All ${#VIDEO_FILES[@]} streams started. Press Ctrl+C to stop."
wait
@@ -0,0 +1,108 @@
#!/bin/bash
# One-shot JoyVL demo: model -> interaction orchestrator -> JD webui.
#
# The model is the only fixed component; every external module is pluggable and
# configured here via env (unset = disabled), matching JoyVL's design:
#
# TTS_URL voice out (point at our tts_bridge, or any OpenAI-Realtime-style TTS WS)
# ASR_URL voice in (point at our asr_bridge, or any compatible ASR WS)
# DELEGATION_BACKEND_URL orchestrator background brain for </delegation> (OpenAI-compatible; DELEGATION_MODEL/DELEGATION_API_KEY pick model+key)
# DELEGATION_KIND=chat (default) text/VL brain answers; =image a text-to-image model generates a picture
# chat+Claude: DELEGATION_BACKEND_URL=https://api.anthropic.com/v1/ DELEGATION_MODEL=claude-... DELEGATION_API_KEY=sk-ant-...
# image+Qwen-Image: DELEGATION_KIND=image DELEGATION_BACKEND_URL=http://127.0.0.1:8091/v1
# BACKGROUND_CODEX_API_URL webui-side background agent
# ENABLE_MEMORY (default 1) 3-tier memory; SUMMARIZER_BACKEND_URL/MODEL pick the summarizer (default: reuse main)
#
# Usage: bash start_all.sh
# TTS_URL=ws://host/v1/tts ASR_URL=ws://host/v1/asr GPU=0 bash start_all.sh
set -o pipefail
CODE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
VENV="${VENV:-$CODE/.venv}"
WEBUI_DIR="${WEBUI_DIR:-$CODE/joyvl-interaction/joyvl-interaction-webui}"
MODEL="${MODEL:-jdopensource/JoyAI-VL-Interaction-Preview}"
SERVED_NAME="${SERVED_NAME:-JoyAI-VL-Interaction-Preview}"
GPU="${GPU:-0}"
MODEL_PORT="${MODEL_PORT:-8061}"
ORCH_PORT="${ORCH_PORT:-8070}"
WEBUI_PORT="${WEBUI_PORT:-8999}"
WEBUI_USERNAME="${WEBUI_USERNAME:-admin}"
WEBUI_PASSWORD="${WEBUI_PASSWORD:-changeme}" # override via env for any exposed deployment
ENABLE_MEMORY="${ENABLE_MEMORY:-1}"
RESPONSE_DEDUP="${RESPONSE_DEDUP:-0.85}" # demo drops near-duplicate narration; set 1.0 for reference (exact-only)
TTS_URL="${TTS_URL:-}"
ASR_URL="${ASR_URL:-}"
BACKGROUND_CODEX_API_URL="${BACKGROUND_CODEX_API_URL:-}"
LOG=/tmp/joyvl_demo
mkdir -p "$LOG"
[ -d "$WEBUI_DIR" ] || { echo "! WEBUI_DIR not found: $WEBUI_DIR (set WEBUI_DIR=...)"; exit 1; }
kill_port() {
local p
p=$(ss -tlnp 2>/dev/null | grep ":$1 " | grep -oE 'pid=[0-9]+' | head -1 | cut -d= -f2)
[ -n "${p:-}" ] && kill "$p" 2>/dev/null
return 0
}
wait_http() {
for _ in $(seq 1 150); do
curl -sk -m2 "$1" -o /dev/null 2>/dev/null && return 0
sleep 2
done
echo " ! timeout waiting for $1"
return 1
}
echo "Resetting ports ${MODEL_PORT} ${ORCH_PORT} ${WEBUI_PORT}…"
for p in "$MODEL_PORT" "$ORCH_PORT" "$WEBUI_PORT"; do kill_port "$p"; done
sleep 3
[ -f "$VENV/bin/activate" ] && source "$VENV/bin/activate"
# detach into a new session so servers outlive this launcher
spawn() { local log="$1"; shift; setsid "$@" > "$log" 2>&1 < /dev/null & }
echo "[1/3] model on GPU ${GPU}…"
spawn "$LOG/model.log" env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 CUDA_VISIBLE_DEVICES="$GPU" \
vllm serve "$MODEL" --served-model-name "$SERVED_NAME" --port "$MODEL_PORT" \
--gpu-memory-utilization 0.85 --max-model-len 131072 --enable-prefix-caching \
--limit-mm-per-prompt '{"image":256,"video":1}'
wait_http "http://127.0.0.1:${MODEL_PORT}/health" && echo " up :${MODEL_PORT}"
echo "[2/3] orchestrator (memory=${ENABLE_MEMORY})…"
ORCH_ARGS=(--port "$ORCH_PORT" --main-backend-url "http://127.0.0.1:${MODEL_PORT}/v1"
--main-model "$SERVED_NAME"
--response-dedup-threshold "$RESPONSE_DEDUP")
# delegation: point at a stronger background brain to enable it; otherwise keep it off
if [ -n "${DELEGATION_BACKEND_URL:-}" ]; then
ORCH_ARGS+=(--delegation-backend-url "$DELEGATION_BACKEND_URL")
ORCH_ARGS+=(--delegation-kind "${DELEGATION_KIND:-chat}")
[ -n "${DELEGATION_MODEL:-}" ] && ORCH_ARGS+=(--delegation-model "$DELEGATION_MODEL")
[ -n "${DELEGATION_API_KEY:-}" ] && ORCH_ARGS+=(--delegation-api-key "$DELEGATION_API_KEY")
else
ORCH_ARGS+=(--no-delegation)
fi
[ "$ENABLE_MEMORY" = "0" ] && ORCH_ARGS+=(--no-memory)
[ -n "${SUMMARIZER_BACKEND_URL:-}" ] && ORCH_ARGS+=(--summarizer-backend-url "$SUMMARIZER_BACKEND_URL")
[ -n "${SUMMARIZER_MODEL:-}" ] && ORCH_ARGS+=(--summarizer-model "$SUMMARIZER_MODEL")
spawn "$LOG/orch.log" env PYTHONPATH="$CODE" python -m vllm_omni.experimental.fullduplex.joyvl.serving.server "${ORCH_ARGS[@]}"
wait_http "http://127.0.0.1:${ORCH_PORT}/health" && echo " up :${ORCH_PORT}"
echo "[3/3] JD webui…"
spawn "$LOG/webui.log" env WEBUI_USERNAME="$WEBUI_USERNAME" WEBUI_PASSWORD="$WEBUI_PASSWORD" \
TTS_URL="$TTS_URL" ASR_URL="$ASR_URL" BACKGROUND_CODEX_API_URL="$BACKGROUND_CODEX_API_URL" \
bash -c "cd '$WEBUI_DIR' && exec bash scripts/start_server.sh --api-base 'http://127.0.0.1:${ORCH_PORT}/v1'"
webui_up=0
for _ in $(seq 1 60); do grep -q "Server startup complete" "$LOG/webui.log" 2>/dev/null && { webui_up=1; break; }; sleep 1; done
[ "$webui_up" = 1 ] || { echo "! webui did not start — see $LOG/webui.log"; exit 1; }
IP=$(hostname -I 2>/dev/null | awk '{print $1}')
echo
echo "=== JoyVL demo ready ==="
echo " webui : https://${IP}:${WEBUI_PORT} (user ${WEBUI_USERNAME} / pass ${WEBUI_PASSWORD})"
echo " memory : $([ "$ENABLE_MEMORY" = 0 ] && echo off || echo on)"
echo " TTS : ${TTS_URL:-<unset>}"
echo " ASR : ${ASR_URL:-<unset>}"
echo " agent : ${BACKGROUND_CODEX_API_URL:-<unset>}"
echo " logs : ${LOG}/{model,orch,webui}.log"
@@ -0,0 +1,24 @@
#!/bin/bash
# Serve Qwen3-ASR (the same audio-LLM AURA uses as its ASR stage) and run the
# webui<->ASR bridge. Then start the webui with:
# ASR_URL=ws://127.0.0.1:8093/v1/asr bash scripts/start_server.sh
set -euo pipefail
ASR_MODEL="${ASR_MODEL:-Qwen/Qwen3-ASR-1.7B}"
ASR_PORT="${ASR_PORT:-8094}"
BRIDGE_PORT="${BRIDGE_PORT:-8093}"
GPU="${GPU:-2}"
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
# Qwen3-ASR is audio-in / text-out; served as a normal multimodal chat model.
CUDA_VISIBLE_DEVICES="${GPU}" vllm serve "${ASR_MODEL}" \
--host 0.0.0.0 --port "${ASR_PORT}" \
--gpu-memory-utilization 0.3 --trust-remote-code &
ASR_PID=$!
trap 'kill ${ASR_PID} 2>/dev/null || true' EXIT
until curl -s "http://127.0.0.1:${ASR_PORT}/health" >/dev/null 2>&1; do sleep 2; done
python "${SCRIPT_DIR}/../bridges/asr_bridge.py" \
--port "${BRIDGE_PORT}" \
--backend-url "http://127.0.0.1:${ASR_PORT}" \
--model "${ASR_MODEL}"
@@ -0,0 +1,23 @@
#!/bin/bash
# Launch the interaction orchestrator in front of a running model server.
set -euo pipefail
PORT="${PORT:-8070}"
MAIN_BACKEND_URL="${MAIN_BACKEND_URL:-http://127.0.0.1:8061/v1}"
MAIN_MODEL="${MAIN_MODEL:-JoyAI-VL-Interaction-Preview}"
PERSONA="${PERSONA:-default}"
# By default memory reuses the main model as its own summarizer. Set
# SUMMARIZER_BACKEND_URL/SUMMARIZER_MODEL to a dedicated Qwen3-VL-4B server,
# or pass --no-memory for the lightest setup.
EXTRA=()
[[ -n "${SUMMARIZER_BACKEND_URL:-}" ]] && EXTRA+=(--summarizer-backend-url "${SUMMARIZER_BACKEND_URL}")
[[ -n "${SUMMARIZER_MODEL:-}" ]] && EXTRA+=(--summarizer-model "${SUMMARIZER_MODEL}")
[[ "${NO_MEMORY:-0}" == "1" ]] && EXTRA+=(--no-memory)
python -m vllm_omni.experimental.fullduplex.joyvl.serving.server \
--port "${PORT}" \
--main-backend-url "${MAIN_BACKEND_URL}" \
--main-model "${MAIN_MODEL}" \
--persona "${PERSONA}" \
"${EXTRA[@]}"
@@ -0,0 +1,22 @@
#!/bin/bash
# Serve the JoyVL interaction model as a plain VLM (NOT --omni: it is a
# standard Qwen3-VL autoregressive model, not an omni/diffusion model).
set -euo pipefail
MODEL="${MODEL:-jdopensource/JoyAI-VL-Interaction-Preview}"
SERVED_NAME="${SERVED_NAME:-JoyAI-VL-Interaction-Preview}"
PORT="${PORT:-8061}"
GPU="${GPU:-0}"
# Covers the default short-term window (chunk_frames=100) + key_frames=0 (all frames
# summarized). Lower MAX_MODEL_LEN / IMAGE_LIMIT (and --chunk-frames) together for ~24GB.
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
IMAGE_LIMIT="${IMAGE_LIMIT:-256}"
CUDA_VISIBLE_DEVICES="${GPU}" vllm serve "${MODEL}" \
--served-model-name "${SERVED_NAME}" \
--port "${PORT}" \
--max-model-len "${MAX_MODEL_LEN}" \
--gpu-memory-utilization 0.85 \
--enable-prefix-caching \
--enable-chunked-prefill \
--limit-mm-per-prompt "{\"image\":${IMAGE_LIMIT},\"video\":1}"
@@ -0,0 +1,25 @@
#!/bin/bash
# Serve Qwen3-TTS (vLLM-Omni) and run the webui<->TTS bridge.
# Then start the webui with: TTS_URL=ws://127.0.0.1:8092/v1/tts bash scripts/start_server.sh
set -euo pipefail
TTS_MODEL="${TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}"
TTS_PORT="${TTS_PORT:-8091}"
BRIDGE_PORT="${BRIDGE_PORT:-8092}"
GPU="${GPU:-1}"
VOICE="${VOICE:-vivian}"
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
CUDA_VISIBLE_DEVICES="${GPU}" vllm-omni serve "${TTS_MODEL}" \
--deploy-config vllm_omni/deploy/qwen3_tts.yaml \
--host 0.0.0.0 --port "${TTS_PORT}" \
--gpu-memory-utilization 0.9 --trust-remote-code --omni &
TTS_PID=$!
trap 'kill ${TTS_PID} 2>/dev/null || true' EXIT
# wait for the TTS server, then run the bridge in the foreground
until curl -s "http://127.0.0.1:${TTS_PORT}/health" >/dev/null 2>&1; do sleep 2; done
python "${SCRIPT_DIR}/../bridges/tts_bridge.py" \
--port "${BRIDGE_PORT}" \
--backend-url "ws://127.0.0.1:${TTS_PORT}/v1/audio/speech/stream" \
--voice "${VOICE}"