[Feature] JoyAI-VL-Interaction streaming interaction serving layer (#4575)
Signed-off-by: lishunyang12 <125541396+lishunyang12@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: lishunyang <lishunyang12@163.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
45 files changed
+3699
-1
No files matched your search
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import struct
|
||||
import wave
|
||||
|
||||
import aiohttp
|
||||
from aiohttp import web
|
||||
|
||||
_HEADER = struct.Struct(">iii")
|
||||
_SAMPLE_RATE = 16000
|
||||
_DEFAULT_PROMPT = "Transcribe the audio."
|
||||
_ASR_TEXT_TAG = "<asr_text>"
|
||||
|
||||
|
||||
def _clean_transcript(raw: str) -> str:
|
||||
"""Qwen3-ASR emits ``language <Lang><asr_text><transcript>``; keep the transcript."""
|
||||
text = raw or ""
|
||||
marker = text.rfind(_ASR_TEXT_TAG)
|
||||
if marker != -1:
|
||||
text = text[marker + len(_ASR_TEXT_TAG) :]
|
||||
return text.replace("</asr_text>", "").strip()
|
||||
|
||||
|
||||
def _pcm_to_wav_b64(pcm: bytes, sample_rate: int) -> str:
|
||||
buf = io.BytesIO()
|
||||
with wave.open(buf, "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(sample_rate)
|
||||
wav.writeframes(pcm)
|
||||
return base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def _result(text: str, *, final: bool) -> dict:
|
||||
return {
|
||||
"asr_response": {
|
||||
"event_type": "IS_FINAL" if final else "IS_PARTIAL",
|
||||
"recognition_result": {"hypothesis": [{"text": text, "confidence": None}]},
|
||||
},
|
||||
"mid": "",
|
||||
"code": 0,
|
||||
"msg": "ok",
|
||||
}
|
||||
|
||||
|
||||
async def _transcribe(cfg: dict, pcm: bytes) -> str:
|
||||
if not pcm:
|
||||
return ""
|
||||
wav_b64 = _pcm_to_wav_b64(pcm, cfg["sample_rate"])
|
||||
body = {
|
||||
"model": cfg["model"],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_audio", "input_audio": {"data": wav_b64, "format": "wav"}},
|
||||
{"type": "text", "text": cfg["prompt"]},
|
||||
],
|
||||
}
|
||||
],
|
||||
"temperature": 0.0,
|
||||
"max_tokens": cfg["max_tokens"],
|
||||
}
|
||||
url = cfg["backend_url"].rstrip("/") + "/v1/chat/completions"
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(url, json=body) as resp:
|
||||
payload = await resp.json()
|
||||
choices = payload.get("choices") or []
|
||||
if not choices:
|
||||
return ""
|
||||
return _clean_transcript(choices[0].get("message", {}).get("content") or "")
|
||||
|
||||
|
||||
async def _handle(request: web.Request) -> web.WebSocketResponse:
|
||||
ws = web.WebSocketResponse(max_msg_size=0)
|
||||
await ws.prepare(request)
|
||||
cfg = request.app["cfg"]
|
||||
pcm = bytearray()
|
||||
try:
|
||||
async for msg in ws:
|
||||
if msg.type == aiohttp.WSMsgType.BINARY and len(msg.data) >= _HEADER.size:
|
||||
seqid, _, _ = _HEADER.unpack(msg.data[: _HEADER.size])
|
||||
pcm.extend(msg.data[_HEADER.size :])
|
||||
if seqid < 0:
|
||||
text = await _transcribe(cfg, bytes(pcm))
|
||||
await ws.send_str(json.dumps(_result(text, final=True), ensure_ascii=False))
|
||||
pcm.clear()
|
||||
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR):
|
||||
break
|
||||
finally:
|
||||
if not ws.closed:
|
||||
await ws.close()
|
||||
return ws
|
||||
|
||||
|
||||
def create_app(backend_url: str, model: str, sample_rate: int, prompt: str, max_tokens: int) -> web.Application:
|
||||
app = web.Application()
|
||||
app["cfg"] = {
|
||||
"backend_url": backend_url,
|
||||
"model": model,
|
||||
"sample_rate": sample_rate,
|
||||
"prompt": prompt,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
app.router.add_get("/v1/asr", _handle)
|
||||
return app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="JoyVL webui <-> Qwen3-ASR (chat input_audio) bridge")
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=8093)
|
||||
parser.add_argument("--backend-url", default="http://127.0.0.1:8094")
|
||||
parser.add_argument("--model", default="Qwen/Qwen3-ASR-1.7B")
|
||||
parser.add_argument("--sample-rate", type=int, default=_SAMPLE_RATE)
|
||||
parser.add_argument("--prompt", default=_DEFAULT_PROMPT)
|
||||
parser.add_argument("--max-tokens", type=int, default=256)
|
||||
args = parser.parse_args()
|
||||
app = create_app(args.backend_url, args.model, args.sample_rate, args.prompt, args.max_tokens)
|
||||
web.run_app(app, host=args.host, port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,117 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
from aiohttp import web
|
||||
|
||||
try:
|
||||
import cn2an # optional: Arabic-digit -> Chinese number normalization
|
||||
except ImportError:
|
||||
cn2an = None
|
||||
|
||||
logger = logging.getLogger("tts_bridge")
|
||||
|
||||
|
||||
def _normalize(text: str) -> str:
|
||||
"""Spell out Arabic digits in Chinese so short counting outputs synthesize
|
||||
intelligibly: bare "1次" garbles on Qwen3-TTS (and others), while "一次"
|
||||
is clean. ``cn2an.transform`` rewrites digits in place ("第3次"->"第三次")
|
||||
and is pure-Python (~microseconds). No-op if cn2an is unavailable."""
|
||||
if not cn2an or not text:
|
||||
return text
|
||||
try:
|
||||
return cn2an.transform(text, "an2cn")
|
||||
except Exception as err: # never let TN break synthesis
|
||||
logger.debug("cn2an normalize skipped: %s", err)
|
||||
return text
|
||||
|
||||
|
||||
async def _pump_backend_to_front(back: aiohttp.ClientWebSocketResponse, front: web.WebSocketResponse) -> None:
|
||||
async for msg in back:
|
||||
if msg.type == aiohttp.WSMsgType.BINARY:
|
||||
await front.send_bytes(msg.data)
|
||||
elif msg.type == aiohttp.WSMsgType.TEXT:
|
||||
event = json.loads(msg.data).get("type")
|
||||
if event == "session.done":
|
||||
await front.send_str(json.dumps({"type": "response.done"}))
|
||||
return
|
||||
if event == "error":
|
||||
await front.send_str(json.dumps({"type": "error", "error": "tts backend error"}))
|
||||
return
|
||||
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR):
|
||||
return
|
||||
|
||||
|
||||
async def _handle(request: web.Request) -> web.WebSocketResponse:
|
||||
front = web.WebSocketResponse(heartbeat=20, max_msg_size=0)
|
||||
await front.prepare(request)
|
||||
cfg = request.app["cfg"]
|
||||
|
||||
session = aiohttp.ClientSession()
|
||||
back: aiohttp.ClientWebSocketResponse | None = None
|
||||
pump: asyncio.Task | None = None
|
||||
parts: list[str] = []
|
||||
try:
|
||||
async for msg in front:
|
||||
if msg.type != web.WSMsgType.TEXT:
|
||||
continue
|
||||
data = json.loads(msg.data)
|
||||
|
||||
if "config" in data and back is None:
|
||||
voice = (data["config"] or {}).get("voice") or cfg["voice"]
|
||||
back = await session.ws_connect(cfg["backend_url"], max_msg_size=0)
|
||||
await back.send_str(
|
||||
json.dumps({"type": "session.config", "response_format": cfg["response_format"], "voice": voice})
|
||||
)
|
||||
pump = asyncio.create_task(_pump_backend_to_front(back, front))
|
||||
elif data.get("type") == "input_text.append" and back is not None:
|
||||
parts.append(data.get("text", ""))
|
||||
elif data.get("type") == "input_text.commit" and back is not None:
|
||||
# Normalize the full utterance once (digits may span chunks), then send.
|
||||
await back.send_str(json.dumps({"type": "input.text", "text": _normalize("".join(parts))}))
|
||||
parts = []
|
||||
await back.send_str(json.dumps({"type": "input.done"}))
|
||||
if pump is not None:
|
||||
await pump
|
||||
break
|
||||
except Exception as err:
|
||||
logger.warning("tts bridge error: %s", err)
|
||||
finally:
|
||||
if pump is not None and not pump.done():
|
||||
pump.cancel()
|
||||
if back is not None and not back.closed:
|
||||
await back.close()
|
||||
await session.close()
|
||||
if not front.closed:
|
||||
await front.close()
|
||||
return front
|
||||
|
||||
|
||||
def create_app(backend_url: str, voice: str, response_format: str) -> web.Application:
|
||||
app = web.Application()
|
||||
app["cfg"] = {"backend_url": backend_url, "voice": voice, "response_format": response_format}
|
||||
app.router.add_get("/v1/tts", _handle)
|
||||
return app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="JoyVL webui <-> Qwen3-TTS websocket bridge")
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=8092)
|
||||
parser.add_argument("--backend-url", default="ws://127.0.0.1:8091/v1/audio/speech/stream")
|
||||
parser.add_argument("--voice", default="vivian")
|
||||
parser.add_argument("--response-format", default="pcm")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
web.run_app(create_app(args.backend_url, args.voice, args.response_format), host=args.host, port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
from stream_client import stream
|
||||
|
||||
ICON = {"response": "SPEAK ", "silence": "silence ", "delegate": "DELEGATE"}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("video")
|
||||
parser.add_argument("--server", default="http://127.0.0.1:8070")
|
||||
parser.add_argument("--query", default=None)
|
||||
parser.add_argument("--fps", type=float, default=1.0)
|
||||
parser.add_argument("--session-id", default="cli")
|
||||
args = parser.parse_args()
|
||||
|
||||
for tick in stream(args.video, args.server, args.session_id, args.query, args.fps):
|
||||
if tick.action == "silence":
|
||||
continue
|
||||
line = f"[{tick.t:6.1f}s] {ICON.get(tick.action)} {tick.text}"
|
||||
if tick.delegation:
|
||||
line += f" (-> {tick.delegation.get('question', '')})"
|
||||
print(line, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
import cv2
|
||||
import requests
|
||||
|
||||
|
||||
@dataclass
|
||||
class Tick:
|
||||
index: int
|
||||
t: float
|
||||
action: str
|
||||
text: str
|
||||
skipped: bool
|
||||
latency_ms: float
|
||||
delegation: dict | None
|
||||
|
||||
|
||||
def iter_frames(video_path: str, fps: float) -> Iterator[tuple[float, bytes]]:
|
||||
cap = cv2.VideoCapture(video_path)
|
||||
src_fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
||||
step = max(1, round(src_fps / fps))
|
||||
idx = 0
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
break
|
||||
if idx % step == 0:
|
||||
ok, buf = cv2.imencode(".jpg", frame)
|
||||
if ok:
|
||||
yield idx / src_fps, buf.tobytes()
|
||||
idx += 1
|
||||
cap.release()
|
||||
|
||||
|
||||
def stream(
|
||||
video_path: str,
|
||||
server: str,
|
||||
session_id: str,
|
||||
query: str | None = None,
|
||||
fps: float = 1.0,
|
||||
realtime: bool = False,
|
||||
) -> Iterator[Tick]:
|
||||
"""Send each sampled frame as one chat completion; yield the parsed action.
|
||||
|
||||
``query`` is attached to the first tick to arm the session; inject later
|
||||
queries by posting another frame turn with a new text part.
|
||||
"""
|
||||
url = server.rstrip("/") + "/v1/chat/completions"
|
||||
requests.post(server.rstrip("/") + "/reset", json={"session_id": session_id}, timeout=10)
|
||||
for i, (t, jpeg) in enumerate(iter_frames(video_path, fps)):
|
||||
data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
|
||||
content = [{"type": "image_url", "image_url": {"url": data_url}}]
|
||||
if i == 0 and query:
|
||||
content.insert(0, {"type": "text", "text": query})
|
||||
resp = requests.post(
|
||||
url,
|
||||
json={"session_id": session_id, "messages": [{"role": "user", "content": content}]},
|
||||
timeout=120,
|
||||
).json()
|
||||
info = resp.get("interaction", {})
|
||||
yield Tick(
|
||||
index=info.get("frame_index", i + 1),
|
||||
t=t,
|
||||
action=info.get("action", "?"),
|
||||
text=resp["choices"][0]["message"]["content"],
|
||||
skipped=info.get("inference_skipped", False),
|
||||
latency_ms=info.get("latency_ms", 0.0),
|
||||
delegation=info.get("delegation"),
|
||||
)
|
||||
if realtime:
|
||||
time.sleep(1.0 / fps)
|
||||
@@ -0,0 +1,46 @@
|
||||
# RTSP local streaming
|
||||
|
||||
Simulate an RTSP camera from a local video file so the WebUI (or any RTSP client) can
|
||||
test the RTSP input path without a physical IP camera. RTSP is a **webui-side input** —
|
||||
the browser pulls the stream and feeds frames to the orchestrator over the normal API;
|
||||
no serving-layer code is involved.
|
||||
|
||||
Needs [`ffmpeg`](https://ffmpeg.org/) and [MediaMTX](https://github.com/bluenviron/mediamtx/releases)
|
||||
(download the binary + `mediamtx.yml` and keep them next to these scripts, or point
|
||||
`MEDIAMTX_BIN` / `MEDIAMTX_CONFIG` at them).
|
||||
|
||||
## 1. Start the RTSP server
|
||||
|
||||
```bash
|
||||
bash ./mediamtx.sh # MediaMTX, listens on :8554
|
||||
```
|
||||
|
||||
## 2. Push a local video as an RTSP stream
|
||||
|
||||
```bash
|
||||
bash ./rtsp.sh ./videos/example.mp4 rtsp://127.0.0.1:8554/fire1
|
||||
```
|
||||
|
||||
`rtsp.sh` takes `[video-path] [rtsp-url]` (or set `VIDEO_PATH` / `RTSP_URL`). To stream a
|
||||
whole directory of clips, one stream per file:
|
||||
|
||||
```bash
|
||||
bash ./rtsp_all.sh ./videos rtsp://127.0.0.1:8554
|
||||
```
|
||||
|
||||
## 3. Use it in the WebUI
|
||||
|
||||
Enter the URL in the WebUI RTSP box:
|
||||
|
||||
```
|
||||
rtsp://127.0.0.1:8554/fire1
|
||||
```
|
||||
|
||||
If the WebUI runs on another machine, replace `127.0.0.1` with the MediaMTX host's IP.
|
||||
|
||||
## Notes
|
||||
|
||||
- `rtsp.sh` transcodes audio to AAC; replace `-c:a aac -b:a 128k -ar 44100` with `-an`
|
||||
if the source clip has no audio track and your `ffmpeg` errors on it.
|
||||
- If a client cannot connect, check that port `8554` is reachable and not blocked by a
|
||||
firewall, container network, or security group.
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Start a local MediaMTX RTSP server (listens on :8554 by default).
|
||||
# Download the binary + mediamtx.yml from https://github.com/bluenviron/mediamtx/releases
|
||||
# and either keep them next to this script or point the vars below at them.
|
||||
|
||||
MEDIAMTX_BIN="${MEDIAMTX_BIN:-./mediamtx}"
|
||||
MEDIAMTX_CONFIG="${MEDIAMTX_CONFIG:-./mediamtx.yml}"
|
||||
|
||||
exec "$MEDIAMTX_BIN" "$MEDIAMTX_CONFIG"
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Loop a local video file and push it to an RTSP URL, simulating an RTSP camera.
|
||||
# bash ./rtsp.sh [video-path] [rtsp-output-url]
|
||||
# Defaults stream ./videos/example.mp4 to rtsp://127.0.0.1:8554/fire1
|
||||
|
||||
VIDEO_PATH="${1:-${VIDEO_PATH:-./videos/example.mp4}}"
|
||||
RTSP_URL="${2:-${RTSP_URL:-rtsp://127.0.0.1:8554/fire1}}"
|
||||
|
||||
if [[ ! -f "$VIDEO_PATH" ]]; then
|
||||
echo "Video file not found: $VIDEO_PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Input video: $VIDEO_PATH"
|
||||
echo "RTSP output: $RTSP_URL"
|
||||
|
||||
# Drop "-c:a aac -b:a 128k -ar 44100" for "-an" if the source has no audio track.
|
||||
exec ffmpeg \
|
||||
-hide_banner \
|
||||
-loglevel info \
|
||||
-re \
|
||||
-stream_loop -1 \
|
||||
-i "$VIDEO_PATH" \
|
||||
-vf "scale='min(1280,iw)':-2" \
|
||||
-c:v libx264 \
|
||||
-preset veryfast \
|
||||
-tune zerolatency \
|
||||
-b:v 2500k \
|
||||
-maxrate 2500k \
|
||||
-bufsize 5000k \
|
||||
-c:a aac \
|
||||
-b:a 128k \
|
||||
-ar 44100 \
|
||||
-f rtsp \
|
||||
-rtsp_transport udp \
|
||||
"$RTSP_URL"
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Push every .mp4 in a directory as its own RTSP stream (one ffmpeg per file),
|
||||
# named rtsp://<base>/<filename-without-extension>.
|
||||
# bash ./rtsp_all.sh [video-dir] [rtsp-base-url]
|
||||
|
||||
VIDEO_DIR="${1:-./videos}"
|
||||
RTSP_BASE="${2:-rtsp://127.0.0.1:8554}"
|
||||
|
||||
if [[ ! -d "$VIDEO_DIR" ]]; then
|
||||
echo "Video directory not found: $VIDEO_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
shopt -s nullglob
|
||||
VIDEO_FILES=("$VIDEO_DIR"/*.mp4)
|
||||
shopt -u nullglob
|
||||
|
||||
if [[ ${#VIDEO_FILES[@]} -eq 0 ]]; then
|
||||
echo "No video files found in: $VIDEO_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found ${#VIDEO_FILES[@]} video(s) in $VIDEO_DIR"
|
||||
|
||||
PIDS=()
|
||||
cleanup() {
|
||||
echo "Stopping all streams..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
wait 2>/dev/null
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
for video in "${VIDEO_FILES[@]}"; do
|
||||
stream_name="$(basename "${video%.*}")"
|
||||
rtsp_url="${RTSP_BASE}/${stream_name}"
|
||||
echo "Starting stream: $rtsp_url <- $(basename "$video")"
|
||||
ffmpeg \
|
||||
-hide_banner \
|
||||
-loglevel error \
|
||||
-re \
|
||||
-stream_loop -1 \
|
||||
-i "$video" \
|
||||
-vf "format=yuv420p,scale='min(1280,iw)':'min(720,ih)':force_original_aspect_ratio=decrease:force_divisible_by=2" \
|
||||
-c:v libx264 \
|
||||
-preset veryfast \
|
||||
-tune zerolatency \
|
||||
-b:v 4000k \
|
||||
-maxrate 4500k \
|
||||
-bufsize 9000k \
|
||||
-c:a aac \
|
||||
-b:a 128k \
|
||||
-ar 44100 \
|
||||
-f rtsp \
|
||||
-rtsp_transport tcp \
|
||||
"$rtsp_url" &
|
||||
PIDS+=($!)
|
||||
done
|
||||
|
||||
echo "All ${#VIDEO_FILES[@]} streams started. Press Ctrl+C to stop."
|
||||
wait
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/bin/bash
|
||||
# One-shot JoyVL demo: model -> interaction orchestrator -> JD webui.
|
||||
#
|
||||
# The model is the only fixed component; every external module is pluggable and
|
||||
# configured here via env (unset = disabled), matching JoyVL's design:
|
||||
#
|
||||
# TTS_URL voice out (point at our tts_bridge, or any OpenAI-Realtime-style TTS WS)
|
||||
# ASR_URL voice in (point at our asr_bridge, or any compatible ASR WS)
|
||||
# DELEGATION_BACKEND_URL orchestrator background brain for </delegation> (OpenAI-compatible; DELEGATION_MODEL/DELEGATION_API_KEY pick model+key)
|
||||
# DELEGATION_KIND=chat (default) text/VL brain answers; =image a text-to-image model generates a picture
|
||||
# chat+Claude: DELEGATION_BACKEND_URL=https://api.anthropic.com/v1/ DELEGATION_MODEL=claude-... DELEGATION_API_KEY=sk-ant-...
|
||||
# image+Qwen-Image: DELEGATION_KIND=image DELEGATION_BACKEND_URL=http://127.0.0.1:8091/v1
|
||||
# BACKGROUND_CODEX_API_URL webui-side background agent
|
||||
# ENABLE_MEMORY (default 1) 3-tier memory; SUMMARIZER_BACKEND_URL/MODEL pick the summarizer (default: reuse main)
|
||||
#
|
||||
# Usage: bash start_all.sh
|
||||
# TTS_URL=ws://host/v1/tts ASR_URL=ws://host/v1/asr GPU=0 bash start_all.sh
|
||||
set -o pipefail
|
||||
|
||||
CODE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
VENV="${VENV:-$CODE/.venv}"
|
||||
WEBUI_DIR="${WEBUI_DIR:-$CODE/joyvl-interaction/joyvl-interaction-webui}"
|
||||
MODEL="${MODEL:-jdopensource/JoyAI-VL-Interaction-Preview}"
|
||||
SERVED_NAME="${SERVED_NAME:-JoyAI-VL-Interaction-Preview}"
|
||||
GPU="${GPU:-0}"
|
||||
MODEL_PORT="${MODEL_PORT:-8061}"
|
||||
ORCH_PORT="${ORCH_PORT:-8070}"
|
||||
WEBUI_PORT="${WEBUI_PORT:-8999}"
|
||||
WEBUI_USERNAME="${WEBUI_USERNAME:-admin}"
|
||||
WEBUI_PASSWORD="${WEBUI_PASSWORD:-changeme}" # override via env for any exposed deployment
|
||||
ENABLE_MEMORY="${ENABLE_MEMORY:-1}"
|
||||
RESPONSE_DEDUP="${RESPONSE_DEDUP:-0.85}" # demo drops near-duplicate narration; set 1.0 for reference (exact-only)
|
||||
TTS_URL="${TTS_URL:-}"
|
||||
ASR_URL="${ASR_URL:-}"
|
||||
BACKGROUND_CODEX_API_URL="${BACKGROUND_CODEX_API_URL:-}"
|
||||
|
||||
LOG=/tmp/joyvl_demo
|
||||
mkdir -p "$LOG"
|
||||
|
||||
[ -d "$WEBUI_DIR" ] || { echo "! WEBUI_DIR not found: $WEBUI_DIR (set WEBUI_DIR=...)"; exit 1; }
|
||||
|
||||
kill_port() {
|
||||
local p
|
||||
p=$(ss -tlnp 2>/dev/null | grep ":$1 " | grep -oE 'pid=[0-9]+' | head -1 | cut -d= -f2)
|
||||
[ -n "${p:-}" ] && kill "$p" 2>/dev/null
|
||||
return 0
|
||||
}
|
||||
wait_http() {
|
||||
for _ in $(seq 1 150); do
|
||||
curl -sk -m2 "$1" -o /dev/null 2>/dev/null && return 0
|
||||
sleep 2
|
||||
done
|
||||
echo " ! timeout waiting for $1"
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "Resetting ports ${MODEL_PORT} ${ORCH_PORT} ${WEBUI_PORT}…"
|
||||
for p in "$MODEL_PORT" "$ORCH_PORT" "$WEBUI_PORT"; do kill_port "$p"; done
|
||||
sleep 3
|
||||
|
||||
[ -f "$VENV/bin/activate" ] && source "$VENV/bin/activate"
|
||||
|
||||
# detach into a new session so servers outlive this launcher
|
||||
spawn() { local log="$1"; shift; setsid "$@" > "$log" 2>&1 < /dev/null & }
|
||||
|
||||
echo "[1/3] model on GPU ${GPU}…"
|
||||
spawn "$LOG/model.log" env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 CUDA_VISIBLE_DEVICES="$GPU" \
|
||||
vllm serve "$MODEL" --served-model-name "$SERVED_NAME" --port "$MODEL_PORT" \
|
||||
--gpu-memory-utilization 0.85 --max-model-len 131072 --enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":256,"video":1}'
|
||||
wait_http "http://127.0.0.1:${MODEL_PORT}/health" && echo " up :${MODEL_PORT}"
|
||||
|
||||
echo "[2/3] orchestrator (memory=${ENABLE_MEMORY})…"
|
||||
ORCH_ARGS=(--port "$ORCH_PORT" --main-backend-url "http://127.0.0.1:${MODEL_PORT}/v1"
|
||||
--main-model "$SERVED_NAME"
|
||||
--response-dedup-threshold "$RESPONSE_DEDUP")
|
||||
# delegation: point at a stronger background brain to enable it; otherwise keep it off
|
||||
if [ -n "${DELEGATION_BACKEND_URL:-}" ]; then
|
||||
ORCH_ARGS+=(--delegation-backend-url "$DELEGATION_BACKEND_URL")
|
||||
ORCH_ARGS+=(--delegation-kind "${DELEGATION_KIND:-chat}")
|
||||
[ -n "${DELEGATION_MODEL:-}" ] && ORCH_ARGS+=(--delegation-model "$DELEGATION_MODEL")
|
||||
[ -n "${DELEGATION_API_KEY:-}" ] && ORCH_ARGS+=(--delegation-api-key "$DELEGATION_API_KEY")
|
||||
else
|
||||
ORCH_ARGS+=(--no-delegation)
|
||||
fi
|
||||
[ "$ENABLE_MEMORY" = "0" ] && ORCH_ARGS+=(--no-memory)
|
||||
[ -n "${SUMMARIZER_BACKEND_URL:-}" ] && ORCH_ARGS+=(--summarizer-backend-url "$SUMMARIZER_BACKEND_URL")
|
||||
[ -n "${SUMMARIZER_MODEL:-}" ] && ORCH_ARGS+=(--summarizer-model "$SUMMARIZER_MODEL")
|
||||
spawn "$LOG/orch.log" env PYTHONPATH="$CODE" python -m vllm_omni.experimental.fullduplex.joyvl.serving.server "${ORCH_ARGS[@]}"
|
||||
wait_http "http://127.0.0.1:${ORCH_PORT}/health" && echo " up :${ORCH_PORT}"
|
||||
|
||||
echo "[3/3] JD webui…"
|
||||
spawn "$LOG/webui.log" env WEBUI_USERNAME="$WEBUI_USERNAME" WEBUI_PASSWORD="$WEBUI_PASSWORD" \
|
||||
TTS_URL="$TTS_URL" ASR_URL="$ASR_URL" BACKGROUND_CODEX_API_URL="$BACKGROUND_CODEX_API_URL" \
|
||||
bash -c "cd '$WEBUI_DIR' && exec bash scripts/start_server.sh --api-base 'http://127.0.0.1:${ORCH_PORT}/v1'"
|
||||
webui_up=0
|
||||
for _ in $(seq 1 60); do grep -q "Server startup complete" "$LOG/webui.log" 2>/dev/null && { webui_up=1; break; }; sleep 1; done
|
||||
[ "$webui_up" = 1 ] || { echo "! webui did not start — see $LOG/webui.log"; exit 1; }
|
||||
|
||||
IP=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||
echo
|
||||
echo "=== JoyVL demo ready ==="
|
||||
echo " webui : https://${IP}:${WEBUI_PORT} (user ${WEBUI_USERNAME} / pass ${WEBUI_PASSWORD})"
|
||||
echo " memory : $([ "$ENABLE_MEMORY" = 0 ] && echo off || echo on)"
|
||||
echo " TTS : ${TTS_URL:-<unset>}"
|
||||
echo " ASR : ${ASR_URL:-<unset>}"
|
||||
echo " agent : ${BACKGROUND_CODEX_API_URL:-<unset>}"
|
||||
echo " logs : ${LOG}/{model,orch,webui}.log"
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
# Serve Qwen3-ASR (the same audio-LLM AURA uses as its ASR stage) and run the
|
||||
# webui<->ASR bridge. Then start the webui with:
|
||||
# ASR_URL=ws://127.0.0.1:8093/v1/asr bash scripts/start_server.sh
|
||||
set -euo pipefail
|
||||
|
||||
ASR_MODEL="${ASR_MODEL:-Qwen/Qwen3-ASR-1.7B}"
|
||||
ASR_PORT="${ASR_PORT:-8094}"
|
||||
BRIDGE_PORT="${BRIDGE_PORT:-8093}"
|
||||
GPU="${GPU:-2}"
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Qwen3-ASR is audio-in / text-out; served as a normal multimodal chat model.
|
||||
CUDA_VISIBLE_DEVICES="${GPU}" vllm serve "${ASR_MODEL}" \
|
||||
--host 0.0.0.0 --port "${ASR_PORT}" \
|
||||
--gpu-memory-utilization 0.3 --trust-remote-code &
|
||||
ASR_PID=$!
|
||||
trap 'kill ${ASR_PID} 2>/dev/null || true' EXIT
|
||||
|
||||
until curl -s "http://127.0.0.1:${ASR_PORT}/health" >/dev/null 2>&1; do sleep 2; done
|
||||
python "${SCRIPT_DIR}/../bridges/asr_bridge.py" \
|
||||
--port "${BRIDGE_PORT}" \
|
||||
--backend-url "http://127.0.0.1:${ASR_PORT}" \
|
||||
--model "${ASR_MODEL}"
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
# Launch the interaction orchestrator in front of a running model server.
|
||||
set -euo pipefail
|
||||
|
||||
PORT="${PORT:-8070}"
|
||||
MAIN_BACKEND_URL="${MAIN_BACKEND_URL:-http://127.0.0.1:8061/v1}"
|
||||
MAIN_MODEL="${MAIN_MODEL:-JoyAI-VL-Interaction-Preview}"
|
||||
PERSONA="${PERSONA:-default}"
|
||||
|
||||
# By default memory reuses the main model as its own summarizer. Set
|
||||
# SUMMARIZER_BACKEND_URL/SUMMARIZER_MODEL to a dedicated Qwen3-VL-4B server,
|
||||
# or pass --no-memory for the lightest setup.
|
||||
EXTRA=()
|
||||
[[ -n "${SUMMARIZER_BACKEND_URL:-}" ]] && EXTRA+=(--summarizer-backend-url "${SUMMARIZER_BACKEND_URL}")
|
||||
[[ -n "${SUMMARIZER_MODEL:-}" ]] && EXTRA+=(--summarizer-model "${SUMMARIZER_MODEL}")
|
||||
[[ "${NO_MEMORY:-0}" == "1" ]] && EXTRA+=(--no-memory)
|
||||
|
||||
python -m vllm_omni.experimental.fullduplex.joyvl.serving.server \
|
||||
--port "${PORT}" \
|
||||
--main-backend-url "${MAIN_BACKEND_URL}" \
|
||||
--main-model "${MAIN_MODEL}" \
|
||||
--persona "${PERSONA}" \
|
||||
"${EXTRA[@]}"
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
# Serve the JoyVL interaction model as a plain VLM (NOT --omni: it is a
|
||||
# standard Qwen3-VL autoregressive model, not an omni/diffusion model).
|
||||
set -euo pipefail
|
||||
|
||||
MODEL="${MODEL:-jdopensource/JoyAI-VL-Interaction-Preview}"
|
||||
SERVED_NAME="${SERVED_NAME:-JoyAI-VL-Interaction-Preview}"
|
||||
PORT="${PORT:-8061}"
|
||||
GPU="${GPU:-0}"
|
||||
# Covers the default short-term window (chunk_frames=100) + key_frames=0 (all frames
|
||||
# summarized). Lower MAX_MODEL_LEN / IMAGE_LIMIT (and --chunk-frames) together for ~24GB.
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
|
||||
IMAGE_LIMIT="${IMAGE_LIMIT:-256}"
|
||||
|
||||
CUDA_VISIBLE_DEVICES="${GPU}" vllm serve "${MODEL}" \
|
||||
--served-model-name "${SERVED_NAME}" \
|
||||
--port "${PORT}" \
|
||||
--max-model-len "${MAX_MODEL_LEN}" \
|
||||
--gpu-memory-utilization 0.85 \
|
||||
--enable-prefix-caching \
|
||||
--enable-chunked-prefill \
|
||||
--limit-mm-per-prompt "{\"image\":${IMAGE_LIMIT},\"video\":1}"
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Serve Qwen3-TTS (vLLM-Omni) and run the webui<->TTS bridge.
|
||||
# Then start the webui with: TTS_URL=ws://127.0.0.1:8092/v1/tts bash scripts/start_server.sh
|
||||
set -euo pipefail
|
||||
|
||||
TTS_MODEL="${TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}"
|
||||
TTS_PORT="${TTS_PORT:-8091}"
|
||||
BRIDGE_PORT="${BRIDGE_PORT:-8092}"
|
||||
GPU="${GPU:-1}"
|
||||
VOICE="${VOICE:-vivian}"
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
CUDA_VISIBLE_DEVICES="${GPU}" vllm-omni serve "${TTS_MODEL}" \
|
||||
--deploy-config vllm_omni/deploy/qwen3_tts.yaml \
|
||||
--host 0.0.0.0 --port "${TTS_PORT}" \
|
||||
--gpu-memory-utilization 0.9 --trust-remote-code --omni &
|
||||
TTS_PID=$!
|
||||
trap 'kill ${TTS_PID} 2>/dev/null || true' EXIT
|
||||
|
||||
# wait for the TTS server, then run the bridge in the foreground
|
||||
until curl -s "http://127.0.0.1:${TTS_PORT}/health" >/dev/null 2>&1; do sleep 2; done
|
||||
python "${SCRIPT_DIR}/../bridges/tts_bridge.py" \
|
||||
--port "${BRIDGE_PORT}" \
|
||||
--backend-url "ws://127.0.0.1:${TTS_PORT}/v1/audio/speech/stream" \
|
||||
--voice "${VOICE}"
|
||||
Reference in new issue
Block a user