[Feature] JoyAI-VL-Interaction streaming interaction serving layer (#4575)
Signed-off-by: lishunyang12 <125541396+lishunyang12@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: lishunyang <lishunyang12@163.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
45 files changed
+3699
-1
No files matched your search
@@ -0,0 +1,108 @@
|
||||
#!/bin/bash
|
||||
# One-shot JoyVL demo: model -> interaction orchestrator -> JD webui.
|
||||
#
|
||||
# The model is the only fixed component; every external module is pluggable and
|
||||
# configured here via env (unset = disabled), matching JoyVL's design:
|
||||
#
|
||||
# TTS_URL voice out (point at our tts_bridge, or any OpenAI-Realtime-style TTS WS)
|
||||
# ASR_URL voice in (point at our asr_bridge, or any compatible ASR WS)
|
||||
# DELEGATION_BACKEND_URL orchestrator background brain for </delegation> (OpenAI-compatible; DELEGATION_MODEL/DELEGATION_API_KEY pick model+key)
|
||||
# DELEGATION_KIND=chat (default) text/VL brain answers; =image a text-to-image model generates a picture
|
||||
# chat+Claude: DELEGATION_BACKEND_URL=https://api.anthropic.com/v1/ DELEGATION_MODEL=claude-... DELEGATION_API_KEY=sk-ant-...
|
||||
# image+Qwen-Image: DELEGATION_KIND=image DELEGATION_BACKEND_URL=http://127.0.0.1:8091/v1
|
||||
# BACKGROUND_CODEX_API_URL webui-side background agent
|
||||
# ENABLE_MEMORY (default 1) 3-tier memory; SUMMARIZER_BACKEND_URL/MODEL pick the summarizer (default: reuse main)
|
||||
#
|
||||
# Usage: bash start_all.sh
|
||||
# TTS_URL=ws://host/v1/tts ASR_URL=ws://host/v1/asr GPU=0 bash start_all.sh
|
||||
set -o pipefail
|
||||
|
||||
CODE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
VENV="${VENV:-$CODE/.venv}"
|
||||
WEBUI_DIR="${WEBUI_DIR:-$CODE/joyvl-interaction/joyvl-interaction-webui}"
|
||||
MODEL="${MODEL:-jdopensource/JoyAI-VL-Interaction-Preview}"
|
||||
SERVED_NAME="${SERVED_NAME:-JoyAI-VL-Interaction-Preview}"
|
||||
GPU="${GPU:-0}"
|
||||
MODEL_PORT="${MODEL_PORT:-8061}"
|
||||
ORCH_PORT="${ORCH_PORT:-8070}"
|
||||
WEBUI_PORT="${WEBUI_PORT:-8999}"
|
||||
WEBUI_USERNAME="${WEBUI_USERNAME:-admin}"
|
||||
WEBUI_PASSWORD="${WEBUI_PASSWORD:-changeme}" # override via env for any exposed deployment
|
||||
ENABLE_MEMORY="${ENABLE_MEMORY:-1}"
|
||||
RESPONSE_DEDUP="${RESPONSE_DEDUP:-0.85}" # demo drops near-duplicate narration; set 1.0 for reference (exact-only)
|
||||
TTS_URL="${TTS_URL:-}"
|
||||
ASR_URL="${ASR_URL:-}"
|
||||
BACKGROUND_CODEX_API_URL="${BACKGROUND_CODEX_API_URL:-}"
|
||||
|
||||
LOG=/tmp/joyvl_demo
|
||||
mkdir -p "$LOG"
|
||||
|
||||
[ -d "$WEBUI_DIR" ] || { echo "! WEBUI_DIR not found: $WEBUI_DIR (set WEBUI_DIR=...)"; exit 1; }
|
||||
|
||||
kill_port() {
|
||||
local p
|
||||
p=$(ss -tlnp 2>/dev/null | grep ":$1 " | grep -oE 'pid=[0-9]+' | head -1 | cut -d= -f2)
|
||||
[ -n "${p:-}" ] && kill "$p" 2>/dev/null
|
||||
return 0
|
||||
}
|
||||
wait_http() {
|
||||
for _ in $(seq 1 150); do
|
||||
curl -sk -m2 "$1" -o /dev/null 2>/dev/null && return 0
|
||||
sleep 2
|
||||
done
|
||||
echo " ! timeout waiting for $1"
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "Resetting ports ${MODEL_PORT} ${ORCH_PORT} ${WEBUI_PORT}…"
|
||||
for p in "$MODEL_PORT" "$ORCH_PORT" "$WEBUI_PORT"; do kill_port "$p"; done
|
||||
sleep 3
|
||||
|
||||
[ -f "$VENV/bin/activate" ] && source "$VENV/bin/activate"
|
||||
|
||||
# detach into a new session so servers outlive this launcher
|
||||
spawn() { local log="$1"; shift; setsid "$@" > "$log" 2>&1 < /dev/null & }
|
||||
|
||||
echo "[1/3] model on GPU ${GPU}…"
|
||||
spawn "$LOG/model.log" env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 CUDA_VISIBLE_DEVICES="$GPU" \
|
||||
vllm serve "$MODEL" --served-model-name "$SERVED_NAME" --port "$MODEL_PORT" \
|
||||
--gpu-memory-utilization 0.85 --max-model-len 131072 --enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":256,"video":1}'
|
||||
wait_http "http://127.0.0.1:${MODEL_PORT}/health" && echo " up :${MODEL_PORT}"
|
||||
|
||||
echo "[2/3] orchestrator (memory=${ENABLE_MEMORY})…"
|
||||
ORCH_ARGS=(--port "$ORCH_PORT" --main-backend-url "http://127.0.0.1:${MODEL_PORT}/v1"
|
||||
--main-model "$SERVED_NAME"
|
||||
--response-dedup-threshold "$RESPONSE_DEDUP")
|
||||
# delegation: point at a stronger background brain to enable it; otherwise keep it off
|
||||
if [ -n "${DELEGATION_BACKEND_URL:-}" ]; then
|
||||
ORCH_ARGS+=(--delegation-backend-url "$DELEGATION_BACKEND_URL")
|
||||
ORCH_ARGS+=(--delegation-kind "${DELEGATION_KIND:-chat}")
|
||||
[ -n "${DELEGATION_MODEL:-}" ] && ORCH_ARGS+=(--delegation-model "$DELEGATION_MODEL")
|
||||
[ -n "${DELEGATION_API_KEY:-}" ] && ORCH_ARGS+=(--delegation-api-key "$DELEGATION_API_KEY")
|
||||
else
|
||||
ORCH_ARGS+=(--no-delegation)
|
||||
fi
|
||||
[ "$ENABLE_MEMORY" = "0" ] && ORCH_ARGS+=(--no-memory)
|
||||
[ -n "${SUMMARIZER_BACKEND_URL:-}" ] && ORCH_ARGS+=(--summarizer-backend-url "$SUMMARIZER_BACKEND_URL")
|
||||
[ -n "${SUMMARIZER_MODEL:-}" ] && ORCH_ARGS+=(--summarizer-model "$SUMMARIZER_MODEL")
|
||||
spawn "$LOG/orch.log" env PYTHONPATH="$CODE" python -m vllm_omni.experimental.fullduplex.joyvl.serving.server "${ORCH_ARGS[@]}"
|
||||
wait_http "http://127.0.0.1:${ORCH_PORT}/health" && echo " up :${ORCH_PORT}"
|
||||
|
||||
echo "[3/3] JD webui…"
|
||||
spawn "$LOG/webui.log" env WEBUI_USERNAME="$WEBUI_USERNAME" WEBUI_PASSWORD="$WEBUI_PASSWORD" \
|
||||
TTS_URL="$TTS_URL" ASR_URL="$ASR_URL" BACKGROUND_CODEX_API_URL="$BACKGROUND_CODEX_API_URL" \
|
||||
bash -c "cd '$WEBUI_DIR' && exec bash scripts/start_server.sh --api-base 'http://127.0.0.1:${ORCH_PORT}/v1'"
|
||||
webui_up=0
|
||||
for _ in $(seq 1 60); do grep -q "Server startup complete" "$LOG/webui.log" 2>/dev/null && { webui_up=1; break; }; sleep 1; done
|
||||
[ "$webui_up" = 1 ] || { echo "! webui did not start — see $LOG/webui.log"; exit 1; }
|
||||
|
||||
IP=$(hostname -I 2>/dev/null | awk '{print $1}')
|
||||
echo
|
||||
echo "=== JoyVL demo ready ==="
|
||||
echo " webui : https://${IP}:${WEBUI_PORT} (user ${WEBUI_USERNAME} / pass ${WEBUI_PASSWORD})"
|
||||
echo " memory : $([ "$ENABLE_MEMORY" = 0 ] && echo off || echo on)"
|
||||
echo " TTS : ${TTS_URL:-<unset>}"
|
||||
echo " ASR : ${ASR_URL:-<unset>}"
|
||||
echo " agent : ${BACKGROUND_CODEX_API_URL:-<unset>}"
|
||||
echo " logs : ${LOG}/{model,orch,webui}.log"
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
# Serve Qwen3-ASR (the same audio-LLM AURA uses as its ASR stage) and run the
|
||||
# webui<->ASR bridge. Then start the webui with:
|
||||
# ASR_URL=ws://127.0.0.1:8093/v1/asr bash scripts/start_server.sh
|
||||
set -euo pipefail
|
||||
|
||||
ASR_MODEL="${ASR_MODEL:-Qwen/Qwen3-ASR-1.7B}"
|
||||
ASR_PORT="${ASR_PORT:-8094}"
|
||||
BRIDGE_PORT="${BRIDGE_PORT:-8093}"
|
||||
GPU="${GPU:-2}"
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Qwen3-ASR is audio-in / text-out; served as a normal multimodal chat model.
|
||||
CUDA_VISIBLE_DEVICES="${GPU}" vllm serve "${ASR_MODEL}" \
|
||||
--host 0.0.0.0 --port "${ASR_PORT}" \
|
||||
--gpu-memory-utilization 0.3 --trust-remote-code &
|
||||
ASR_PID=$!
|
||||
trap 'kill ${ASR_PID} 2>/dev/null || true' EXIT
|
||||
|
||||
until curl -s "http://127.0.0.1:${ASR_PORT}/health" >/dev/null 2>&1; do sleep 2; done
|
||||
python "${SCRIPT_DIR}/../bridges/asr_bridge.py" \
|
||||
--port "${BRIDGE_PORT}" \
|
||||
--backend-url "http://127.0.0.1:${ASR_PORT}" \
|
||||
--model "${ASR_MODEL}"
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
# Launch the interaction orchestrator in front of a running model server.
|
||||
set -euo pipefail
|
||||
|
||||
PORT="${PORT:-8070}"
|
||||
MAIN_BACKEND_URL="${MAIN_BACKEND_URL:-http://127.0.0.1:8061/v1}"
|
||||
MAIN_MODEL="${MAIN_MODEL:-JoyAI-VL-Interaction-Preview}"
|
||||
PERSONA="${PERSONA:-default}"
|
||||
|
||||
# By default memory reuses the main model as its own summarizer. Set
|
||||
# SUMMARIZER_BACKEND_URL/SUMMARIZER_MODEL to a dedicated Qwen3-VL-4B server,
|
||||
# or pass --no-memory for the lightest setup.
|
||||
EXTRA=()
|
||||
[[ -n "${SUMMARIZER_BACKEND_URL:-}" ]] && EXTRA+=(--summarizer-backend-url "${SUMMARIZER_BACKEND_URL}")
|
||||
[[ -n "${SUMMARIZER_MODEL:-}" ]] && EXTRA+=(--summarizer-model "${SUMMARIZER_MODEL}")
|
||||
[[ "${NO_MEMORY:-0}" == "1" ]] && EXTRA+=(--no-memory)
|
||||
|
||||
python -m vllm_omni.experimental.fullduplex.joyvl.serving.server \
|
||||
--port "${PORT}" \
|
||||
--main-backend-url "${MAIN_BACKEND_URL}" \
|
||||
--main-model "${MAIN_MODEL}" \
|
||||
--persona "${PERSONA}" \
|
||||
"${EXTRA[@]}"
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
# Serve the JoyVL interaction model as a plain VLM (NOT --omni: it is a
|
||||
# standard Qwen3-VL autoregressive model, not an omni/diffusion model).
|
||||
set -euo pipefail
|
||||
|
||||
MODEL="${MODEL:-jdopensource/JoyAI-VL-Interaction-Preview}"
|
||||
SERVED_NAME="${SERVED_NAME:-JoyAI-VL-Interaction-Preview}"
|
||||
PORT="${PORT:-8061}"
|
||||
GPU="${GPU:-0}"
|
||||
# Covers the default short-term window (chunk_frames=100) + key_frames=0 (all frames
|
||||
# summarized). Lower MAX_MODEL_LEN / IMAGE_LIMIT (and --chunk-frames) together for ~24GB.
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
|
||||
IMAGE_LIMIT="${IMAGE_LIMIT:-256}"
|
||||
|
||||
CUDA_VISIBLE_DEVICES="${GPU}" vllm serve "${MODEL}" \
|
||||
--served-model-name "${SERVED_NAME}" \
|
||||
--port "${PORT}" \
|
||||
--max-model-len "${MAX_MODEL_LEN}" \
|
||||
--gpu-memory-utilization 0.85 \
|
||||
--enable-prefix-caching \
|
||||
--enable-chunked-prefill \
|
||||
--limit-mm-per-prompt "{\"image\":${IMAGE_LIMIT},\"video\":1}"
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Serve Qwen3-TTS (vLLM-Omni) and run the webui<->TTS bridge.
|
||||
# Then start the webui with: TTS_URL=ws://127.0.0.1:8092/v1/tts bash scripts/start_server.sh
|
||||
set -euo pipefail
|
||||
|
||||
TTS_MODEL="${TTS_MODEL:-Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice}"
|
||||
TTS_PORT="${TTS_PORT:-8091}"
|
||||
BRIDGE_PORT="${BRIDGE_PORT:-8092}"
|
||||
GPU="${GPU:-1}"
|
||||
VOICE="${VOICE:-vivian}"
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
CUDA_VISIBLE_DEVICES="${GPU}" vllm-omni serve "${TTS_MODEL}" \
|
||||
--deploy-config vllm_omni/deploy/qwen3_tts.yaml \
|
||||
--host 0.0.0.0 --port "${TTS_PORT}" \
|
||||
--gpu-memory-utilization 0.9 --trust-remote-code --omni &
|
||||
TTS_PID=$!
|
||||
trap 'kill ${TTS_PID} 2>/dev/null || true' EXIT
|
||||
|
||||
# wait for the TTS server, then run the bridge in the foreground
|
||||
until curl -s "http://127.0.0.1:${TTS_PORT}/health" >/dev/null 2>&1; do sleep 2; done
|
||||
python "${SCRIPT_DIR}/../bridges/tts_bridge.py" \
|
||||
--port "${BRIDGE_PORT}" \
|
||||
--backend-url "ws://127.0.0.1:${TTS_PORT}/v1/audio/speech/stream" \
|
||||
--voice "${VOICE}"
|
||||
Reference in new issue
Block a user